diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index c8f0159c6..ac5445dd5 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -1758,7 +1758,19 @@ func (d *Daemon) acquireTaskContainer( revokeRun := revokeCredential revokeCredential = func() { closeModel(); revokeRun() } - ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(d.containerEnv(task, credential), model.Env...)) + // The container lives in its own network namespace, so a loopback State + // Store target names the container itself. Give this task an address on + // the host bridge that forwards to it, closed with the task. + stateStoreURL, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential) + if err != nil { + revokeCredential() + d.teardownStorage(ctx, task, repo, workDir) + park("state store relay failed", err) + return nil, "", nil, false + } + revokeCredential = release + + ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(d.containerEnv(task, credential, stateStoreURL), model.Env...)) if err != nil { revokeCredential() // Roll back the storage we just set up: the mounts were created @@ -2190,7 +2202,47 @@ func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte) } } -func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken string) []string { +// taskStateStoreEndpoint is the State Store address one task container is +// handed, and the release that ends the relay carrying it. The container lives +// in its own network namespace, so a loopback target names the container +// itself; behind a relay, the address is the host bridge's. +func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, func(), error) { + relay, err := d.openTaskStateStoreRelay(ctx) + if err != nil { + return "", release, err + } + if relay == nil { + return d.ConnectedStateStore.URL, release, nil + } + return relay.Addr(), func() { relay.Close(); release() }, nil +} + +// openTaskStateStoreRelay starts the relay that carries one task container's +// State Store traffic to the daemon's target. The target stays bound where the +// operator put it; the container is handed an address on the host bridge, the +// same way it is handed NATS's. Nil when the target already reaches the +// containers, or when no bridge address is known. +func (d *Daemon) openTaskStateStoreRelay(ctx context.Context) (*stateStoreRelay, error) { + addr, loopback := loopbackStateStoreTarget(d.ConnectedStateStore.URL) + if !loopback { + return nil, nil + } + gateway := "" + if d.ContainerPool != nil { + gateway = d.ContainerPool.HostGateway() + } + if gateway == "" { + d.Log.Warn("state store target is loopback and the container pool has no host bridge address; task containers cannot reach it", + "target", d.ConnectedStateStore.URL) + return nil, nil + } + return openStateStoreRelay(ctx, gateway, addr) +} + +// containerEnv is the environment a task container starts with. stateStoreURL +// is the address the container can reach the State Store by, which is the +// relay's rather than the daemon's own target. +func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken, stateStoreURL string) []string { var env []string // The endpoint the daemon's own client connected with at startup, not // the live config: a reloaded [nats] section must not point new @@ -2199,10 +2251,10 @@ func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken string) []str if token := d.ConnectedNATS.ContainerToken(); token != "" { env = append(env, "NATS_TOKEN="+token) } - // Pass the State Store target with this task's scoped token, never the - // daemon's own. - if d.ConnectedStateStore.URL != "" { - env = append(env, "STATE_STORE_URL="+d.ConnectedStateStore.URL) + // Pass the container-reachable endpoint with this task's scoped token, + // never the daemon's own. + if stateStoreURL != "" { + env = append(env, "STATE_STORE_URL="+stateStoreURL) if len(d.ConnectedStateStore.CA) > 0 { env = append(env, "STATE_STORE_CA="+string(d.ConnectedStateStore.CA)) } diff --git a/internal/daemon/statestore_relay.go b/internal/daemon/statestore_relay.go new file mode 100644 index 000000000..567adeaad --- /dev/null +++ b/internal/daemon/statestore_relay.go @@ -0,0 +1,119 @@ +package daemon + +import ( + "context" + "fmt" + "io" + "net" + "strings" + "sync" + "time" +) + +// stateStoreDialTimeout bounds one forwarded connection's dial. The target is +// local, so a dial that has not connected by then is a dead State Store rather +// than a slow one. +const stateStoreDialTimeout = 10 * time.Second + +// stateStoreRelay carries one task's State Store traffic from an address on +// the host's container bridge to the target the daemon dialed. A container has +// its own network namespace, so a loopback target there names the container +// itself; the relay address is what the container is handed instead. The State +// Store keeps listening exactly where the operator bound it. +type stateStoreRelay struct { + target string + ln net.Listener +} + +// openStateStoreRelay listens on gateway and forwards to addr. The listener is +// per task: it accepts only from the bridge the container joins, and closing +// it is what ends the task's access. +func openStateStoreRelay(ctx context.Context, gateway, addr string) (*stateStoreRelay, error) { + ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", net.JoinHostPort(gateway, "0")) + if err != nil { + return nil, fmt.Errorf("state store relay: listen on %s: %w", gateway, err) + } + r := &stateStoreRelay{target: addr, ln: ln} + go r.serve(ctx) + return r, nil +} + +// Addr is the address the task container is handed. +func (r *stateStoreRelay) Addr() string { return r.ln.Addr().String() } + +// Close stops accepting. Streams already open end when the container that owns +// them goes away, and a restarted daemon leaves no listener behind: the +// sockets belong to its process. +func (r *stateStoreRelay) Close() { _ = r.ln.Close() } + +func (r *stateStoreRelay) serve(ctx context.Context) { + for { + downstream, err := r.ln.Accept() + if err != nil { + return + } + go r.forward(ctx, downstream) + } +} + +// forward joins one container stream to the target, half-closing each side so +// a gRPC stream drains in both directions before the pair ends. +func (r *stateStoreRelay) forward(ctx context.Context, downstream net.Conn) { + ctx, cancel := context.WithTimeout(ctx, stateStoreDialTimeout) + defer cancel() + upstream, err := (&net.Dialer{}).DialContext(ctx, "tcp", r.target) + if err != nil { + _ = downstream.Close() + return + } + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); copyAndClose(upstream, downstream) }() + go func() { defer wg.Done(); copyAndClose(downstream, upstream) }() + wg.Wait() + _ = downstream.Close() + _ = upstream.Close() +} + +func copyAndClose(dst, src net.Conn) { + _, _ = io.Copy(dst, src) + if half, ok := dst.(interface{ CloseWrite() error }); ok { + _ = half.CloseWrite() + return + } + _ = dst.Close() +} + +// loopbackStateStoreTarget splits target into the address to forward to, and +// reports whether it is this host's loopback: the one case a task container +// cannot reach on its own. A target already addressed for the containers' +// network needs no relay. +func loopbackStateStoreTarget(target string) (addr string, loopback bool) { + addr, host, ok := stateStoreTargetAddr(target) + if !ok || !loopbackHost(host) { + return "", false + } + return addr, true +} + +// stateStoreTargetAddr is target as host:port, accepting the gRPC scheme form +// ("dns:///host:port") the configuration allows. +func stateStoreTargetAddr(target string) (addr, host string, ok bool) { + target = strings.TrimSpace(target) + if _, rest, found := strings.Cut(target, ":///"); found { + target = rest + } + host, _, err := net.SplitHostPort(target) + if err != nil { + return "", "", false + } + return target, host, true +} + +func loopbackHost(host string) bool { + if host == "localhost" { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} diff --git a/internal/daemon/statestore_relay_test.go b/internal/daemon/statestore_relay_test.go new file mode 100644 index 000000000..015d1c623 --- /dev/null +++ b/internal/daemon/statestore_relay_test.go @@ -0,0 +1,161 @@ +package daemon + +import ( + "context" + "io" + "net" + "strings" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/domain/workflow" +) + +// The relay is what makes a loopback State Store reachable from a container: +// bytes written to its address arrive at the target. +func TestStateStoreRelayForwards(t *testing.T) { + t.Parallel() + + upstream := echoListener(t) + relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String()) + if err != nil { + t.Fatalf("open relay: %v", err) + } + t.Cleanup(relay.Close) + if relay.Addr() == upstream.Addr().String() { + t.Fatalf("relay listens on the target's own address %s", relay.Addr()) + } + + dialer := &net.Dialer{} + conn, err := dialer.DialContext(context.Background(), "tcp", relay.Addr()) + if err != nil { + t.Fatalf("dial relay: %v", err) + } + defer func() { _ = conn.Close() }() + if _, err := conn.Write([]byte("ping")); err != nil { + t.Fatalf("write through relay: %v", err) + } + got := make([]byte, 4) + if _, err := io.ReadFull(conn, got); err != nil { + t.Fatalf("read through relay: %v", err) + } + if string(got) != "ping" { + t.Fatalf("relay carried %q, want %q", got, "ping") + } +} + +// The relay is per task, so releasing the task has to end the access it gave. +func TestStateStoreRelayClosesWithTask(t *testing.T) { + t.Parallel() + + upstream := echoListener(t) + relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String()) + if err != nil { + t.Fatalf("open relay: %v", err) + } + addr := relay.Addr() + relay.Close() + + if conn, err := (&net.Dialer{Timeout: time.Second}).DialContext(context.Background(), "tcp", addr); err == nil { + _ = conn.Close() + t.Fatalf("relay still accepts at %s after the task released it", addr) + } +} + +// An ordinary image task is handed the relay's address, never the loopback +// target the daemon dialed, with the task-scoped credentials unchanged. +func TestContainerEnvStateStore(t *testing.T) { + t.Parallel() + + upstream := echoListener(t) + relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String()) + if err != nil { + t.Fatalf("open relay: %v", err) + } + t.Cleanup(relay.Close) + + d := &Daemon{ + ConnectedNATS: NATSEndpoint{URL: "nats://10.0.0.5:4222"}, + ConnectedStateStore: StateStoreEndpoint{URL: upstream.Addr().String(), CA: []byte("ca-pem")}, + } + env := envValues(d.containerEnv(&workflow.Task{ID: 7}, "task-token", relay.Addr())) + + if got := env["STATE_STORE_URL"]; got != relay.Addr() { + t.Fatalf("STATE_STORE_URL = %q, want the relay address %q", got, relay.Addr()) + } + if env["STATE_STORE_TOKEN"] != "task-token" { + t.Fatalf("STATE_STORE_TOKEN = %q, want the task's own token", env["STATE_STORE_TOKEN"]) + } + if env["STATE_STORE_CA"] != "ca-pem" { + t.Fatalf("STATE_STORE_CA = %q, want the CA passed through", env["STATE_STORE_CA"]) + } + if got := env["NATS_URL"]; got != "nats://10.0.0.5:4222" { + t.Fatalf("NATS_URL = %q, want the connected broker", got) + } +} + +// Only a target the container cannot reach is relayed. An address already +// configured for the containers' network is passed through untouched, and a +// target the relay cannot forward is left alone rather than guessed at. +func TestLoopbackStateStoreTarget(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + target string + addr string + loopback bool + }{ + {name: "loopback ipv4", target: "127.0.0.1:9090", addr: "127.0.0.1:9090", loopback: true}, + {name: "loopback ipv6", target: "[::1]:9090", addr: "[::1]:9090", loopback: true}, + {name: "localhost", target: "localhost:9090", addr: "localhost:9090", loopback: true}, + {name: "grpc scheme form", target: "dns:///127.0.0.1:9090", addr: "127.0.0.1:9090", loopback: true}, + {name: "host address", target: "10.0.0.5:9090"}, + {name: "service name", target: "state-store:9090"}, + {name: "unset", target: ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + addr, loopback := loopbackStateStoreTarget(tc.target) + if loopback != tc.loopback { + t.Fatalf("loopbackStateStoreTarget(%q) loopback = %t, want %t", tc.target, loopback, tc.loopback) + } + if addr != tc.addr { + t.Fatalf("loopbackStateStoreTarget(%q) addr = %q, want %q", tc.target, addr, tc.addr) + } + }) + } +} + +// echoListener accepts connections on loopback and echoes what it reads. +func echoListener(t *testing.T) net.Listener { + t.Helper() + ln, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + defer func() { _ = conn.Close() }() + _, _ = io.Copy(conn, conn) + }() + } + }() + return ln +} + +func envValues(env []string) map[string]string { + out := make(map[string]string, len(env)) + for _, entry := range env { + key, value, _ := strings.Cut(entry, "=") + out[key] = value + } + return out +} diff --git a/internal/infrastructure/kitrun/env_test.go b/internal/infrastructure/kitrun/env_test.go new file mode 100644 index 000000000..b0c14a046 --- /dev/null +++ b/internal/infrastructure/kitrun/env_test.go @@ -0,0 +1,43 @@ +package kitrun + +import ( + "net" + "strconv" + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/infrastructure/egress" +) + +// A Kit worker reaches the State Store through its relay, never through the +// daemon's own target: ordinary image tasks got a per-task relay instead, and +// this path must not follow them onto it. +func TestWorkerEnvKeepsRelayAddress(t *testing.T) { + t.Parallel() + + env, err := WorkerEnv(Endpoints{ + NATS: "nats://127.0.0.1:4222", + StateStore: "127.0.0.1:9090", + StateStoreToken: "task-token", + StateStoreCA: []byte("ca-pem"), + }, 0, 0) + if err != nil { + t.Fatalf("WorkerEnv: %v", err) + } + values := make(map[string]string, len(env)) + for _, entry := range env { + key, value, _ := strings.Cut(entry, "=") + values[key] = value + } + + want := net.JoinHostPort(egress.RelayAlias, strconv.Itoa(egress.RelayStateStorePort)) + if got := values["STATE_STORE_URL"]; got != want { + t.Fatalf("STATE_STORE_URL = %q, want the relay address %q", got, want) + } + if values["STATE_STORE_TOKEN"] != "task-token" { + t.Fatalf("STATE_STORE_TOKEN = %q, want the task's own token", values["STATE_STORE_TOKEN"]) + } + if values["STATE_STORE_CA"] != "ca-pem" { + t.Fatalf("STATE_STORE_CA = %q, want the CA passed through", values["STATE_STORE_CA"]) + } +}