fix(daemon): give a task container a reachable State Store endpoint

This commit is contained in:
sam committed 2026-10-20 03:12:21 +11:00
1 parent 89cbbfc871
commit 92bc0f692f
4 files changed
+381 -6

No files matched your search

+58 -6
View File
@@ -1758,7 +1758,19 @@ func (d *Daemon) acquireTaskContainer(
revokeRun := revokeCredential
revokeCredential = func() { closeModel(); revokeRun() }
ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(d.containerEnv(task, credential), model.Env...))
// The container lives in its own network namespace, so a loopback State
// Store target names the container itself. Give this task an address on
// the host bridge that forwards to it, closed with the task.
stateStoreURL, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential)
if err != nil {
revokeCredential()
d.teardownStorage(ctx, task, repo, workDir)
park("state store relay failed", err)
return nil, "", nil, false
}
revokeCredential = release
ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(d.containerEnv(task, credential, stateStoreURL), model.Env...))
if err != nil {
revokeCredential()
// Roll back the storage we just set up: the mounts were created
@@ -2190,7 +2202,47 @@ func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte)
}
}
func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken string) []string {
// taskStateStoreEndpoint is the State Store address one task container is
// handed, and the release that ends the relay carrying it. The container lives
// in its own network namespace, so a loopback target names the container
// itself; behind a relay, the address is the host bridge's.
func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, func(), error) {
relay, err := d.openTaskStateStoreRelay(ctx)
if err != nil {
return "", release, err
}
if relay == nil {
return d.ConnectedStateStore.URL, release, nil
}
return relay.Addr(), func() { relay.Close(); release() }, nil
}
// openTaskStateStoreRelay starts the relay that carries one task container's
// State Store traffic to the daemon's target. The target stays bound where the
// operator put it; the container is handed an address on the host bridge, the
// same way it is handed NATS's. Nil when the target already reaches the
// containers, or when no bridge address is known.
func (d *Daemon) openTaskStateStoreRelay(ctx context.Context) (*stateStoreRelay, error) {
addr, loopback := loopbackStateStoreTarget(d.ConnectedStateStore.URL)
if !loopback {
return nil, nil
}
gateway := ""
if d.ContainerPool != nil {
gateway = d.ContainerPool.HostGateway()
}
if gateway == "" {
d.Log.Warn("state store target is loopback and the container pool has no host bridge address; task containers cannot reach it",
"target", d.ConnectedStateStore.URL)
return nil, nil
}
return openStateStoreRelay(ctx, gateway, addr)
}
// containerEnv is the environment a task container starts with. stateStoreURL
// is the address the container can reach the State Store by, which is the
// relay's rather than the daemon's own target.
func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken, stateStoreURL string) []string {
var env []string
// The endpoint the daemon's own client connected with at startup, not
// the live config: a reloaded [nats] section must not point new
@@ -2199,10 +2251,10 @@ func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken string) []str
if token := d.ConnectedNATS.ContainerToken(); token != "" {
env = append(env, "NATS_TOKEN="+token)
}
// Pass the State Store target with this task's scoped token, never the
// daemon's own.
if d.ConnectedStateStore.URL != "" {
env = append(env, "STATE_STORE_URL="+d.ConnectedStateStore.URL)
// Pass the container-reachable endpoint with this task's scoped token,
// never the daemon's own.
if stateStoreURL != "" {
env = append(env, "STATE_STORE_URL="+stateStoreURL)
if len(d.ConnectedStateStore.CA) > 0 {
env = append(env, "STATE_STORE_CA="+string(d.ConnectedStateStore.CA))
}
+119
View File
@@ -0,0 +1,119 @@
package daemon
import (
"context"
"fmt"
"io"
"net"
"strings"
"sync"
"time"
)
// stateStoreDialTimeout bounds one forwarded connection's dial. The target is
// local, so a dial that has not connected by then is a dead State Store rather
// than a slow one.
const stateStoreDialTimeout = 10 * time.Second
// stateStoreRelay carries one task's State Store traffic from an address on
// the host's container bridge to the target the daemon dialed. A container has
// its own network namespace, so a loopback target there names the container
// itself; the relay address is what the container is handed instead. The State
// Store keeps listening exactly where the operator bound it.
type stateStoreRelay struct {
target string
ln net.Listener
}
// openStateStoreRelay listens on gateway and forwards to addr. The listener is
// per task: it accepts only from the bridge the container joins, and closing
// it is what ends the task's access.
func openStateStoreRelay(ctx context.Context, gateway, addr string) (*stateStoreRelay, error) {
ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", net.JoinHostPort(gateway, "0"))
if err != nil {
return nil, fmt.Errorf("state store relay: listen on %s: %w", gateway, err)
}
r := &stateStoreRelay{target: addr, ln: ln}
go r.serve(ctx)
return r, nil
}
// Addr is the address the task container is handed.
func (r *stateStoreRelay) Addr() string { return r.ln.Addr().String() }
// Close stops accepting. Streams already open end when the container that owns
// them goes away, and a restarted daemon leaves no listener behind: the
// sockets belong to its process.
func (r *stateStoreRelay) Close() { _ = r.ln.Close() }
func (r *stateStoreRelay) serve(ctx context.Context) {
for {
downstream, err := r.ln.Accept()
if err != nil {
return
}
go r.forward(ctx, downstream)
}
}
// forward joins one container stream to the target, half-closing each side so
// a gRPC stream drains in both directions before the pair ends.
func (r *stateStoreRelay) forward(ctx context.Context, downstream net.Conn) {
ctx, cancel := context.WithTimeout(ctx, stateStoreDialTimeout)
defer cancel()
upstream, err := (&net.Dialer{}).DialContext(ctx, "tcp", r.target)
if err != nil {
_ = downstream.Close()
return
}
var wg sync.WaitGroup
wg.Add(2)
go func() { defer wg.Done(); copyAndClose(upstream, downstream) }()
go func() { defer wg.Done(); copyAndClose(downstream, upstream) }()
wg.Wait()
_ = downstream.Close()
_ = upstream.Close()
}
func copyAndClose(dst, src net.Conn) {
_, _ = io.Copy(dst, src)
if half, ok := dst.(interface{ CloseWrite() error }); ok {
_ = half.CloseWrite()
return
}
_ = dst.Close()
}
// loopbackStateStoreTarget splits target into the address to forward to, and
// reports whether it is this host's loopback: the one case a task container
// cannot reach on its own. A target already addressed for the containers'
// network needs no relay.
func loopbackStateStoreTarget(target string) (addr string, loopback bool) {
addr, host, ok := stateStoreTargetAddr(target)
if !ok || !loopbackHost(host) {
return "", false
}
return addr, true
}
// stateStoreTargetAddr is target as host:port, accepting the gRPC scheme form
// ("dns:///host:port") the configuration allows.
func stateStoreTargetAddr(target string) (addr, host string, ok bool) {
target = strings.TrimSpace(target)
if _, rest, found := strings.Cut(target, ":///"); found {
target = rest
}
host, _, err := net.SplitHostPort(target)
if err != nil {
return "", "", false
}
return target, host, true
}
func loopbackHost(host string) bool {
if host == "localhost" {
return true
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
+161
View File
@@ -0,0 +1,161 @@
package daemon
import (
"context"
"io"
"net"
"strings"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/domain/workflow"
)
// The relay is what makes a loopback State Store reachable from a container:
// bytes written to its address arrive at the target.
func TestStateStoreRelayForwards(t *testing.T) {
t.Parallel()
upstream := echoListener(t)
relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String())
if err != nil {
t.Fatalf("open relay: %v", err)
}
t.Cleanup(relay.Close)
if relay.Addr() == upstream.Addr().String() {
t.Fatalf("relay listens on the target's own address %s", relay.Addr())
}
dialer := &net.Dialer{}
conn, err := dialer.DialContext(context.Background(), "tcp", relay.Addr())
if err != nil {
t.Fatalf("dial relay: %v", err)
}
defer func() { _ = conn.Close() }()
if _, err := conn.Write([]byte("ping")); err != nil {
t.Fatalf("write through relay: %v", err)
}
got := make([]byte, 4)
if _, err := io.ReadFull(conn, got); err != nil {
t.Fatalf("read through relay: %v", err)
}
if string(got) != "ping" {
t.Fatalf("relay carried %q, want %q", got, "ping")
}
}
// The relay is per task, so releasing the task has to end the access it gave.
func TestStateStoreRelayClosesWithTask(t *testing.T) {
t.Parallel()
upstream := echoListener(t)
relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String())
if err != nil {
t.Fatalf("open relay: %v", err)
}
addr := relay.Addr()
relay.Close()
if conn, err := (&net.Dialer{Timeout: time.Second}).DialContext(context.Background(), "tcp", addr); err == nil {
_ = conn.Close()
t.Fatalf("relay still accepts at %s after the task released it", addr)
}
}
// An ordinary image task is handed the relay's address, never the loopback
// target the daemon dialed, with the task-scoped credentials unchanged.
func TestContainerEnvStateStore(t *testing.T) {
t.Parallel()
upstream := echoListener(t)
relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String())
if err != nil {
t.Fatalf("open relay: %v", err)
}
t.Cleanup(relay.Close)
d := &Daemon{
ConnectedNATS: NATSEndpoint{URL: "nats://10.0.0.5:4222"},
ConnectedStateStore: StateStoreEndpoint{URL: upstream.Addr().String(), CA: []byte("ca-pem")},
}
env := envValues(d.containerEnv(&workflow.Task{ID: 7}, "task-token", relay.Addr()))
if got := env["STATE_STORE_URL"]; got != relay.Addr() {
t.Fatalf("STATE_STORE_URL = %q, want the relay address %q", got, relay.Addr())
}
if env["STATE_STORE_TOKEN"] != "task-token" {
t.Fatalf("STATE_STORE_TOKEN = %q, want the task's own token", env["STATE_STORE_TOKEN"])
}
if env["STATE_STORE_CA"] != "ca-pem" {
t.Fatalf("STATE_STORE_CA = %q, want the CA passed through", env["STATE_STORE_CA"])
}
if got := env["NATS_URL"]; got != "nats://10.0.0.5:4222" {
t.Fatalf("NATS_URL = %q, want the connected broker", got)
}
}
// Only a target the container cannot reach is relayed. An address already
// configured for the containers' network is passed through untouched, and a
// target the relay cannot forward is left alone rather than guessed at.
func TestLoopbackStateStoreTarget(t *testing.T) {
t.Parallel()
cases := []struct {
name string
target string
addr string
loopback bool
}{
{name: "loopback ipv4", target: "127.0.0.1:9090", addr: "127.0.0.1:9090", loopback: true},
{name: "loopback ipv6", target: "[::1]:9090", addr: "[::1]:9090", loopback: true},
{name: "localhost", target: "localhost:9090", addr: "localhost:9090", loopback: true},
{name: "grpc scheme form", target: "dns:///127.0.0.1:9090", addr: "127.0.0.1:9090", loopback: true},
{name: "host address", target: "10.0.0.5:9090"},
{name: "service name", target: "state-store:9090"},
{name: "unset", target: ""},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
addr, loopback := loopbackStateStoreTarget(tc.target)
if loopback != tc.loopback {
t.Fatalf("loopbackStateStoreTarget(%q) loopback = %t, want %t", tc.target, loopback, tc.loopback)
}
if addr != tc.addr {
t.Fatalf("loopbackStateStoreTarget(%q) addr = %q, want %q", tc.target, addr, tc.addr)
}
})
}
}
// echoListener accepts connections on loopback and echoes what it reads.
func echoListener(t *testing.T) net.Listener {
t.Helper()
ln, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
t.Cleanup(func() { _ = ln.Close() })
go func() {
for {
conn, err := ln.Accept()
if err != nil {
return
}
go func() {
defer func() { _ = conn.Close() }()
_, _ = io.Copy(conn, conn)
}()
}
}()
return ln
}
func envValues(env []string) map[string]string {
out := make(map[string]string, len(env))
for _, entry := range env {
key, value, _ := strings.Cut(entry, "=")
out[key] = value
}
return out
}
@@ -0,0 +1,43 @@
package kitrun
import (
"net"
"strconv"
"strings"
"testing"
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
)
// A Kit worker reaches the State Store through its relay, never through the
// daemon's own target: ordinary image tasks got a per-task relay instead, and
// this path must not follow them onto it.
func TestWorkerEnvKeepsRelayAddress(t *testing.T) {
t.Parallel()
env, err := WorkerEnv(Endpoints{
NATS: "nats://127.0.0.1:4222",
StateStore: "127.0.0.1:9090",
StateStoreToken: "task-token",
StateStoreCA: []byte("ca-pem"),
}, 0, 0)
if err != nil {
t.Fatalf("WorkerEnv: %v", err)
}
values := make(map[string]string, len(env))
for _, entry := range env {
key, value, _ := strings.Cut(entry, "=")
values[key] = value
}
want := net.JoinHostPort(egress.RelayAlias, strconv.Itoa(egress.RelayStateStorePort))
if got := values["STATE_STORE_URL"]; got != want {
t.Fatalf("STATE_STORE_URL = %q, want the relay address %q", got, want)
}
if values["STATE_STORE_TOKEN"] != "task-token" {
t.Fatalf("STATE_STORE_TOKEN = %q, want the task's own token", values["STATE_STORE_TOKEN"])
}
if values["STATE_STORE_CA"] != "ca-pem" {
t.Fatalf("STATE_STORE_CA = %q, want the CA passed through", values["STATE_STORE_CA"])
}
}