feat(egress): session-authenticated proxy terminating TLS under the Kit network policy

CONNECT tunnels and absolute-form http requests authenticate to a
per-container session, are checked against the session's phase policy,
and are forwarded with verified upstream TLS. A tunnel serves only its
admitted host: mismatched SNI and Host headers are refused. Upstream dials
refuse loopback, and refuse internal addresses unless the policy names the
exact IP, so an allowed name cannot resolve back into the host.
This commit is contained in:
sam committed 2026-09-26 02:31:00 +10:00
1 parent 6f0540ba16
commit d1e5c628d0
3 files changed
+649

No files matched your search

+13
View File
@@ -91,3 +91,16 @@ func (r rules) allows(host string, port int) bool {
}
return false
}
// namesIP reports whether an allow entry names ip exactly. Only an explicit
// IP entry opens an internal address; a wildcard or everything never does.
func (r rules) namesIP(ip net.IP) bool {
for _, p := range r.allow {
if !p.everything && p.suffix == "" {
if named := net.ParseIP(p.host); named != nil && named.Equal(ip) {
return true
}
}
}
return false
}
+361
View File
@@ -0,0 +1,361 @@
package egress
import (
"context"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"net"
"net/http"
"net/http/httputil"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/docker/sandbox-kit-spec/v3/spec"
)
// proxyUser is the fixed user name in a sandbox's proxy URL; the session
// token is the password.
const proxyUser = "archie"
const handshakeTimeout = 10 * time.Second
// cgnat is shared address space (RFC 6598), which overlay networks use for
// host addresses; it is as internal as RFC 1918 space.
var cgnat = &net.IPNet{IP: net.IPv4(100, 64, 0, 0), Mask: net.CIDRMask(10, 32)}
// Session is one sandbox container's standing with the proxy: its token and
// its Kit's network policy. It starts in the install phase; the runner moves
// it to runtime before the workload's entrypoint starts.
type Session struct {
token string
install rules
runtime rules
atRun atomic.Bool
}
// Token is the secret the container presents as its proxy password.
func (s *Session) Token() string { return s.token }
// EnterRuntime switches the session from install-phase to runtime-phase
// egress. It is one-way: install egress never reopens.
func (s *Session) EnterRuntime() { s.atRun.Store(true) }
func (s *Session) rules() rules {
if s.atRun.Load() {
return s.runtime
}
return s.install
}
// ProxyOptions are the proxy's upstream dependencies. Both are optional:
// the zero value dials the network through the upstream guard and verifies
// upstream TLS against the system roots.
type ProxyOptions struct {
UpstreamRoots *x509.CertPool
Dial func(ctx context.Context, network, addr string) (net.Conn, error)
}
// Proxy is the egress proxy sandbox containers reach through their relay.
// It authenticates each request to a session, applies that session's
// network policy, terminates TLS with the daemon CA, and forwards to the
// real upstream with verified TLS.
type Proxy struct {
ca *CA
dial func(ctx context.Context, network, addr string) (net.Conn, error)
transport *http.Transport
// rewrite runs on every forwarded request after policy has admitted it.
rewrite func(*Session, *http.Request) error
mu sync.RWMutex
sessions map[string]*Session
}
type sessionKey struct{}
func NewProxy(ca *CA, opts ProxyOptions) *Proxy {
p := &Proxy{ca: ca, dial: opts.Dial, sessions: map[string]*Session{}}
p.transport = &http.Transport{
DialContext: p.dialUpstream,
TLSClientConfig: &tls.Config{RootCAs: opts.UpstreamRoots, MinVersion: tls.VersionTLS12},
ForceAttemptHTTP2: true,
TLSHandshakeTimeout: handshakeTimeout,
}
return p
}
// Register opens a session for one container under its Kit's policy. A nil
// policy allows no egress in either phase.
func (p *Proxy) Register(policy *spec.PhasedNetwork) (*Session, error) {
raw := make([]byte, 32)
if _, err := rand.Read(raw); err != nil {
return nil, err
}
s := &Session{token: hex.EncodeToString(raw)}
if policy != nil {
s.install, s.runtime = compileRules(policy.Install), compileRules(policy.Runtime)
}
p.mu.Lock()
p.sessions[s.token] = s
p.mu.Unlock()
return s, nil
}
// Revoke ends a session. Requests already forwarded finish; new ones fail.
func (p *Proxy) Revoke(token string) {
p.mu.Lock()
delete(p.sessions, token)
p.mu.Unlock()
}
func (p *Proxy) session(r *http.Request) *Session {
encoded, ok := strings.CutPrefix(r.Header.Get("Proxy-Authorization"), "Basic ")
if !ok {
return nil
}
decoded, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return nil
}
user, token, ok := strings.Cut(string(decoded), ":")
if !ok || user != proxyUser {
return nil
}
p.mu.RLock()
defer p.mu.RUnlock()
return p.sessions[token]
}
func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s := p.session(r)
if s == nil {
w.Header().Set("Proxy-Authenticate", `Basic realm="archie"`)
http.Error(w, "proxy authentication required", http.StatusProxyAuthRequired)
return
}
switch {
case r.Method == http.MethodConnect:
p.serveConnect(w, r, s)
case r.URL.IsAbs() && r.URL.Scheme == "http":
host, port, err := splitTarget(r.URL.Host, 80)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if !s.rules().allows(host, port) {
http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is not allowed", http.StatusForbidden)
return
}
p.forward(r.Context(), w, r, s, "http", host, port)
default:
http.Error(w, "only CONNECT and absolute-form http requests are proxied", http.StatusBadRequest)
}
}
func (p *Proxy) serveConnect(w http.ResponseWriter, r *http.Request, s *Session) {
host, port, err := splitTarget(r.Host, 443)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if !s.rules().allows(host, port) {
http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is not allowed", http.StatusForbidden)
return
}
hijacker, ok := w.(http.Hijacker)
if !ok {
http.Error(w, "connection cannot be tunnelled", http.StatusInternalServerError)
return
}
conn, _, err := hijacker.Hijack()
if err != nil {
return
}
if _, err := conn.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil {
_ = conn.Close()
return
}
tlsConn := tls.Server(conn, &tls.Config{
MinVersion: tls.VersionTLS12,
NextProtos: []string{"http/1.1"},
GetCertificate: func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
// The tunnel was admitted for host; a handshake naming another
// server would let the client reach it under host's permission.
if hello.ServerName != "" && normalizeHost(hello.ServerName) != host {
return nil, fmt.Errorf("TLS server name %q does not match tunnel host %q", hello.ServerName, host)
}
return p.ca.leaf(host)
},
})
handshakeCtx, cancel := context.WithTimeout(r.Context(), handshakeTimeout)
err = tlsConn.HandshakeContext(handshakeCtx)
cancel()
if err != nil {
_ = tlsConn.Close()
return
}
tunnel := &http.Server{
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reqHost, _, err := splitTarget(r.Host, port)
if err != nil || reqHost != host {
http.Error(w, "request host does not match its tunnel", http.StatusMisdirectedRequest)
return
}
if !s.rules().allows(host, port) {
http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is no longer allowed", http.StatusForbidden)
return
}
p.forward(r.Context(), w, r, s, "https", host, port)
}),
ReadHeaderTimeout: handshakeTimeout,
}
_ = tunnel.Serve(newOneConnListener(tlsConn))
}
func (p *Proxy) forward(ctx context.Context, w http.ResponseWriter, r *http.Request, s *Session, scheme, host string, port int) {
target := net.JoinHostPort(host, strconv.Itoa(port))
// The hook runs before anything is sent, so a failed rewrite (a missing
// credential binding, say) never reaches upstream half-applied.
if p.rewrite != nil {
if err := p.rewrite(s, r); err != nil {
http.Error(w, "egress to "+target+": "+err.Error(), http.StatusBadGateway)
return
}
}
rp := &httputil.ReverseProxy{
Transport: p.transport,
FlushInterval: -1,
Rewrite: func(pr *httputil.ProxyRequest) {
pr.Out.URL.Scheme = scheme
pr.Out.URL.Host = target
pr.Out.Host = pr.In.Host
pr.Out.Header.Del("Proxy-Authorization")
pr.Out.Header.Del("Proxy-Connection")
pr.Out = pr.Out.WithContext(context.WithValue(ctx, sessionKey{}, s))
},
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
http.Error(w, "upstream "+target+": "+err.Error(), http.StatusBadGateway)
},
}
rp.ServeHTTP(w, r)
}
// dialUpstream resolves and dials an upstream, refusing addresses that
// would turn the proxy into a path back into the host or its networks.
func (p *Proxy) dialUpstream(ctx context.Context, network, addr string) (net.Conn, error) {
if p.dial != nil {
return p.dial(ctx, network, addr)
}
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
s, _ := ctx.Value(sessionKey{}).(*Session)
var ips []net.IP
if ip := net.ParseIP(host); ip != nil {
ips = []net.IP{ip}
} else {
addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
if err != nil {
return nil, err
}
for _, a := range addrs {
ips = append(ips, a.IP)
}
}
var guardErr error
for _, ip := range ips {
named := s != nil && s.rules().namesIP(ip)
if guardErr = guardUpstream(ip, named); guardErr != nil {
continue
}
return (&net.Dialer{Timeout: handshakeTimeout}).DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
}
if guardErr == nil {
guardErr = fmt.Errorf("%s resolved to no address", host)
}
return nil, guardErr
}
// guardUpstream refuses loopback and unspecified addresses always, and
// private, link-local, shared and multicast addresses unless the policy
// named that exact IP.
func guardUpstream(ip net.IP, namedByPolicy bool) error {
if ip.IsLoopback() || ip.IsUnspecified() {
return fmt.Errorf("upstream address %s is the host itself", ip)
}
internal := ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
ip.IsInterfaceLocalMulticast() || ip.IsMulticast() || cgnat.Contains(ip)
if internal && !namedByPolicy {
return fmt.Errorf("upstream address %s is internal and not named by the network policy", ip)
}
return nil
}
func splitTarget(hostport string, defaultPort int) (string, int, error) {
if !hasPort(hostport) {
return normalizeHost(strings.Trim(hostport, "[]")), defaultPort, nil
}
host, portText, err := net.SplitHostPort(hostport)
if err != nil {
return "", 0, err
}
port, err := strconv.Atoi(portText)
if err != nil || port <= 0 || port > 65535 {
return "", 0, fmt.Errorf("invalid port in %q", hostport)
}
return normalizeHost(host), port, nil
}
// hasPort reports whether hostport ends in a port: a bracketed IPv6 literal
// followed by one, or a host with exactly one colon.
func hasPort(hostport string) bool {
if strings.HasPrefix(hostport, "[") {
return strings.Contains(hostport, "]:")
}
return strings.Count(hostport, ":") == 1
}
// oneConnListener serves exactly one already-accepted connection, so an
// http.Server can drive a hijacked tunnel.
type oneConnListener struct {
conn net.Conn
once sync.Once
done chan struct{}
}
func newOneConnListener(conn net.Conn) *oneConnListener {
return &oneConnListener{conn: conn, done: make(chan struct{})}
}
func (l *oneConnListener) Accept() (net.Conn, error) {
var c net.Conn
l.once.Do(func() { c = &closeNotifyConn{Conn: l.conn, done: l.done} })
if c != nil {
return c, nil
}
<-l.done
return nil, errors.New("tunnel closed")
}
func (l *oneConnListener) Close() error { return nil }
func (l *oneConnListener) Addr() net.Addr { return l.conn.LocalAddr() }
type closeNotifyConn struct {
net.Conn
once sync.Once
done chan struct{}
}
func (c *closeNotifyConn) Close() error {
c.once.Do(func() { close(c.done) })
return c.Conn.Close()
}
@@ -0,0 +1,275 @@
package egress
import (
"context"
"crypto/tls"
"crypto/x509"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"sync/atomic"
"testing"
"github.com/docker/sandbox-kit-spec/v3/spec"
)
// harness wires a proxy to local upstreams: the proxy's dialer maps the
// public names a policy uses onto httptest listeners, so the tests exercise
// the real CONNECT, TLS termination and forwarding paths hermetically.
type harness struct {
proxy *Proxy
server *httptest.Server
upstream *httptest.Server
plain *httptest.Server
hits atomic.Int32
lastAuth atomic.Value
caPool *x509.CertPool
}
func newHarness(t *testing.T) *harness {
t.Helper()
h := &harness{}
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
h.hits.Add(1)
h.lastAuth.Store(r.Header.Get("Authorization"))
_, _ = io.WriteString(w, "upstream:"+r.Host+r.URL.Path)
})
h.upstream = httptest.NewTLSServer(handler)
h.plain = httptest.NewServer(handler)
t.Cleanup(h.upstream.Close)
t.Cleanup(h.plain.Close)
ca, err := LoadOrCreateCA(t.TempDir())
if err != nil {
t.Fatal(err)
}
upstreamRoots := x509.NewCertPool()
upstreamRoots.AddCert(h.upstream.Certificate())
routes := map[string]string{
"api.example.com:443": h.upstream.Listener.Addr().String(),
"install.example.com:443": h.upstream.Listener.Addr().String(),
"denied.example.com:443": h.upstream.Listener.Addr().String(),
"plain.example.com:80": h.plain.Listener.Addr().String(),
}
h.proxy = NewProxy(ca, ProxyOptions{
UpstreamRoots: upstreamRoots,
Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
target, ok := routes[addr]
if !ok {
t.Errorf("proxy dialled unexpected upstream %s", addr)
return nil, &net.OpError{Op: "dial", Net: network, Err: net.UnknownNetworkError("unrouted")}
}
return (&net.Dialer{}).DialContext(ctx, network, target)
},
})
h.server = httptest.NewServer(h.proxy)
t.Cleanup(h.server.Close)
h.caPool = x509.NewCertPool()
h.caPool.AppendCertsFromPEM(ca.CertPEM())
return h
}
func (h *harness) register(t *testing.T) *Session {
t.Helper()
s, err := h.proxy.Register(&spec.PhasedNetwork{
Install: &spec.NetworkRules{Allow: []string{"install.example.com"}},
Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443", "plain.example.com:80"}},
})
if err != nil {
t.Fatal(err)
}
return s
}
func (h *harness) client(token string) *http.Client {
proxyURL, _ := url.Parse(h.server.URL)
if token != "" {
proxyURL.User = url.UserPassword(proxyUser, token)
}
return &http.Client{Transport: &http.Transport{
Proxy: http.ProxyURL(proxyURL),
TLSClientConfig: &tls.Config{RootCAs: h.caPool},
}}
}
func get(t *testing.T, c *http.Client, target string) (int, string, error) {
t.Helper()
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
if err != nil {
t.Fatal(err)
}
resp, err := c.Do(req)
if err != nil {
return 0, "", err
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(body), nil
}
func TestProxyForwardsAllowedHTTPS(t *testing.T) {
h := newHarness(t)
s := h.register(t)
s.EnterRuntime()
status, body, err := get(t, h.client(s.Token()), "https://api.example.com/v1/messages")
if err != nil {
t.Fatalf("request through proxy: %v", err)
}
if status != http.StatusOK || body != "upstream:api.example.com/v1/messages" {
t.Fatalf("got %d %q", status, body)
}
}
func TestProxyRefusesWithoutAValidToken(t *testing.T) {
h := newHarness(t)
s := h.register(t)
s.EnterRuntime()
revoked := h.register(t)
revoked.EnterRuntime()
h.proxy.Revoke(revoked.Token())
for name, token := range map[string]string{"no token": "", "unknown token": "not-a-token", "revoked token": revoked.Token()} {
t.Run(name, func(t *testing.T) {
if _, _, err := get(t, h.client(token), "https://api.example.com/"); err == nil {
t.Fatal("request succeeded without a valid session")
}
status, _, err := get(t, h.client(token), "http://plain.example.com/")
if err != nil {
t.Fatal(err)
}
if status != http.StatusProxyAuthRequired {
t.Fatalf("plain request got %d, want 407", status)
}
})
}
if h.hits.Load() != 0 {
t.Fatalf("upstream saw %d requests from unauthenticated clients", h.hits.Load())
}
}
func TestProxyEnforcesThePhasePolicy(t *testing.T) {
h := newHarness(t)
s := h.register(t)
c := h.client(s.Token())
if status, _, err := get(t, c, "https://install.example.com/"); err != nil || status != http.StatusOK {
t.Fatalf("install-phase host during install: %d %v", status, err)
}
if _, _, err := get(t, c, "https://api.example.com/"); err == nil {
t.Fatal("runtime-only host reachable during install")
}
s.EnterRuntime()
if _, _, err := get(t, h.client(s.Token()), "https://install.example.com/"); err == nil {
t.Fatal("install-phase host still reachable at runtime")
}
if _, _, err := get(t, h.client(s.Token()), "https://denied.example.com/"); err == nil {
t.Fatal("host outside every allow list reachable")
}
}
func TestProxyForwardsAllowedPlainHTTP(t *testing.T) {
h := newHarness(t)
s := h.register(t)
s.EnterRuntime()
status, body, err := get(t, h.client(s.Token()), "http://plain.example.com/x")
if err != nil || status != http.StatusOK || body != "upstream:plain.example.com/x" {
t.Fatalf("got %d %q %v", status, body, err)
}
status, _, err = get(t, h.client(s.Token()), "http://denied.example.com/")
if err != nil || status != http.StatusForbidden {
t.Fatalf("denied plain request got %d %v, want 403", status, err)
}
}
func TestProxyRefusesAHostHeaderThatLeavesTheTunnel(t *testing.T) {
h := newHarness(t)
s := h.register(t)
s.EnterRuntime()
c := h.client(s.Token())
req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://api.example.com/", nil)
req.Host = "denied.example.com"
resp, err := c.Do(req)
if err == nil {
defer resp.Body.Close()
if resp.StatusCode != http.StatusMisdirectedRequest {
t.Fatalf("smuggled Host got %d, want 421", resp.StatusCode)
}
}
if h.hits.Load() != 0 {
t.Fatal("a request whose Host differs from its tunnel reached upstream")
}
}
func TestProxyRefusesMismatchedSNI(t *testing.T) {
h := newHarness(t)
s := h.register(t)
s.EnterRuntime()
proxyURL, _ := url.Parse(h.server.URL)
proxyURL.User = url.UserPassword(proxyUser, s.Token())
tr := &http.Transport{
Proxy: http.ProxyURL(proxyURL),
TLSClientConfig: &tls.Config{RootCAs: h.caPool, ServerName: "denied.example.com"},
}
req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://api.example.com/", nil)
resp, err := (&http.Client{Transport: tr}).Do(req)
if err == nil {
_ = resp.Body.Close()
t.Fatal("a TLS handshake naming another host completed inside the tunnel")
}
if h.hits.Load() != 0 {
t.Fatal("mismatched SNI reached upstream")
}
}
func TestProxyAppliesTheRewriteHook(t *testing.T) {
h := newHarness(t)
var seen atomic.Pointer[Session]
h.proxy.rewrite = func(s *Session, r *http.Request) error {
seen.Store(s)
r.Header.Set("Authorization", "Bearer injected")
return nil
}
s := h.register(t)
s.EnterRuntime()
if _, _, err := get(t, h.client(s.Token()), "https://api.example.com/"); err != nil {
t.Fatal(err)
}
if got := h.lastAuth.Load(); got != "Bearer injected" {
t.Fatalf("upstream Authorization %v, want the rewritten value", got)
}
if seen.Load() != s {
t.Fatal("rewrite hook did not receive the request's session")
}
}
func TestUpstreamGuard(t *testing.T) {
tests := []struct {
ip string
literal bool
want bool
}{
{"93.184.216.34", false, true},
{"127.0.0.1", false, false},
{"127.0.0.1", true, false},
{"::1", false, false},
{"0.0.0.0", true, false},
{"10.1.2.3", false, false},
{"10.1.2.3", true, true},
{"172.17.0.1", false, false},
{"192.168.1.10", false, false},
{"169.254.169.254", false, false},
{"fd00::1", false, false},
{"fe80::1", false, false},
}
for _, tt := range tests {
err := guardUpstream(net.ParseIP(tt.ip), tt.literal)
if got := err == nil; got != tt.want {
t.Errorf("guardUpstream(%s, literal=%v) allowed=%v, want %v (%v)", tt.ip, tt.literal, got, tt.want, err)
}
}
if err := guardUpstream(net.ParseIP("127.0.0.1"), false); err == nil || !strings.Contains(err.Error(), "127.0.0.1") {
t.Fatalf("guard error %v does not name the refused address", err)
}
}