diff --git a/internal/infrastructure/egress/policy.go b/internal/infrastructure/egress/policy.go index 508bf189a..109788cee 100644 --- a/internal/infrastructure/egress/policy.go +++ b/internal/infrastructure/egress/policy.go @@ -91,3 +91,16 @@ func (r rules) allows(host string, port int) bool { } return false } + +// namesIP reports whether an allow entry names ip exactly. Only an explicit +// IP entry opens an internal address; a wildcard or everything never does. +func (r rules) namesIP(ip net.IP) bool { + for _, p := range r.allow { + if !p.everything && p.suffix == "" { + if named := net.ParseIP(p.host); named != nil && named.Equal(ip) { + return true + } + } + } + return false +} diff --git a/internal/infrastructure/egress/proxy.go b/internal/infrastructure/egress/proxy.go new file mode 100644 index 000000000..ed7dd2077 --- /dev/null +++ b/internal/infrastructure/egress/proxy.go @@ -0,0 +1,361 @@ +package egress + +import ( + "context" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "net" + "net/http" + "net/http/httputil" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/docker/sandbox-kit-spec/v3/spec" +) + +// proxyUser is the fixed user name in a sandbox's proxy URL; the session +// token is the password. +const proxyUser = "archie" + +const handshakeTimeout = 10 * time.Second + +// cgnat is shared address space (RFC 6598), which overlay networks use for +// host addresses; it is as internal as RFC 1918 space. +var cgnat = &net.IPNet{IP: net.IPv4(100, 64, 0, 0), Mask: net.CIDRMask(10, 32)} + +// Session is one sandbox container's standing with the proxy: its token and +// its Kit's network policy. It starts in the install phase; the runner moves +// it to runtime before the workload's entrypoint starts. +type Session struct { + token string + install rules + runtime rules + atRun atomic.Bool +} + +// Token is the secret the container presents as its proxy password. +func (s *Session) Token() string { return s.token } + +// EnterRuntime switches the session from install-phase to runtime-phase +// egress. It is one-way: install egress never reopens. +func (s *Session) EnterRuntime() { s.atRun.Store(true) } + +func (s *Session) rules() rules { + if s.atRun.Load() { + return s.runtime + } + return s.install +} + +// ProxyOptions are the proxy's upstream dependencies. Both are optional: +// the zero value dials the network through the upstream guard and verifies +// upstream TLS against the system roots. +type ProxyOptions struct { + UpstreamRoots *x509.CertPool + Dial func(ctx context.Context, network, addr string) (net.Conn, error) +} + +// Proxy is the egress proxy sandbox containers reach through their relay. +// It authenticates each request to a session, applies that session's +// network policy, terminates TLS with the daemon CA, and forwards to the +// real upstream with verified TLS. +type Proxy struct { + ca *CA + dial func(ctx context.Context, network, addr string) (net.Conn, error) + transport *http.Transport + + // rewrite runs on every forwarded request after policy has admitted it. + rewrite func(*Session, *http.Request) error + + mu sync.RWMutex + sessions map[string]*Session +} + +type sessionKey struct{} + +func NewProxy(ca *CA, opts ProxyOptions) *Proxy { + p := &Proxy{ca: ca, dial: opts.Dial, sessions: map[string]*Session{}} + p.transport = &http.Transport{ + DialContext: p.dialUpstream, + TLSClientConfig: &tls.Config{RootCAs: opts.UpstreamRoots, MinVersion: tls.VersionTLS12}, + ForceAttemptHTTP2: true, + TLSHandshakeTimeout: handshakeTimeout, + } + return p +} + +// Register opens a session for one container under its Kit's policy. A nil +// policy allows no egress in either phase. +func (p *Proxy) Register(policy *spec.PhasedNetwork) (*Session, error) { + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return nil, err + } + s := &Session{token: hex.EncodeToString(raw)} + if policy != nil { + s.install, s.runtime = compileRules(policy.Install), compileRules(policy.Runtime) + } + p.mu.Lock() + p.sessions[s.token] = s + p.mu.Unlock() + return s, nil +} + +// Revoke ends a session. Requests already forwarded finish; new ones fail. +func (p *Proxy) Revoke(token string) { + p.mu.Lock() + delete(p.sessions, token) + p.mu.Unlock() +} + +func (p *Proxy) session(r *http.Request) *Session { + encoded, ok := strings.CutPrefix(r.Header.Get("Proxy-Authorization"), "Basic ") + if !ok { + return nil + } + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return nil + } + user, token, ok := strings.Cut(string(decoded), ":") + if !ok || user != proxyUser { + return nil + } + p.mu.RLock() + defer p.mu.RUnlock() + return p.sessions[token] +} + +func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s := p.session(r) + if s == nil { + w.Header().Set("Proxy-Authenticate", `Basic realm="archie"`) + http.Error(w, "proxy authentication required", http.StatusProxyAuthRequired) + return + } + switch { + case r.Method == http.MethodConnect: + p.serveConnect(w, r, s) + case r.URL.IsAbs() && r.URL.Scheme == "http": + host, port, err := splitTarget(r.URL.Host, 80) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if !s.rules().allows(host, port) { + http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is not allowed", http.StatusForbidden) + return + } + p.forward(r.Context(), w, r, s, "http", host, port) + default: + http.Error(w, "only CONNECT and absolute-form http requests are proxied", http.StatusBadRequest) + } +} + +func (p *Proxy) serveConnect(w http.ResponseWriter, r *http.Request, s *Session) { + host, port, err := splitTarget(r.Host, 443) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if !s.rules().allows(host, port) { + http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is not allowed", http.StatusForbidden) + return + } + hijacker, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "connection cannot be tunnelled", http.StatusInternalServerError) + return + } + conn, _, err := hijacker.Hijack() + if err != nil { + return + } + if _, err := conn.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil { + _ = conn.Close() + return + } + tlsConn := tls.Server(conn, &tls.Config{ + MinVersion: tls.VersionTLS12, + NextProtos: []string{"http/1.1"}, + GetCertificate: func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + // The tunnel was admitted for host; a handshake naming another + // server would let the client reach it under host's permission. + if hello.ServerName != "" && normalizeHost(hello.ServerName) != host { + return nil, fmt.Errorf("TLS server name %q does not match tunnel host %q", hello.ServerName, host) + } + return p.ca.leaf(host) + }, + }) + handshakeCtx, cancel := context.WithTimeout(r.Context(), handshakeTimeout) + err = tlsConn.HandshakeContext(handshakeCtx) + cancel() + if err != nil { + _ = tlsConn.Close() + return + } + tunnel := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reqHost, _, err := splitTarget(r.Host, port) + if err != nil || reqHost != host { + http.Error(w, "request host does not match its tunnel", http.StatusMisdirectedRequest) + return + } + if !s.rules().allows(host, port) { + http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is no longer allowed", http.StatusForbidden) + return + } + p.forward(r.Context(), w, r, s, "https", host, port) + }), + ReadHeaderTimeout: handshakeTimeout, + } + _ = tunnel.Serve(newOneConnListener(tlsConn)) +} + +func (p *Proxy) forward(ctx context.Context, w http.ResponseWriter, r *http.Request, s *Session, scheme, host string, port int) { + target := net.JoinHostPort(host, strconv.Itoa(port)) + // The hook runs before anything is sent, so a failed rewrite (a missing + // credential binding, say) never reaches upstream half-applied. + if p.rewrite != nil { + if err := p.rewrite(s, r); err != nil { + http.Error(w, "egress to "+target+": "+err.Error(), http.StatusBadGateway) + return + } + } + rp := &httputil.ReverseProxy{ + Transport: p.transport, + FlushInterval: -1, + Rewrite: func(pr *httputil.ProxyRequest) { + pr.Out.URL.Scheme = scheme + pr.Out.URL.Host = target + pr.Out.Host = pr.In.Host + pr.Out.Header.Del("Proxy-Authorization") + pr.Out.Header.Del("Proxy-Connection") + pr.Out = pr.Out.WithContext(context.WithValue(ctx, sessionKey{}, s)) + }, + ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) { + http.Error(w, "upstream "+target+": "+err.Error(), http.StatusBadGateway) + }, + } + rp.ServeHTTP(w, r) +} + +// dialUpstream resolves and dials an upstream, refusing addresses that +// would turn the proxy into a path back into the host or its networks. +func (p *Proxy) dialUpstream(ctx context.Context, network, addr string) (net.Conn, error) { + if p.dial != nil { + return p.dial(ctx, network, addr) + } + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + s, _ := ctx.Value(sessionKey{}).(*Session) + var ips []net.IP + if ip := net.ParseIP(host); ip != nil { + ips = []net.IP{ip} + } else { + addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host) + if err != nil { + return nil, err + } + for _, a := range addrs { + ips = append(ips, a.IP) + } + } + var guardErr error + for _, ip := range ips { + named := s != nil && s.rules().namesIP(ip) + if guardErr = guardUpstream(ip, named); guardErr != nil { + continue + } + return (&net.Dialer{Timeout: handshakeTimeout}).DialContext(ctx, network, net.JoinHostPort(ip.String(), port)) + } + if guardErr == nil { + guardErr = fmt.Errorf("%s resolved to no address", host) + } + return nil, guardErr +} + +// guardUpstream refuses loopback and unspecified addresses always, and +// private, link-local, shared and multicast addresses unless the policy +// named that exact IP. +func guardUpstream(ip net.IP, namedByPolicy bool) error { + if ip.IsLoopback() || ip.IsUnspecified() { + return fmt.Errorf("upstream address %s is the host itself", ip) + } + internal := ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || + ip.IsInterfaceLocalMulticast() || ip.IsMulticast() || cgnat.Contains(ip) + if internal && !namedByPolicy { + return fmt.Errorf("upstream address %s is internal and not named by the network policy", ip) + } + return nil +} + +func splitTarget(hostport string, defaultPort int) (string, int, error) { + if !hasPort(hostport) { + return normalizeHost(strings.Trim(hostport, "[]")), defaultPort, nil + } + host, portText, err := net.SplitHostPort(hostport) + if err != nil { + return "", 0, err + } + port, err := strconv.Atoi(portText) + if err != nil || port <= 0 || port > 65535 { + return "", 0, fmt.Errorf("invalid port in %q", hostport) + } + return normalizeHost(host), port, nil +} + +// hasPort reports whether hostport ends in a port: a bracketed IPv6 literal +// followed by one, or a host with exactly one colon. +func hasPort(hostport string) bool { + if strings.HasPrefix(hostport, "[") { + return strings.Contains(hostport, "]:") + } + return strings.Count(hostport, ":") == 1 +} + +// oneConnListener serves exactly one already-accepted connection, so an +// http.Server can drive a hijacked tunnel. +type oneConnListener struct { + conn net.Conn + once sync.Once + done chan struct{} +} + +func newOneConnListener(conn net.Conn) *oneConnListener { + return &oneConnListener{conn: conn, done: make(chan struct{})} +} + +func (l *oneConnListener) Accept() (net.Conn, error) { + var c net.Conn + l.once.Do(func() { c = &closeNotifyConn{Conn: l.conn, done: l.done} }) + if c != nil { + return c, nil + } + <-l.done + return nil, errors.New("tunnel closed") +} + +func (l *oneConnListener) Close() error { return nil } +func (l *oneConnListener) Addr() net.Addr { return l.conn.LocalAddr() } + +type closeNotifyConn struct { + net.Conn + once sync.Once + done chan struct{} +} + +func (c *closeNotifyConn) Close() error { + c.once.Do(func() { close(c.done) }) + return c.Conn.Close() +} diff --git a/internal/infrastructure/egress/proxy_test.go b/internal/infrastructure/egress/proxy_test.go new file mode 100644 index 000000000..1081fe621 --- /dev/null +++ b/internal/infrastructure/egress/proxy_test.go @@ -0,0 +1,275 @@ +package egress + +import ( + "context" + "crypto/tls" + "crypto/x509" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + + "github.com/docker/sandbox-kit-spec/v3/spec" +) + +// harness wires a proxy to local upstreams: the proxy's dialer maps the +// public names a policy uses onto httptest listeners, so the tests exercise +// the real CONNECT, TLS termination and forwarding paths hermetically. +type harness struct { + proxy *Proxy + server *httptest.Server + upstream *httptest.Server + plain *httptest.Server + hits atomic.Int32 + lastAuth atomic.Value + caPool *x509.CertPool +} + +func newHarness(t *testing.T) *harness { + t.Helper() + h := &harness{} + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h.hits.Add(1) + h.lastAuth.Store(r.Header.Get("Authorization")) + _, _ = io.WriteString(w, "upstream:"+r.Host+r.URL.Path) + }) + h.upstream = httptest.NewTLSServer(handler) + h.plain = httptest.NewServer(handler) + t.Cleanup(h.upstream.Close) + t.Cleanup(h.plain.Close) + + ca, err := LoadOrCreateCA(t.TempDir()) + if err != nil { + t.Fatal(err) + } + upstreamRoots := x509.NewCertPool() + upstreamRoots.AddCert(h.upstream.Certificate()) + routes := map[string]string{ + "api.example.com:443": h.upstream.Listener.Addr().String(), + "install.example.com:443": h.upstream.Listener.Addr().String(), + "denied.example.com:443": h.upstream.Listener.Addr().String(), + "plain.example.com:80": h.plain.Listener.Addr().String(), + } + h.proxy = NewProxy(ca, ProxyOptions{ + UpstreamRoots: upstreamRoots, + Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { + target, ok := routes[addr] + if !ok { + t.Errorf("proxy dialled unexpected upstream %s", addr) + return nil, &net.OpError{Op: "dial", Net: network, Err: net.UnknownNetworkError("unrouted")} + } + return (&net.Dialer{}).DialContext(ctx, network, target) + }, + }) + h.server = httptest.NewServer(h.proxy) + t.Cleanup(h.server.Close) + h.caPool = x509.NewCertPool() + h.caPool.AppendCertsFromPEM(ca.CertPEM()) + return h +} + +func (h *harness) register(t *testing.T) *Session { + t.Helper() + s, err := h.proxy.Register(&spec.PhasedNetwork{ + Install: &spec.NetworkRules{Allow: []string{"install.example.com"}}, + Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443", "plain.example.com:80"}}, + }) + if err != nil { + t.Fatal(err) + } + return s +} + +func (h *harness) client(token string) *http.Client { + proxyURL, _ := url.Parse(h.server.URL) + if token != "" { + proxyURL.User = url.UserPassword(proxyUser, token) + } + return &http.Client{Transport: &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + TLSClientConfig: &tls.Config{RootCAs: h.caPool}, + }} +} + +func get(t *testing.T, c *http.Client, target string) (int, string, error) { + t.Helper() + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, target, nil) + if err != nil { + t.Fatal(err) + } + resp, err := c.Do(req) + if err != nil { + return 0, "", err + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(body), nil +} + +func TestProxyForwardsAllowedHTTPS(t *testing.T) { + h := newHarness(t) + s := h.register(t) + s.EnterRuntime() + status, body, err := get(t, h.client(s.Token()), "https://api.example.com/v1/messages") + if err != nil { + t.Fatalf("request through proxy: %v", err) + } + if status != http.StatusOK || body != "upstream:api.example.com/v1/messages" { + t.Fatalf("got %d %q", status, body) + } +} + +func TestProxyRefusesWithoutAValidToken(t *testing.T) { + h := newHarness(t) + s := h.register(t) + s.EnterRuntime() + revoked := h.register(t) + revoked.EnterRuntime() + h.proxy.Revoke(revoked.Token()) + for name, token := range map[string]string{"no token": "", "unknown token": "not-a-token", "revoked token": revoked.Token()} { + t.Run(name, func(t *testing.T) { + if _, _, err := get(t, h.client(token), "https://api.example.com/"); err == nil { + t.Fatal("request succeeded without a valid session") + } + status, _, err := get(t, h.client(token), "http://plain.example.com/") + if err != nil { + t.Fatal(err) + } + if status != http.StatusProxyAuthRequired { + t.Fatalf("plain request got %d, want 407", status) + } + }) + } + if h.hits.Load() != 0 { + t.Fatalf("upstream saw %d requests from unauthenticated clients", h.hits.Load()) + } +} + +func TestProxyEnforcesThePhasePolicy(t *testing.T) { + h := newHarness(t) + s := h.register(t) + c := h.client(s.Token()) + + if status, _, err := get(t, c, "https://install.example.com/"); err != nil || status != http.StatusOK { + t.Fatalf("install-phase host during install: %d %v", status, err) + } + if _, _, err := get(t, c, "https://api.example.com/"); err == nil { + t.Fatal("runtime-only host reachable during install") + } + s.EnterRuntime() + if _, _, err := get(t, h.client(s.Token()), "https://install.example.com/"); err == nil { + t.Fatal("install-phase host still reachable at runtime") + } + if _, _, err := get(t, h.client(s.Token()), "https://denied.example.com/"); err == nil { + t.Fatal("host outside every allow list reachable") + } +} + +func TestProxyForwardsAllowedPlainHTTP(t *testing.T) { + h := newHarness(t) + s := h.register(t) + s.EnterRuntime() + status, body, err := get(t, h.client(s.Token()), "http://plain.example.com/x") + if err != nil || status != http.StatusOK || body != "upstream:plain.example.com/x" { + t.Fatalf("got %d %q %v", status, body, err) + } + status, _, err = get(t, h.client(s.Token()), "http://denied.example.com/") + if err != nil || status != http.StatusForbidden { + t.Fatalf("denied plain request got %d %v, want 403", status, err) + } +} + +func TestProxyRefusesAHostHeaderThatLeavesTheTunnel(t *testing.T) { + h := newHarness(t) + s := h.register(t) + s.EnterRuntime() + c := h.client(s.Token()) + req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://api.example.com/", nil) + req.Host = "denied.example.com" + resp, err := c.Do(req) + if err == nil { + defer resp.Body.Close() + if resp.StatusCode != http.StatusMisdirectedRequest { + t.Fatalf("smuggled Host got %d, want 421", resp.StatusCode) + } + } + if h.hits.Load() != 0 { + t.Fatal("a request whose Host differs from its tunnel reached upstream") + } +} + +func TestProxyRefusesMismatchedSNI(t *testing.T) { + h := newHarness(t) + s := h.register(t) + s.EnterRuntime() + proxyURL, _ := url.Parse(h.server.URL) + proxyURL.User = url.UserPassword(proxyUser, s.Token()) + tr := &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + TLSClientConfig: &tls.Config{RootCAs: h.caPool, ServerName: "denied.example.com"}, + } + req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://api.example.com/", nil) + resp, err := (&http.Client{Transport: tr}).Do(req) + if err == nil { + _ = resp.Body.Close() + t.Fatal("a TLS handshake naming another host completed inside the tunnel") + } + if h.hits.Load() != 0 { + t.Fatal("mismatched SNI reached upstream") + } +} + +func TestProxyAppliesTheRewriteHook(t *testing.T) { + h := newHarness(t) + var seen atomic.Pointer[Session] + h.proxy.rewrite = func(s *Session, r *http.Request) error { + seen.Store(s) + r.Header.Set("Authorization", "Bearer injected") + return nil + } + s := h.register(t) + s.EnterRuntime() + if _, _, err := get(t, h.client(s.Token()), "https://api.example.com/"); err != nil { + t.Fatal(err) + } + if got := h.lastAuth.Load(); got != "Bearer injected" { + t.Fatalf("upstream Authorization %v, want the rewritten value", got) + } + if seen.Load() != s { + t.Fatal("rewrite hook did not receive the request's session") + } +} + +func TestUpstreamGuard(t *testing.T) { + tests := []struct { + ip string + literal bool + want bool + }{ + {"93.184.216.34", false, true}, + {"127.0.0.1", false, false}, + {"127.0.0.1", true, false}, + {"::1", false, false}, + {"0.0.0.0", true, false}, + {"10.1.2.3", false, false}, + {"10.1.2.3", true, true}, + {"172.17.0.1", false, false}, + {"192.168.1.10", false, false}, + {"169.254.169.254", false, false}, + {"fd00::1", false, false}, + {"fe80::1", false, false}, + } + for _, tt := range tests { + err := guardUpstream(net.ParseIP(tt.ip), tt.literal) + if got := err == nil; got != tt.want { + t.Errorf("guardUpstream(%s, literal=%v) allowed=%v, want %v (%v)", tt.ip, tt.literal, got, tt.want, err) + } + } + if err := guardUpstream(net.ParseIP("127.0.0.1"), false); err == nil || !strings.Contains(err.Error(), "127.0.0.1") { + t.Fatalf("guard error %v does not name the refused address", err) + } +}