406 lines
12 KiB
Go
406 lines
12 KiB
Go
package egress
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httputil"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/docker/sandbox-kit-spec/v3/spec"
|
|
)
|
|
|
|
// proxyUser is the fixed user name in a sandbox's proxy URL; the session
|
|
// token is the password.
|
|
const proxyUser = "archie"
|
|
|
|
const handshakeTimeout = 10 * time.Second
|
|
|
|
// cgnat is shared address space (RFC 6598), which overlay networks use for
|
|
// host addresses; it is as internal as RFC 1918 space.
|
|
var cgnat = &net.IPNet{IP: net.IPv4(100, 64, 0, 0), Mask: net.CIDRMask(10, 32)}
|
|
|
|
// Session is one sandbox container's standing with the proxy: its token and
|
|
// its Kit's network policy. It starts in the install phase; the runner moves
|
|
// it to runtime before the workload's entrypoint starts.
|
|
type Session struct {
|
|
token string
|
|
run string
|
|
org string
|
|
install rules
|
|
runtime rules
|
|
injections []injection
|
|
oauth []oauthRule
|
|
atRun atomic.Bool
|
|
}
|
|
|
|
// Token is the secret the container presents as its proxy password.
|
|
func (s *Session) Token() string { return s.token }
|
|
|
|
// EnterRuntime switches the session from install-phase to runtime-phase
|
|
// egress. It is one-way: install egress never reopens.
|
|
func (s *Session) EnterRuntime() { s.atRun.Store(true) }
|
|
|
|
func (s *Session) rules() rules {
|
|
if s.atRun.Load() {
|
|
return s.runtime
|
|
}
|
|
return s.install
|
|
}
|
|
|
|
// ProxyOptions are the proxy's upstream dependencies. Both are optional:
|
|
// the zero value dials the network through the upstream guard and verifies
|
|
// upstream TLS against the system roots.
|
|
type ProxyOptions struct {
|
|
UpstreamRoots *x509.CertPool
|
|
Dial func(ctx context.Context, network, addr string) (net.Conn, error)
|
|
// Resolver supplies credentials for injection. Without one, a request
|
|
// that needs a required credential is refused.
|
|
Resolver Resolver
|
|
// OAuthStore holds the org token sets OAuth credentials use. Without
|
|
// one, Register refuses a session with a required OAuth credential.
|
|
OAuthStore OAuthStore
|
|
}
|
|
|
|
// SessionOptions describe one container's run: the run credential it acts
|
|
// under, the org whose OAuth token sets it uses, and its Kit's network
|
|
// policy and credential requests.
|
|
type SessionOptions struct {
|
|
Run string
|
|
Org string
|
|
Network *spec.PhasedNetwork
|
|
// Credentials are what the Kit declares; Bound maps each service the run
|
|
// carries to its kind. A rule fires only for a service bound with the same
|
|
// kind.
|
|
Credentials []spec.CredentialCapability
|
|
Bound map[string]CredentialKind
|
|
}
|
|
|
|
// Proxy is the egress proxy sandbox containers reach through their relay.
|
|
// It authenticates each request to a session, applies that session's
|
|
// network policy, terminates TLS with the daemon CA, and forwards to the
|
|
// real upstream with verified TLS.
|
|
type Proxy struct {
|
|
ca *CA
|
|
dial func(ctx context.Context, network, addr string) (net.Conn, error)
|
|
transport *http.Transport
|
|
resolver Resolver
|
|
oauth OAuthStore
|
|
|
|
mu sync.RWMutex
|
|
sessions map[string]*Session
|
|
}
|
|
|
|
type sessionKey struct{}
|
|
|
|
func NewProxy(ca *CA, opts ProxyOptions) *Proxy {
|
|
p := &Proxy{ca: ca, dial: opts.Dial, resolver: opts.Resolver, oauth: opts.OAuthStore, sessions: map[string]*Session{}}
|
|
p.transport = &http.Transport{
|
|
DialContext: p.dialUpstream,
|
|
TLSClientConfig: &tls.Config{RootCAs: opts.UpstreamRoots, MinVersion: tls.VersionTLS12},
|
|
ForceAttemptHTTP2: true,
|
|
TLSHandshakeTimeout: handshakeTimeout,
|
|
}
|
|
return p
|
|
}
|
|
|
|
// Register opens a session for one container. A nil network policy allows
|
|
// no egress in either phase.
|
|
func (p *Proxy) Register(opts SessionOptions) (*Session, error) {
|
|
raw := make([]byte, 32)
|
|
if _, err := rand.Read(raw); err != nil {
|
|
return nil, err
|
|
}
|
|
s := &Session{
|
|
token: hex.EncodeToString(raw), run: opts.Run, org: opts.Org,
|
|
injections: compileInjections(opts.Credentials, opts.Bound), oauth: compileOAuthRules(opts.Credentials, opts.Bound),
|
|
}
|
|
for _, rule := range s.oauth {
|
|
if rule.required && p.oauth == nil {
|
|
return nil, fmt.Errorf("credential %q is OAuth-managed and no harness secret store is configured", rule.service)
|
|
}
|
|
}
|
|
if opts.Network != nil {
|
|
s.install, s.runtime = compileRules(opts.Network.Install), compileRules(opts.Network.Runtime)
|
|
}
|
|
p.mu.Lock()
|
|
p.sessions[s.token] = s
|
|
p.mu.Unlock()
|
|
return s, nil
|
|
}
|
|
|
|
// Revoke ends a session. Requests already forwarded finish; new ones fail.
|
|
func (p *Proxy) Revoke(token string) {
|
|
p.mu.Lock()
|
|
delete(p.sessions, token)
|
|
p.mu.Unlock()
|
|
}
|
|
|
|
func (p *Proxy) session(r *http.Request) *Session {
|
|
encoded, ok := strings.CutPrefix(r.Header.Get("Proxy-Authorization"), "Basic ")
|
|
if !ok {
|
|
return nil
|
|
}
|
|
decoded, err := base64.StdEncoding.DecodeString(encoded)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
user, token, ok := strings.Cut(string(decoded), ":")
|
|
if !ok || user != proxyUser {
|
|
return nil
|
|
}
|
|
p.mu.RLock()
|
|
defer p.mu.RUnlock()
|
|
return p.sessions[token]
|
|
}
|
|
|
|
func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
s := p.session(r)
|
|
if s == nil {
|
|
w.Header().Set("Proxy-Authenticate", `Basic realm="archie"`)
|
|
http.Error(w, "proxy authentication required", http.StatusProxyAuthRequired)
|
|
return
|
|
}
|
|
switch {
|
|
case r.Method == http.MethodConnect:
|
|
p.serveConnect(w, r, s)
|
|
case r.URL.IsAbs() && r.URL.Scheme == "http":
|
|
host, port, err := splitTarget(r.URL.Host, 80)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if !s.rules().allows(host, port) {
|
|
http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is not allowed", http.StatusForbidden)
|
|
return
|
|
}
|
|
p.forward(r.Context(), w, r, s, "http", host, port)
|
|
default:
|
|
http.Error(w, "only CONNECT and absolute-form http requests are proxied", http.StatusBadRequest)
|
|
}
|
|
}
|
|
|
|
func (p *Proxy) serveConnect(w http.ResponseWriter, r *http.Request, s *Session) {
|
|
host, port, err := splitTarget(r.Host, 443)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if !s.rules().allows(host, port) {
|
|
http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is not allowed", http.StatusForbidden)
|
|
return
|
|
}
|
|
hijacker, ok := w.(http.Hijacker)
|
|
if !ok {
|
|
http.Error(w, "connection cannot be tunnelled", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
conn, _, err := hijacker.Hijack()
|
|
if err != nil {
|
|
return
|
|
}
|
|
if _, err := conn.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil {
|
|
_ = conn.Close()
|
|
return
|
|
}
|
|
tlsConn := tls.Server(conn, &tls.Config{
|
|
MinVersion: tls.VersionTLS12,
|
|
NextProtos: []string{"http/1.1"},
|
|
GetCertificate: func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
|
// The tunnel was admitted for host; a handshake naming another
|
|
// server would let the client reach it under host's permission.
|
|
if hello.ServerName != "" && normalizeHost(hello.ServerName) != host {
|
|
return nil, fmt.Errorf("TLS server name %q does not match tunnel host %q", hello.ServerName, host)
|
|
}
|
|
return p.ca.leaf(host)
|
|
},
|
|
})
|
|
handshakeCtx, cancel := context.WithTimeout(r.Context(), handshakeTimeout)
|
|
err = tlsConn.HandshakeContext(handshakeCtx)
|
|
cancel()
|
|
if err != nil {
|
|
_ = tlsConn.Close()
|
|
return
|
|
}
|
|
tunnel := &http.Server{
|
|
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
reqHost, _, err := splitTarget(r.Host, port)
|
|
if err != nil || reqHost != host {
|
|
http.Error(w, "request host does not match its tunnel", http.StatusMisdirectedRequest)
|
|
return
|
|
}
|
|
if !s.rules().allows(host, port) {
|
|
http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is no longer allowed", http.StatusForbidden)
|
|
return
|
|
}
|
|
p.forward(r.Context(), w, r, s, "https", host, port)
|
|
}),
|
|
ReadHeaderTimeout: handshakeTimeout,
|
|
}
|
|
_ = tunnel.Serve(newOneConnListener(tlsConn))
|
|
}
|
|
|
|
func (p *Proxy) forward(ctx context.Context, w http.ResponseWriter, r *http.Request, s *Session, scheme, host string, port int) {
|
|
target := net.JoinHostPort(host, strconv.Itoa(port))
|
|
if rule, ok := tokenEndpointRule(s, r, host, port); ok {
|
|
granted, err := p.granted(ctx, s, rule)
|
|
if err != nil {
|
|
http.Error(w, "egress to "+target+": "+err.Error(), http.StatusBadGateway)
|
|
return
|
|
}
|
|
if granted {
|
|
p.interceptOAuth(ctx, w, r, s, rule, scheme, target)
|
|
return
|
|
}
|
|
}
|
|
// Injection happens before anything is sent, so a credential that
|
|
// cannot be resolved never reaches upstream half-applied.
|
|
err := p.inject(ctx, s, r, host, port)
|
|
if err == nil {
|
|
err = p.injectOAuth(ctx, s, r, host, port)
|
|
}
|
|
if err != nil {
|
|
http.Error(w, "egress to "+target+": "+err.Error(), http.StatusBadGateway)
|
|
return
|
|
}
|
|
rp := &httputil.ReverseProxy{
|
|
Transport: p.transport,
|
|
FlushInterval: -1,
|
|
Rewrite: func(pr *httputil.ProxyRequest) {
|
|
pr.Out.URL.Scheme = scheme
|
|
pr.Out.URL.Host = target
|
|
pr.Out.Host = pr.In.Host
|
|
pr.Out.Header.Del("Proxy-Authorization")
|
|
pr.Out.Header.Del("Proxy-Connection")
|
|
pr.Out = pr.Out.WithContext(context.WithValue(ctx, sessionKey{}, s))
|
|
},
|
|
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
|
|
http.Error(w, "upstream "+target+": "+err.Error(), http.StatusBadGateway)
|
|
},
|
|
}
|
|
rp.ServeHTTP(w, r)
|
|
}
|
|
|
|
// dialUpstream resolves and dials an upstream, refusing addresses that
|
|
// would turn the proxy into a path back into the host or its networks.
|
|
func (p *Proxy) dialUpstream(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
if p.dial != nil {
|
|
return p.dial(ctx, network, addr)
|
|
}
|
|
host, port, err := net.SplitHostPort(addr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
s, _ := ctx.Value(sessionKey{}).(*Session)
|
|
var ips []net.IP
|
|
if ip := net.ParseIP(host); ip != nil {
|
|
ips = []net.IP{ip}
|
|
} else {
|
|
addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, a := range addrs {
|
|
ips = append(ips, a.IP)
|
|
}
|
|
}
|
|
var guardErr error
|
|
for _, ip := range ips {
|
|
named := s != nil && s.rules().namesIP(ip)
|
|
if guardErr = guardUpstream(ip, named); guardErr != nil {
|
|
continue
|
|
}
|
|
return (&net.Dialer{Timeout: handshakeTimeout}).DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
|
|
}
|
|
if guardErr == nil {
|
|
guardErr = fmt.Errorf("%s resolved to no address", host)
|
|
}
|
|
return nil, guardErr
|
|
}
|
|
|
|
// guardUpstream refuses loopback and unspecified addresses always, and
|
|
// private, link-local, shared and multicast addresses unless the policy
|
|
// named that exact IP.
|
|
func guardUpstream(ip net.IP, namedByPolicy bool) error {
|
|
if ip.IsLoopback() || ip.IsUnspecified() {
|
|
return fmt.Errorf("upstream address %s is the host itself", ip)
|
|
}
|
|
internal := ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
|
|
ip.IsInterfaceLocalMulticast() || ip.IsMulticast() || cgnat.Contains(ip)
|
|
if internal && !namedByPolicy {
|
|
return fmt.Errorf("upstream address %s is internal and not named by the network policy", ip)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func splitTarget(hostport string, defaultPort int) (string, int, error) {
|
|
if !hasPort(hostport) {
|
|
return normalizeHost(strings.Trim(hostport, "[]")), defaultPort, nil
|
|
}
|
|
host, portText, err := net.SplitHostPort(hostport)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
port, err := strconv.Atoi(portText)
|
|
if err != nil || port <= 0 || port > 65535 {
|
|
return "", 0, fmt.Errorf("invalid port in %q", hostport)
|
|
}
|
|
return normalizeHost(host), port, nil
|
|
}
|
|
|
|
// hasPort reports whether hostport ends in a port: a bracketed IPv6 literal
|
|
// followed by one, or a host with exactly one colon.
|
|
func hasPort(hostport string) bool {
|
|
if strings.HasPrefix(hostport, "[") {
|
|
return strings.Contains(hostport, "]:")
|
|
}
|
|
return strings.Count(hostport, ":") == 1
|
|
}
|
|
|
|
// oneConnListener serves exactly one already-accepted connection, so an
|
|
// http.Server can drive a hijacked tunnel.
|
|
type oneConnListener struct {
|
|
conn net.Conn
|
|
once sync.Once
|
|
done chan struct{}
|
|
}
|
|
|
|
func newOneConnListener(conn net.Conn) *oneConnListener {
|
|
return &oneConnListener{conn: conn, done: make(chan struct{})}
|
|
}
|
|
|
|
func (l *oneConnListener) Accept() (net.Conn, error) {
|
|
var c net.Conn
|
|
l.once.Do(func() { c = &closeNotifyConn{Conn: l.conn, done: l.done} })
|
|
if c != nil {
|
|
return c, nil
|
|
}
|
|
<-l.done
|
|
return nil, errors.New("tunnel closed")
|
|
}
|
|
|
|
func (l *oneConnListener) Close() error { return nil }
|
|
func (l *oneConnListener) Addr() net.Addr { return l.conn.LocalAddr() }
|
|
|
|
type closeNotifyConn struct {
|
|
net.Conn
|
|
once sync.Once
|
|
done chan struct{}
|
|
}
|
|
|
|
func (c *closeNotifyConn) Close() error {
|
|
c.once.Do(func() { close(c.done) })
|
|
return c.Conn.Close()
|
|
}
|