The egress proxy now holds a Kit's OAuth tokens on the host. A refresh carrying the sentinel refresh token goes upstream with the org's stored one; a successful response is written to the State Store and returned to the container with sentinels only. Requests to the Kit's resource hosts carrying the sentinel access token get the stored one. The org's token set is read or written only for a run whose credential carries the service, so a Kit declaring a service never reaches the org secret by itself. The launcher grants a bound OAuth service without a config secret, and Register refuses a required OAuth credential when no harness secret store is wired. A provider that does not rotate refresh tokens keeps the stored one. Part of archie-core-egkf.7 (#1208). Credential-file rendering and per-binding concurrency remain.
402 lines
12 KiB
Go
402 lines
12 KiB
Go
package egress
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httputil"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/docker/sandbox-kit-spec/v3/spec"
|
|
)
|
|
|
|
// proxyUser is the fixed user name in a sandbox's proxy URL; the session
|
|
// token is the password.
|
|
const proxyUser = "archie"
|
|
|
|
const handshakeTimeout = 10 * time.Second
|
|
|
|
// cgnat is shared address space (RFC 6598), which overlay networks use for
|
|
// host addresses; it is as internal as RFC 1918 space.
|
|
var cgnat = &net.IPNet{IP: net.IPv4(100, 64, 0, 0), Mask: net.CIDRMask(10, 32)}
|
|
|
|
// Session is one sandbox container's standing with the proxy: its token and
|
|
// its Kit's network policy. It starts in the install phase; the runner moves
|
|
// it to runtime before the workload's entrypoint starts.
|
|
type Session struct {
|
|
token string
|
|
run string
|
|
org string
|
|
install rules
|
|
runtime rules
|
|
injections []injection
|
|
oauth []oauthRule
|
|
atRun atomic.Bool
|
|
}
|
|
|
|
// Token is the secret the container presents as its proxy password.
|
|
func (s *Session) Token() string { return s.token }
|
|
|
|
// EnterRuntime switches the session from install-phase to runtime-phase
|
|
// egress. It is one-way: install egress never reopens.
|
|
func (s *Session) EnterRuntime() { s.atRun.Store(true) }
|
|
|
|
func (s *Session) rules() rules {
|
|
if s.atRun.Load() {
|
|
return s.runtime
|
|
}
|
|
return s.install
|
|
}
|
|
|
|
// ProxyOptions are the proxy's upstream dependencies. Both are optional:
|
|
// the zero value dials the network through the upstream guard and verifies
|
|
// upstream TLS against the system roots.
|
|
type ProxyOptions struct {
|
|
UpstreamRoots *x509.CertPool
|
|
Dial func(ctx context.Context, network, addr string) (net.Conn, error)
|
|
// Resolver supplies credentials for injection. Without one, a request
|
|
// that needs a required credential is refused.
|
|
Resolver Resolver
|
|
// OAuthStore holds the org token sets OAuth credentials use. Without
|
|
// one, Register refuses a session with a required OAuth credential.
|
|
OAuthStore OAuthStore
|
|
}
|
|
|
|
// SessionOptions describe one container's run: the run credential it acts
|
|
// under, the org whose OAuth token sets it uses, and its Kit's network
|
|
// policy and credential requests.
|
|
type SessionOptions struct {
|
|
Run string
|
|
Org string
|
|
Network *spec.PhasedNetwork
|
|
Credentials []spec.CredentialCapability
|
|
}
|
|
|
|
// Proxy is the egress proxy sandbox containers reach through their relay.
|
|
// It authenticates each request to a session, applies that session's
|
|
// network policy, terminates TLS with the daemon CA, and forwards to the
|
|
// real upstream with verified TLS.
|
|
type Proxy struct {
|
|
ca *CA
|
|
dial func(ctx context.Context, network, addr string) (net.Conn, error)
|
|
transport *http.Transport
|
|
resolver Resolver
|
|
oauth OAuthStore
|
|
|
|
mu sync.RWMutex
|
|
sessions map[string]*Session
|
|
}
|
|
|
|
type sessionKey struct{}
|
|
|
|
func NewProxy(ca *CA, opts ProxyOptions) *Proxy {
|
|
p := &Proxy{ca: ca, dial: opts.Dial, resolver: opts.Resolver, oauth: opts.OAuthStore, sessions: map[string]*Session{}}
|
|
p.transport = &http.Transport{
|
|
DialContext: p.dialUpstream,
|
|
TLSClientConfig: &tls.Config{RootCAs: opts.UpstreamRoots, MinVersion: tls.VersionTLS12},
|
|
ForceAttemptHTTP2: true,
|
|
TLSHandshakeTimeout: handshakeTimeout,
|
|
}
|
|
return p
|
|
}
|
|
|
|
// Register opens a session for one container. A nil network policy allows
|
|
// no egress in either phase.
|
|
func (p *Proxy) Register(opts SessionOptions) (*Session, error) {
|
|
raw := make([]byte, 32)
|
|
if _, err := rand.Read(raw); err != nil {
|
|
return nil, err
|
|
}
|
|
s := &Session{
|
|
token: hex.EncodeToString(raw), run: opts.Run, org: opts.Org,
|
|
injections: compileInjections(opts.Credentials), oauth: compileOAuthRules(opts.Credentials),
|
|
}
|
|
for _, rule := range s.oauth {
|
|
if rule.required && p.oauth == nil {
|
|
return nil, fmt.Errorf("credential %q is OAuth-managed and no harness secret store is configured", rule.service)
|
|
}
|
|
}
|
|
if opts.Network != nil {
|
|
s.install, s.runtime = compileRules(opts.Network.Install), compileRules(opts.Network.Runtime)
|
|
}
|
|
p.mu.Lock()
|
|
p.sessions[s.token] = s
|
|
p.mu.Unlock()
|
|
return s, nil
|
|
}
|
|
|
|
// Revoke ends a session. Requests already forwarded finish; new ones fail.
|
|
func (p *Proxy) Revoke(token string) {
|
|
p.mu.Lock()
|
|
delete(p.sessions, token)
|
|
p.mu.Unlock()
|
|
}
|
|
|
|
func (p *Proxy) session(r *http.Request) *Session {
|
|
encoded, ok := strings.CutPrefix(r.Header.Get("Proxy-Authorization"), "Basic ")
|
|
if !ok {
|
|
return nil
|
|
}
|
|
decoded, err := base64.StdEncoding.DecodeString(encoded)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
user, token, ok := strings.Cut(string(decoded), ":")
|
|
if !ok || user != proxyUser {
|
|
return nil
|
|
}
|
|
p.mu.RLock()
|
|
defer p.mu.RUnlock()
|
|
return p.sessions[token]
|
|
}
|
|
|
|
func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
s := p.session(r)
|
|
if s == nil {
|
|
w.Header().Set("Proxy-Authenticate", `Basic realm="archie"`)
|
|
http.Error(w, "proxy authentication required", http.StatusProxyAuthRequired)
|
|
return
|
|
}
|
|
switch {
|
|
case r.Method == http.MethodConnect:
|
|
p.serveConnect(w, r, s)
|
|
case r.URL.IsAbs() && r.URL.Scheme == "http":
|
|
host, port, err := splitTarget(r.URL.Host, 80)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if !s.rules().allows(host, port) {
|
|
http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is not allowed", http.StatusForbidden)
|
|
return
|
|
}
|
|
p.forward(r.Context(), w, r, s, "http", host, port)
|
|
default:
|
|
http.Error(w, "only CONNECT and absolute-form http requests are proxied", http.StatusBadRequest)
|
|
}
|
|
}
|
|
|
|
func (p *Proxy) serveConnect(w http.ResponseWriter, r *http.Request, s *Session) {
|
|
host, port, err := splitTarget(r.Host, 443)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if !s.rules().allows(host, port) {
|
|
http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is not allowed", http.StatusForbidden)
|
|
return
|
|
}
|
|
hijacker, ok := w.(http.Hijacker)
|
|
if !ok {
|
|
http.Error(w, "connection cannot be tunnelled", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
conn, _, err := hijacker.Hijack()
|
|
if err != nil {
|
|
return
|
|
}
|
|
if _, err := conn.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil {
|
|
_ = conn.Close()
|
|
return
|
|
}
|
|
tlsConn := tls.Server(conn, &tls.Config{
|
|
MinVersion: tls.VersionTLS12,
|
|
NextProtos: []string{"http/1.1"},
|
|
GetCertificate: func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
|
// The tunnel was admitted for host; a handshake naming another
|
|
// server would let the client reach it under host's permission.
|
|
if hello.ServerName != "" && normalizeHost(hello.ServerName) != host {
|
|
return nil, fmt.Errorf("TLS server name %q does not match tunnel host %q", hello.ServerName, host)
|
|
}
|
|
return p.ca.leaf(host)
|
|
},
|
|
})
|
|
handshakeCtx, cancel := context.WithTimeout(r.Context(), handshakeTimeout)
|
|
err = tlsConn.HandshakeContext(handshakeCtx)
|
|
cancel()
|
|
if err != nil {
|
|
_ = tlsConn.Close()
|
|
return
|
|
}
|
|
tunnel := &http.Server{
|
|
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
reqHost, _, err := splitTarget(r.Host, port)
|
|
if err != nil || reqHost != host {
|
|
http.Error(w, "request host does not match its tunnel", http.StatusMisdirectedRequest)
|
|
return
|
|
}
|
|
if !s.rules().allows(host, port) {
|
|
http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is no longer allowed", http.StatusForbidden)
|
|
return
|
|
}
|
|
p.forward(r.Context(), w, r, s, "https", host, port)
|
|
}),
|
|
ReadHeaderTimeout: handshakeTimeout,
|
|
}
|
|
_ = tunnel.Serve(newOneConnListener(tlsConn))
|
|
}
|
|
|
|
func (p *Proxy) forward(ctx context.Context, w http.ResponseWriter, r *http.Request, s *Session, scheme, host string, port int) {
|
|
target := net.JoinHostPort(host, strconv.Itoa(port))
|
|
if rule, ok := tokenEndpointRule(s, r, host, port); ok {
|
|
granted, err := p.granted(ctx, s, rule)
|
|
if err != nil {
|
|
http.Error(w, "egress to "+target+": "+err.Error(), http.StatusBadGateway)
|
|
return
|
|
}
|
|
if granted {
|
|
p.interceptOAuth(ctx, w, r, s, rule, scheme, target)
|
|
return
|
|
}
|
|
}
|
|
// Injection happens before anything is sent, so a credential that
|
|
// cannot be resolved never reaches upstream half-applied.
|
|
err := p.inject(ctx, s, r, host, port)
|
|
if err == nil {
|
|
err = p.injectOAuth(ctx, s, r, host, port)
|
|
}
|
|
if err != nil {
|
|
http.Error(w, "egress to "+target+": "+err.Error(), http.StatusBadGateway)
|
|
return
|
|
}
|
|
rp := &httputil.ReverseProxy{
|
|
Transport: p.transport,
|
|
FlushInterval: -1,
|
|
Rewrite: func(pr *httputil.ProxyRequest) {
|
|
pr.Out.URL.Scheme = scheme
|
|
pr.Out.URL.Host = target
|
|
pr.Out.Host = pr.In.Host
|
|
pr.Out.Header.Del("Proxy-Authorization")
|
|
pr.Out.Header.Del("Proxy-Connection")
|
|
pr.Out = pr.Out.WithContext(context.WithValue(ctx, sessionKey{}, s))
|
|
},
|
|
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
|
|
http.Error(w, "upstream "+target+": "+err.Error(), http.StatusBadGateway)
|
|
},
|
|
}
|
|
rp.ServeHTTP(w, r)
|
|
}
|
|
|
|
// dialUpstream resolves and dials an upstream, refusing addresses that
|
|
// would turn the proxy into a path back into the host or its networks.
|
|
func (p *Proxy) dialUpstream(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
if p.dial != nil {
|
|
return p.dial(ctx, network, addr)
|
|
}
|
|
host, port, err := net.SplitHostPort(addr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
s, _ := ctx.Value(sessionKey{}).(*Session)
|
|
var ips []net.IP
|
|
if ip := net.ParseIP(host); ip != nil {
|
|
ips = []net.IP{ip}
|
|
} else {
|
|
addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, a := range addrs {
|
|
ips = append(ips, a.IP)
|
|
}
|
|
}
|
|
var guardErr error
|
|
for _, ip := range ips {
|
|
named := s != nil && s.rules().namesIP(ip)
|
|
if guardErr = guardUpstream(ip, named); guardErr != nil {
|
|
continue
|
|
}
|
|
return (&net.Dialer{Timeout: handshakeTimeout}).DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
|
|
}
|
|
if guardErr == nil {
|
|
guardErr = fmt.Errorf("%s resolved to no address", host)
|
|
}
|
|
return nil, guardErr
|
|
}
|
|
|
|
// guardUpstream refuses loopback and unspecified addresses always, and
|
|
// private, link-local, shared and multicast addresses unless the policy
|
|
// named that exact IP.
|
|
func guardUpstream(ip net.IP, namedByPolicy bool) error {
|
|
if ip.IsLoopback() || ip.IsUnspecified() {
|
|
return fmt.Errorf("upstream address %s is the host itself", ip)
|
|
}
|
|
internal := ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
|
|
ip.IsInterfaceLocalMulticast() || ip.IsMulticast() || cgnat.Contains(ip)
|
|
if internal && !namedByPolicy {
|
|
return fmt.Errorf("upstream address %s is internal and not named by the network policy", ip)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func splitTarget(hostport string, defaultPort int) (string, int, error) {
|
|
if !hasPort(hostport) {
|
|
return normalizeHost(strings.Trim(hostport, "[]")), defaultPort, nil
|
|
}
|
|
host, portText, err := net.SplitHostPort(hostport)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
port, err := strconv.Atoi(portText)
|
|
if err != nil || port <= 0 || port > 65535 {
|
|
return "", 0, fmt.Errorf("invalid port in %q", hostport)
|
|
}
|
|
return normalizeHost(host), port, nil
|
|
}
|
|
|
|
// hasPort reports whether hostport ends in a port: a bracketed IPv6 literal
|
|
// followed by one, or a host with exactly one colon.
|
|
func hasPort(hostport string) bool {
|
|
if strings.HasPrefix(hostport, "[") {
|
|
return strings.Contains(hostport, "]:")
|
|
}
|
|
return strings.Count(hostport, ":") == 1
|
|
}
|
|
|
|
// oneConnListener serves exactly one already-accepted connection, so an
|
|
// http.Server can drive a hijacked tunnel.
|
|
type oneConnListener struct {
|
|
conn net.Conn
|
|
once sync.Once
|
|
done chan struct{}
|
|
}
|
|
|
|
func newOneConnListener(conn net.Conn) *oneConnListener {
|
|
return &oneConnListener{conn: conn, done: make(chan struct{})}
|
|
}
|
|
|
|
func (l *oneConnListener) Accept() (net.Conn, error) {
|
|
var c net.Conn
|
|
l.once.Do(func() { c = &closeNotifyConn{Conn: l.conn, done: l.done} })
|
|
if c != nil {
|
|
return c, nil
|
|
}
|
|
<-l.done
|
|
return nil, errors.New("tunnel closed")
|
|
}
|
|
|
|
func (l *oneConnListener) Close() error { return nil }
|
|
func (l *oneConnListener) Addr() net.Addr { return l.conn.LocalAddr() }
|
|
|
|
type closeNotifyConn struct {
|
|
net.Conn
|
|
once sync.Once
|
|
done chan struct{}
|
|
}
|
|
|
|
func (c *closeNotifyConn) Close() error {
|
|
c.once.Do(func() { close(c.done) })
|
|
return c.Conn.Close()
|
|
}
|