Archie runs its own agent loop only. Removes Kit resolution and launch, the harness runner and adapters, the setup terminal and sandbox view, harness secrets and OAuth capture (table dropped in migration 0075), the task-tools contract that only Kit containers reached, and the sandbox-kit-spec dependency. Agent profiles carry image and tools only. The egress proxy keeps the model-session path; the sandbox relay code stays for #1867.
355 lines
11 KiB
Go
355 lines
11 KiB
Go
package egress
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"encoding/base64"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httputil"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// proxyUser is the fixed user name in a sandbox's proxy URL; the session
|
|
// token is the password.
|
|
const proxyUser = "archie"
|
|
|
|
const handshakeTimeout = 10 * time.Second
|
|
|
|
// cgnat is shared address space (RFC 6598), which overlay networks use for
|
|
// host addresses; it is as internal as RFC 1918 space.
|
|
var cgnat = &net.IPNet{IP: net.IPv4(100, 64, 0, 0), Mask: net.CIDRMask(10, 32)}
|
|
|
|
// Session is one container's standing with the proxy: its token, the hosts
|
|
// it may reach, and the keys it may have substituted on the way.
|
|
type Session struct {
|
|
token string
|
|
allow rules
|
|
substitutions []substitution
|
|
secrets map[string]string
|
|
}
|
|
|
|
// ProxyOptions are the proxy's upstream dependencies. All are optional: the
|
|
// zero value dials the network through the upstream guard and verifies
|
|
// upstream TLS against the system roots.
|
|
type ProxyOptions struct {
|
|
UpstreamRoots *x509.CertPool
|
|
Dial func(ctx context.Context, network, addr string) (net.Conn, error)
|
|
// Resolver supplies the keys a session does not carry itself. Without
|
|
// one, a substitution for such a key is refused.
|
|
Resolver Resolver
|
|
}
|
|
|
|
// SessionOptions describe one container's session.
|
|
type SessionOptions struct {
|
|
// Token is the container's proxy password, and the run credential the
|
|
// Resolver narrows keys by.
|
|
Token string
|
|
// Allow lists the hosts the container may reach. Nothing else is allowed.
|
|
Allow []string
|
|
// Substitutions swap a service's sentinel for its key on requests to the
|
|
// service's host.
|
|
Substitutions []Substitution
|
|
// Secrets are keys the run itself carries, keyed by service. A
|
|
// substitution prefers them over the Resolver, so a run-supplied key needs
|
|
// no credential binding.
|
|
Secrets map[string]string
|
|
}
|
|
|
|
// Proxy is the egress proxy task containers send their model calls through.
|
|
// It authenticates each request to a session, applies that session's allow
|
|
// list, terminates TLS with the daemon CA, substitutes keys, and forwards to
|
|
// the real upstream with verified TLS.
|
|
type Proxy struct {
|
|
ca *CA
|
|
dial func(ctx context.Context, network, addr string) (net.Conn, error)
|
|
transport *http.Transport
|
|
resolver Resolver
|
|
|
|
mu sync.RWMutex
|
|
sessions map[string]*Session
|
|
}
|
|
|
|
type sessionKey struct{}
|
|
|
|
func NewProxy(ca *CA, opts ProxyOptions) *Proxy {
|
|
p := &Proxy{ca: ca, dial: opts.Dial, resolver: opts.Resolver, sessions: map[string]*Session{}}
|
|
p.transport = &http.Transport{
|
|
DialContext: p.dialUpstream,
|
|
TLSClientConfig: &tls.Config{RootCAs: opts.UpstreamRoots, MinVersion: tls.VersionTLS12},
|
|
ForceAttemptHTTP2: true,
|
|
TLSHandshakeTimeout: handshakeTimeout,
|
|
}
|
|
return p
|
|
}
|
|
|
|
// Register opens a session for one container.
|
|
func (p *Proxy) Register(opts SessionOptions) (*Session, error) {
|
|
if opts.Token == "" {
|
|
return nil, errors.New("egress session needs the run credential")
|
|
}
|
|
s := &Session{
|
|
token: opts.Token, allow: rules{allow: compilePatterns(opts.Allow)},
|
|
substitutions: compileSubstitutions(opts.Substitutions), secrets: opts.Secrets,
|
|
}
|
|
p.mu.Lock()
|
|
p.sessions[s.token] = s
|
|
p.mu.Unlock()
|
|
return s, nil
|
|
}
|
|
|
|
// Revoke ends a session. Requests already forwarded finish; new ones fail.
|
|
func (p *Proxy) Revoke(token string) {
|
|
p.mu.Lock()
|
|
delete(p.sessions, token)
|
|
p.mu.Unlock()
|
|
}
|
|
|
|
func (p *Proxy) session(r *http.Request) *Session {
|
|
encoded, ok := strings.CutPrefix(r.Header.Get("Proxy-Authorization"), "Basic ")
|
|
if !ok {
|
|
return nil
|
|
}
|
|
decoded, err := base64.StdEncoding.DecodeString(encoded)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
user, token, ok := strings.Cut(string(decoded), ":")
|
|
if !ok || user != proxyUser {
|
|
return nil
|
|
}
|
|
p.mu.RLock()
|
|
defer p.mu.RUnlock()
|
|
return p.sessions[token]
|
|
}
|
|
|
|
func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
s := p.session(r)
|
|
if s == nil {
|
|
w.Header().Set("Proxy-Authenticate", `Basic realm="archie"`)
|
|
http.Error(w, "proxy authentication required", http.StatusProxyAuthRequired)
|
|
return
|
|
}
|
|
switch {
|
|
case r.Method == http.MethodConnect:
|
|
p.serveConnect(w, r, s)
|
|
case r.URL.IsAbs() && r.URL.Scheme == "http":
|
|
host, port, err := splitTarget(r.URL.Host, 80)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if !s.allow.allows(host, port) {
|
|
http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is not allowed", http.StatusForbidden)
|
|
return
|
|
}
|
|
p.forward(r.Context(), w, r, s, "http", host, port)
|
|
default:
|
|
http.Error(w, "only CONNECT and absolute-form http requests are proxied", http.StatusBadRequest)
|
|
}
|
|
}
|
|
|
|
func (p *Proxy) serveConnect(w http.ResponseWriter, r *http.Request, s *Session) {
|
|
host, port, err := splitTarget(r.Host, 443)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if !s.allow.allows(host, port) {
|
|
http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is not allowed", http.StatusForbidden)
|
|
return
|
|
}
|
|
hijacker, ok := w.(http.Hijacker)
|
|
if !ok {
|
|
http.Error(w, "connection cannot be tunnelled", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
conn, _, err := hijacker.Hijack()
|
|
if err != nil {
|
|
return
|
|
}
|
|
if _, err := conn.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil {
|
|
_ = conn.Close()
|
|
return
|
|
}
|
|
tlsConn := tls.Server(conn, &tls.Config{
|
|
MinVersion: tls.VersionTLS12,
|
|
NextProtos: []string{"http/1.1"},
|
|
GetCertificate: func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
|
// The tunnel was admitted for host; a handshake naming another
|
|
// server would let the client reach it under host's permission.
|
|
if hello.ServerName != "" && normalizeHost(hello.ServerName) != host {
|
|
return nil, fmt.Errorf("TLS server name %q does not match tunnel host %q", hello.ServerName, host)
|
|
}
|
|
return p.ca.leaf(host)
|
|
},
|
|
})
|
|
handshakeCtx, cancel := context.WithTimeout(r.Context(), handshakeTimeout)
|
|
err = tlsConn.HandshakeContext(handshakeCtx)
|
|
cancel()
|
|
if err != nil {
|
|
_ = tlsConn.Close()
|
|
return
|
|
}
|
|
tunnel := &http.Server{
|
|
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
reqHost, _, err := splitTarget(r.Host, port)
|
|
if err != nil || reqHost != host {
|
|
http.Error(w, "request host does not match its tunnel", http.StatusMisdirectedRequest)
|
|
return
|
|
}
|
|
if !s.allow.allows(host, port) {
|
|
http.Error(w, "egress to "+net.JoinHostPort(host, strconv.Itoa(port))+" is no longer allowed", http.StatusForbidden)
|
|
return
|
|
}
|
|
p.forward(r.Context(), w, r, s, "https", host, port)
|
|
}),
|
|
ReadHeaderTimeout: handshakeTimeout,
|
|
}
|
|
_ = tunnel.Serve(newOneConnListener(tlsConn))
|
|
}
|
|
|
|
func (p *Proxy) forward(ctx context.Context, w http.ResponseWriter, r *http.Request, s *Session, scheme, host string, port int) {
|
|
target := net.JoinHostPort(host, strconv.Itoa(port))
|
|
// Substitution happens before anything is sent, so a key that cannot be
|
|
// resolved never reaches upstream half-applied.
|
|
if err := p.substitute(ctx, s, r, host, port); err != nil {
|
|
http.Error(w, "egress to "+target+": "+err.Error(), http.StatusBadGateway)
|
|
return
|
|
}
|
|
rp := &httputil.ReverseProxy{
|
|
Transport: p.transport,
|
|
FlushInterval: -1,
|
|
Rewrite: func(pr *httputil.ProxyRequest) {
|
|
pr.Out.URL.Scheme = scheme
|
|
pr.Out.URL.Host = target
|
|
pr.Out.Host = pr.In.Host
|
|
pr.Out.Header.Del("Proxy-Authorization")
|
|
pr.Out.Header.Del("Proxy-Connection")
|
|
pr.Out = pr.Out.WithContext(context.WithValue(ctx, sessionKey{}, s))
|
|
},
|
|
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
|
|
http.Error(w, "upstream "+target+": "+err.Error(), http.StatusBadGateway)
|
|
},
|
|
}
|
|
rp.ServeHTTP(w, r)
|
|
}
|
|
|
|
// dialUpstream resolves and dials an upstream, refusing addresses that
|
|
// would turn the proxy into a path back into the host or its networks.
|
|
func (p *Proxy) dialUpstream(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
if p.dial != nil {
|
|
return p.dial(ctx, network, addr)
|
|
}
|
|
host, port, err := net.SplitHostPort(addr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
s, _ := ctx.Value(sessionKey{}).(*Session)
|
|
var ips []net.IP
|
|
if ip := net.ParseIP(host); ip != nil {
|
|
ips = []net.IP{ip}
|
|
} else {
|
|
addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, a := range addrs {
|
|
ips = append(ips, a.IP)
|
|
}
|
|
}
|
|
var guardErr error
|
|
for _, ip := range ips {
|
|
named := s != nil && s.allow.namesIP(ip)
|
|
if guardErr = guardUpstream(ip, named); guardErr != nil {
|
|
continue
|
|
}
|
|
return (&net.Dialer{Timeout: handshakeTimeout}).DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
|
|
}
|
|
if guardErr == nil {
|
|
guardErr = fmt.Errorf("%s resolved to no address", host)
|
|
}
|
|
return nil, guardErr
|
|
}
|
|
|
|
// guardUpstream refuses loopback and unspecified addresses always, and
|
|
// private, link-local, shared and multicast addresses unless the policy
|
|
// named that exact IP.
|
|
func guardUpstream(ip net.IP, namedByPolicy bool) error {
|
|
if ip.IsLoopback() || ip.IsUnspecified() {
|
|
return fmt.Errorf("upstream address %s is the host itself", ip)
|
|
}
|
|
internal := ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
|
|
ip.IsInterfaceLocalMulticast() || ip.IsMulticast() || cgnat.Contains(ip)
|
|
if internal && !namedByPolicy {
|
|
return fmt.Errorf("upstream address %s is internal and not named by the network policy", ip)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func splitTarget(hostport string, defaultPort int) (string, int, error) {
|
|
if !hasPort(hostport) {
|
|
return normalizeHost(strings.Trim(hostport, "[]")), defaultPort, nil
|
|
}
|
|
host, portText, err := net.SplitHostPort(hostport)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
port, err := strconv.Atoi(portText)
|
|
if err != nil || port <= 0 || port > 65535 {
|
|
return "", 0, fmt.Errorf("invalid port in %q", hostport)
|
|
}
|
|
return normalizeHost(host), port, nil
|
|
}
|
|
|
|
// hasPort reports whether hostport ends in a port: a bracketed IPv6 literal
|
|
// followed by one, or a host with exactly one colon.
|
|
func hasPort(hostport string) bool {
|
|
if strings.HasPrefix(hostport, "[") {
|
|
return strings.Contains(hostport, "]:")
|
|
}
|
|
return strings.Count(hostport, ":") == 1
|
|
}
|
|
|
|
// oneConnListener serves exactly one already-accepted connection, so an
|
|
// http.Server can drive a hijacked tunnel.
|
|
type oneConnListener struct {
|
|
conn net.Conn
|
|
once sync.Once
|
|
done chan struct{}
|
|
}
|
|
|
|
func newOneConnListener(conn net.Conn) *oneConnListener {
|
|
return &oneConnListener{conn: conn, done: make(chan struct{})}
|
|
}
|
|
|
|
func (l *oneConnListener) Accept() (net.Conn, error) {
|
|
var c net.Conn
|
|
l.once.Do(func() { c = &closeNotifyConn{Conn: l.conn, done: l.done} })
|
|
if c != nil {
|
|
return c, nil
|
|
}
|
|
<-l.done
|
|
return nil, errors.New("tunnel closed")
|
|
}
|
|
|
|
func (l *oneConnListener) Close() error { return nil }
|
|
func (l *oneConnListener) Addr() net.Addr { return l.conn.LocalAddr() }
|
|
|
|
type closeNotifyConn struct {
|
|
net.Conn
|
|
once sync.Once
|
|
done chan struct{}
|
|
}
|
|
|
|
func (c *closeNotifyConn) Close() error {
|
|
c.once.Do(func() { close(c.done) })
|
|
return c.Conn.Close()
|
|
}
|