feat(egress): inject Kit API-key credentials at the proxy

A session carries its run and its Kit's credential@1 requests. The proxy
applies apiKey inject rules only in the credential's phase and only to
hosts its rule names, after the network policy has admitted the request,
so a key never reaches a host its Kit did not declare. The real value
comes from a Resolver that returns only secrets the run credential
carries: a required credential that is unbound or unresolvable stops the
request before anything is sent, and an optional one passes only when it
is genuinely unbound. Named keys appear in the sandbox as a sentinel;
inject-only keys have no presence.
This commit is contained in:
sam committed 2026-09-26 02:46:40 +10:00
1 parent 6eb5708914
commit 266dac9e5c
5 files changed
+351 -61

No files matched your search

@@ -0,0 +1,117 @@
package egress
import (
"context"
"errors"
"fmt"
"net/http"
"strings"
"github.com/docker/sandbox-kit-spec/v3/spec"
)
// Sentinel is the value a sandbox sees in place of a proxy-managed key, so
// it can tell a credential is wired without ever holding it.
const Sentinel = "archie-proxy-managed"
// ErrUnbound reports that the run has no binding for a service. It is the
// only resolver error an optional credential may pass over; any other error
// stops the request.
var ErrUnbound = errors.New("egress: credential not bound")
// Resolver supplies a credential's real value for one run. An implementation
// returns only secrets the run's credential carries, so what a Kit declares
// is always narrowed by what the org granted the run's identity.
type Resolver interface {
Resolve(ctx context.Context, run, service string) (string, error)
}
// ResolverFunc adapts a function to Resolver.
type ResolverFunc func(ctx context.Context, run, service string) (string, error)
func (f ResolverFunc) Resolve(ctx context.Context, run, service string) (string, error) {
return f(ctx, run, service)
}
// injection is one compiled credential@1 apiKey inject rule.
type injection struct {
service string
required bool
runtime bool
domain pattern
header string
format string
basic bool
username string
}
func compileInjections(creds []spec.CredentialCapability) []injection {
var out []injection
for _, c := range creds {
if c.APIKey == nil {
continue
}
for _, rule := range c.APIKey.Inject {
format := rule.Format
if format == "" {
format = "%s"
}
out = append(out, injection{
service: c.Service,
required: c.Required,
runtime: c.Phase == "runtime",
domain: compilePattern(rule.Domain),
header: rule.Header,
format: format,
basic: strings.EqualFold(rule.Scheme, "basic"),
username: rule.Username,
})
}
}
return out
}
// inject presents the session's credentials on a request policy has already
// admitted. Only rules for this phase and this host apply, so a key is never
// sent anywhere its Kit did not name.
func (p *Proxy) inject(ctx context.Context, s *Session, r *http.Request, host string, port int) error {
atRuntime := s.atRun.Load()
for _, rule := range s.injections {
if rule.runtime != atRuntime || !rule.domain.matches(normalizeHost(host), port) {
continue
}
if p.resolver == nil {
if rule.required {
return fmt.Errorf("credential %q is required and no resolver is configured", rule.service)
}
continue
}
secret, err := p.resolver.Resolve(ctx, s.run, rule.service)
switch {
case errors.Is(err, ErrUnbound) && !rule.required:
continue
case errors.Is(err, ErrUnbound):
return fmt.Errorf("credential %q is required and not bound for this run", rule.service)
case err != nil:
return fmt.Errorf("resolve credential %q: %w", rule.service, err)
}
if rule.basic {
r.SetBasicAuth(rule.username, secret)
continue
}
r.Header.Set(rule.header, fmt.Sprintf(rule.format, secret))
}
return nil
}
// SentinelEnv is the environment a sandbox gets for its Kit's API keys: the
// sentinel under each named key. An inject-only key has no presence at all.
func SentinelEnv(creds []spec.CredentialCapability) []string {
var env []string
for _, c := range creds {
if c.APIKey != nil && c.APIKey.Name != "" {
env = append(env, c.APIKey.Name+"="+Sentinel)
}
}
return env
}
@@ -0,0 +1,185 @@
package egress
import (
"context"
"encoding/base64"
"errors"
"net/http"
"slices"
"testing"
"github.com/docker/sandbox-kit-spec/v3/spec"
)
func apiKey(service, phase string, required bool, name string, inject ...spec.Inject) spec.CredentialCapability {
return spec.CredentialCapability{
Credential: spec.Credential{Service: service, Phase: phase, APIKey: &spec.APIKey{Name: name, ProxyManaged: true, Inject: inject}},
Required: required,
}
}
func (h *harness) registerWith(t *testing.T, creds ...spec.CredentialCapability) *Session {
t.Helper()
s, err := h.proxy.Register(SessionOptions{
Run: "run-7",
Network: &spec.PhasedNetwork{
Install: &spec.NetworkRules{Allow: []string{"install.example.com"}},
Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443", "plain.example.com:80"}},
},
Credentials: creds,
})
if err != nil {
t.Fatal(err)
}
return s
}
func TestInjectsAnAPIKeyHeader(t *testing.T) {
h := newHarness(t)
h.bind("run-7/example", "real-secret")
s := h.registerWith(t, apiKey("example", "runtime", true, "EXAMPLE_KEY",
spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"}))
s.EnterRuntime()
req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://api.example.com/", nil)
req.Header.Set("Authorization", "Bearer "+Sentinel)
resp, err := h.client(s.Token()).Do(req)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if got := h.lastAuth.Load(); got != "Bearer real-secret" {
t.Fatalf("upstream Authorization %v, want the real key in place of the sentinel", got)
}
}
func TestInjectsBasicAuth(t *testing.T) {
h := newHarness(t)
h.bind("run-7/example", "real-secret")
s := h.registerWith(t, apiKey("example", "runtime", true, "",
spec.Inject{Domain: "api.example.com", Scheme: "basic", Username: "x-access-token"}))
s.EnterRuntime()
if _, _, err := get(t, h.client(s.Token()), "https://api.example.com/"); err != nil {
t.Fatal(err)
}
want := "Basic " + base64.StdEncoding.EncodeToString([]byte("x-access-token:real-secret"))
if got := h.lastAuth.Load(); got != want {
t.Fatalf("upstream Authorization %v, want %q", got, want)
}
}
func TestInjectionIsScopedToItsDomainAndPhase(t *testing.T) {
h := newHarness(t)
h.bind("run-7/example", "real-secret")
h.bind("run-7/installer", "install-secret")
s := h.registerWith(t,
apiKey("example", "runtime", true, "EXAMPLE_KEY", spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"}),
apiKey("installer", "install", true, "", spec.Inject{Domain: "plain.example.com", Header: "X-Install", Format: "%s"}),
)
s.EnterRuntime()
if _, _, err := get(t, h.client(s.Token()), "http://plain.example.com/"); err != nil {
t.Fatal(err)
}
if got := h.lastAuth.Load(); got != "" {
t.Fatalf("a key for api.example.com was sent to plain.example.com: %v", got)
}
if h.lastInstall.Load() != "" {
t.Fatal("an install-phase credential was injected at runtime")
}
}
func TestAnUnboundCredential(t *testing.T) {
t.Run("required stops the request before it reaches upstream", func(t *testing.T) {
h := newHarness(t)
s := h.registerWith(t, apiKey("example", "runtime", true, "EXAMPLE_KEY",
spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"}))
s.EnterRuntime()
status, _, err := get(t, h.client(s.Token()), "https://api.example.com/")
if err != nil {
t.Fatal(err)
}
if status != http.StatusBadGateway || h.hits.Load() != 0 {
t.Fatalf("status %d with %d upstream hits; want 502 and none", status, h.hits.Load())
}
})
t.Run("optional passes without one", func(t *testing.T) {
h := newHarness(t)
s := h.registerWith(t, apiKey("example", "runtime", false, "EXAMPLE_KEY",
spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"}))
s.EnterRuntime()
status, _, err := get(t, h.client(s.Token()), "https://api.example.com/")
if err != nil || status != http.StatusOK {
t.Fatalf("got %d %v", status, err)
}
if got := h.lastAuth.Load(); got != "" {
t.Fatalf("upstream saw Authorization %v with no binding", got)
}
})
t.Run("a resolver failure is never treated as unbound", func(t *testing.T) {
h := newHarness(t)
h.failResolves(errors.New("state store unreachable"))
s := h.registerWith(t, apiKey("example", "runtime", false, "EXAMPLE_KEY",
spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"}))
s.EnterRuntime()
status, _, err := get(t, h.client(s.Token()), "https://api.example.com/")
if err != nil {
t.Fatal(err)
}
if status != http.StatusBadGateway || h.hits.Load() != 0 {
t.Fatalf("status %d with %d upstream hits; want 502 and none", status, h.hits.Load())
}
})
}
func TestTheResolverSeesTheRunAndService(t *testing.T) {
h := newHarness(t)
h.bind("run-7/example", "real-secret")
s := h.registerWith(t, apiKey("example", "runtime", true, "EXAMPLE_KEY",
spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"}))
s.EnterRuntime()
if _, _, err := get(t, h.client(s.Token()), "https://api.example.com/"); err != nil {
t.Fatal(err)
}
h.mu.Lock()
defer h.mu.Unlock()
if !slices.Contains(h.resolved, "run-7/example") {
t.Fatalf("resolver calls %v, want run-7/example", h.resolved)
}
}
func TestSentinelEnv(t *testing.T) {
env := SentinelEnv([]spec.CredentialCapability{
apiKey("named", "runtime", true, "NAMED_KEY", spec.Inject{Domain: "a.example.com", Header: "X", Format: "%s"}),
apiKey("injectonly", "runtime", true, "", spec.Inject{Domain: "b.example.com", Header: "X", Format: "%s"}),
})
if !slices.Equal(env, []string{"NAMED_KEY=" + Sentinel}) {
t.Fatalf("SentinelEnv = %v; a named key gets the sentinel, an inject-only key gets nothing", env)
}
}
// fakeResolver lets the harness stand in for the run credential's secrets.
func (h *harness) resolve(_ context.Context, run, service string) (string, error) {
key := run + "/" + service
h.mu.Lock()
defer h.mu.Unlock()
h.resolved = append(h.resolved, key)
if h.resolveErr != nil {
return "", h.resolveErr
}
v, ok := h.secrets[key]
if !ok {
return "", ErrUnbound
}
return v, nil
}
func (h *harness) bind(key, value string) {
h.mu.Lock()
defer h.mu.Unlock()
h.secrets[key] = value
}
func (h *harness) failResolves(err error) {
h.mu.Lock()
defer h.mu.Unlock()
h.resolveErr = err
}
@@ -201,7 +201,7 @@ func TestSandboxNetworkReachesOnlyTheProxy(t *testing.T) {
// isolated sandbox network leaves it unassigned.
sandboxGateway := sandboxNet.Network.IPAM.Config[0].Subnet.Masked().Addr().Next().String()
session, err := proxy.Register(&spec.PhasedNetwork{Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443"}}})
session, err := proxy.Register(SessionOptions{Run: "integration", Network: &spec.PhasedNetwork{Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443"}}}})
if err != nil {
t.Fatal(err)
}
+30 -21
View File
@@ -35,10 +35,12 @@ var cgnat = &net.IPNet{IP: net.IPv4(100, 64, 0, 0), Mask: net.CIDRMask(10, 32)}
// its Kit's network policy. It starts in the install phase; the runner moves
// it to runtime before the workload's entrypoint starts.
type Session struct {
token string
install rules
runtime rules
atRun atomic.Bool
token string
run string
install rules
runtime rules
injections []injection
atRun atomic.Bool
}
// Token is the secret the container presents as its proxy password.
@@ -61,6 +63,17 @@ func (s *Session) rules() rules {
type ProxyOptions struct {
UpstreamRoots *x509.CertPool
Dial func(ctx context.Context, network, addr string) (net.Conn, error)
// Resolver supplies credentials for injection. Without one, a request
// that needs a required credential is refused.
Resolver Resolver
}
// SessionOptions describe one container's run: the run credential it acts
// under, and its Kit's network policy and credential requests.
type SessionOptions struct {
Run string
Network *spec.PhasedNetwork
Credentials []spec.CredentialCapability
}
// Proxy is the egress proxy sandbox containers reach through their relay.
@@ -71,9 +84,7 @@ type Proxy struct {
ca *CA
dial func(ctx context.Context, network, addr string) (net.Conn, error)
transport *http.Transport
// rewrite runs on every forwarded request after policy has admitted it.
rewrite func(*Session, *http.Request) error
resolver Resolver
mu sync.RWMutex
sessions map[string]*Session
@@ -82,7 +93,7 @@ type Proxy struct {
type sessionKey struct{}
func NewProxy(ca *CA, opts ProxyOptions) *Proxy {
p := &Proxy{ca: ca, dial: opts.Dial, sessions: map[string]*Session{}}
p := &Proxy{ca: ca, dial: opts.Dial, resolver: opts.Resolver, sessions: map[string]*Session{}}
p.transport = &http.Transport{
DialContext: p.dialUpstream,
TLSClientConfig: &tls.Config{RootCAs: opts.UpstreamRoots, MinVersion: tls.VersionTLS12},
@@ -92,16 +103,16 @@ func NewProxy(ca *CA, opts ProxyOptions) *Proxy {
return p
}
// Register opens a session for one container under its Kit's policy. A nil
// policy allows no egress in either phase.
func (p *Proxy) Register(policy *spec.PhasedNetwork) (*Session, error) {
// Register opens a session for one container. A nil network policy allows
// no egress in either phase.
func (p *Proxy) Register(opts SessionOptions) (*Session, error) {
raw := make([]byte, 32)
if _, err := rand.Read(raw); err != nil {
return nil, err
}
s := &Session{token: hex.EncodeToString(raw)}
if policy != nil {
s.install, s.runtime = compileRules(policy.Install), compileRules(policy.Runtime)
s := &Session{token: hex.EncodeToString(raw), run: opts.Run, injections: compileInjections(opts.Credentials)}
if opts.Network != nil {
s.install, s.runtime = compileRules(opts.Network.Install), compileRules(opts.Network.Runtime)
}
p.mu.Lock()
p.sessions[s.token] = s
@@ -222,13 +233,11 @@ func (p *Proxy) serveConnect(w http.ResponseWriter, r *http.Request, s *Session)
func (p *Proxy) forward(ctx context.Context, w http.ResponseWriter, r *http.Request, s *Session, scheme, host string, port int) {
target := net.JoinHostPort(host, strconv.Itoa(port))
// The hook runs before anything is sent, so a failed rewrite (a missing
// credential binding, say) never reaches upstream half-applied.
if p.rewrite != nil {
if err := p.rewrite(s, r); err != nil {
http.Error(w, "egress to "+target+": "+err.Error(), http.StatusBadGateway)
return
}
// Injection happens before anything is sent, so a credential that
// cannot be resolved never reaches upstream half-applied.
if err := p.inject(ctx, s, r, host, port); err != nil {
http.Error(w, "egress to "+target+": "+err.Error(), http.StatusBadGateway)
return
}
rp := &httputil.ReverseProxy{
Transport: p.transport,
+18 -39
View File
@@ -10,31 +10,37 @@ import (
"net/http/httptest"
"net/url"
"strings"
"sync"
"sync/atomic"
"testing"
"github.com/docker/sandbox-kit-spec/v3/spec"
)
// harness wires a proxy to local upstreams: the proxy's dialer maps the
// public names a policy uses onto httptest listeners, so the tests exercise
// the real CONNECT, TLS termination and forwarding paths hermetically.
type harness struct {
proxy *Proxy
server *httptest.Server
upstream *httptest.Server
plain *httptest.Server
hits atomic.Int32
lastAuth atomic.Value
caPool *x509.CertPool
proxy *Proxy
server *httptest.Server
upstream *httptest.Server
plain *httptest.Server
hits atomic.Int32
lastAuth atomic.Value
lastInstall atomic.Value
caPool *x509.CertPool
mu sync.Mutex
secrets map[string]string
resolved []string
resolveErr error
}
func newHarness(t *testing.T) *harness {
t.Helper()
h := &harness{}
h := &harness{secrets: map[string]string{}}
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
h.hits.Add(1)
h.lastAuth.Store(r.Header.Get("Authorization"))
h.lastInstall.Store(r.Header.Get("X-Install"))
_, _ = io.WriteString(w, "upstream:"+r.Host+r.URL.Path)
})
h.upstream = httptest.NewTLSServer(handler)
@@ -56,6 +62,7 @@ func newHarness(t *testing.T) *harness {
}
h.proxy = NewProxy(ca, ProxyOptions{
UpstreamRoots: upstreamRoots,
Resolver: ResolverFunc(h.resolve),
Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
target, ok := routes[addr]
if !ok {
@@ -74,14 +81,7 @@ func newHarness(t *testing.T) *harness {
func (h *harness) register(t *testing.T) *Session {
t.Helper()
s, err := h.proxy.Register(&spec.PhasedNetwork{
Install: &spec.NetworkRules{Allow: []string{"install.example.com"}},
Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443", "plain.example.com:80"}},
})
if err != nil {
t.Fatal(err)
}
return s
return h.registerWith(t)
}
func (h *harness) client(token string) *http.Client {
@@ -223,27 +223,6 @@ func TestProxyRefusesMismatchedSNI(t *testing.T) {
}
}
func TestProxyAppliesTheRewriteHook(t *testing.T) {
h := newHarness(t)
var seen atomic.Pointer[Session]
h.proxy.rewrite = func(s *Session, r *http.Request) error {
seen.Store(s)
r.Header.Set("Authorization", "Bearer injected")
return nil
}
s := h.register(t)
s.EnterRuntime()
if _, _, err := get(t, h.client(s.Token()), "https://api.example.com/"); err != nil {
t.Fatal(err)
}
if got := h.lastAuth.Load(); got != "Bearer injected" {
t.Fatalf("upstream Authorization %v, want the rewritten value", got)
}
if seen.Load() != s {
t.Fatal("rewrite hook did not receive the request's session")
}
}
func TestUpstreamGuard(t *testing.T) {
tests := []struct {
ip string