From 266dac9e5c7cd2c0d9f12f67ee29caadd8504d5e Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 26 Sep 2026 02:46:40 +1000 Subject: [PATCH] feat(egress): inject Kit API-key credentials at the proxy A session carries its run and its Kit's credential@1 requests. The proxy applies apiKey inject rules only in the credential's phase and only to hosts its rule names, after the network policy has admitted the request, so a key never reaches a host its Kit did not declare. The real value comes from a Resolver that returns only secrets the run credential carries: a required credential that is unbound or unresolvable stops the request before anything is sent, and an optional one passes only when it is genuinely unbound. Named keys appear in the sandbox as a sentinel; inject-only keys have no presence. --- internal/infrastructure/egress/credentials.go | 117 +++++++++++ .../infrastructure/egress/credentials_test.go | 185 ++++++++++++++++++ .../infrastructure/egress/networks_test.go | 2 +- internal/infrastructure/egress/proxy.go | 51 +++-- internal/infrastructure/egress/proxy_test.go | 57 ++---- 5 files changed, 351 insertions(+), 61 deletions(-) create mode 100644 internal/infrastructure/egress/credentials.go create mode 100644 internal/infrastructure/egress/credentials_test.go diff --git a/internal/infrastructure/egress/credentials.go b/internal/infrastructure/egress/credentials.go new file mode 100644 index 000000000..1efa9a24b --- /dev/null +++ b/internal/infrastructure/egress/credentials.go @@ -0,0 +1,117 @@ +package egress + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/docker/sandbox-kit-spec/v3/spec" +) + +// Sentinel is the value a sandbox sees in place of a proxy-managed key, so +// it can tell a credential is wired without ever holding it. +const Sentinel = "archie-proxy-managed" + +// ErrUnbound reports that the run has no binding for a service. It is the +// only resolver error an optional credential may pass over; any other error +// stops the request. +var ErrUnbound = errors.New("egress: credential not bound") + +// Resolver supplies a credential's real value for one run. An implementation +// returns only secrets the run's credential carries, so what a Kit declares +// is always narrowed by what the org granted the run's identity. +type Resolver interface { + Resolve(ctx context.Context, run, service string) (string, error) +} + +// ResolverFunc adapts a function to Resolver. +type ResolverFunc func(ctx context.Context, run, service string) (string, error) + +func (f ResolverFunc) Resolve(ctx context.Context, run, service string) (string, error) { + return f(ctx, run, service) +} + +// injection is one compiled credential@1 apiKey inject rule. +type injection struct { + service string + required bool + runtime bool + domain pattern + header string + format string + basic bool + username string +} + +func compileInjections(creds []spec.CredentialCapability) []injection { + var out []injection + for _, c := range creds { + if c.APIKey == nil { + continue + } + for _, rule := range c.APIKey.Inject { + format := rule.Format + if format == "" { + format = "%s" + } + out = append(out, injection{ + service: c.Service, + required: c.Required, + runtime: c.Phase == "runtime", + domain: compilePattern(rule.Domain), + header: rule.Header, + format: format, + basic: strings.EqualFold(rule.Scheme, "basic"), + username: rule.Username, + }) + } + } + return out +} + +// inject presents the session's credentials on a request policy has already +// admitted. Only rules for this phase and this host apply, so a key is never +// sent anywhere its Kit did not name. +func (p *Proxy) inject(ctx context.Context, s *Session, r *http.Request, host string, port int) error { + atRuntime := s.atRun.Load() + for _, rule := range s.injections { + if rule.runtime != atRuntime || !rule.domain.matches(normalizeHost(host), port) { + continue + } + if p.resolver == nil { + if rule.required { + return fmt.Errorf("credential %q is required and no resolver is configured", rule.service) + } + continue + } + secret, err := p.resolver.Resolve(ctx, s.run, rule.service) + switch { + case errors.Is(err, ErrUnbound) && !rule.required: + continue + case errors.Is(err, ErrUnbound): + return fmt.Errorf("credential %q is required and not bound for this run", rule.service) + case err != nil: + return fmt.Errorf("resolve credential %q: %w", rule.service, err) + } + if rule.basic { + r.SetBasicAuth(rule.username, secret) + continue + } + r.Header.Set(rule.header, fmt.Sprintf(rule.format, secret)) + } + return nil +} + +// SentinelEnv is the environment a sandbox gets for its Kit's API keys: the +// sentinel under each named key. An inject-only key has no presence at all. +func SentinelEnv(creds []spec.CredentialCapability) []string { + var env []string + for _, c := range creds { + if c.APIKey != nil && c.APIKey.Name != "" { + env = append(env, c.APIKey.Name+"="+Sentinel) + } + } + return env +} diff --git a/internal/infrastructure/egress/credentials_test.go b/internal/infrastructure/egress/credentials_test.go new file mode 100644 index 000000000..e2805a5ce --- /dev/null +++ b/internal/infrastructure/egress/credentials_test.go @@ -0,0 +1,185 @@ +package egress + +import ( + "context" + "encoding/base64" + "errors" + "net/http" + "slices" + "testing" + + "github.com/docker/sandbox-kit-spec/v3/spec" +) + +func apiKey(service, phase string, required bool, name string, inject ...spec.Inject) spec.CredentialCapability { + return spec.CredentialCapability{ + Credential: spec.Credential{Service: service, Phase: phase, APIKey: &spec.APIKey{Name: name, ProxyManaged: true, Inject: inject}}, + Required: required, + } +} + +func (h *harness) registerWith(t *testing.T, creds ...spec.CredentialCapability) *Session { + t.Helper() + s, err := h.proxy.Register(SessionOptions{ + Run: "run-7", + Network: &spec.PhasedNetwork{ + Install: &spec.NetworkRules{Allow: []string{"install.example.com"}}, + Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443", "plain.example.com:80"}}, + }, + Credentials: creds, + }) + if err != nil { + t.Fatal(err) + } + return s +} + +func TestInjectsAnAPIKeyHeader(t *testing.T) { + h := newHarness(t) + h.bind("run-7/example", "real-secret") + s := h.registerWith(t, apiKey("example", "runtime", true, "EXAMPLE_KEY", + spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"})) + s.EnterRuntime() + req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://api.example.com/", nil) + req.Header.Set("Authorization", "Bearer "+Sentinel) + resp, err := h.client(s.Token()).Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if got := h.lastAuth.Load(); got != "Bearer real-secret" { + t.Fatalf("upstream Authorization %v, want the real key in place of the sentinel", got) + } +} + +func TestInjectsBasicAuth(t *testing.T) { + h := newHarness(t) + h.bind("run-7/example", "real-secret") + s := h.registerWith(t, apiKey("example", "runtime", true, "", + spec.Inject{Domain: "api.example.com", Scheme: "basic", Username: "x-access-token"})) + s.EnterRuntime() + if _, _, err := get(t, h.client(s.Token()), "https://api.example.com/"); err != nil { + t.Fatal(err) + } + want := "Basic " + base64.StdEncoding.EncodeToString([]byte("x-access-token:real-secret")) + if got := h.lastAuth.Load(); got != want { + t.Fatalf("upstream Authorization %v, want %q", got, want) + } +} + +func TestInjectionIsScopedToItsDomainAndPhase(t *testing.T) { + h := newHarness(t) + h.bind("run-7/example", "real-secret") + h.bind("run-7/installer", "install-secret") + s := h.registerWith(t, + apiKey("example", "runtime", true, "EXAMPLE_KEY", spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"}), + apiKey("installer", "install", true, "", spec.Inject{Domain: "plain.example.com", Header: "X-Install", Format: "%s"}), + ) + s.EnterRuntime() + if _, _, err := get(t, h.client(s.Token()), "http://plain.example.com/"); err != nil { + t.Fatal(err) + } + if got := h.lastAuth.Load(); got != "" { + t.Fatalf("a key for api.example.com was sent to plain.example.com: %v", got) + } + if h.lastInstall.Load() != "" { + t.Fatal("an install-phase credential was injected at runtime") + } +} + +func TestAnUnboundCredential(t *testing.T) { + t.Run("required stops the request before it reaches upstream", func(t *testing.T) { + h := newHarness(t) + s := h.registerWith(t, apiKey("example", "runtime", true, "EXAMPLE_KEY", + spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"})) + s.EnterRuntime() + status, _, err := get(t, h.client(s.Token()), "https://api.example.com/") + if err != nil { + t.Fatal(err) + } + if status != http.StatusBadGateway || h.hits.Load() != 0 { + t.Fatalf("status %d with %d upstream hits; want 502 and none", status, h.hits.Load()) + } + }) + t.Run("optional passes without one", func(t *testing.T) { + h := newHarness(t) + s := h.registerWith(t, apiKey("example", "runtime", false, "EXAMPLE_KEY", + spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"})) + s.EnterRuntime() + status, _, err := get(t, h.client(s.Token()), "https://api.example.com/") + if err != nil || status != http.StatusOK { + t.Fatalf("got %d %v", status, err) + } + if got := h.lastAuth.Load(); got != "" { + t.Fatalf("upstream saw Authorization %v with no binding", got) + } + }) + t.Run("a resolver failure is never treated as unbound", func(t *testing.T) { + h := newHarness(t) + h.failResolves(errors.New("state store unreachable")) + s := h.registerWith(t, apiKey("example", "runtime", false, "EXAMPLE_KEY", + spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"})) + s.EnterRuntime() + status, _, err := get(t, h.client(s.Token()), "https://api.example.com/") + if err != nil { + t.Fatal(err) + } + if status != http.StatusBadGateway || h.hits.Load() != 0 { + t.Fatalf("status %d with %d upstream hits; want 502 and none", status, h.hits.Load()) + } + }) +} + +func TestTheResolverSeesTheRunAndService(t *testing.T) { + h := newHarness(t) + h.bind("run-7/example", "real-secret") + s := h.registerWith(t, apiKey("example", "runtime", true, "EXAMPLE_KEY", + spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"})) + s.EnterRuntime() + if _, _, err := get(t, h.client(s.Token()), "https://api.example.com/"); err != nil { + t.Fatal(err) + } + h.mu.Lock() + defer h.mu.Unlock() + if !slices.Contains(h.resolved, "run-7/example") { + t.Fatalf("resolver calls %v, want run-7/example", h.resolved) + } +} + +func TestSentinelEnv(t *testing.T) { + env := SentinelEnv([]spec.CredentialCapability{ + apiKey("named", "runtime", true, "NAMED_KEY", spec.Inject{Domain: "a.example.com", Header: "X", Format: "%s"}), + apiKey("injectonly", "runtime", true, "", spec.Inject{Domain: "b.example.com", Header: "X", Format: "%s"}), + }) + if !slices.Equal(env, []string{"NAMED_KEY=" + Sentinel}) { + t.Fatalf("SentinelEnv = %v; a named key gets the sentinel, an inject-only key gets nothing", env) + } +} + +// fakeResolver lets the harness stand in for the run credential's secrets. +func (h *harness) resolve(_ context.Context, run, service string) (string, error) { + key := run + "/" + service + h.mu.Lock() + defer h.mu.Unlock() + h.resolved = append(h.resolved, key) + if h.resolveErr != nil { + return "", h.resolveErr + } + v, ok := h.secrets[key] + if !ok { + return "", ErrUnbound + } + return v, nil +} + +func (h *harness) bind(key, value string) { + h.mu.Lock() + defer h.mu.Unlock() + h.secrets[key] = value +} + +func (h *harness) failResolves(err error) { + h.mu.Lock() + defer h.mu.Unlock() + h.resolveErr = err +} diff --git a/internal/infrastructure/egress/networks_test.go b/internal/infrastructure/egress/networks_test.go index 00631aa64..bb9bdba18 100644 --- a/internal/infrastructure/egress/networks_test.go +++ b/internal/infrastructure/egress/networks_test.go @@ -201,7 +201,7 @@ func TestSandboxNetworkReachesOnlyTheProxy(t *testing.T) { // isolated sandbox network leaves it unassigned. sandboxGateway := sandboxNet.Network.IPAM.Config[0].Subnet.Masked().Addr().Next().String() - session, err := proxy.Register(&spec.PhasedNetwork{Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443"}}}) + session, err := proxy.Register(SessionOptions{Run: "integration", Network: &spec.PhasedNetwork{Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443"}}}}) if err != nil { t.Fatal(err) } diff --git a/internal/infrastructure/egress/proxy.go b/internal/infrastructure/egress/proxy.go index ed7dd2077..9f69c8eb6 100644 --- a/internal/infrastructure/egress/proxy.go +++ b/internal/infrastructure/egress/proxy.go @@ -35,10 +35,12 @@ var cgnat = &net.IPNet{IP: net.IPv4(100, 64, 0, 0), Mask: net.CIDRMask(10, 32)} // its Kit's network policy. It starts in the install phase; the runner moves // it to runtime before the workload's entrypoint starts. type Session struct { - token string - install rules - runtime rules - atRun atomic.Bool + token string + run string + install rules + runtime rules + injections []injection + atRun atomic.Bool } // Token is the secret the container presents as its proxy password. @@ -61,6 +63,17 @@ func (s *Session) rules() rules { type ProxyOptions struct { UpstreamRoots *x509.CertPool Dial func(ctx context.Context, network, addr string) (net.Conn, error) + // Resolver supplies credentials for injection. Without one, a request + // that needs a required credential is refused. + Resolver Resolver +} + +// SessionOptions describe one container's run: the run credential it acts +// under, and its Kit's network policy and credential requests. +type SessionOptions struct { + Run string + Network *spec.PhasedNetwork + Credentials []spec.CredentialCapability } // Proxy is the egress proxy sandbox containers reach through their relay. @@ -71,9 +84,7 @@ type Proxy struct { ca *CA dial func(ctx context.Context, network, addr string) (net.Conn, error) transport *http.Transport - - // rewrite runs on every forwarded request after policy has admitted it. - rewrite func(*Session, *http.Request) error + resolver Resolver mu sync.RWMutex sessions map[string]*Session @@ -82,7 +93,7 @@ type Proxy struct { type sessionKey struct{} func NewProxy(ca *CA, opts ProxyOptions) *Proxy { - p := &Proxy{ca: ca, dial: opts.Dial, sessions: map[string]*Session{}} + p := &Proxy{ca: ca, dial: opts.Dial, resolver: opts.Resolver, sessions: map[string]*Session{}} p.transport = &http.Transport{ DialContext: p.dialUpstream, TLSClientConfig: &tls.Config{RootCAs: opts.UpstreamRoots, MinVersion: tls.VersionTLS12}, @@ -92,16 +103,16 @@ func NewProxy(ca *CA, opts ProxyOptions) *Proxy { return p } -// Register opens a session for one container under its Kit's policy. A nil -// policy allows no egress in either phase. -func (p *Proxy) Register(policy *spec.PhasedNetwork) (*Session, error) { +// Register opens a session for one container. A nil network policy allows +// no egress in either phase. +func (p *Proxy) Register(opts SessionOptions) (*Session, error) { raw := make([]byte, 32) if _, err := rand.Read(raw); err != nil { return nil, err } - s := &Session{token: hex.EncodeToString(raw)} - if policy != nil { - s.install, s.runtime = compileRules(policy.Install), compileRules(policy.Runtime) + s := &Session{token: hex.EncodeToString(raw), run: opts.Run, injections: compileInjections(opts.Credentials)} + if opts.Network != nil { + s.install, s.runtime = compileRules(opts.Network.Install), compileRules(opts.Network.Runtime) } p.mu.Lock() p.sessions[s.token] = s @@ -222,13 +233,11 @@ func (p *Proxy) serveConnect(w http.ResponseWriter, r *http.Request, s *Session) func (p *Proxy) forward(ctx context.Context, w http.ResponseWriter, r *http.Request, s *Session, scheme, host string, port int) { target := net.JoinHostPort(host, strconv.Itoa(port)) - // The hook runs before anything is sent, so a failed rewrite (a missing - // credential binding, say) never reaches upstream half-applied. - if p.rewrite != nil { - if err := p.rewrite(s, r); err != nil { - http.Error(w, "egress to "+target+": "+err.Error(), http.StatusBadGateway) - return - } + // Injection happens before anything is sent, so a credential that + // cannot be resolved never reaches upstream half-applied. + if err := p.inject(ctx, s, r, host, port); err != nil { + http.Error(w, "egress to "+target+": "+err.Error(), http.StatusBadGateway) + return } rp := &httputil.ReverseProxy{ Transport: p.transport, diff --git a/internal/infrastructure/egress/proxy_test.go b/internal/infrastructure/egress/proxy_test.go index 1081fe621..73869f725 100644 --- a/internal/infrastructure/egress/proxy_test.go +++ b/internal/infrastructure/egress/proxy_test.go @@ -10,31 +10,37 @@ import ( "net/http/httptest" "net/url" "strings" + "sync" "sync/atomic" "testing" - - "github.com/docker/sandbox-kit-spec/v3/spec" ) // harness wires a proxy to local upstreams: the proxy's dialer maps the // public names a policy uses onto httptest listeners, so the tests exercise // the real CONNECT, TLS termination and forwarding paths hermetically. type harness struct { - proxy *Proxy - server *httptest.Server - upstream *httptest.Server - plain *httptest.Server - hits atomic.Int32 - lastAuth atomic.Value - caPool *x509.CertPool + proxy *Proxy + server *httptest.Server + upstream *httptest.Server + plain *httptest.Server + hits atomic.Int32 + lastAuth atomic.Value + lastInstall atomic.Value + caPool *x509.CertPool + + mu sync.Mutex + secrets map[string]string + resolved []string + resolveErr error } func newHarness(t *testing.T) *harness { t.Helper() - h := &harness{} + h := &harness{secrets: map[string]string{}} handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { h.hits.Add(1) h.lastAuth.Store(r.Header.Get("Authorization")) + h.lastInstall.Store(r.Header.Get("X-Install")) _, _ = io.WriteString(w, "upstream:"+r.Host+r.URL.Path) }) h.upstream = httptest.NewTLSServer(handler) @@ -56,6 +62,7 @@ func newHarness(t *testing.T) *harness { } h.proxy = NewProxy(ca, ProxyOptions{ UpstreamRoots: upstreamRoots, + Resolver: ResolverFunc(h.resolve), Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { target, ok := routes[addr] if !ok { @@ -74,14 +81,7 @@ func newHarness(t *testing.T) *harness { func (h *harness) register(t *testing.T) *Session { t.Helper() - s, err := h.proxy.Register(&spec.PhasedNetwork{ - Install: &spec.NetworkRules{Allow: []string{"install.example.com"}}, - Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443", "plain.example.com:80"}}, - }) - if err != nil { - t.Fatal(err) - } - return s + return h.registerWith(t) } func (h *harness) client(token string) *http.Client { @@ -223,27 +223,6 @@ func TestProxyRefusesMismatchedSNI(t *testing.T) { } } -func TestProxyAppliesTheRewriteHook(t *testing.T) { - h := newHarness(t) - var seen atomic.Pointer[Session] - h.proxy.rewrite = func(s *Session, r *http.Request) error { - seen.Store(s) - r.Header.Set("Authorization", "Bearer injected") - return nil - } - s := h.register(t) - s.EnterRuntime() - if _, _, err := get(t, h.client(s.Token()), "https://api.example.com/"); err != nil { - t.Fatal(err) - } - if got := h.lastAuth.Load(); got != "Bearer injected" { - t.Fatalf("upstream Authorization %v, want the rewritten value", got) - } - if seen.Load() != s { - t.Fatal("rewrite hook did not receive the request's session") - } -} - func TestUpstreamGuard(t *testing.T) { tests := []struct { ip string