diff --git a/internal/infrastructure/egress/credentials.go b/internal/infrastructure/egress/credentials.go new file mode 100644 index 000000000..1efa9a24b --- /dev/null +++ b/internal/infrastructure/egress/credentials.go @@ -0,0 +1,117 @@ +package egress + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/docker/sandbox-kit-spec/v3/spec" +) + +// Sentinel is the value a sandbox sees in place of a proxy-managed key, so +// it can tell a credential is wired without ever holding it. +const Sentinel = "archie-proxy-managed" + +// ErrUnbound reports that the run has no binding for a service. It is the +// only resolver error an optional credential may pass over; any other error +// stops the request. +var ErrUnbound = errors.New("egress: credential not bound") + +// Resolver supplies a credential's real value for one run. An implementation +// returns only secrets the run's credential carries, so what a Kit declares +// is always narrowed by what the org granted the run's identity. +type Resolver interface { + Resolve(ctx context.Context, run, service string) (string, error) +} + +// ResolverFunc adapts a function to Resolver. +type ResolverFunc func(ctx context.Context, run, service string) (string, error) + +func (f ResolverFunc) Resolve(ctx context.Context, run, service string) (string, error) { + return f(ctx, run, service) +} + +// injection is one compiled credential@1 apiKey inject rule. +type injection struct { + service string + required bool + runtime bool + domain pattern + header string + format string + basic bool + username string +} + +func compileInjections(creds []spec.CredentialCapability) []injection { + var out []injection + for _, c := range creds { + if c.APIKey == nil { + continue + } + for _, rule := range c.APIKey.Inject { + format := rule.Format + if format == "" { + format = "%s" + } + out = append(out, injection{ + service: c.Service, + required: c.Required, + runtime: c.Phase == "runtime", + domain: compilePattern(rule.Domain), + header: rule.Header, + format: format, + basic: strings.EqualFold(rule.Scheme, "basic"), + username: rule.Username, + }) + } + } + return out +} + +// inject presents the session's credentials on a request policy has already +// admitted. Only rules for this phase and this host apply, so a key is never +// sent anywhere its Kit did not name. +func (p *Proxy) inject(ctx context.Context, s *Session, r *http.Request, host string, port int) error { + atRuntime := s.atRun.Load() + for _, rule := range s.injections { + if rule.runtime != atRuntime || !rule.domain.matches(normalizeHost(host), port) { + continue + } + if p.resolver == nil { + if rule.required { + return fmt.Errorf("credential %q is required and no resolver is configured", rule.service) + } + continue + } + secret, err := p.resolver.Resolve(ctx, s.run, rule.service) + switch { + case errors.Is(err, ErrUnbound) && !rule.required: + continue + case errors.Is(err, ErrUnbound): + return fmt.Errorf("credential %q is required and not bound for this run", rule.service) + case err != nil: + return fmt.Errorf("resolve credential %q: %w", rule.service, err) + } + if rule.basic { + r.SetBasicAuth(rule.username, secret) + continue + } + r.Header.Set(rule.header, fmt.Sprintf(rule.format, secret)) + } + return nil +} + +// SentinelEnv is the environment a sandbox gets for its Kit's API keys: the +// sentinel under each named key. An inject-only key has no presence at all. +func SentinelEnv(creds []spec.CredentialCapability) []string { + var env []string + for _, c := range creds { + if c.APIKey != nil && c.APIKey.Name != "" { + env = append(env, c.APIKey.Name+"="+Sentinel) + } + } + return env +} diff --git a/internal/infrastructure/egress/credentials_test.go b/internal/infrastructure/egress/credentials_test.go new file mode 100644 index 000000000..e2805a5ce --- /dev/null +++ b/internal/infrastructure/egress/credentials_test.go @@ -0,0 +1,185 @@ +package egress + +import ( + "context" + "encoding/base64" + "errors" + "net/http" + "slices" + "testing" + + "github.com/docker/sandbox-kit-spec/v3/spec" +) + +func apiKey(service, phase string, required bool, name string, inject ...spec.Inject) spec.CredentialCapability { + return spec.CredentialCapability{ + Credential: spec.Credential{Service: service, Phase: phase, APIKey: &spec.APIKey{Name: name, ProxyManaged: true, Inject: inject}}, + Required: required, + } +} + +func (h *harness) registerWith(t *testing.T, creds ...spec.CredentialCapability) *Session { + t.Helper() + s, err := h.proxy.Register(SessionOptions{ + Run: "run-7", + Network: &spec.PhasedNetwork{ + Install: &spec.NetworkRules{Allow: []string{"install.example.com"}}, + Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443", "plain.example.com:80"}}, + }, + Credentials: creds, + }) + if err != nil { + t.Fatal(err) + } + return s +} + +func TestInjectsAnAPIKeyHeader(t *testing.T) { + h := newHarness(t) + h.bind("run-7/example", "real-secret") + s := h.registerWith(t, apiKey("example", "runtime", true, "EXAMPLE_KEY", + spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"})) + s.EnterRuntime() + req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://api.example.com/", nil) + req.Header.Set("Authorization", "Bearer "+Sentinel) + resp, err := h.client(s.Token()).Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if got := h.lastAuth.Load(); got != "Bearer real-secret" { + t.Fatalf("upstream Authorization %v, want the real key in place of the sentinel", got) + } +} + +func TestInjectsBasicAuth(t *testing.T) { + h := newHarness(t) + h.bind("run-7/example", "real-secret") + s := h.registerWith(t, apiKey("example", "runtime", true, "", + spec.Inject{Domain: "api.example.com", Scheme: "basic", Username: "x-access-token"})) + s.EnterRuntime() + if _, _, err := get(t, h.client(s.Token()), "https://api.example.com/"); err != nil { + t.Fatal(err) + } + want := "Basic " + base64.StdEncoding.EncodeToString([]byte("x-access-token:real-secret")) + if got := h.lastAuth.Load(); got != want { + t.Fatalf("upstream Authorization %v, want %q", got, want) + } +} + +func TestInjectionIsScopedToItsDomainAndPhase(t *testing.T) { + h := newHarness(t) + h.bind("run-7/example", "real-secret") + h.bind("run-7/installer", "install-secret") + s := h.registerWith(t, + apiKey("example", "runtime", true, "EXAMPLE_KEY", spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"}), + apiKey("installer", "install", true, "", spec.Inject{Domain: "plain.example.com", Header: "X-Install", Format: "%s"}), + ) + s.EnterRuntime() + if _, _, err := get(t, h.client(s.Token()), "http://plain.example.com/"); err != nil { + t.Fatal(err) + } + if got := h.lastAuth.Load(); got != "" { + t.Fatalf("a key for api.example.com was sent to plain.example.com: %v", got) + } + if h.lastInstall.Load() != "" { + t.Fatal("an install-phase credential was injected at runtime") + } +} + +func TestAnUnboundCredential(t *testing.T) { + t.Run("required stops the request before it reaches upstream", func(t *testing.T) { + h := newHarness(t) + s := h.registerWith(t, apiKey("example", "runtime", true, "EXAMPLE_KEY", + spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"})) + s.EnterRuntime() + status, _, err := get(t, h.client(s.Token()), "https://api.example.com/") + if err != nil { + t.Fatal(err) + } + if status != http.StatusBadGateway || h.hits.Load() != 0 { + t.Fatalf("status %d with %d upstream hits; want 502 and none", status, h.hits.Load()) + } + }) + t.Run("optional passes without one", func(t *testing.T) { + h := newHarness(t) + s := h.registerWith(t, apiKey("example", "runtime", false, "EXAMPLE_KEY", + spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"})) + s.EnterRuntime() + status, _, err := get(t, h.client(s.Token()), "https://api.example.com/") + if err != nil || status != http.StatusOK { + t.Fatalf("got %d %v", status, err) + } + if got := h.lastAuth.Load(); got != "" { + t.Fatalf("upstream saw Authorization %v with no binding", got) + } + }) + t.Run("a resolver failure is never treated as unbound", func(t *testing.T) { + h := newHarness(t) + h.failResolves(errors.New("state store unreachable")) + s := h.registerWith(t, apiKey("example", "runtime", false, "EXAMPLE_KEY", + spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"})) + s.EnterRuntime() + status, _, err := get(t, h.client(s.Token()), "https://api.example.com/") + if err != nil { + t.Fatal(err) + } + if status != http.StatusBadGateway || h.hits.Load() != 0 { + t.Fatalf("status %d with %d upstream hits; want 502 and none", status, h.hits.Load()) + } + }) +} + +func TestTheResolverSeesTheRunAndService(t *testing.T) { + h := newHarness(t) + h.bind("run-7/example", "real-secret") + s := h.registerWith(t, apiKey("example", "runtime", true, "EXAMPLE_KEY", + spec.Inject{Domain: "api.example.com", Header: "Authorization", Format: "Bearer %s"})) + s.EnterRuntime() + if _, _, err := get(t, h.client(s.Token()), "https://api.example.com/"); err != nil { + t.Fatal(err) + } + h.mu.Lock() + defer h.mu.Unlock() + if !slices.Contains(h.resolved, "run-7/example") { + t.Fatalf("resolver calls %v, want run-7/example", h.resolved) + } +} + +func TestSentinelEnv(t *testing.T) { + env := SentinelEnv([]spec.CredentialCapability{ + apiKey("named", "runtime", true, "NAMED_KEY", spec.Inject{Domain: "a.example.com", Header: "X", Format: "%s"}), + apiKey("injectonly", "runtime", true, "", spec.Inject{Domain: "b.example.com", Header: "X", Format: "%s"}), + }) + if !slices.Equal(env, []string{"NAMED_KEY=" + Sentinel}) { + t.Fatalf("SentinelEnv = %v; a named key gets the sentinel, an inject-only key gets nothing", env) + } +} + +// fakeResolver lets the harness stand in for the run credential's secrets. +func (h *harness) resolve(_ context.Context, run, service string) (string, error) { + key := run + "/" + service + h.mu.Lock() + defer h.mu.Unlock() + h.resolved = append(h.resolved, key) + if h.resolveErr != nil { + return "", h.resolveErr + } + v, ok := h.secrets[key] + if !ok { + return "", ErrUnbound + } + return v, nil +} + +func (h *harness) bind(key, value string) { + h.mu.Lock() + defer h.mu.Unlock() + h.secrets[key] = value +} + +func (h *harness) failResolves(err error) { + h.mu.Lock() + defer h.mu.Unlock() + h.resolveErr = err +} diff --git a/internal/infrastructure/egress/networks_test.go b/internal/infrastructure/egress/networks_test.go index 00631aa64..bb9bdba18 100644 --- a/internal/infrastructure/egress/networks_test.go +++ b/internal/infrastructure/egress/networks_test.go @@ -201,7 +201,7 @@ func TestSandboxNetworkReachesOnlyTheProxy(t *testing.T) { // isolated sandbox network leaves it unassigned. sandboxGateway := sandboxNet.Network.IPAM.Config[0].Subnet.Masked().Addr().Next().String() - session, err := proxy.Register(&spec.PhasedNetwork{Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443"}}}) + session, err := proxy.Register(SessionOptions{Run: "integration", Network: &spec.PhasedNetwork{Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443"}}}}) if err != nil { t.Fatal(err) } diff --git a/internal/infrastructure/egress/proxy.go b/internal/infrastructure/egress/proxy.go index ed7dd2077..9f69c8eb6 100644 --- a/internal/infrastructure/egress/proxy.go +++ b/internal/infrastructure/egress/proxy.go @@ -35,10 +35,12 @@ var cgnat = &net.IPNet{IP: net.IPv4(100, 64, 0, 0), Mask: net.CIDRMask(10, 32)} // its Kit's network policy. It starts in the install phase; the runner moves // it to runtime before the workload's entrypoint starts. type Session struct { - token string - install rules - runtime rules - atRun atomic.Bool + token string + run string + install rules + runtime rules + injections []injection + atRun atomic.Bool } // Token is the secret the container presents as its proxy password. @@ -61,6 +63,17 @@ func (s *Session) rules() rules { type ProxyOptions struct { UpstreamRoots *x509.CertPool Dial func(ctx context.Context, network, addr string) (net.Conn, error) + // Resolver supplies credentials for injection. Without one, a request + // that needs a required credential is refused. + Resolver Resolver +} + +// SessionOptions describe one container's run: the run credential it acts +// under, and its Kit's network policy and credential requests. +type SessionOptions struct { + Run string + Network *spec.PhasedNetwork + Credentials []spec.CredentialCapability } // Proxy is the egress proxy sandbox containers reach through their relay. @@ -71,9 +84,7 @@ type Proxy struct { ca *CA dial func(ctx context.Context, network, addr string) (net.Conn, error) transport *http.Transport - - // rewrite runs on every forwarded request after policy has admitted it. - rewrite func(*Session, *http.Request) error + resolver Resolver mu sync.RWMutex sessions map[string]*Session @@ -82,7 +93,7 @@ type Proxy struct { type sessionKey struct{} func NewProxy(ca *CA, opts ProxyOptions) *Proxy { - p := &Proxy{ca: ca, dial: opts.Dial, sessions: map[string]*Session{}} + p := &Proxy{ca: ca, dial: opts.Dial, resolver: opts.Resolver, sessions: map[string]*Session{}} p.transport = &http.Transport{ DialContext: p.dialUpstream, TLSClientConfig: &tls.Config{RootCAs: opts.UpstreamRoots, MinVersion: tls.VersionTLS12}, @@ -92,16 +103,16 @@ func NewProxy(ca *CA, opts ProxyOptions) *Proxy { return p } -// Register opens a session for one container under its Kit's policy. A nil -// policy allows no egress in either phase. -func (p *Proxy) Register(policy *spec.PhasedNetwork) (*Session, error) { +// Register opens a session for one container. A nil network policy allows +// no egress in either phase. +func (p *Proxy) Register(opts SessionOptions) (*Session, error) { raw := make([]byte, 32) if _, err := rand.Read(raw); err != nil { return nil, err } - s := &Session{token: hex.EncodeToString(raw)} - if policy != nil { - s.install, s.runtime = compileRules(policy.Install), compileRules(policy.Runtime) + s := &Session{token: hex.EncodeToString(raw), run: opts.Run, injections: compileInjections(opts.Credentials)} + if opts.Network != nil { + s.install, s.runtime = compileRules(opts.Network.Install), compileRules(opts.Network.Runtime) } p.mu.Lock() p.sessions[s.token] = s @@ -222,13 +233,11 @@ func (p *Proxy) serveConnect(w http.ResponseWriter, r *http.Request, s *Session) func (p *Proxy) forward(ctx context.Context, w http.ResponseWriter, r *http.Request, s *Session, scheme, host string, port int) { target := net.JoinHostPort(host, strconv.Itoa(port)) - // The hook runs before anything is sent, so a failed rewrite (a missing - // credential binding, say) never reaches upstream half-applied. - if p.rewrite != nil { - if err := p.rewrite(s, r); err != nil { - http.Error(w, "egress to "+target+": "+err.Error(), http.StatusBadGateway) - return - } + // Injection happens before anything is sent, so a credential that + // cannot be resolved never reaches upstream half-applied. + if err := p.inject(ctx, s, r, host, port); err != nil { + http.Error(w, "egress to "+target+": "+err.Error(), http.StatusBadGateway) + return } rp := &httputil.ReverseProxy{ Transport: p.transport, diff --git a/internal/infrastructure/egress/proxy_test.go b/internal/infrastructure/egress/proxy_test.go index 1081fe621..73869f725 100644 --- a/internal/infrastructure/egress/proxy_test.go +++ b/internal/infrastructure/egress/proxy_test.go @@ -10,31 +10,37 @@ import ( "net/http/httptest" "net/url" "strings" + "sync" "sync/atomic" "testing" - - "github.com/docker/sandbox-kit-spec/v3/spec" ) // harness wires a proxy to local upstreams: the proxy's dialer maps the // public names a policy uses onto httptest listeners, so the tests exercise // the real CONNECT, TLS termination and forwarding paths hermetically. type harness struct { - proxy *Proxy - server *httptest.Server - upstream *httptest.Server - plain *httptest.Server - hits atomic.Int32 - lastAuth atomic.Value - caPool *x509.CertPool + proxy *Proxy + server *httptest.Server + upstream *httptest.Server + plain *httptest.Server + hits atomic.Int32 + lastAuth atomic.Value + lastInstall atomic.Value + caPool *x509.CertPool + + mu sync.Mutex + secrets map[string]string + resolved []string + resolveErr error } func newHarness(t *testing.T) *harness { t.Helper() - h := &harness{} + h := &harness{secrets: map[string]string{}} handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { h.hits.Add(1) h.lastAuth.Store(r.Header.Get("Authorization")) + h.lastInstall.Store(r.Header.Get("X-Install")) _, _ = io.WriteString(w, "upstream:"+r.Host+r.URL.Path) }) h.upstream = httptest.NewTLSServer(handler) @@ -56,6 +62,7 @@ func newHarness(t *testing.T) *harness { } h.proxy = NewProxy(ca, ProxyOptions{ UpstreamRoots: upstreamRoots, + Resolver: ResolverFunc(h.resolve), Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { target, ok := routes[addr] if !ok { @@ -74,14 +81,7 @@ func newHarness(t *testing.T) *harness { func (h *harness) register(t *testing.T) *Session { t.Helper() - s, err := h.proxy.Register(&spec.PhasedNetwork{ - Install: &spec.NetworkRules{Allow: []string{"install.example.com"}}, - Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443", "plain.example.com:80"}}, - }) - if err != nil { - t.Fatal(err) - } - return s + return h.registerWith(t) } func (h *harness) client(token string) *http.Client { @@ -223,27 +223,6 @@ func TestProxyRefusesMismatchedSNI(t *testing.T) { } } -func TestProxyAppliesTheRewriteHook(t *testing.T) { - h := newHarness(t) - var seen atomic.Pointer[Session] - h.proxy.rewrite = func(s *Session, r *http.Request) error { - seen.Store(s) - r.Header.Set("Authorization", "Bearer injected") - return nil - } - s := h.register(t) - s.EnterRuntime() - if _, _, err := get(t, h.client(s.Token()), "https://api.example.com/"); err != nil { - t.Fatal(err) - } - if got := h.lastAuth.Load(); got != "Bearer injected" { - t.Fatalf("upstream Authorization %v, want the rewritten value", got) - } - if seen.Load() != s { - t.Fatal("rewrite hook did not receive the request's session") - } -} - func TestUpstreamGuard(t *testing.T) { tests := []struct { ip string