feat(egress): sandbox networks that reach only the relay
Each sandbox gets an internal bridge on which the host holds no address, so host services listening on every interface are unreachable; the relay is attached under a fixed alias and is the one way out. ProxyEnv gives a sandbox container its proxy URL and CA trust. The Docker integration test proves allowed egress works and that the proxy listener, host services and the internet are unreachable directly; it fails if the host-address inhibition is removed.
This commit is contained in:
1 parent
3394d36086
commit
cfa5c472dc
3 files changed
+460
No files matched your search
@@ -0,0 +1,162 @@
|
||||
package egress
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/api/types/network"
|
||||
"github.com/moby/moby/client"
|
||||
)
|
||||
|
||||
const (
|
||||
// RelayAlias is the name a sandbox container reaches its relay by.
|
||||
RelayAlias = "archie-egress"
|
||||
// RelayPort is the port the relay listens on inside sandbox networks.
|
||||
RelayPort = 3128
|
||||
|
||||
relayLabel = "archie-egress-relay"
|
||||
sandboxLabel = "archie-egress-sandbox"
|
||||
)
|
||||
|
||||
// RelaySpec is how the relay container is started. Network is the
|
||||
// host-reachable network it joins to reach the proxy; Entrypoint and Cmd
|
||||
// run archie-agent relay forwarding to the proxy's address on that network.
|
||||
type RelaySpec struct {
|
||||
// Name is the relay container's name. It also scopes the labels Start
|
||||
// prunes by, so two Networks on one host never remove each other's.
|
||||
Name string
|
||||
Image string
|
||||
Entrypoint []string
|
||||
Cmd []string
|
||||
Network string
|
||||
// Binds are host paths in Docker's "src:dst[:opts]" form. The relay
|
||||
// needs none when its image carries archie-agent.
|
||||
Binds []string
|
||||
}
|
||||
|
||||
// Networks provisions sandbox networks: internal bridges on which the host
|
||||
// holds no address, so the only reachable endpoint is the relay. Every
|
||||
// other host service, including ones listening on all interfaces, is out of
|
||||
// reach.
|
||||
type Networks struct {
|
||||
cli *client.Client
|
||||
relay RelaySpec
|
||||
}
|
||||
|
||||
func NewNetworks(cli *client.Client, relay RelaySpec) *Networks {
|
||||
return &Networks{cli: cli, relay: relay}
|
||||
}
|
||||
|
||||
// Start removes sandbox networks a previous daemon left behind and starts a
|
||||
// fresh relay. Work interrupted by a restart is requeued, so no live sandbox
|
||||
// depends on the old ones.
|
||||
func (n *Networks) Start(ctx context.Context) error {
|
||||
if err := n.removeRelay(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
list, err := n.cli.NetworkList(ctx, client.NetworkListOptions{Filters: client.Filters{}.Add("label", sandboxLabel+"="+n.relay.Name)})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list sandbox networks: %w", err)
|
||||
}
|
||||
for _, stale := range list.Items {
|
||||
if _, err := n.cli.NetworkRemove(ctx, stale.ID, client.NetworkRemoveOptions{}); err != nil {
|
||||
return fmt.Errorf("remove stale sandbox network %s: %w", stale.Name, err)
|
||||
}
|
||||
}
|
||||
created, err := n.cli.ContainerCreate(ctx, client.ContainerCreateOptions{
|
||||
Name: n.relay.Name,
|
||||
Config: &container.Config{
|
||||
Image: n.relay.Image,
|
||||
Entrypoint: n.relay.Entrypoint,
|
||||
Cmd: n.relay.Cmd,
|
||||
Labels: map[string]string{relayLabel: n.relay.Name},
|
||||
},
|
||||
HostConfig: &container.HostConfig{
|
||||
NetworkMode: container.NetworkMode(n.relay.Network),
|
||||
Binds: n.relay.Binds,
|
||||
RestartPolicy: container.RestartPolicy{Name: container.RestartPolicyUnlessStopped},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("create egress relay: %w", err)
|
||||
}
|
||||
if _, err := n.cli.ContainerStart(ctx, created.ID, client.ContainerStartOptions{}); err != nil {
|
||||
return errors.Join(fmt.Errorf("start egress relay: %w", err), n.removeRelay(context.WithoutCancel(ctx)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close removes the relay. Sandbox networks are removed by their owners.
|
||||
func (n *Networks) Close(ctx context.Context) error { return n.removeRelay(ctx) }
|
||||
|
||||
func (n *Networks) removeRelay(ctx context.Context) error {
|
||||
list, err := n.cli.ContainerList(ctx, client.ContainerListOptions{
|
||||
All: true,
|
||||
Filters: client.Filters{}.Add("label", relayLabel+"="+n.relay.Name),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list egress relays: %w", err)
|
||||
}
|
||||
for _, c := range list.Items {
|
||||
if _, err := n.cli.ContainerRemove(ctx, c.ID, client.ContainerRemoveOptions{Force: true}); err != nil {
|
||||
return fmt.Errorf("remove egress relay: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create makes one sandbox's network and attaches the relay to it.
|
||||
func (n *Networks) Create(ctx context.Context, name string) error {
|
||||
if _, err := n.cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
|
||||
Driver: "bridge",
|
||||
Internal: true,
|
||||
// With no host address on the bridge, nothing on the host is
|
||||
// routable from the sandbox; internal alone still exposes every
|
||||
// host service through the gateway.
|
||||
Options: map[string]string{"com.docker.network.bridge.inhibit_ipv4": "true"},
|
||||
Labels: map[string]string{sandboxLabel: n.relay.Name},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("create sandbox network %s: %w", name, err)
|
||||
}
|
||||
if _, err := n.cli.NetworkConnect(ctx, name, client.NetworkConnectOptions{
|
||||
Container: n.relay.Name,
|
||||
EndpointConfig: &network.EndpointSettings{Aliases: []string{RelayAlias}},
|
||||
}); err != nil {
|
||||
return errors.Join(fmt.Errorf("attach egress relay to %s: %w", name, err), n.Remove(context.WithoutCancel(ctx), name))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove detaches the relay from a sandbox network and deletes it.
|
||||
func (n *Networks) Remove(ctx context.Context, name string) error {
|
||||
_, detachErr := n.cli.NetworkDisconnect(ctx, name, client.NetworkDisconnectOptions{Container: n.relay.Name, Force: true})
|
||||
if _, err := n.cli.NetworkRemove(ctx, name, client.NetworkRemoveOptions{}); err != nil {
|
||||
return errors.Join(detachErr, fmt.Errorf("remove sandbox network %s: %w", name, err))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProxyEnv is the environment a sandbox container needs to send its egress
|
||||
// through the relay under session token and to trust the daemon CA mounted
|
||||
// at caPath. Both spellings are set because tools disagree on case.
|
||||
func ProxyEnv(token, caPath string) []string {
|
||||
proxy := (&url.URL{
|
||||
Scheme: "http",
|
||||
User: url.UserPassword(proxyUser, token),
|
||||
Host: net.JoinHostPort(RelayAlias, strconv.Itoa(RelayPort)),
|
||||
}).String()
|
||||
env := []string{"NO_PROXY=", "no_proxy="}
|
||||
for _, name := range []string{"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY"} {
|
||||
env = append(env, name+"="+proxy, strings.ToLower(name)+"="+proxy)
|
||||
}
|
||||
for _, name := range []string{"SSL_CERT_FILE", "NODE_EXTRA_CA_CERTS", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"} {
|
||||
env = append(env, name+"="+caPath)
|
||||
}
|
||||
return env
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
package egress
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/docker/sandbox-kit-spec/v3/spec"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/client"
|
||||
)
|
||||
|
||||
const probeImage = "alpine:3"
|
||||
|
||||
func buildStatic(t *testing.T, dir, pkg, name string) {
|
||||
t.Helper()
|
||||
cmd := exec.CommandContext(t.Context(), "go", "build", "-o", filepath.Join(dir, name), pkg)
|
||||
cmd.Env = append(os.Environ(), "CGO_ENABLED=0")
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("build %s: %v\n%s", pkg, err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func ensureImage(t *testing.T, cli *client.Client, ref string) {
|
||||
t.Helper()
|
||||
if _, err := cli.ImageInspect(t.Context(), ref); err == nil {
|
||||
return
|
||||
}
|
||||
rc, err := cli.ImagePull(t.Context(), ref, client.ImagePullOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("pull %s: %v", ref, err)
|
||||
}
|
||||
defer rc.Close()
|
||||
_, _ = io.Copy(io.Discard, rc)
|
||||
}
|
||||
|
||||
func bridgeGateway(t *testing.T, cli *client.Client) string {
|
||||
t.Helper()
|
||||
res, err := cli.NetworkInspect(t.Context(), "bridge", client.NetworkInspectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range res.Network.IPAM.Config {
|
||||
if c.Gateway.IsValid() && c.Gateway.Is4() {
|
||||
return c.Gateway.String()
|
||||
}
|
||||
}
|
||||
t.Fatal("default bridge has no IPv4 gateway")
|
||||
return ""
|
||||
}
|
||||
|
||||
// runProbe runs egressclient in a container on the sandbox network and
|
||||
// returns one result line per probe.
|
||||
func runProbe(t *testing.T, cli *client.Client, networkName, bin, caPath, token string, probes ...string) []string {
|
||||
t.Helper()
|
||||
outDir := t.TempDir()
|
||||
if err := os.Chmod(outDir, 0o777); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
created, err := cli.ContainerCreate(t.Context(), client.ContainerCreateOptions{
|
||||
Config: &container.Config{
|
||||
Image: probeImage,
|
||||
Entrypoint: []string{"/probe/egressclient"},
|
||||
Cmd: append([]string{"/out/result"}, probes...),
|
||||
Env: ProxyEnv(token, "/ca/ca.pem"),
|
||||
},
|
||||
HostConfig: &container.HostConfig{
|
||||
NetworkMode: container.NetworkMode(networkName),
|
||||
// z relabels for SELinux hosts, which otherwise refuse the
|
||||
// container access to freshly created host files.
|
||||
Binds: []string{bin + ":/probe:ro,z", caPath + ":/ca:ro,z", outDir + ":/out:z"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = cli.ContainerRemove(context.WithoutCancel(t.Context()), created.ID, client.ContainerRemoveOptions{Force: true})
|
||||
}()
|
||||
if _, err := cli.ContainerStart(t.Context(), created.ID, client.ContainerStartOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wait := cli.ContainerWait(t.Context(), created.ID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
|
||||
select {
|
||||
case <-wait.Result:
|
||||
case err := <-wait.Error:
|
||||
t.Fatal(err)
|
||||
case <-time.After(90 * time.Second):
|
||||
t.Fatal("probe container did not finish")
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(outDir, "result"))
|
||||
if err != nil {
|
||||
t.Fatalf("probe wrote no result: %v", err)
|
||||
}
|
||||
return strings.Split(strings.TrimSpace(string(raw)), "\n")
|
||||
}
|
||||
|
||||
// TestSandboxNetworkReachesOnlyTheProxy runs real containers: a sandbox on
|
||||
// an isolated network, the relay, and the proxy on the host. It proves the
|
||||
// PRD's egress properties end to end rather than by construction.
|
||||
func TestSandboxNetworkReachesOnlyTheProxy(t *testing.T) {
|
||||
cli, err := client.New(client.FromEnv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = cli.Close() })
|
||||
ensureImage(t, cli, probeImage)
|
||||
|
||||
bin := t.TempDir()
|
||||
buildStatic(t, bin, "github.com/samcharles93/archie-core/cmd/archie-agent", "archie-agent")
|
||||
buildStatic(t, bin, "./testdata/egressclient", "egressclient")
|
||||
|
||||
upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.WriteString(w, "upstream:"+r.Host+r.URL.Path)
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
roots := x509.NewCertPool()
|
||||
roots.AddCert(upstream.Certificate())
|
||||
|
||||
caDir := t.TempDir()
|
||||
ca, err := LoadOrCreateCA(caDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
proxy := NewProxy(ca, ProxyOptions{
|
||||
UpstreamRoots: roots,
|
||||
Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
if addr != "api.example.com:443" {
|
||||
return nil, fmt.Errorf("unrouted upstream %s", addr)
|
||||
}
|
||||
return (&net.Dialer{}).DialContext(ctx, network, upstream.Listener.Addr().String())
|
||||
},
|
||||
})
|
||||
gateway := bridgeGateway(t, cli)
|
||||
ln, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", net.JoinHostPort(gateway, "0"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
server := &http.Server{Handler: proxy, ReadHeaderTimeout: 10 * time.Second}
|
||||
go func() { _ = server.Serve(ln) }()
|
||||
t.Cleanup(func() { _ = server.Close() })
|
||||
proxyAddr := ln.Addr().String()
|
||||
|
||||
name := fmt.Sprintf("archie-egress-test-%d", time.Now().UnixNano())
|
||||
networks := NewNetworks(cli, RelaySpec{
|
||||
Name: name + "-relay",
|
||||
Image: probeImage,
|
||||
Entrypoint: []string{"/bin/archie-agent"},
|
||||
Cmd: []string{"relay", "-listen", ":" + strconv.Itoa(RelayPort), "-to", proxyAddr},
|
||||
Network: "bridge",
|
||||
Binds: []string{filepath.Join(bin, "archie-agent") + ":/bin/archie-agent:ro,z"},
|
||||
})
|
||||
if err := networks.Start(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = networks.Close(context.WithoutCancel(t.Context())) })
|
||||
if err := networks.Create(t.Context(), name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = networks.Remove(context.WithoutCancel(t.Context()), name) })
|
||||
|
||||
// A host service listening on every interface, the way NATS and
|
||||
// Postgres do. The sandbox must not reach it by any address.
|
||||
hostService, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", ":0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = hostService.Close() })
|
||||
go func() {
|
||||
for {
|
||||
c, err := hostService.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = c.Close()
|
||||
}
|
||||
}()
|
||||
_, hostPort, err := net.SplitHostPort(hostService.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sandboxNet, err := cli.NetworkInspect(t.Context(), name, client.NetworkInspectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(sandboxNet.Network.IPAM.Config) == 0 || !sandboxNet.Network.IPAM.Config[0].Subnet.IsValid() {
|
||||
t.Fatal("sandbox network has no subnet")
|
||||
}
|
||||
// The address a bridge gateway would hold: the subnet's first host. An
|
||||
// isolated sandbox network leaves it unassigned.
|
||||
sandboxGateway := sandboxNet.Network.IPAM.Config[0].Subnet.Masked().Addr().Next().String()
|
||||
|
||||
session, err := proxy.Register(&spec.PhasedNetwork{Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
session.EnterRuntime()
|
||||
|
||||
results := runProbe(t, cli, name, bin, caDir, session.Token(),
|
||||
"get", "https://api.example.com/ok",
|
||||
"get", "https://denied.example.com/",
|
||||
"dial", proxyAddr,
|
||||
"dial", net.JoinHostPort(sandboxGateway, hostPort),
|
||||
"dial", net.JoinHostPort(gateway, hostPort),
|
||||
"dial", "1.1.1.1:443",
|
||||
)
|
||||
want := []struct{ prefix, why string }{
|
||||
{"OK 200 upstream:api.example.com/ok", "an allowed host is reachable through the relay and proxy"},
|
||||
{"ERR", "a host outside the policy is refused"},
|
||||
{"BLOCKED", "the proxy's own listener is unreachable except through the relay"},
|
||||
{"BLOCKED", "a host service on all interfaces is unreachable through the sandbox's own gateway address"},
|
||||
{"BLOCKED", "a host service on all interfaces is unreachable through the docker bridge"},
|
||||
{"BLOCKED", "the internet is unreachable directly"},
|
||||
}
|
||||
if len(results) != len(want) {
|
||||
t.Fatalf("probe returned %d results, want %d: %q", len(results), len(want), results)
|
||||
}
|
||||
for i, w := range want {
|
||||
if !strings.HasPrefix(results[i], w.prefix) {
|
||||
t.Errorf("%s: got %q, want prefix %q", w.why, results[i], w.prefix)
|
||||
}
|
||||
}
|
||||
|
||||
wrong := runProbe(t, cli, name, bin, caDir, "not-a-token", "get", "https://api.example.com/ok")
|
||||
if len(wrong) != 1 || !strings.HasPrefix(wrong[0], "ERR") {
|
||||
t.Errorf("a sandbox with the wrong token reached upstream: %q", wrong)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
// Command egressclient runs inside a sandbox container in the egress
|
||||
// integration test. Each argument pair is a probe; one result line per
|
||||
// probe goes to the file named by the first argument.
|
||||
//
|
||||
// get <url> OK <status> <body> | ERR <error>
|
||||
// dial <host:port> OPEN | BLOCKED
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
out, err := os.Create(os.Args[1])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer out.Close()
|
||||
c := &http.Client{Timeout: 10 * time.Second, Transport: &http.Transport{Proxy: http.ProxyFromEnvironment}}
|
||||
args := os.Args[2:]
|
||||
for i := 0; i+1 < len(args); i += 2 {
|
||||
verb, target := args[i], args[i+1]
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
switch verb {
|
||||
case "get":
|
||||
fmt.Fprintln(out, get(ctx, c, target))
|
||||
case "dial":
|
||||
conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", target)
|
||||
if err != nil {
|
||||
fmt.Fprintln(out, "BLOCKED")
|
||||
} else {
|
||||
_ = conn.Close()
|
||||
fmt.Fprintln(out, "OPEN")
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func get(ctx context.Context, c *http.Client, target string) string {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||||
if err != nil {
|
||||
return "ERR " + err.Error()
|
||||
}
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
return "ERR " + strings.ReplaceAll(err.Error(), "\n", " ")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Sprintf("OK %d %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
Reference in new issue
Block a user