From 7b103b5af4e7d2cea55eb7ca3be564acaedd13bd Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 26 Sep 2026 02:38:12 +1000 Subject: [PATCH] feat(egress): sandbox networks that reach only the relay Each sandbox gets an internal bridge on which the host holds no address, so host services listening on every interface are unreachable; the relay is attached under a fixed alias and is the one way out. ProxyEnv gives a sandbox container its proxy URL and CA trust. The Docker integration test proves allowed egress works and that the proxy listener, host services and the internet are unreachable directly; it fails if the host-address inhibition is removed. --- internal/infrastructure/egress/networks.go | 162 ++++++++++++ .../infrastructure/egress/networks_test.go | 239 ++++++++++++++++++ .../egress/testdata/egressclient/main.go | 59 +++++ 3 files changed, 460 insertions(+) create mode 100644 internal/infrastructure/egress/networks.go create mode 100644 internal/infrastructure/egress/networks_test.go create mode 100644 internal/infrastructure/egress/testdata/egressclient/main.go diff --git a/internal/infrastructure/egress/networks.go b/internal/infrastructure/egress/networks.go new file mode 100644 index 000000000..39cf1fd70 --- /dev/null +++ b/internal/infrastructure/egress/networks.go @@ -0,0 +1,162 @@ +package egress + +import ( + "context" + "errors" + "fmt" + "net" + "net/url" + "strconv" + "strings" + + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/network" + "github.com/moby/moby/client" +) + +const ( + // RelayAlias is the name a sandbox container reaches its relay by. + RelayAlias = "archie-egress" + // RelayPort is the port the relay listens on inside sandbox networks. + RelayPort = 3128 + + relayLabel = "archie-egress-relay" + sandboxLabel = "archie-egress-sandbox" +) + +// RelaySpec is how the relay container is started. Network is the +// host-reachable network it joins to reach the proxy; Entrypoint and Cmd +// run archie-agent relay forwarding to the proxy's address on that network. +type RelaySpec struct { + // Name is the relay container's name. It also scopes the labels Start + // prunes by, so two Networks on one host never remove each other's. + Name string + Image string + Entrypoint []string + Cmd []string + Network string + // Binds are host paths in Docker's "src:dst[:opts]" form. The relay + // needs none when its image carries archie-agent. + Binds []string +} + +// Networks provisions sandbox networks: internal bridges on which the host +// holds no address, so the only reachable endpoint is the relay. Every +// other host service, including ones listening on all interfaces, is out of +// reach. +type Networks struct { + cli *client.Client + relay RelaySpec +} + +func NewNetworks(cli *client.Client, relay RelaySpec) *Networks { + return &Networks{cli: cli, relay: relay} +} + +// Start removes sandbox networks a previous daemon left behind and starts a +// fresh relay. Work interrupted by a restart is requeued, so no live sandbox +// depends on the old ones. +func (n *Networks) Start(ctx context.Context) error { + if err := n.removeRelay(ctx); err != nil { + return err + } + list, err := n.cli.NetworkList(ctx, client.NetworkListOptions{Filters: client.Filters{}.Add("label", sandboxLabel+"="+n.relay.Name)}) + if err != nil { + return fmt.Errorf("list sandbox networks: %w", err) + } + for _, stale := range list.Items { + if _, err := n.cli.NetworkRemove(ctx, stale.ID, client.NetworkRemoveOptions{}); err != nil { + return fmt.Errorf("remove stale sandbox network %s: %w", stale.Name, err) + } + } + created, err := n.cli.ContainerCreate(ctx, client.ContainerCreateOptions{ + Name: n.relay.Name, + Config: &container.Config{ + Image: n.relay.Image, + Entrypoint: n.relay.Entrypoint, + Cmd: n.relay.Cmd, + Labels: map[string]string{relayLabel: n.relay.Name}, + }, + HostConfig: &container.HostConfig{ + NetworkMode: container.NetworkMode(n.relay.Network), + Binds: n.relay.Binds, + RestartPolicy: container.RestartPolicy{Name: container.RestartPolicyUnlessStopped}, + }, + }) + if err != nil { + return fmt.Errorf("create egress relay: %w", err) + } + if _, err := n.cli.ContainerStart(ctx, created.ID, client.ContainerStartOptions{}); err != nil { + return errors.Join(fmt.Errorf("start egress relay: %w", err), n.removeRelay(context.WithoutCancel(ctx))) + } + return nil +} + +// Close removes the relay. Sandbox networks are removed by their owners. +func (n *Networks) Close(ctx context.Context) error { return n.removeRelay(ctx) } + +func (n *Networks) removeRelay(ctx context.Context) error { + list, err := n.cli.ContainerList(ctx, client.ContainerListOptions{ + All: true, + Filters: client.Filters{}.Add("label", relayLabel+"="+n.relay.Name), + }) + if err != nil { + return fmt.Errorf("list egress relays: %w", err) + } + for _, c := range list.Items { + if _, err := n.cli.ContainerRemove(ctx, c.ID, client.ContainerRemoveOptions{Force: true}); err != nil { + return fmt.Errorf("remove egress relay: %w", err) + } + } + return nil +} + +// Create makes one sandbox's network and attaches the relay to it. +func (n *Networks) Create(ctx context.Context, name string) error { + if _, err := n.cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{ + Driver: "bridge", + Internal: true, + // With no host address on the bridge, nothing on the host is + // routable from the sandbox; internal alone still exposes every + // host service through the gateway. + Options: map[string]string{"com.docker.network.bridge.inhibit_ipv4": "true"}, + Labels: map[string]string{sandboxLabel: n.relay.Name}, + }); err != nil { + return fmt.Errorf("create sandbox network %s: %w", name, err) + } + if _, err := n.cli.NetworkConnect(ctx, name, client.NetworkConnectOptions{ + Container: n.relay.Name, + EndpointConfig: &network.EndpointSettings{Aliases: []string{RelayAlias}}, + }); err != nil { + return errors.Join(fmt.Errorf("attach egress relay to %s: %w", name, err), n.Remove(context.WithoutCancel(ctx), name)) + } + return nil +} + +// Remove detaches the relay from a sandbox network and deletes it. +func (n *Networks) Remove(ctx context.Context, name string) error { + _, detachErr := n.cli.NetworkDisconnect(ctx, name, client.NetworkDisconnectOptions{Container: n.relay.Name, Force: true}) + if _, err := n.cli.NetworkRemove(ctx, name, client.NetworkRemoveOptions{}); err != nil { + return errors.Join(detachErr, fmt.Errorf("remove sandbox network %s: %w", name, err)) + } + return nil +} + +// ProxyEnv is the environment a sandbox container needs to send its egress +// through the relay under session token and to trust the daemon CA mounted +// at caPath. Both spellings are set because tools disagree on case. +func ProxyEnv(token, caPath string) []string { + proxy := (&url.URL{ + Scheme: "http", + User: url.UserPassword(proxyUser, token), + Host: net.JoinHostPort(RelayAlias, strconv.Itoa(RelayPort)), + }).String() + env := []string{"NO_PROXY=", "no_proxy="} + for _, name := range []string{"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY"} { + env = append(env, name+"="+proxy, strings.ToLower(name)+"="+proxy) + } + for _, name := range []string{"SSL_CERT_FILE", "NODE_EXTRA_CA_CERTS", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"} { + env = append(env, name+"="+caPath) + } + return env +} diff --git a/internal/infrastructure/egress/networks_test.go b/internal/infrastructure/egress/networks_test.go new file mode 100644 index 000000000..00631aa64 --- /dev/null +++ b/internal/infrastructure/egress/networks_test.go @@ -0,0 +1,239 @@ +package egress + +import ( + "context" + "crypto/x509" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/docker/sandbox-kit-spec/v3/spec" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/client" +) + +const probeImage = "alpine:3" + +func buildStatic(t *testing.T, dir, pkg, name string) { + t.Helper() + cmd := exec.CommandContext(t.Context(), "go", "build", "-o", filepath.Join(dir, name), pkg) + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build %s: %v\n%s", pkg, err, out) + } +} + +func ensureImage(t *testing.T, cli *client.Client, ref string) { + t.Helper() + if _, err := cli.ImageInspect(t.Context(), ref); err == nil { + return + } + rc, err := cli.ImagePull(t.Context(), ref, client.ImagePullOptions{}) + if err != nil { + t.Fatalf("pull %s: %v", ref, err) + } + defer rc.Close() + _, _ = io.Copy(io.Discard, rc) +} + +func bridgeGateway(t *testing.T, cli *client.Client) string { + t.Helper() + res, err := cli.NetworkInspect(t.Context(), "bridge", client.NetworkInspectOptions{}) + if err != nil { + t.Fatal(err) + } + for _, c := range res.Network.IPAM.Config { + if c.Gateway.IsValid() && c.Gateway.Is4() { + return c.Gateway.String() + } + } + t.Fatal("default bridge has no IPv4 gateway") + return "" +} + +// runProbe runs egressclient in a container on the sandbox network and +// returns one result line per probe. +func runProbe(t *testing.T, cli *client.Client, networkName, bin, caPath, token string, probes ...string) []string { + t.Helper() + outDir := t.TempDir() + if err := os.Chmod(outDir, 0o777); err != nil { + t.Fatal(err) + } + created, err := cli.ContainerCreate(t.Context(), client.ContainerCreateOptions{ + Config: &container.Config{ + Image: probeImage, + Entrypoint: []string{"/probe/egressclient"}, + Cmd: append([]string{"/out/result"}, probes...), + Env: ProxyEnv(token, "/ca/ca.pem"), + }, + HostConfig: &container.HostConfig{ + NetworkMode: container.NetworkMode(networkName), + // z relabels for SELinux hosts, which otherwise refuse the + // container access to freshly created host files. + Binds: []string{bin + ":/probe:ro,z", caPath + ":/ca:ro,z", outDir + ":/out:z"}, + }, + }) + if err != nil { + t.Fatal(err) + } + defer func() { + _, _ = cli.ContainerRemove(context.WithoutCancel(t.Context()), created.ID, client.ContainerRemoveOptions{Force: true}) + }() + if _, err := cli.ContainerStart(t.Context(), created.ID, client.ContainerStartOptions{}); err != nil { + t.Fatal(err) + } + wait := cli.ContainerWait(t.Context(), created.ID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}) + select { + case <-wait.Result: + case err := <-wait.Error: + t.Fatal(err) + case <-time.After(90 * time.Second): + t.Fatal("probe container did not finish") + } + raw, err := os.ReadFile(filepath.Join(outDir, "result")) + if err != nil { + t.Fatalf("probe wrote no result: %v", err) + } + return strings.Split(strings.TrimSpace(string(raw)), "\n") +} + +// TestSandboxNetworkReachesOnlyTheProxy runs real containers: a sandbox on +// an isolated network, the relay, and the proxy on the host. It proves the +// PRD's egress properties end to end rather than by construction. +func TestSandboxNetworkReachesOnlyTheProxy(t *testing.T) { + cli, err := client.New(client.FromEnv) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cli.Close() }) + ensureImage(t, cli, probeImage) + + bin := t.TempDir() + buildStatic(t, bin, "github.com/samcharles93/archie-core/cmd/archie-agent", "archie-agent") + buildStatic(t, bin, "./testdata/egressclient", "egressclient") + + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "upstream:"+r.Host+r.URL.Path) + })) + t.Cleanup(upstream.Close) + roots := x509.NewCertPool() + roots.AddCert(upstream.Certificate()) + + caDir := t.TempDir() + ca, err := LoadOrCreateCA(caDir) + if err != nil { + t.Fatal(err) + } + proxy := NewProxy(ca, ProxyOptions{ + UpstreamRoots: roots, + Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { + if addr != "api.example.com:443" { + return nil, fmt.Errorf("unrouted upstream %s", addr) + } + return (&net.Dialer{}).DialContext(ctx, network, upstream.Listener.Addr().String()) + }, + }) + gateway := bridgeGateway(t, cli) + ln, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", net.JoinHostPort(gateway, "0")) + if err != nil { + t.Fatal(err) + } + server := &http.Server{Handler: proxy, ReadHeaderTimeout: 10 * time.Second} + go func() { _ = server.Serve(ln) }() + t.Cleanup(func() { _ = server.Close() }) + proxyAddr := ln.Addr().String() + + name := fmt.Sprintf("archie-egress-test-%d", time.Now().UnixNano()) + networks := NewNetworks(cli, RelaySpec{ + Name: name + "-relay", + Image: probeImage, + Entrypoint: []string{"/bin/archie-agent"}, + Cmd: []string{"relay", "-listen", ":" + strconv.Itoa(RelayPort), "-to", proxyAddr}, + Network: "bridge", + Binds: []string{filepath.Join(bin, "archie-agent") + ":/bin/archie-agent:ro,z"}, + }) + if err := networks.Start(t.Context()); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = networks.Close(context.WithoutCancel(t.Context())) }) + if err := networks.Create(t.Context(), name); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = networks.Remove(context.WithoutCancel(t.Context()), name) }) + + // A host service listening on every interface, the way NATS and + // Postgres do. The sandbox must not reach it by any address. + hostService, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", ":0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = hostService.Close() }) + go func() { + for { + c, err := hostService.Accept() + if err != nil { + return + } + _ = c.Close() + } + }() + _, hostPort, err := net.SplitHostPort(hostService.Addr().String()) + if err != nil { + t.Fatal(err) + } + sandboxNet, err := cli.NetworkInspect(t.Context(), name, client.NetworkInspectOptions{}) + if err != nil { + t.Fatal(err) + } + if len(sandboxNet.Network.IPAM.Config) == 0 || !sandboxNet.Network.IPAM.Config[0].Subnet.IsValid() { + t.Fatal("sandbox network has no subnet") + } + // The address a bridge gateway would hold: the subnet's first host. An + // isolated sandbox network leaves it unassigned. + sandboxGateway := sandboxNet.Network.IPAM.Config[0].Subnet.Masked().Addr().Next().String() + + session, err := proxy.Register(&spec.PhasedNetwork{Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443"}}}) + if err != nil { + t.Fatal(err) + } + session.EnterRuntime() + + results := runProbe(t, cli, name, bin, caDir, session.Token(), + "get", "https://api.example.com/ok", + "get", "https://denied.example.com/", + "dial", proxyAddr, + "dial", net.JoinHostPort(sandboxGateway, hostPort), + "dial", net.JoinHostPort(gateway, hostPort), + "dial", "1.1.1.1:443", + ) + want := []struct{ prefix, why string }{ + {"OK 200 upstream:api.example.com/ok", "an allowed host is reachable through the relay and proxy"}, + {"ERR", "a host outside the policy is refused"}, + {"BLOCKED", "the proxy's own listener is unreachable except through the relay"}, + {"BLOCKED", "a host service on all interfaces is unreachable through the sandbox's own gateway address"}, + {"BLOCKED", "a host service on all interfaces is unreachable through the docker bridge"}, + {"BLOCKED", "the internet is unreachable directly"}, + } + if len(results) != len(want) { + t.Fatalf("probe returned %d results, want %d: %q", len(results), len(want), results) + } + for i, w := range want { + if !strings.HasPrefix(results[i], w.prefix) { + t.Errorf("%s: got %q, want prefix %q", w.why, results[i], w.prefix) + } + } + + wrong := runProbe(t, cli, name, bin, caDir, "not-a-token", "get", "https://api.example.com/ok") + if len(wrong) != 1 || !strings.HasPrefix(wrong[0], "ERR") { + t.Errorf("a sandbox with the wrong token reached upstream: %q", wrong) + } +} diff --git a/internal/infrastructure/egress/testdata/egressclient/main.go b/internal/infrastructure/egress/testdata/egressclient/main.go new file mode 100644 index 000000000..b4cd2a93d --- /dev/null +++ b/internal/infrastructure/egress/testdata/egressclient/main.go @@ -0,0 +1,59 @@ +// Command egressclient runs inside a sandbox container in the egress +// integration test. Each argument pair is a probe; one result line per +// probe goes to the file named by the first argument. +// +// get OK | ERR +// dial OPEN | BLOCKED +package main + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "os" + "strings" + "time" +) + +func main() { + out, err := os.Create(os.Args[1]) + if err != nil { + panic(err) + } + defer out.Close() + c := &http.Client{Timeout: 10 * time.Second, Transport: &http.Transport{Proxy: http.ProxyFromEnvironment}} + args := os.Args[2:] + for i := 0; i+1 < len(args); i += 2 { + verb, target := args[i], args[i+1] + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + switch verb { + case "get": + fmt.Fprintln(out, get(ctx, c, target)) + case "dial": + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", target) + if err != nil { + fmt.Fprintln(out, "BLOCKED") + } else { + _ = conn.Close() + fmt.Fprintln(out, "OPEN") + } + } + cancel() + } +} + +func get(ctx context.Context, c *http.Client, target string) string { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) + if err != nil { + return "ERR " + err.Error() + } + resp, err := c.Do(req) + if err != nil { + return "ERR " + strings.ReplaceAll(err.Error(), "\n", " ") + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + return fmt.Sprintf("OK %d %s", resp.StatusCode, strings.TrimSpace(string(body))) +}