feat(egress): sandbox networks that reach only the relay

Each sandbox gets an internal bridge on which the host holds no address,
so host services listening on every interface are unreachable; the relay
is attached under a fixed alias and is the one way out. ProxyEnv gives a
sandbox container its proxy URL and CA trust. The Docker integration test
proves allowed egress works and that the proxy listener, host services and
the internet are unreachable directly; it fails if the host-address
inhibition is removed.
This commit is contained in:
sam committed 2026-09-26 02:38:12 +10:00
1 parent 3394d36086
commit cfa5c472dc
3 files changed
+460

No files matched your search

+162
View File
@@ -0,0 +1,162 @@
package egress
import (
"context"
"errors"
"fmt"
"net"
"net/url"
"strconv"
"strings"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/network"
"github.com/moby/moby/client"
)
const (
// RelayAlias is the name a sandbox container reaches its relay by.
RelayAlias = "archie-egress"
// RelayPort is the port the relay listens on inside sandbox networks.
RelayPort = 3128
relayLabel = "archie-egress-relay"
sandboxLabel = "archie-egress-sandbox"
)
// RelaySpec is how the relay container is started. Network is the
// host-reachable network it joins to reach the proxy; Entrypoint and Cmd
// run archie-agent relay forwarding to the proxy's address on that network.
type RelaySpec struct {
// Name is the relay container's name. It also scopes the labels Start
// prunes by, so two Networks on one host never remove each other's.
Name string
Image string
Entrypoint []string
Cmd []string
Network string
// Binds are host paths in Docker's "src:dst[:opts]" form. The relay
// needs none when its image carries archie-agent.
Binds []string
}
// Networks provisions sandbox networks: internal bridges on which the host
// holds no address, so the only reachable endpoint is the relay. Every
// other host service, including ones listening on all interfaces, is out of
// reach.
type Networks struct {
cli *client.Client
relay RelaySpec
}
func NewNetworks(cli *client.Client, relay RelaySpec) *Networks {
return &Networks{cli: cli, relay: relay}
}
// Start removes sandbox networks a previous daemon left behind and starts a
// fresh relay. Work interrupted by a restart is requeued, so no live sandbox
// depends on the old ones.
func (n *Networks) Start(ctx context.Context) error {
if err := n.removeRelay(ctx); err != nil {
return err
}
list, err := n.cli.NetworkList(ctx, client.NetworkListOptions{Filters: client.Filters{}.Add("label", sandboxLabel+"="+n.relay.Name)})
if err != nil {
return fmt.Errorf("list sandbox networks: %w", err)
}
for _, stale := range list.Items {
if _, err := n.cli.NetworkRemove(ctx, stale.ID, client.NetworkRemoveOptions{}); err != nil {
return fmt.Errorf("remove stale sandbox network %s: %w", stale.Name, err)
}
}
created, err := n.cli.ContainerCreate(ctx, client.ContainerCreateOptions{
Name: n.relay.Name,
Config: &container.Config{
Image: n.relay.Image,
Entrypoint: n.relay.Entrypoint,
Cmd: n.relay.Cmd,
Labels: map[string]string{relayLabel: n.relay.Name},
},
HostConfig: &container.HostConfig{
NetworkMode: container.NetworkMode(n.relay.Network),
Binds: n.relay.Binds,
RestartPolicy: container.RestartPolicy{Name: container.RestartPolicyUnlessStopped},
},
})
if err != nil {
return fmt.Errorf("create egress relay: %w", err)
}
if _, err := n.cli.ContainerStart(ctx, created.ID, client.ContainerStartOptions{}); err != nil {
return errors.Join(fmt.Errorf("start egress relay: %w", err), n.removeRelay(context.WithoutCancel(ctx)))
}
return nil
}
// Close removes the relay. Sandbox networks are removed by their owners.
func (n *Networks) Close(ctx context.Context) error { return n.removeRelay(ctx) }
func (n *Networks) removeRelay(ctx context.Context) error {
list, err := n.cli.ContainerList(ctx, client.ContainerListOptions{
All: true,
Filters: client.Filters{}.Add("label", relayLabel+"="+n.relay.Name),
})
if err != nil {
return fmt.Errorf("list egress relays: %w", err)
}
for _, c := range list.Items {
if _, err := n.cli.ContainerRemove(ctx, c.ID, client.ContainerRemoveOptions{Force: true}); err != nil {
return fmt.Errorf("remove egress relay: %w", err)
}
}
return nil
}
// Create makes one sandbox's network and attaches the relay to it.
func (n *Networks) Create(ctx context.Context, name string) error {
if _, err := n.cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
Driver: "bridge",
Internal: true,
// With no host address on the bridge, nothing on the host is
// routable from the sandbox; internal alone still exposes every
// host service through the gateway.
Options: map[string]string{"com.docker.network.bridge.inhibit_ipv4": "true"},
Labels: map[string]string{sandboxLabel: n.relay.Name},
}); err != nil {
return fmt.Errorf("create sandbox network %s: %w", name, err)
}
if _, err := n.cli.NetworkConnect(ctx, name, client.NetworkConnectOptions{
Container: n.relay.Name,
EndpointConfig: &network.EndpointSettings{Aliases: []string{RelayAlias}},
}); err != nil {
return errors.Join(fmt.Errorf("attach egress relay to %s: %w", name, err), n.Remove(context.WithoutCancel(ctx), name))
}
return nil
}
// Remove detaches the relay from a sandbox network and deletes it.
func (n *Networks) Remove(ctx context.Context, name string) error {
_, detachErr := n.cli.NetworkDisconnect(ctx, name, client.NetworkDisconnectOptions{Container: n.relay.Name, Force: true})
if _, err := n.cli.NetworkRemove(ctx, name, client.NetworkRemoveOptions{}); err != nil {
return errors.Join(detachErr, fmt.Errorf("remove sandbox network %s: %w", name, err))
}
return nil
}
// ProxyEnv is the environment a sandbox container needs to send its egress
// through the relay under session token and to trust the daemon CA mounted
// at caPath. Both spellings are set because tools disagree on case.
func ProxyEnv(token, caPath string) []string {
proxy := (&url.URL{
Scheme: "http",
User: url.UserPassword(proxyUser, token),
Host: net.JoinHostPort(RelayAlias, strconv.Itoa(RelayPort)),
}).String()
env := []string{"NO_PROXY=", "no_proxy="}
for _, name := range []string{"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY"} {
env = append(env, name+"="+proxy, strings.ToLower(name)+"="+proxy)
}
for _, name := range []string{"SSL_CERT_FILE", "NODE_EXTRA_CA_CERTS", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"} {
env = append(env, name+"="+caPath)
}
return env
}
@@ -0,0 +1,239 @@
package egress
import (
"context"
"crypto/x509"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
"github.com/docker/sandbox-kit-spec/v3/spec"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/client"
)
const probeImage = "alpine:3"
func buildStatic(t *testing.T, dir, pkg, name string) {
t.Helper()
cmd := exec.CommandContext(t.Context(), "go", "build", "-o", filepath.Join(dir, name), pkg)
cmd.Env = append(os.Environ(), "CGO_ENABLED=0")
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("build %s: %v\n%s", pkg, err, out)
}
}
func ensureImage(t *testing.T, cli *client.Client, ref string) {
t.Helper()
if _, err := cli.ImageInspect(t.Context(), ref); err == nil {
return
}
rc, err := cli.ImagePull(t.Context(), ref, client.ImagePullOptions{})
if err != nil {
t.Fatalf("pull %s: %v", ref, err)
}
defer rc.Close()
_, _ = io.Copy(io.Discard, rc)
}
func bridgeGateway(t *testing.T, cli *client.Client) string {
t.Helper()
res, err := cli.NetworkInspect(t.Context(), "bridge", client.NetworkInspectOptions{})
if err != nil {
t.Fatal(err)
}
for _, c := range res.Network.IPAM.Config {
if c.Gateway.IsValid() && c.Gateway.Is4() {
return c.Gateway.String()
}
}
t.Fatal("default bridge has no IPv4 gateway")
return ""
}
// runProbe runs egressclient in a container on the sandbox network and
// returns one result line per probe.
func runProbe(t *testing.T, cli *client.Client, networkName, bin, caPath, token string, probes ...string) []string {
t.Helper()
outDir := t.TempDir()
if err := os.Chmod(outDir, 0o777); err != nil {
t.Fatal(err)
}
created, err := cli.ContainerCreate(t.Context(), client.ContainerCreateOptions{
Config: &container.Config{
Image: probeImage,
Entrypoint: []string{"/probe/egressclient"},
Cmd: append([]string{"/out/result"}, probes...),
Env: ProxyEnv(token, "/ca/ca.pem"),
},
HostConfig: &container.HostConfig{
NetworkMode: container.NetworkMode(networkName),
// z relabels for SELinux hosts, which otherwise refuse the
// container access to freshly created host files.
Binds: []string{bin + ":/probe:ro,z", caPath + ":/ca:ro,z", outDir + ":/out:z"},
},
})
if err != nil {
t.Fatal(err)
}
defer func() {
_, _ = cli.ContainerRemove(context.WithoutCancel(t.Context()), created.ID, client.ContainerRemoveOptions{Force: true})
}()
if _, err := cli.ContainerStart(t.Context(), created.ID, client.ContainerStartOptions{}); err != nil {
t.Fatal(err)
}
wait := cli.ContainerWait(t.Context(), created.ID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
select {
case <-wait.Result:
case err := <-wait.Error:
t.Fatal(err)
case <-time.After(90 * time.Second):
t.Fatal("probe container did not finish")
}
raw, err := os.ReadFile(filepath.Join(outDir, "result"))
if err != nil {
t.Fatalf("probe wrote no result: %v", err)
}
return strings.Split(strings.TrimSpace(string(raw)), "\n")
}
// TestSandboxNetworkReachesOnlyTheProxy runs real containers: a sandbox on
// an isolated network, the relay, and the proxy on the host. It proves the
// PRD's egress properties end to end rather than by construction.
func TestSandboxNetworkReachesOnlyTheProxy(t *testing.T) {
cli, err := client.New(client.FromEnv)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = cli.Close() })
ensureImage(t, cli, probeImage)
bin := t.TempDir()
buildStatic(t, bin, "github.com/samcharles93/archie-core/cmd/archie-agent", "archie-agent")
buildStatic(t, bin, "./testdata/egressclient", "egressclient")
upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.WriteString(w, "upstream:"+r.Host+r.URL.Path)
}))
t.Cleanup(upstream.Close)
roots := x509.NewCertPool()
roots.AddCert(upstream.Certificate())
caDir := t.TempDir()
ca, err := LoadOrCreateCA(caDir)
if err != nil {
t.Fatal(err)
}
proxy := NewProxy(ca, ProxyOptions{
UpstreamRoots: roots,
Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
if addr != "api.example.com:443" {
return nil, fmt.Errorf("unrouted upstream %s", addr)
}
return (&net.Dialer{}).DialContext(ctx, network, upstream.Listener.Addr().String())
},
})
gateway := bridgeGateway(t, cli)
ln, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", net.JoinHostPort(gateway, "0"))
if err != nil {
t.Fatal(err)
}
server := &http.Server{Handler: proxy, ReadHeaderTimeout: 10 * time.Second}
go func() { _ = server.Serve(ln) }()
t.Cleanup(func() { _ = server.Close() })
proxyAddr := ln.Addr().String()
name := fmt.Sprintf("archie-egress-test-%d", time.Now().UnixNano())
networks := NewNetworks(cli, RelaySpec{
Name: name + "-relay",
Image: probeImage,
Entrypoint: []string{"/bin/archie-agent"},
Cmd: []string{"relay", "-listen", ":" + strconv.Itoa(RelayPort), "-to", proxyAddr},
Network: "bridge",
Binds: []string{filepath.Join(bin, "archie-agent") + ":/bin/archie-agent:ro,z"},
})
if err := networks.Start(t.Context()); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = networks.Close(context.WithoutCancel(t.Context())) })
if err := networks.Create(t.Context(), name); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = networks.Remove(context.WithoutCancel(t.Context()), name) })
// A host service listening on every interface, the way NATS and
// Postgres do. The sandbox must not reach it by any address.
hostService, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", ":0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = hostService.Close() })
go func() {
for {
c, err := hostService.Accept()
if err != nil {
return
}
_ = c.Close()
}
}()
_, hostPort, err := net.SplitHostPort(hostService.Addr().String())
if err != nil {
t.Fatal(err)
}
sandboxNet, err := cli.NetworkInspect(t.Context(), name, client.NetworkInspectOptions{})
if err != nil {
t.Fatal(err)
}
if len(sandboxNet.Network.IPAM.Config) == 0 || !sandboxNet.Network.IPAM.Config[0].Subnet.IsValid() {
t.Fatal("sandbox network has no subnet")
}
// The address a bridge gateway would hold: the subnet's first host. An
// isolated sandbox network leaves it unassigned.
sandboxGateway := sandboxNet.Network.IPAM.Config[0].Subnet.Masked().Addr().Next().String()
session, err := proxy.Register(&spec.PhasedNetwork{Runtime: &spec.NetworkRules{Allow: []string{"api.example.com:443"}}})
if err != nil {
t.Fatal(err)
}
session.EnterRuntime()
results := runProbe(t, cli, name, bin, caDir, session.Token(),
"get", "https://api.example.com/ok",
"get", "https://denied.example.com/",
"dial", proxyAddr,
"dial", net.JoinHostPort(sandboxGateway, hostPort),
"dial", net.JoinHostPort(gateway, hostPort),
"dial", "1.1.1.1:443",
)
want := []struct{ prefix, why string }{
{"OK 200 upstream:api.example.com/ok", "an allowed host is reachable through the relay and proxy"},
{"ERR", "a host outside the policy is refused"},
{"BLOCKED", "the proxy's own listener is unreachable except through the relay"},
{"BLOCKED", "a host service on all interfaces is unreachable through the sandbox's own gateway address"},
{"BLOCKED", "a host service on all interfaces is unreachable through the docker bridge"},
{"BLOCKED", "the internet is unreachable directly"},
}
if len(results) != len(want) {
t.Fatalf("probe returned %d results, want %d: %q", len(results), len(want), results)
}
for i, w := range want {
if !strings.HasPrefix(results[i], w.prefix) {
t.Errorf("%s: got %q, want prefix %q", w.why, results[i], w.prefix)
}
}
wrong := runProbe(t, cli, name, bin, caDir, "not-a-token", "get", "https://api.example.com/ok")
if len(wrong) != 1 || !strings.HasPrefix(wrong[0], "ERR") {
t.Errorf("a sandbox with the wrong token reached upstream: %q", wrong)
}
}
@@ -0,0 +1,59 @@
// Command egressclient runs inside a sandbox container in the egress
// integration test. Each argument pair is a probe; one result line per
// probe goes to the file named by the first argument.
//
// get <url> OK <status> <body> | ERR <error>
// dial <host:port> OPEN | BLOCKED
package main
import (
"context"
"fmt"
"io"
"net"
"net/http"
"os"
"strings"
"time"
)
func main() {
out, err := os.Create(os.Args[1])
if err != nil {
panic(err)
}
defer out.Close()
c := &http.Client{Timeout: 10 * time.Second, Transport: &http.Transport{Proxy: http.ProxyFromEnvironment}}
args := os.Args[2:]
for i := 0; i+1 < len(args); i += 2 {
verb, target := args[i], args[i+1]
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
switch verb {
case "get":
fmt.Fprintln(out, get(ctx, c, target))
case "dial":
conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", target)
if err != nil {
fmt.Fprintln(out, "BLOCKED")
} else {
_ = conn.Close()
fmt.Fprintln(out, "OPEN")
}
}
cancel()
}
}
func get(ctx context.Context, c *http.Client, target string) string {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
if err != nil {
return "ERR " + err.Error()
}
resp, err := c.Do(req)
if err != nil {
return "ERR " + strings.ReplaceAll(err.Error(), "\n", " ")
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
return fmt.Sprintf("OK %d %s", resp.StatusCode, strings.TrimSpace(string(body)))
}