From 3a965e2a4f36bb9f2dd60d69cfe167157ed19069 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 14 Nov 2026 03:59:48 +1100 Subject: [PATCH] feat(nats): an external broker defers logins to archie through auth callout (#1866) Task containers log in with their run credential on an external broker and reach only their own subjects, decided by the embedded broker's authenticator. The broker token is never put in a container environment. --- Dockerfile | 2 +- cmd/archie-agent/main.go | 14 +-- config.example.toml | 3 +- deployments/docker-nats-stack.toml | 2 + deployments/nats-callout.conf | 10 ++ deployments/systemd-user-service.md | 8 ++ docker-compose.yml | 14 +-- go.mod | 5 +- go.sum | 12 +++ internal/app/agentworker/worker.go | 10 +- internal/app/archied/bootstrap.go | 30 +++++- internal/app/archied/readiness.go | 13 --- internal/config/config.go | 11 +- internal/daemon/container_run.go | 1 - internal/daemon/daemon.go | 22 +--- .../agenttransport/nats/transport.go | 11 +- .../infrastructure/configuration/validate.go | 3 + internal/infrastructure/egress/sandbox_env.go | 4 - internal/infrastructure/eventbus/nats/auth.go | 66 ++++++++---- .../infrastructure/eventbus/nats/auth_test.go | 56 +++++++++- .../infrastructure/eventbus/nats/callout.go | 101 ++++++++++++++++++ .../infrastructure/openshell/policy_test.go | 4 +- 22 files changed, 299 insertions(+), 103 deletions(-) create mode 100644 deployments/nats-callout.conf create mode 100644 internal/infrastructure/eventbus/nats/callout.go diff --git a/Dockerfile b/Dockerfile index 2069bf70b..f46e89020 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,5 +103,5 @@ ENV GOMODCACHE=/data/cache/go/mod \ COPY --from=builder /usr/local/bin/archie-agent /usr/local/bin/archie-agent -# The daemon injects NATS_URL and, when configured, NATS_TOKEN. +# The daemon injects NATS_URL. ENTRYPOINT ["archie-agent"] diff --git a/cmd/archie-agent/main.go b/cmd/archie-agent/main.go index 10fb807c5..676332f7b 100644 --- a/cmd/archie-agent/main.go +++ b/cmd/archie-agent/main.go @@ -52,7 +52,10 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run return 0 } - natsURL, natsToken := natsConnectionSettings(*natsURLFlag, getenv) + natsURL := *natsURLFlag + if natsURL == "" { + natsURL = getenv("NATS_URL") + } if natsURL == "" { _, _ = fmt.Fprintln(stderr, "error: -nats-url or NATS_URL is required") flags.Usage() @@ -79,7 +82,6 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run if err := runWorker(ctx, agentworker.Settings{ NATSURL: natsURL, - NATSToken: natsToken, StateStoreTarget: stateStoreURL, StateStoreToken: stateStoreToken, StateStoreCA: []byte(getenv("STATE_STORE_CA")), @@ -92,14 +94,6 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run return 0 } -func natsConnectionSettings(flagURL string, getenv func(string) string) (url, token string) { - url = flagURL - if url == "" { - url = getenv("NATS_URL") - } - return url, getenv("NATS_TOKEN") -} - type relayServer func(ctx context.Context, forwards map[string]string) error // runRelay is the sandbox egress relay: the container on both a sandbox's diff --git a/config.example.toml b/config.example.toml index d226f32c3..27265da02 100644 --- a/config.example.toml +++ b/config.example.toml @@ -74,7 +74,8 @@ target = "127.0.0.1:9090" # [nats] # mode = "embedded" # or "external" with url # url = "nats://localhost:4222" -# token_env = "NATS_TOKEN" +# token_env = "NATS_TOKEN" # external: the archie-callout user's password +# callout_issuer_seed_env = "NATS_CALLOUT_SEED" # external: account nkey seed; the broker's auth_callout issuer is its public key # [log] # file = "~/.local/share/archie/logs/archied.log" diff --git a/deployments/docker-nats-stack.toml b/deployments/docker-nats-stack.toml index 644a7dddf..c38c2db56 100644 --- a/deployments/docker-nats-stack.toml +++ b/deployments/docker-nats-stack.toml @@ -59,6 +59,8 @@ mode = "external" # The pinned Compose-network gateway is reachable from both the host daemon # and the task containers it launches on archie-core_default. url = "nats://172.19.0.1:4222" +token_env = "NATS_TOKEN" +callout_issuer_seed_env = "NATS_CALLOUT_SEED" [containers] network = "archie-core_default" diff --git a/deployments/nats-callout.conf b/deployments/nats-callout.conf new file mode 100644 index 000000000..d4d842681 --- /dev/null +++ b/deployments/nats-callout.conf @@ -0,0 +1,10 @@ +jetstream { store_dir: /data } +http_port: 8222 + +authorization { + users: [{ user: "archie-callout", password: $NATS_TOKEN }] + auth_callout { + issuer: $NATS_CALLOUT_ISSUER + auth_users: ["archie-callout"] + } +} diff --git a/deployments/systemd-user-service.md b/deployments/systemd-user-service.md index 5f9169d2c..d5779dbba 100644 --- a/deployments/systemd-user-service.md +++ b/deployments/systemd-user-service.md @@ -225,11 +225,19 @@ docker compose up -d nats [nats] mode = "external" url = "nats://172.19.0.1:4222" +token_env = "NATS_TOKEN" +callout_issuer_seed_env = "NATS_CALLOUT_SEED" [containers] network = "archie-core_default" ``` +The broker defers every login to archied (NATS auth callout), so a worker logs +in with its run credential and reaches only its own task's subjects. Generate +an account nkey (`nk -gen account -pubout`), export its seed as +`NATS_CALLOUT_SEED` for archied and its public key as `NATS_CALLOUT_ISSUER` for +Compose, and set `NATS_TOKEN` to the same value in both. + The daemon hands workers the URL its own client connected to. The Compose service name is not resolvable from the native host, and localhost inside a worker is the worker itself; the pinned network gateway is reachable from both. diff --git a/docker-compose.yml b/docker-compose.yml index 4bc5958ba..aab8017ee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,19 +46,19 @@ services: nats: image: nats:2.14-alpine3.22 # The client port is published on every interface (agent containers reach - # it over the bridge), so it refuses to start without a token. Set - # NATS_TOKEN here and name the same value in [nats] token_env. - entrypoint: - - sh - - -c - - '[ -n "$$NATS_TOKEN" ] || { echo "NATS_TOKEN is required" >&2; exit 1; }; exec nats-server -js -m 8222 --auth "$$NATS_TOKEN"' + # it over the bridge), so it defers every login to archied. Set NATS_TOKEN + # (name it in [nats] token_env) and NATS_CALLOUT_ISSUER, the public key of + # the account nkey seed named in [nats] callout_issuer_seed_env. + command: ["-c", "/etc/nats/nats.conf"] environment: - NATS_TOKEN: ${NATS_TOKEN:-} + NATS_TOKEN: ${NATS_TOKEN:?NATS_TOKEN is required} + NATS_CALLOUT_ISSUER: ${NATS_CALLOUT_ISSUER:?NATS_CALLOUT_ISSUER is required} ports: - "4222:4222" # client - "127.0.0.1:8222:8222" # monitoring (http://localhost:8222) volumes: - nats_data:/data + - ./deployments/nats-callout.conf:/etc/nats/nats.conf:ro restart: unless-stopped healthcheck: test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:8222/healthz"] diff --git a/go.mod b/go.mod index 345bd886e..1cf695e30 100644 --- a/go.mod +++ b/go.mod @@ -38,8 +38,6 @@ require ( github.com/mfridman/interpolate v0.2.0 // indirect github.com/minio/highwayhash v1.0.4 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/nats-io/jwt/v2 v2.8.3 // indirect - github.com/nats-io/nkeys v0.4.17 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/oklog/run v1.2.0 // indirect github.com/philhofer/fwd v1.2.1 // indirect @@ -86,12 +84,15 @@ require ( github.com/moby/moby/api v1.56.1 github.com/moby/moby/client v0.6.2 github.com/modelcontextprotocol/go-sdk v1.8.0 + github.com/nats-io/jwt/v2 v2.8.3 github.com/nats-io/nats-server/v2 v2.15.3 github.com/nats-io/nats.go v1.55.0 + github.com/nats-io/nkeys v0.4.17 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.1 github.com/pressly/goose/v3 v3.30.1 github.com/samcharles93/ai-sdk v0.6.0 + github.com/synadia-io/callout.go v0.2.3 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 go.opentelemetry.io/otel v1.47.0 diff --git a/go.sum b/go.sum index 3a79133a7..8507ef279 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cel.dev/cel-go v0.33.2 h1:eAcEdBguc6AFrGbPOyE0g168Uey4kKFW9J6qsBLSeMU= cel.dev/cel-go v0.33.2/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU= cel.dev/expr v0.26.0 h1:oLXvOF0jiYA5A/tmzU91pLEWZkKynhLJlC9Gw7PrKvs= cel.dev/expr v0.26.0/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Microsoft/go-winio v0.6.4 h1:kJF2pamtYCWXf0xAT8sAkwKC0pMl2HYBrRPHAqyUc+g= @@ -16,6 +18,8 @@ github.com/antithesishq/antithesis-sdk-go v0.9.0 h1:JQpWOgYSYBb9HqgPtaXBswHxZFmO github.com/antithesishq/antithesis-sdk-go v0.9.0/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/aricart/nst.go v0.2.0 h1:vfC+upJofU7OFJZM5Ph+HNko1XfJtVuSE6Q7FCQmT3g= +github.com/aricart/nst.go v0.2.0/go.mod h1:13ztPOEw3Jz0/LT8yDi5wZB+T4Lsj3Uhkc17CyZthII= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= @@ -135,6 +139,8 @@ github.com/mfridman/interpolate v0.2.0 h1:HM9ID8p6CUxoMJX+fECOtGQ2Mu7TW/GIMPyu48 github.com/mfridman/interpolate v0.2.0/go.mod h1:8SfDy95IaPNnIokZh9mvIppmc3fGl8fx+Gu7uusuO9Q= github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/moby/api v1.56.1 h1:PpWkvVPB7Fr/No8w+TfyJ/I6rWZ2YPZhT3JorIg+06c= @@ -151,6 +157,8 @@ github.com/nats-io/nats.go v1.55.0 h1:Ng8QKHdvdHIWoaiq0+d/aH1BWkxvcJxWJ0Y/9BPKx3 github.com/nats-io/nats.go v1.55.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= github.com/nats-io/nkeys v0.4.17 h1:JAJrlDhOyQqOt0a1eg4cbIZ1Kc7+vmkCigAltpwQpVA= github.com/nats-io/nkeys v0.4.17/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nsc/v2 v2.10.3-0.20250110165315-eeda721ecff6 h1:V1uh9L3rGIUeYoMd0/EZY2Tvy5+1CkgGpDRx8t3+PNY= +github.com/nats-io/nsc/v2 v2.10.3-0.20250110165315-eeda721ecff6/go.mod h1:ScomAvx1cgjiXzW3WpGo9x/lLENkwELhewCcok/GTU8= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk= @@ -191,6 +199,10 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/synadia-io/callout.go v0.2.3 h1:22u8sIIdpkKuyySXqEWvGAkAQ4tMBnvLtpVPNBdbj6Y= +github.com/synadia-io/callout.go v0.2.3/go.mod h1:IDVofaME9xzbBO5nUvEL59Ml0jGd6gAsF1qC4Nm6Pn8= +github.com/synadia-io/jwt-auth-builder.go v0.0.9 h1:UmXQ0JsqiA7sboE0VC46YEqrGAFA4MYHnhw54uYKRi8= +github.com/synadia-io/jwt-auth-builder.go v0.0.9/go.mod h1:0XPmvG3TihNkiM3ZFVtcCAzv0oBgFtjnw/lb/h8NHs0= github.com/tinylib/msgp v1.6.5 h1:iAH6XTP7BpzErjBZlujJQXxA+GmRi2YWs6M4KdLtNbE= github.com/tinylib/msgp v1.6.5/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/internal/app/agentworker/worker.go b/internal/app/agentworker/worker.go index 0a93a483e..06644db77 100644 --- a/internal/app/agentworker/worker.go +++ b/internal/app/agentworker/worker.go @@ -35,9 +35,8 @@ func (e *StartupError) Unwrap() error { return e.Err } // Settings are process-derived inputs for one worker runtime. type Settings struct { - NATSURL string - NATSToken string - WorkDir string + NATSURL string + WorkDir string // StateStoreTarget and StateStoreToken are the daemon-injected State Store // gRPC endpoint and bearer token. StateStoreTarget is required: the legacy @@ -128,8 +127,8 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies } dependencies.markSafe(ctx, workDir, log) - // The task is known before connecting: without a broker token the - // worker logs in as its task, with its run credential. + // The task is known before connecting: the worker logs in as its task, + // with its run credential. taskID, err := dependencies.bootID(workDir) if err != nil { log.ErrorContext(ctx, "task boot failed", "err", err) @@ -137,7 +136,6 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies } transport, err := dependencies.connect(ctx, agentnats.Config{ URL: settings.NATSURL, - Token: settings.NATSToken, TaskID: taskID, StateStoreURL: settings.StateStoreTarget, StateStoreToken: settings.StateStoreToken, diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 9e828725b..a2120b7e2 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -155,9 +155,6 @@ type boot struct { // the embedded server's ClientURL(). Recorded so Daemon.ConnectedNATS // carries the real endpoint rather than an empty config URL. natsURL string - // natsToken is the credential the daemon's client connected with at - // startup. Embedded mode generates it; external mode resolves it once. - natsToken string containerPool *container.Pool // taskRuntime is what task agents run in: the container pool, or an @@ -388,6 +385,9 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco log.ErrorContext(ctx, "nats credentials", "err", err) return err } + if err := b.startCallout(ctx, url, natsToken); err != nil { + return err + } } else { endpoint, readErr := servicekit.ReadNATSEndpoint(cfg.StateDir) if readErr == nil { @@ -445,7 +445,6 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco } b.natsClient = natsClient b.natsURL = url - b.natsToken = natsToken b.addCleanup(func() { natsClient.Close() }) b.addCleanup(func() { reactionClient.Close() }) b.reactionClient = reactionClient @@ -453,6 +452,27 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco return nil } +// startCallout makes the external broker defer every login to archie, so a task +// container logs in with its run credential and reaches only its own subjects. +// It runs before any other login, because those are decided here. +func (b *boot) startCallout(ctx context.Context, url, token string) error { + seed := os.Getenv(b.cfg.NATS.CalloutIssuerSeedEnv) + if seed == "" { + return fmt.Errorf("%s is required when nats.callout_issuer_seed_env is configured", b.cfg.NATS.CalloutIssuerSeedEnv) + } + opts := nats.CalloutOptions{URL: url, Token: token, IssuerSeed: seed} + if b.stateStoreGrants != nil { + opts.Tasks = taskBroker{runs: b.stateStoreGrants.Client} + } + svc, err := nats.StartCallout(ctx, opts, b.log) + if err != nil { + b.log.ErrorContext(ctx, "nats callout start failed", "err", err) + return err + } + b.addCleanup(svc.Close) + return nil +} + func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) { cfg, log := b.cfg, b.log if cfg.StateDir == "" { @@ -627,7 +647,7 @@ func (b *boot) buildDaemon(ctx context.Context) { d := &daemon.Daemon{ ResolveRepository: b.repositoryAccess, Cfg: b.cfgHolder, - ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken, Scoped: b.cfg.NATS.Mode != config.NATSModeExternal}, + ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL}, ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), CA: b.stateStoreCA(ctx)}, Store: b.stateStore, Identity: daemon.NewIdentityGate(b.stateClient, b.stateClient, b.accessChain, b.stateClient, b.stateClient, b.orgPrincipals, workspaces, log), diff --git a/internal/app/archied/readiness.go b/internal/app/archied/readiness.go index 22146f6e9..01cd7919d 100644 --- a/internal/app/archied/readiness.go +++ b/internal/app/archied/readiness.go @@ -53,9 +53,6 @@ func (b *boot) setupReadinessProbes() { if b.accessLive != nil { probes = append(probes, readiness.NewProblemProbe("access_policies", b.accessLive)) } - if cfg.NATS.Mode == config.NATSModeExternal { - probes = append(probes, unscopedBroker{}) - } b.healthRegistry = health.NewRegistry(probes...) } @@ -97,16 +94,6 @@ func diskProbePath(cfg config.Config) string { } } -// unscopedBroker reports that containers hold the external broker's own -// token: only the embedded broker can limit a container to its task. -type unscopedBroker struct{} - -func (unscopedBroker) Name() string { return "nats_credential_scope" } - -func (unscopedBroker) Check(context.Context) health.Result { - return health.Result{Status: health.StatusDegraded, Detail: "external NATS: containers get a broker token that reaches every subject; use the embedded broker"} -} - // stateCallDeadline bounds each State Store call the daemon makes without a // deadline of its own. const stateCallDeadline = 20 * time.Second diff --git a/internal/config/config.go b/internal/config/config.go index f9107f85d..80087ee68 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -647,9 +647,16 @@ type NATSConfig struct { // URL is the NATS server address, e.g. "nats://localhost:4222". // Required when Mode is "external"; must be empty otherwise. URL string `toml:"url" yaml:"url"` - // TokenEnv optionally names an env var holding an external NATS auth token. - // Empty means no authentication. Embedded mode generates a per-start token. + // TokenEnv names an env var holding the external broker's instance token: + // the password of the "archie-callout" auth user, and the token archie's own + // services log in with. Required when Mode is "external". Embedded mode + // generates a per-start token. TokenEnv string `toml:"token_env" yaml:"token_env"` + // CalloutIssuerSeedEnv names an env var holding the account nkey seed whose + // public key the broker lists as its auth_callout issuer. Required when + // Mode is "external": the broker defers every login to archied, so a task + // container holds only its run credential. + CalloutIssuerSeedEnv string `toml:"callout_issuer_seed_env" yaml:"callout_issuer_seed_env"` } // CaptureConfig configures the unbound webhook capture endpoint. diff --git a/internal/daemon/container_run.go b/internal/daemon/container_run.go index ab20eb78a..7c5542371 100644 --- a/internal/daemon/container_run.go +++ b/internal/daemon/container_run.go @@ -132,7 +132,6 @@ func (d *Daemon) acquireTaskContainer( func (d *Daemon) workerEndpoints(stateStoreToken string) egress.Endpoints { return egress.Endpoints{ NATS: d.ConnectedNATS.URL, - NATSToken: d.ConnectedNATS.ContainerToken(), StateStore: d.ConnectedStateStore.URL, StateStoreToken: stateStoreToken, StateStoreCA: d.ConnectedStateStore.CA, diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index da1c16894..cc7fa941b 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -41,25 +41,11 @@ type RunCredentialIssuer interface { Issue(ctx context.Context, task *workflow.Task, limit time.Duration) (token string, revoke func(), err error) } -// NATSEndpoint is the broker address and credential archied connected with at -// startup. It is runtime state rather than configuration: embedded mode -// generates both values, and external secret references are resolved once. +// NATSEndpoint is the broker address archied connected with at startup. It is +// runtime state rather than configuration: embedded mode generates it. A +// container logs in with its run credential, never the broker's token. type NATSEndpoint struct { - URL string - Token string - // Scoped is true on the embedded broker, where a container logs in with - // its run credential and reaches only its own task's subjects. - Scoped bool -} - -// ContainerToken is the broker token a container is handed: none on a -// scoped broker. An external broker cannot scope a container, so it still -// gets the broker's token (reported as a health issue by archied). -func (e NATSEndpoint) ContainerToken() string { - if e.Scoped { - return "" - } - return e.Token + URL string } // StateStoreEndpoint is the State Store target and the CA its certificate diff --git a/internal/infrastructure/agenttransport/nats/transport.go b/internal/infrastructure/agenttransport/nats/transport.go index 312a7fc3b..8af68d8f3 100644 --- a/internal/infrastructure/agenttransport/nats/transport.go +++ b/internal/infrastructure/agenttransport/nats/transport.go @@ -35,10 +35,8 @@ import ( // deleted type Config struct { URL string - // Token is a broker-wide token, set only on an external broker. Without - // it the worker logs in as TaskID with StateStoreToken, its run - // credential, which reaches only its own task's subjects. - Token string + // The worker logs in as TaskID with StateStoreToken, its run credential, + // which reaches only its own task's subjects. TaskID int64 StateStoreURL string @@ -82,10 +80,7 @@ func Connect(ctx context.Context, config Config, log *slog.Logger) (*Transport, return nil, err } options := []natsio.Option{natsio.Name("archie-agent")} - switch { - case config.Token != "": - options = append(options, natsio.Token(config.Token)) - case config.StateStoreToken != "": + if config.StateStoreToken != "" { options = append(options, natsio.UserInfo(fmt.Sprintf("%s%d", eventbus.TaskUserPrefix, config.TaskID), config.StateStoreToken), natsio.CustomInboxPrefix(eventbus.TaskInboxPrefix(config.TaskID))) diff --git a/internal/infrastructure/configuration/validate.go b/internal/infrastructure/configuration/validate.go index b9f294748..d1bb0e03a 100644 --- a/internal/infrastructure/configuration/validate.go +++ b/internal/infrastructure/configuration/validate.go @@ -310,6 +310,9 @@ func validateNATS(cfg *config.Config) error { if cfg.NATS.URL == "" { return fmt.Errorf("%w: nats.url is required when nats.mode is %q", ErrInvalidInput, config.NATSModeExternal) } + if cfg.NATS.TokenEnv == "" || cfg.NATS.CalloutIssuerSeedEnv == "" { + return fmt.Errorf("%w: nats.token_env and nats.callout_issuer_seed_env are required when nats.mode is %q", ErrInvalidInput, config.NATSModeExternal) + } case config.NATSModeEmbedded: if cfg.NATS.URL != "" { return fmt.Errorf("%w: nats.url must be empty when nats.mode is %q", ErrInvalidInput, mode) diff --git a/internal/infrastructure/egress/sandbox_env.go b/internal/infrastructure/egress/sandbox_env.go index 9dda57fbf..6992582fd 100644 --- a/internal/infrastructure/egress/sandbox_env.go +++ b/internal/infrastructure/egress/sandbox_env.go @@ -12,7 +12,6 @@ import ( // needs, with the task's own credentials for them. type Endpoints struct { NATS string - NATSToken string StateStore string StateStoreToken string StateStoreCA []byte @@ -34,9 +33,6 @@ func WorkerEnv(e Endpoints) ([]string, error) { // credentials from e. The State Store is left out when e names none. func DirectWorkerEnv(nats, stateStore string, e Endpoints) []string { env := []string{"NATS_URL=" + nats} - if e.NATSToken != "" { - env = append(env, "NATS_TOKEN="+e.NATSToken) - } if e.StateStore != "" { env = append(env, "STATE_STORE_URL="+stateStore) if e.StateStoreToken != "" { diff --git a/internal/infrastructure/eventbus/nats/auth.go b/internal/infrastructure/eventbus/nats/auth.go index 578225c65..3087d25ac 100644 --- a/internal/infrastructure/eventbus/nats/auth.go +++ b/internal/infrastructure/eventbus/nats/auth.go @@ -3,6 +3,7 @@ package nats import ( "context" "crypto/subtle" + "slices" "strconv" "strings" "time" @@ -38,34 +39,61 @@ const taskAuthTimeout = 2 * time.Second // authenticator admits the instance token with every permission, and a task // user only to its own task's subjects, so no container holds a credential -// that reaches past its run. +// that reaches past its run. The embedded broker and the auth callout of an +// external broker both decide through it. type authenticator struct { token string tasks TaskAuthority } +// grant is what a login is allowed. Full is the instance token's reach; a task +// login lists its subjects instead. +type grant struct { + Username string + Full bool + Publish []string + Subscribe []string +} + +// authorize decides one login: the instance token, or "task-" with that +// task's run credential. +func (a authenticator) authorize(ctx context.Context, token, username, password string) (grant, bool) { + if token != "" { + if subtle.ConstantTimeCompare([]byte(token), []byte(a.token)) != 1 { + return grant{}, false + } + return grant{Username: "archie", Full: true}, true + } + taskID, ok := taskUser(username) + if !ok || password == "" || a.tasks == nil { + return grant{}, false + } + ctx, cancel := context.WithTimeout(ctx, taskAuthTimeout) + defer cancel() + subjects, err := a.tasks.TaskSubjects(ctx, taskID, password) + if err != nil { + return grant{}, false + } + return grant{ + Username: username, + Publish: slices.Clone(subjects.Publish), + Subscribe: append(slices.Clone(subjects.Subscribe), TaskInboxPrefix(taskID)+".>"), + }, true +} + func (a authenticator) Check(c server.ClientAuthentication) bool { opts := c.GetOpts() - if opts.Token != "" { - if subtle.ConstantTimeCompare([]byte(opts.Token), []byte(a.token)) != 1 { - return false - } - c.RegisterUser(&server.User{Username: "archie"}) + g, ok := a.authorize(context.Background(), opts.Token, opts.Username, opts.Password) + if !ok { + return false + } + if g.Full { + c.RegisterUser(&server.User{Username: g.Username}) return true } - taskID, ok := taskUser(opts.Username) - if !ok || opts.Password == "" || a.tasks == nil { - return false - } - ctx, cancel := context.WithTimeout(context.Background(), taskAuthTimeout) - defer cancel() - subjects, err := a.tasks.TaskSubjects(ctx, taskID, opts.Password) - if err != nil { - return false - } - c.RegisterUser(&server.User{Username: opts.Username, Permissions: &server.Permissions{ - Publish: &server.SubjectPermission{Allow: subjects.Publish}, - Subscribe: &server.SubjectPermission{Allow: append(subjects.Subscribe, TaskInboxPrefix(taskID)+".>")}, + c.RegisterUser(&server.User{Username: g.Username, Permissions: &server.Permissions{ + Publish: &server.SubjectPermission{Allow: g.Publish}, + Subscribe: &server.SubjectPermission{Allow: g.Subscribe}, // Lets the task answer the one request it is sent, on the requester's // reply subject. Response: &server.ResponsePermission{MaxMsgs: 1}, diff --git a/internal/infrastructure/eventbus/nats/auth_test.go b/internal/infrastructure/eventbus/nats/auth_test.go index d84c19db7..7e67b8101 100644 --- a/internal/infrastructure/eventbus/nats/auth_test.go +++ b/internal/infrastructure/eventbus/nats/auth_test.go @@ -8,7 +8,9 @@ import ( "testing" "time" + "github.com/nats-io/nats-server/v2/server" natsio "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" ) type taskSeven struct{} @@ -22,14 +24,60 @@ func (taskSeven) TaskSubjects(_ context.Context, taskID int64, credential string // TestBrokerScopesTaskCredentials pins that the instance token reaches // everything and a task's run credential reaches only its own subjects, on a -// real embedded broker. +// real embedded broker and on an external one that defers logins to the +// callout. It guards a task container being handed more than its own subjects. func TestBrokerScopesTaskCredentials(t *testing.T) { - srv, err := StartEmbedded(context.Background(), EmbeddedOptions{Token: "instance", StoreDir: t.TempDir(), Tasks: taskSeven{}}, nil) + embedded, err := StartEmbedded(context.Background(), EmbeddedOptions{Token: "instance", StoreDir: t.TempDir(), Tasks: taskSeven{}}, nil) if err != nil { t.Fatal(err) } - defer srv.Shutdown() + defer embedded.Shutdown() + for name, url := range map[string]string{"embedded": embedded.ClientURL(), "callout": calloutBroker(t)} { + t.Run(name, func(t *testing.T) { checkScopes(t, url) }) + } +} + +// calloutBroker starts an external-style broker whose logins are decided by +// StartCallout, configured the way an operator would: one auth user for the +// callout itself, every other login deferred. +func calloutBroker(t *testing.T) string { + t.Helper() + issuer, err := nkeys.CreateAccount() + if err != nil { + t.Fatal(err) + } + pub, err := issuer.PublicKey() + if err != nil { + t.Fatal(err) + } + seed, err := issuer.Seed() + if err != nil { + t.Fatal(err) + } + srv, err := server.NewServer(&server.Options{ + Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true, + Users: []*server.User{{Username: CalloutUser, Password: "instance"}}, + AuthCallout: &server.AuthCallout{Issuer: pub, Account: "$G", AuthUsers: []string{CalloutUser}}, + }) + if err != nil { + t.Fatal(err) + } + srv.Start() + t.Cleanup(srv.Shutdown) + if !srv.ReadyForConnections(10 * time.Second) { + t.Fatal("broker not ready") + } + svc, err := StartCallout(context.Background(), CalloutOptions{URL: srv.ClientURL(), Token: "instance", IssuerSeed: string(seed), Tasks: taskSeven{}}, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(svc.Close) + return srv.ClientURL() +} + +func checkScopes(t *testing.T, url string) { + t.Helper() tests := []struct { name string opts []natsio.Option @@ -61,7 +109,7 @@ func TestBrokerScopesTaskCredentials(t *testing.T) { opts := append(slices.Clone(tt.opts), natsio.ErrorHandler(func(_ *natsio.Conn, _ *natsio.Subscription, err error) { violations <- err.Error() })) - nc, err := natsio.Connect(srv.ClientURL(), opts...) + nc, err := natsio.Connect(url, opts...) if !tt.connects { if err == nil { nc.Close() diff --git a/internal/infrastructure/eventbus/nats/callout.go b/internal/infrastructure/eventbus/nats/callout.go new file mode 100644 index 000000000..0cbde4b30 --- /dev/null +++ b/internal/infrastructure/eventbus/nats/callout.go @@ -0,0 +1,101 @@ +package nats + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/nats-io/jwt/v2" + natsio "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" + "github.com/synadia-io/callout.go" +) + +// CalloutUser is the broker login archied's callout service connects with. The +// external broker lists it in auth_users, so its own login skips the callout; +// its password is the instance token. +const CalloutUser = "archie-callout" + +// calloutAccount is the account a granted login is placed in: the broker's +// global account, which JetStream uses. +const calloutAccount = "$G" + +// CalloutOptions configures StartCallout. +type CalloutOptions struct { + // URL is the external broker. + URL string + // Token is the instance token: the callout user's password, and the token + // that logs in with every permission. + Token string + // IssuerSeed is the account nkey seed whose public key the broker names as + // its auth_callout issuer. + IssuerSeed string + // Tasks admits task users to their own task's subjects. + Tasks TaskAuthority +} + +// Callout is the authorization service of an external broker. +type Callout struct { + svc *callout.AuthorizationService + nc *natsio.Conn +} + +// StartCallout answers an external broker's login requests with the decisions +// the embedded broker makes, so a task container holds only its run credential. +func StartCallout(ctx context.Context, opts CalloutOptions, log *slog.Logger) (*Callout, error) { + if log == nil { + log = slog.New(slog.DiscardHandler) + } + issuer, err := nkeys.FromSeed([]byte(opts.IssuerSeed)) + if err != nil { + return nil, errors.New("nats callout: issuer seed is not an nkey seed") + } + nc, err := natsio.Connect(opts.URL, natsio.UserInfo(CalloutUser, opts.Token), natsio.Name("archie-callout")) + if err != nil { + return nil, fmt.Errorf("nats callout connect: %w", err) + } + svc, err := callout.NewAuthorizationService(nc, + callout.Authorizer(authenticator{token: opts.Token, tasks: opts.Tasks}.calloutAuthorizer(context.WithoutCancel(ctx), issuer)), + callout.ResponseSignerKey(issuer), + callout.Logger(brokerLogger{log: log.With("service", "nats", "component", "callout")}), + ) + if err != nil { + nc.Close() + return nil, fmt.Errorf("nats callout service: %w", err) + } + log.InfoContext(ctx, "nats callout ready", "url", opts.URL) + return &Callout{svc: svc, nc: nc}, nil +} + +// Close stops answering logins. +func (c *Callout) Close() { + _ = c.svc.Stop() + c.nc.Close() +} + +// calloutAuthorizer turns an authenticator decision into a user JWT. An error +// refuses the login. +func (a authenticator) calloutAuthorizer(ctx context.Context, issuer nkeys.KeyPair) callout.AuthorizerFn { + return func(req *jwt.AuthorizationRequest) (string, error) { + g, ok := a.authorize(ctx, req.ConnectOptions.Token, req.ConnectOptions.Username, req.ConnectOptions.Password) + if !ok { + return "", errors.New("login refused") + } + return g.userJWT(req.UserNkey, issuer) + } +} + +// userJWT is the grant as a broker user's permissions, signed by the issuer the +// broker trusts. +func (g grant) userJWT(userNkey string, issuer nkeys.KeyPair) (string, error) { + uc := jwt.NewUserClaims(userNkey) + uc.Name = g.Username + uc.Audience = calloutAccount + if !g.Full { + uc.Pub.Allow.Add(g.Publish...) + uc.Sub.Allow.Add(g.Subscribe...) + uc.Resp = &jwt.ResponsePermission{MaxMsgs: 1} + } + return uc.Encode(issuer) +} diff --git a/internal/infrastructure/openshell/policy_test.go b/internal/infrastructure/openshell/policy_test.go index d2507f39a..1ffe25c0d 100644 --- a/internal/infrastructure/openshell/policy_test.go +++ b/internal/infrastructure/openshell/policy_test.go @@ -30,14 +30,14 @@ func TestSandboxPolicyAdmitsOnlyWhatTheTaskNeeds(t *testing.T) { name: "services by IP and loopback use the services host", servicesHost: "archie.lan", launch: Launch{Worker: egress.Endpoints{ - NATS: "nats://172.17.0.1:4222", NATSToken: "t", StateStore: "dns:///127.0.0.1:50051", StateStoreToken: "run-1", + NATS: "nats://172.17.0.1:4222", StateStore: "dns:///127.0.0.1:50051", StateStoreToken: "run-1", }}, wantRules: map[string]string{ "archie-nats": "archie.lan:4222 " + agentBinary, "archie-state-store": "archie.lan:50051 " + agentBinary, }, wantEnv: map[string]string{ - "NATS_URL": "nats://archie.lan:4222", "NATS_TOKEN": "t", + "NATS_URL": "nats://archie.lan:4222", "STATE_STORE_URL": "archie.lan:50051", "STATE_STORE_TOKEN": "run-1", }, },