feat(nats): an external broker defers logins to archie through auth callout (#1866)

Task containers log in with their run credential on an external broker and
reach only their own subjects, decided by the embedded broker's authenticator.
The broker token is never put in a container environment.
This commit is contained in:
sam committed 2026-11-14 04:04:15 +11:00
1 parent 7d664d7efb
commit 3a965e2a4f
22 files changed
+299 -103

No files matched your search

+1 -1
View File
@@ -103,5 +103,5 @@ ENV GOMODCACHE=/data/cache/go/mod \
COPY --from=builder /usr/local/bin/archie-agent /usr/local/bin/archie-agent COPY --from=builder /usr/local/bin/archie-agent /usr/local/bin/archie-agent
# The daemon injects NATS_URL and, when configured, NATS_TOKEN. # The daemon injects NATS_URL.
ENTRYPOINT ["archie-agent"] ENTRYPOINT ["archie-agent"]
+4 -10
View File
@@ -52,7 +52,10 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run
return 0 return 0
} }
natsURL, natsToken := natsConnectionSettings(*natsURLFlag, getenv) natsURL := *natsURLFlag
if natsURL == "" {
natsURL = getenv("NATS_URL")
}
if natsURL == "" { if natsURL == "" {
_, _ = fmt.Fprintln(stderr, "error: -nats-url or NATS_URL is required") _, _ = fmt.Fprintln(stderr, "error: -nats-url or NATS_URL is required")
flags.Usage() flags.Usage()
@@ -79,7 +82,6 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run
if err := runWorker(ctx, agentworker.Settings{ if err := runWorker(ctx, agentworker.Settings{
NATSURL: natsURL, NATSURL: natsURL,
NATSToken: natsToken,
StateStoreTarget: stateStoreURL, StateStoreTarget: stateStoreURL,
StateStoreToken: stateStoreToken, StateStoreToken: stateStoreToken,
StateStoreCA: []byte(getenv("STATE_STORE_CA")), StateStoreCA: []byte(getenv("STATE_STORE_CA")),
@@ -92,14 +94,6 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run
return 0 return 0
} }
func natsConnectionSettings(flagURL string, getenv func(string) string) (url, token string) {
url = flagURL
if url == "" {
url = getenv("NATS_URL")
}
return url, getenv("NATS_TOKEN")
}
type relayServer func(ctx context.Context, forwards map[string]string) error type relayServer func(ctx context.Context, forwards map[string]string) error
// runRelay is the sandbox egress relay: the container on both a sandbox's // runRelay is the sandbox egress relay: the container on both a sandbox's
+2 -1
View File
@@ -74,7 +74,8 @@ target = "127.0.0.1:9090"
# [nats] # [nats]
# mode = "embedded" # or "external" with url # mode = "embedded" # or "external" with url
# url = "nats://localhost:4222" # url = "nats://localhost:4222"
# token_env = "NATS_TOKEN" # token_env = "NATS_TOKEN" # external: the archie-callout user's password
# callout_issuer_seed_env = "NATS_CALLOUT_SEED" # external: account nkey seed; the broker's auth_callout issuer is its public key
# [log] # [log]
# file = "~/.local/share/archie/logs/archied.log" # file = "~/.local/share/archie/logs/archied.log"
+2
View File
@@ -59,6 +59,8 @@ mode = "external"
# The pinned Compose-network gateway is reachable from both the host daemon # The pinned Compose-network gateway is reachable from both the host daemon
# and the task containers it launches on archie-core_default. # and the task containers it launches on archie-core_default.
url = "nats://172.19.0.1:4222" url = "nats://172.19.0.1:4222"
token_env = "NATS_TOKEN"
callout_issuer_seed_env = "NATS_CALLOUT_SEED"
[containers] [containers]
network = "archie-core_default" network = "archie-core_default"
+10
View File
@@ -0,0 +1,10 @@
jetstream { store_dir: /data }
http_port: 8222
authorization {
users: [{ user: "archie-callout", password: $NATS_TOKEN }]
auth_callout {
issuer: $NATS_CALLOUT_ISSUER
auth_users: ["archie-callout"]
}
}
+8
View File
@@ -225,11 +225,19 @@ docker compose up -d nats
[nats] [nats]
mode = "external" mode = "external"
url = "nats://172.19.0.1:4222" url = "nats://172.19.0.1:4222"
token_env = "NATS_TOKEN"
callout_issuer_seed_env = "NATS_CALLOUT_SEED"
[containers] [containers]
network = "archie-core_default" network = "archie-core_default"
``` ```
The broker defers every login to archied (NATS auth callout), so a worker logs
in with its run credential and reaches only its own task's subjects. Generate
an account nkey (`nk -gen account -pubout`), export its seed as
`NATS_CALLOUT_SEED` for archied and its public key as `NATS_CALLOUT_ISSUER` for
Compose, and set `NATS_TOKEN` to the same value in both.
The daemon hands workers the URL its own client connected to. The Compose The daemon hands workers the URL its own client connected to. The Compose
service name is not resolvable from the native host, and localhost inside a service name is not resolvable from the native host, and localhost inside a
worker is the worker itself; the pinned network gateway is reachable from both. worker is the worker itself; the pinned network gateway is reachable from both.
+7 -7
View File
@@ -46,19 +46,19 @@ services:
nats: nats:
image: nats:2.14-alpine3.22 image: nats:2.14-alpine3.22
# The client port is published on every interface (agent containers reach # The client port is published on every interface (agent containers reach
# it over the bridge), so it refuses to start without a token. Set # it over the bridge), so it defers every login to archied. Set NATS_TOKEN
# NATS_TOKEN here and name the same value in [nats] token_env. # (name it in [nats] token_env) and NATS_CALLOUT_ISSUER, the public key of
entrypoint: # the account nkey seed named in [nats] callout_issuer_seed_env.
- sh command: ["-c", "/etc/nats/nats.conf"]
- -c
- '[ -n "$$NATS_TOKEN" ] || { echo "NATS_TOKEN is required" >&2; exit 1; }; exec nats-server -js -m 8222 --auth "$$NATS_TOKEN"'
environment: environment:
NATS_TOKEN: ${NATS_TOKEN:-} NATS_TOKEN: ${NATS_TOKEN:?NATS_TOKEN is required}
NATS_CALLOUT_ISSUER: ${NATS_CALLOUT_ISSUER:?NATS_CALLOUT_ISSUER is required}
ports: ports:
- "4222:4222" # client - "4222:4222" # client
- "127.0.0.1:8222:8222" # monitoring (http://localhost:8222) - "127.0.0.1:8222:8222" # monitoring (http://localhost:8222)
volumes: volumes:
- nats_data:/data - nats_data:/data
- ./deployments/nats-callout.conf:/etc/nats/nats.conf:ro
restart: unless-stopped restart: unless-stopped
healthcheck: healthcheck:
test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:8222/healthz"] test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:8222/healthz"]
+3 -2
View File
@@ -38,8 +38,6 @@ require (
github.com/mfridman/interpolate v0.2.0 // indirect github.com/mfridman/interpolate v0.2.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect github.com/minio/highwayhash v1.0.4 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/nats-io/jwt/v2 v2.8.3 // indirect
github.com/nats-io/nkeys v0.4.17 // indirect
github.com/nats-io/nuid v1.0.1 // indirect github.com/nats-io/nuid v1.0.1 // indirect
github.com/oklog/run v1.2.0 // indirect github.com/oklog/run v1.2.0 // indirect
github.com/philhofer/fwd v1.2.1 // indirect github.com/philhofer/fwd v1.2.1 // indirect
@@ -86,12 +84,15 @@ require (
github.com/moby/moby/api v1.56.1 github.com/moby/moby/api v1.56.1
github.com/moby/moby/client v0.6.2 github.com/moby/moby/client v0.6.2
github.com/modelcontextprotocol/go-sdk v1.8.0 github.com/modelcontextprotocol/go-sdk v1.8.0
github.com/nats-io/jwt/v2 v2.8.3
github.com/nats-io/nats-server/v2 v2.15.3 github.com/nats-io/nats-server/v2 v2.15.3
github.com/nats-io/nats.go v1.55.0 github.com/nats-io/nats.go v1.55.0
github.com/nats-io/nkeys v0.4.17
github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.1 github.com/opencontainers/image-spec v1.1.1
github.com/pressly/goose/v3 v3.30.1 github.com/pressly/goose/v3 v3.30.1
github.com/samcharles93/ai-sdk v0.6.0 github.com/samcharles93/ai-sdk v0.6.0
github.com/synadia-io/callout.go v0.2.3
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
go.opentelemetry.io/otel v1.47.0 go.opentelemetry.io/otel v1.47.0
+12
View File
@@ -2,6 +2,8 @@ cel.dev/cel-go v0.33.2 h1:eAcEdBguc6AFrGbPOyE0g168Uey4kKFW9J6qsBLSeMU=
cel.dev/cel-go v0.33.2/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU= cel.dev/cel-go v0.33.2/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU=
cel.dev/expr v0.26.0 h1:oLXvOF0jiYA5A/tmzU91pLEWZkKynhLJlC9Gw7PrKvs= cel.dev/expr v0.26.0 h1:oLXvOF0jiYA5A/tmzU91pLEWZkKynhLJlC9Gw7PrKvs=
cel.dev/expr v0.26.0/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cel.dev/expr v0.26.0/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/Microsoft/go-winio v0.6.4 h1:kJF2pamtYCWXf0xAT8sAkwKC0pMl2HYBrRPHAqyUc+g= github.com/Microsoft/go-winio v0.6.4 h1:kJF2pamtYCWXf0xAT8sAkwKC0pMl2HYBrRPHAqyUc+g=
@@ -16,6 +18,8 @@ github.com/antithesishq/antithesis-sdk-go v0.9.0 h1:JQpWOgYSYBb9HqgPtaXBswHxZFmO
github.com/antithesishq/antithesis-sdk-go v0.9.0/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.9.0/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
github.com/aricart/nst.go v0.2.0 h1:vfC+upJofU7OFJZM5Ph+HNko1XfJtVuSE6Q7FCQmT3g=
github.com/aricart/nst.go v0.2.0/go.mod h1:13ztPOEw3Jz0/LT8yDi5wZB+T4Lsj3Uhkc17CyZthII=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw=
@@ -135,6 +139,8 @@ github.com/mfridman/interpolate v0.2.0 h1:HM9ID8p6CUxoMJX+fECOtGQ2Mu7TW/GIMPyu48
github.com/mfridman/interpolate v0.2.0/go.mod h1:8SfDy95IaPNnIokZh9mvIppmc3fGl8fx+Gu7uusuO9Q= github.com/mfridman/interpolate v0.2.0/go.mod h1:8SfDy95IaPNnIokZh9mvIppmc3fGl8fx+Gu7uusuO9Q=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/moby/api v1.56.1 h1:PpWkvVPB7Fr/No8w+TfyJ/I6rWZ2YPZhT3JorIg+06c= github.com/moby/moby/api v1.56.1 h1:PpWkvVPB7Fr/No8w+TfyJ/I6rWZ2YPZhT3JorIg+06c=
@@ -151,6 +157,8 @@ github.com/nats-io/nats.go v1.55.0 h1:Ng8QKHdvdHIWoaiq0+d/aH1BWkxvcJxWJ0Y/9BPKx3
github.com/nats-io/nats.go v1.55.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= github.com/nats-io/nats.go v1.55.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.17 h1:JAJrlDhOyQqOt0a1eg4cbIZ1Kc7+vmkCigAltpwQpVA= github.com/nats-io/nkeys v0.4.17 h1:JAJrlDhOyQqOt0a1eg4cbIZ1Kc7+vmkCigAltpwQpVA=
github.com/nats-io/nkeys v0.4.17/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= github.com/nats-io/nkeys v0.4.17/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nsc/v2 v2.10.3-0.20250110165315-eeda721ecff6 h1:V1uh9L3rGIUeYoMd0/EZY2Tvy5+1CkgGpDRx8t3+PNY=
github.com/nats-io/nsc/v2 v2.10.3-0.20250110165315-eeda721ecff6/go.mod h1:ScomAvx1cgjiXzW3WpGo9x/lLENkwELhewCcok/GTU8=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk= github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk=
@@ -191,6 +199,10 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/synadia-io/callout.go v0.2.3 h1:22u8sIIdpkKuyySXqEWvGAkAQ4tMBnvLtpVPNBdbj6Y=
github.com/synadia-io/callout.go v0.2.3/go.mod h1:IDVofaME9xzbBO5nUvEL59Ml0jGd6gAsF1qC4Nm6Pn8=
github.com/synadia-io/jwt-auth-builder.go v0.0.9 h1:UmXQ0JsqiA7sboE0VC46YEqrGAFA4MYHnhw54uYKRi8=
github.com/synadia-io/jwt-auth-builder.go v0.0.9/go.mod h1:0XPmvG3TihNkiM3ZFVtcCAzv0oBgFtjnw/lb/h8NHs0=
github.com/tinylib/msgp v1.6.5 h1:iAH6XTP7BpzErjBZlujJQXxA+GmRi2YWs6M4KdLtNbE= github.com/tinylib/msgp v1.6.5 h1:iAH6XTP7BpzErjBZlujJQXxA+GmRi2YWs6M4KdLtNbE=
github.com/tinylib/msgp v1.6.5/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tinylib/msgp v1.6.5/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+4 -6
View File
@@ -35,9 +35,8 @@ func (e *StartupError) Unwrap() error { return e.Err }
// Settings are process-derived inputs for one worker runtime. // Settings are process-derived inputs for one worker runtime.
type Settings struct { type Settings struct {
NATSURL string NATSURL string
NATSToken string WorkDir string
WorkDir string
// StateStoreTarget and StateStoreToken are the daemon-injected State Store // StateStoreTarget and StateStoreToken are the daemon-injected State Store
// gRPC endpoint and bearer token. StateStoreTarget is required: the legacy // gRPC endpoint and bearer token. StateStoreTarget is required: the legacy
@@ -128,8 +127,8 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies
} }
dependencies.markSafe(ctx, workDir, log) dependencies.markSafe(ctx, workDir, log)
// The task is known before connecting: without a broker token the // The task is known before connecting: the worker logs in as its task,
// worker logs in as its task, with its run credential. // with its run credential.
taskID, err := dependencies.bootID(workDir) taskID, err := dependencies.bootID(workDir)
if err != nil { if err != nil {
log.ErrorContext(ctx, "task boot failed", "err", err) log.ErrorContext(ctx, "task boot failed", "err", err)
@@ -137,7 +136,6 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies
} }
transport, err := dependencies.connect(ctx, agentnats.Config{ transport, err := dependencies.connect(ctx, agentnats.Config{
URL: settings.NATSURL, URL: settings.NATSURL,
Token: settings.NATSToken,
TaskID: taskID, TaskID: taskID,
StateStoreURL: settings.StateStoreTarget, StateStoreURL: settings.StateStoreTarget,
StateStoreToken: settings.StateStoreToken, StateStoreToken: settings.StateStoreToken,
+25 -5
View File
@@ -155,9 +155,6 @@ type boot struct {
// the embedded server's ClientURL(). Recorded so Daemon.ConnectedNATS // the embedded server's ClientURL(). Recorded so Daemon.ConnectedNATS
// carries the real endpoint rather than an empty config URL. // carries the real endpoint rather than an empty config URL.
natsURL string natsURL string
// natsToken is the credential the daemon's client connected with at
// startup. Embedded mode generates it; external mode resolves it once.
natsToken string
containerPool *container.Pool containerPool *container.Pool
// taskRuntime is what task agents run in: the container pool, or an // taskRuntime is what task agents run in: the container pool, or an
@@ -388,6 +385,9 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco
log.ErrorContext(ctx, "nats credentials", "err", err) log.ErrorContext(ctx, "nats credentials", "err", err)
return err return err
} }
if err := b.startCallout(ctx, url, natsToken); err != nil {
return err
}
} else { } else {
endpoint, readErr := servicekit.ReadNATSEndpoint(cfg.StateDir) endpoint, readErr := servicekit.ReadNATSEndpoint(cfg.StateDir)
if readErr == nil { if readErr == nil {
@@ -445,7 +445,6 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco
} }
b.natsClient = natsClient b.natsClient = natsClient
b.natsURL = url b.natsURL = url
b.natsToken = natsToken
b.addCleanup(func() { natsClient.Close() }) b.addCleanup(func() { natsClient.Close() })
b.addCleanup(func() { reactionClient.Close() }) b.addCleanup(func() { reactionClient.Close() })
b.reactionClient = reactionClient b.reactionClient = reactionClient
@@ -453,6 +452,27 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco
return nil return nil
} }
// startCallout makes the external broker defer every login to archie, so a task
// container logs in with its run credential and reaches only its own subjects.
// It runs before any other login, because those are decided here.
func (b *boot) startCallout(ctx context.Context, url, token string) error {
seed := os.Getenv(b.cfg.NATS.CalloutIssuerSeedEnv)
if seed == "" {
return fmt.Errorf("%s is required when nats.callout_issuer_seed_env is configured", b.cfg.NATS.CalloutIssuerSeedEnv)
}
opts := nats.CalloutOptions{URL: url, Token: token, IssuerSeed: seed}
if b.stateStoreGrants != nil {
opts.Tasks = taskBroker{runs: b.stateStoreGrants.Client}
}
svc, err := nats.StartCallout(ctx, opts, b.log)
if err != nil {
b.log.ErrorContext(ctx, "nats callout start failed", "err", err)
return err
}
b.addCleanup(svc.Close)
return nil
}
func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) { func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) {
cfg, log := b.cfg, b.log cfg, log := b.cfg, b.log
if cfg.StateDir == "" { if cfg.StateDir == "" {
@@ -627,7 +647,7 @@ func (b *boot) buildDaemon(ctx context.Context) {
d := &daemon.Daemon{ d := &daemon.Daemon{
ResolveRepository: b.repositoryAccess, ResolveRepository: b.repositoryAccess,
Cfg: b.cfgHolder, Cfg: b.cfgHolder,
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken, Scoped: b.cfg.NATS.Mode != config.NATSModeExternal}, ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL},
ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), CA: b.stateStoreCA(ctx)}, ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), CA: b.stateStoreCA(ctx)},
Store: b.stateStore, Store: b.stateStore,
Identity: daemon.NewIdentityGate(b.stateClient, b.stateClient, b.accessChain, b.stateClient, b.stateClient, b.orgPrincipals, workspaces, log), Identity: daemon.NewIdentityGate(b.stateClient, b.stateClient, b.accessChain, b.stateClient, b.stateClient, b.orgPrincipals, workspaces, log),
-13
View File
@@ -53,9 +53,6 @@ func (b *boot) setupReadinessProbes() {
if b.accessLive != nil { if b.accessLive != nil {
probes = append(probes, readiness.NewProblemProbe("access_policies", b.accessLive)) probes = append(probes, readiness.NewProblemProbe("access_policies", b.accessLive))
} }
if cfg.NATS.Mode == config.NATSModeExternal {
probes = append(probes, unscopedBroker{})
}
b.healthRegistry = health.NewRegistry(probes...) b.healthRegistry = health.NewRegistry(probes...)
} }
@@ -97,16 +94,6 @@ func diskProbePath(cfg config.Config) string {
} }
} }
// unscopedBroker reports that containers hold the external broker's own
// token: only the embedded broker can limit a container to its task.
type unscopedBroker struct{}
func (unscopedBroker) Name() string { return "nats_credential_scope" }
func (unscopedBroker) Check(context.Context) health.Result {
return health.Result{Status: health.StatusDegraded, Detail: "external NATS: containers get a broker token that reaches every subject; use the embedded broker"}
}
// stateCallDeadline bounds each State Store call the daemon makes without a // stateCallDeadline bounds each State Store call the daemon makes without a
// deadline of its own. // deadline of its own.
const stateCallDeadline = 20 * time.Second const stateCallDeadline = 20 * time.Second
+9 -2
View File
@@ -647,9 +647,16 @@ type NATSConfig struct {
// URL is the NATS server address, e.g. "nats://localhost:4222". // URL is the NATS server address, e.g. "nats://localhost:4222".
// Required when Mode is "external"; must be empty otherwise. // Required when Mode is "external"; must be empty otherwise.
URL string `toml:"url" yaml:"url"` URL string `toml:"url" yaml:"url"`
// TokenEnv optionally names an env var holding an external NATS auth token. // TokenEnv names an env var holding the external broker's instance token:
// Empty means no authentication. Embedded mode generates a per-start token. // the password of the "archie-callout" auth user, and the token archie's own
// services log in with. Required when Mode is "external". Embedded mode
// generates a per-start token.
TokenEnv string `toml:"token_env" yaml:"token_env"` TokenEnv string `toml:"token_env" yaml:"token_env"`
// CalloutIssuerSeedEnv names an env var holding the account nkey seed whose
// public key the broker lists as its auth_callout issuer. Required when
// Mode is "external": the broker defers every login to archied, so a task
// container holds only its run credential.
CalloutIssuerSeedEnv string `toml:"callout_issuer_seed_env" yaml:"callout_issuer_seed_env"`
} }
// CaptureConfig configures the unbound webhook capture endpoint. // CaptureConfig configures the unbound webhook capture endpoint.
-1
View File
@@ -132,7 +132,6 @@ func (d *Daemon) acquireTaskContainer(
func (d *Daemon) workerEndpoints(stateStoreToken string) egress.Endpoints { func (d *Daemon) workerEndpoints(stateStoreToken string) egress.Endpoints {
return egress.Endpoints{ return egress.Endpoints{
NATS: d.ConnectedNATS.URL, NATS: d.ConnectedNATS.URL,
NATSToken: d.ConnectedNATS.ContainerToken(),
StateStore: d.ConnectedStateStore.URL, StateStore: d.ConnectedStateStore.URL,
StateStoreToken: stateStoreToken, StateStoreToken: stateStoreToken,
StateStoreCA: d.ConnectedStateStore.CA, StateStoreCA: d.ConnectedStateStore.CA,
+4 -18
View File
@@ -41,25 +41,11 @@ type RunCredentialIssuer interface {
Issue(ctx context.Context, task *workflow.Task, limit time.Duration) (token string, revoke func(), err error) Issue(ctx context.Context, task *workflow.Task, limit time.Duration) (token string, revoke func(), err error)
} }
// NATSEndpoint is the broker address and credential archied connected with at // NATSEndpoint is the broker address archied connected with at startup. It is
// startup. It is runtime state rather than configuration: embedded mode // runtime state rather than configuration: embedded mode generates it. A
// generates both values, and external secret references are resolved once. // container logs in with its run credential, never the broker's token.
type NATSEndpoint struct { type NATSEndpoint struct {
URL string URL string
Token string
// Scoped is true on the embedded broker, where a container logs in with
// its run credential and reaches only its own task's subjects.
Scoped bool
}
// ContainerToken is the broker token a container is handed: none on a
// scoped broker. An external broker cannot scope a container, so it still
// gets the broker's token (reported as a health issue by archied).
func (e NATSEndpoint) ContainerToken() string {
if e.Scoped {
return ""
}
return e.Token
} }
// StateStoreEndpoint is the State Store target and the CA its certificate // StateStoreEndpoint is the State Store target and the CA its certificate
@@ -35,10 +35,8 @@ import (
// deleted // deleted
type Config struct { type Config struct {
URL string URL string
// Token is a broker-wide token, set only on an external broker. Without // The worker logs in as TaskID with StateStoreToken, its run credential,
// it the worker logs in as TaskID with StateStoreToken, its run // which reaches only its own task's subjects.
// credential, which reaches only its own task's subjects.
Token string
TaskID int64 TaskID int64
StateStoreURL string StateStoreURL string
@@ -82,10 +80,7 @@ func Connect(ctx context.Context, config Config, log *slog.Logger) (*Transport,
return nil, err return nil, err
} }
options := []natsio.Option{natsio.Name("archie-agent")} options := []natsio.Option{natsio.Name("archie-agent")}
switch { if config.StateStoreToken != "" {
case config.Token != "":
options = append(options, natsio.Token(config.Token))
case config.StateStoreToken != "":
options = append(options, options = append(options,
natsio.UserInfo(fmt.Sprintf("%s%d", eventbus.TaskUserPrefix, config.TaskID), config.StateStoreToken), natsio.UserInfo(fmt.Sprintf("%s%d", eventbus.TaskUserPrefix, config.TaskID), config.StateStoreToken),
natsio.CustomInboxPrefix(eventbus.TaskInboxPrefix(config.TaskID))) natsio.CustomInboxPrefix(eventbus.TaskInboxPrefix(config.TaskID)))
@@ -310,6 +310,9 @@ func validateNATS(cfg *config.Config) error {
if cfg.NATS.URL == "" { if cfg.NATS.URL == "" {
return fmt.Errorf("%w: nats.url is required when nats.mode is %q", ErrInvalidInput, config.NATSModeExternal) return fmt.Errorf("%w: nats.url is required when nats.mode is %q", ErrInvalidInput, config.NATSModeExternal)
} }
if cfg.NATS.TokenEnv == "" || cfg.NATS.CalloutIssuerSeedEnv == "" {
return fmt.Errorf("%w: nats.token_env and nats.callout_issuer_seed_env are required when nats.mode is %q", ErrInvalidInput, config.NATSModeExternal)
}
case config.NATSModeEmbedded: case config.NATSModeEmbedded:
if cfg.NATS.URL != "" { if cfg.NATS.URL != "" {
return fmt.Errorf("%w: nats.url must be empty when nats.mode is %q", ErrInvalidInput, mode) return fmt.Errorf("%w: nats.url must be empty when nats.mode is %q", ErrInvalidInput, mode)
@@ -12,7 +12,6 @@ import (
// needs, with the task's own credentials for them. // needs, with the task's own credentials for them.
type Endpoints struct { type Endpoints struct {
NATS string NATS string
NATSToken string
StateStore string StateStore string
StateStoreToken string StateStoreToken string
StateStoreCA []byte StateStoreCA []byte
@@ -34,9 +33,6 @@ func WorkerEnv(e Endpoints) ([]string, error) {
// credentials from e. The State Store is left out when e names none. // credentials from e. The State Store is left out when e names none.
func DirectWorkerEnv(nats, stateStore string, e Endpoints) []string { func DirectWorkerEnv(nats, stateStore string, e Endpoints) []string {
env := []string{"NATS_URL=" + nats} env := []string{"NATS_URL=" + nats}
if e.NATSToken != "" {
env = append(env, "NATS_TOKEN="+e.NATSToken)
}
if e.StateStore != "" { if e.StateStore != "" {
env = append(env, "STATE_STORE_URL="+stateStore) env = append(env, "STATE_STORE_URL="+stateStore)
if e.StateStoreToken != "" { if e.StateStoreToken != "" {
+47 -19
View File
@@ -3,6 +3,7 @@ package nats
import ( import (
"context" "context"
"crypto/subtle" "crypto/subtle"
"slices"
"strconv" "strconv"
"strings" "strings"
"time" "time"
@@ -38,34 +39,61 @@ const taskAuthTimeout = 2 * time.Second
// authenticator admits the instance token with every permission, and a task // authenticator admits the instance token with every permission, and a task
// user only to its own task's subjects, so no container holds a credential // user only to its own task's subjects, so no container holds a credential
// that reaches past its run. // that reaches past its run. The embedded broker and the auth callout of an
// external broker both decide through it.
type authenticator struct { type authenticator struct {
token string token string
tasks TaskAuthority tasks TaskAuthority
} }
// grant is what a login is allowed. Full is the instance token's reach; a task
// login lists its subjects instead.
type grant struct {
Username string
Full bool
Publish []string
Subscribe []string
}
// authorize decides one login: the instance token, or "task-<id>" with that
// task's run credential.
func (a authenticator) authorize(ctx context.Context, token, username, password string) (grant, bool) {
if token != "" {
if subtle.ConstantTimeCompare([]byte(token), []byte(a.token)) != 1 {
return grant{}, false
}
return grant{Username: "archie", Full: true}, true
}
taskID, ok := taskUser(username)
if !ok || password == "" || a.tasks == nil {
return grant{}, false
}
ctx, cancel := context.WithTimeout(ctx, taskAuthTimeout)
defer cancel()
subjects, err := a.tasks.TaskSubjects(ctx, taskID, password)
if err != nil {
return grant{}, false
}
return grant{
Username: username,
Publish: slices.Clone(subjects.Publish),
Subscribe: append(slices.Clone(subjects.Subscribe), TaskInboxPrefix(taskID)+".>"),
}, true
}
func (a authenticator) Check(c server.ClientAuthentication) bool { func (a authenticator) Check(c server.ClientAuthentication) bool {
opts := c.GetOpts() opts := c.GetOpts()
if opts.Token != "" { g, ok := a.authorize(context.Background(), opts.Token, opts.Username, opts.Password)
if subtle.ConstantTimeCompare([]byte(opts.Token), []byte(a.token)) != 1 { if !ok {
return false return false
} }
c.RegisterUser(&server.User{Username: "archie"}) if g.Full {
c.RegisterUser(&server.User{Username: g.Username})
return true return true
} }
taskID, ok := taskUser(opts.Username) c.RegisterUser(&server.User{Username: g.Username, Permissions: &server.Permissions{
if !ok || opts.Password == "" || a.tasks == nil { Publish: &server.SubjectPermission{Allow: g.Publish},
return false Subscribe: &server.SubjectPermission{Allow: g.Subscribe},
}
ctx, cancel := context.WithTimeout(context.Background(), taskAuthTimeout)
defer cancel()
subjects, err := a.tasks.TaskSubjects(ctx, taskID, opts.Password)
if err != nil {
return false
}
c.RegisterUser(&server.User{Username: opts.Username, Permissions: &server.Permissions{
Publish: &server.SubjectPermission{Allow: subjects.Publish},
Subscribe: &server.SubjectPermission{Allow: append(subjects.Subscribe, TaskInboxPrefix(taskID)+".>")},
// Lets the task answer the one request it is sent, on the requester's // Lets the task answer the one request it is sent, on the requester's
// reply subject. // reply subject.
Response: &server.ResponsePermission{MaxMsgs: 1}, Response: &server.ResponsePermission{MaxMsgs: 1},
@@ -8,7 +8,9 @@ import (
"testing" "testing"
"time" "time"
"github.com/nats-io/nats-server/v2/server"
natsio "github.com/nats-io/nats.go" natsio "github.com/nats-io/nats.go"
"github.com/nats-io/nkeys"
) )
type taskSeven struct{} type taskSeven struct{}
@@ -22,14 +24,60 @@ func (taskSeven) TaskSubjects(_ context.Context, taskID int64, credential string
// TestBrokerScopesTaskCredentials pins that the instance token reaches // TestBrokerScopesTaskCredentials pins that the instance token reaches
// everything and a task's run credential reaches only its own subjects, on a // everything and a task's run credential reaches only its own subjects, on a
// real embedded broker. // real embedded broker and on an external one that defers logins to the
// callout. It guards a task container being handed more than its own subjects.
func TestBrokerScopesTaskCredentials(t *testing.T) { func TestBrokerScopesTaskCredentials(t *testing.T) {
srv, err := StartEmbedded(context.Background(), EmbeddedOptions{Token: "instance", StoreDir: t.TempDir(), Tasks: taskSeven{}}, nil) embedded, err := StartEmbedded(context.Background(), EmbeddedOptions{Token: "instance", StoreDir: t.TempDir(), Tasks: taskSeven{}}, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
defer srv.Shutdown() defer embedded.Shutdown()
for name, url := range map[string]string{"embedded": embedded.ClientURL(), "callout": calloutBroker(t)} {
t.Run(name, func(t *testing.T) { checkScopes(t, url) })
}
}
// calloutBroker starts an external-style broker whose logins are decided by
// StartCallout, configured the way an operator would: one auth user for the
// callout itself, every other login deferred.
func calloutBroker(t *testing.T) string {
t.Helper()
issuer, err := nkeys.CreateAccount()
if err != nil {
t.Fatal(err)
}
pub, err := issuer.PublicKey()
if err != nil {
t.Fatal(err)
}
seed, err := issuer.Seed()
if err != nil {
t.Fatal(err)
}
srv, err := server.NewServer(&server.Options{
Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true,
Users: []*server.User{{Username: CalloutUser, Password: "instance"}},
AuthCallout: &server.AuthCallout{Issuer: pub, Account: "$G", AuthUsers: []string{CalloutUser}},
})
if err != nil {
t.Fatal(err)
}
srv.Start()
t.Cleanup(srv.Shutdown)
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("broker not ready")
}
svc, err := StartCallout(context.Background(), CalloutOptions{URL: srv.ClientURL(), Token: "instance", IssuerSeed: string(seed), Tasks: taskSeven{}}, nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(svc.Close)
return srv.ClientURL()
}
func checkScopes(t *testing.T, url string) {
t.Helper()
tests := []struct { tests := []struct {
name string name string
opts []natsio.Option opts []natsio.Option
@@ -61,7 +109,7 @@ func TestBrokerScopesTaskCredentials(t *testing.T) {
opts := append(slices.Clone(tt.opts), natsio.ErrorHandler(func(_ *natsio.Conn, _ *natsio.Subscription, err error) { opts := append(slices.Clone(tt.opts), natsio.ErrorHandler(func(_ *natsio.Conn, _ *natsio.Subscription, err error) {
violations <- err.Error() violations <- err.Error()
})) }))
nc, err := natsio.Connect(srv.ClientURL(), opts...) nc, err := natsio.Connect(url, opts...)
if !tt.connects { if !tt.connects {
if err == nil { if err == nil {
nc.Close() nc.Close()
@@ -0,0 +1,101 @@
package nats
import (
"context"
"errors"
"fmt"
"log/slog"
"github.com/nats-io/jwt/v2"
natsio "github.com/nats-io/nats.go"
"github.com/nats-io/nkeys"
"github.com/synadia-io/callout.go"
)
// CalloutUser is the broker login archied's callout service connects with. The
// external broker lists it in auth_users, so its own login skips the callout;
// its password is the instance token.
const CalloutUser = "archie-callout"
// calloutAccount is the account a granted login is placed in: the broker's
// global account, which JetStream uses.
const calloutAccount = "$G"
// CalloutOptions configures StartCallout.
type CalloutOptions struct {
// URL is the external broker.
URL string
// Token is the instance token: the callout user's password, and the token
// that logs in with every permission.
Token string
// IssuerSeed is the account nkey seed whose public key the broker names as
// its auth_callout issuer.
IssuerSeed string
// Tasks admits task users to their own task's subjects.
Tasks TaskAuthority
}
// Callout is the authorization service of an external broker.
type Callout struct {
svc *callout.AuthorizationService
nc *natsio.Conn
}
// StartCallout answers an external broker's login requests with the decisions
// the embedded broker makes, so a task container holds only its run credential.
func StartCallout(ctx context.Context, opts CalloutOptions, log *slog.Logger) (*Callout, error) {
if log == nil {
log = slog.New(slog.DiscardHandler)
}
issuer, err := nkeys.FromSeed([]byte(opts.IssuerSeed))
if err != nil {
return nil, errors.New("nats callout: issuer seed is not an nkey seed")
}
nc, err := natsio.Connect(opts.URL, natsio.UserInfo(CalloutUser, opts.Token), natsio.Name("archie-callout"))
if err != nil {
return nil, fmt.Errorf("nats callout connect: %w", err)
}
svc, err := callout.NewAuthorizationService(nc,
callout.Authorizer(authenticator{token: opts.Token, tasks: opts.Tasks}.calloutAuthorizer(context.WithoutCancel(ctx), issuer)),
callout.ResponseSignerKey(issuer),
callout.Logger(brokerLogger{log: log.With("service", "nats", "component", "callout")}),
)
if err != nil {
nc.Close()
return nil, fmt.Errorf("nats callout service: %w", err)
}
log.InfoContext(ctx, "nats callout ready", "url", opts.URL)
return &Callout{svc: svc, nc: nc}, nil
}
// Close stops answering logins.
func (c *Callout) Close() {
_ = c.svc.Stop()
c.nc.Close()
}
// calloutAuthorizer turns an authenticator decision into a user JWT. An error
// refuses the login.
func (a authenticator) calloutAuthorizer(ctx context.Context, issuer nkeys.KeyPair) callout.AuthorizerFn {
return func(req *jwt.AuthorizationRequest) (string, error) {
g, ok := a.authorize(ctx, req.ConnectOptions.Token, req.ConnectOptions.Username, req.ConnectOptions.Password)
if !ok {
return "", errors.New("login refused")
}
return g.userJWT(req.UserNkey, issuer)
}
}
// userJWT is the grant as a broker user's permissions, signed by the issuer the
// broker trusts.
func (g grant) userJWT(userNkey string, issuer nkeys.KeyPair) (string, error) {
uc := jwt.NewUserClaims(userNkey)
uc.Name = g.Username
uc.Audience = calloutAccount
if !g.Full {
uc.Pub.Allow.Add(g.Publish...)
uc.Sub.Allow.Add(g.Subscribe...)
uc.Resp = &jwt.ResponsePermission{MaxMsgs: 1}
}
return uc.Encode(issuer)
}
@@ -30,14 +30,14 @@ func TestSandboxPolicyAdmitsOnlyWhatTheTaskNeeds(t *testing.T) {
name: "services by IP and loopback use the services host", name: "services by IP and loopback use the services host",
servicesHost: "archie.lan", servicesHost: "archie.lan",
launch: Launch{Worker: egress.Endpoints{ launch: Launch{Worker: egress.Endpoints{
NATS: "nats://172.17.0.1:4222", NATSToken: "t", StateStore: "dns:///127.0.0.1:50051", StateStoreToken: "run-1", NATS: "nats://172.17.0.1:4222", StateStore: "dns:///127.0.0.1:50051", StateStoreToken: "run-1",
}}, }},
wantRules: map[string]string{ wantRules: map[string]string{
"archie-nats": "archie.lan:4222 " + agentBinary, "archie-nats": "archie.lan:4222 " + agentBinary,
"archie-state-store": "archie.lan:50051 " + agentBinary, "archie-state-store": "archie.lan:50051 " + agentBinary,
}, },
wantEnv: map[string]string{ wantEnv: map[string]string{
"NATS_URL": "nats://archie.lan:4222", "NATS_TOKEN": "t", "NATS_URL": "nats://archie.lan:4222",
"STATE_STORE_URL": "archie.lan:50051", "STATE_STORE_TOKEN": "run-1", "STATE_STORE_URL": "archie.lan:50051", "STATE_STORE_TOKEN": "run-1",
}, },
}, },