feat(nats): an external broker defers logins to archie through auth callout (#1866)

Task containers log in with their run credential on an external broker and
reach only their own subjects, decided by the embedded broker's authenticator.
The broker token is never put in a container environment.
This commit is contained in:
sam committed 2026-11-14 04:04:15 +11:00
1 parent 7d664d7efb
commit 3a965e2a4f
22 files changed
+299 -103

No files matched your search

+1 -1
View File
@@ -103,5 +103,5 @@ ENV GOMODCACHE=/data/cache/go/mod \
COPY --from=builder /usr/local/bin/archie-agent /usr/local/bin/archie-agent
# The daemon injects NATS_URL and, when configured, NATS_TOKEN.
# The daemon injects NATS_URL.
ENTRYPOINT ["archie-agent"]
+4 -10
View File
@@ -52,7 +52,10 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run
return 0
}
natsURL, natsToken := natsConnectionSettings(*natsURLFlag, getenv)
natsURL := *natsURLFlag
if natsURL == "" {
natsURL = getenv("NATS_URL")
}
if natsURL == "" {
_, _ = fmt.Fprintln(stderr, "error: -nats-url or NATS_URL is required")
flags.Usage()
@@ -79,7 +82,6 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run
if err := runWorker(ctx, agentworker.Settings{
NATSURL: natsURL,
NATSToken: natsToken,
StateStoreTarget: stateStoreURL,
StateStoreToken: stateStoreToken,
StateStoreCA: []byte(getenv("STATE_STORE_CA")),
@@ -92,14 +94,6 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run
return 0
}
func natsConnectionSettings(flagURL string, getenv func(string) string) (url, token string) {
url = flagURL
if url == "" {
url = getenv("NATS_URL")
}
return url, getenv("NATS_TOKEN")
}
type relayServer func(ctx context.Context, forwards map[string]string) error
// runRelay is the sandbox egress relay: the container on both a sandbox's
+2 -1
View File
@@ -74,7 +74,8 @@ target = "127.0.0.1:9090"
# [nats]
# mode = "embedded" # or "external" with url
# url = "nats://localhost:4222"
# token_env = "NATS_TOKEN"
# token_env = "NATS_TOKEN" # external: the archie-callout user's password
# callout_issuer_seed_env = "NATS_CALLOUT_SEED" # external: account nkey seed; the broker's auth_callout issuer is its public key
# [log]
# file = "~/.local/share/archie/logs/archied.log"
+2
View File
@@ -59,6 +59,8 @@ mode = "external"
# The pinned Compose-network gateway is reachable from both the host daemon
# and the task containers it launches on archie-core_default.
url = "nats://172.19.0.1:4222"
token_env = "NATS_TOKEN"
callout_issuer_seed_env = "NATS_CALLOUT_SEED"
[containers]
network = "archie-core_default"
+10
View File
@@ -0,0 +1,10 @@
jetstream { store_dir: /data }
http_port: 8222
authorization {
users: [{ user: "archie-callout", password: $NATS_TOKEN }]
auth_callout {
issuer: $NATS_CALLOUT_ISSUER
auth_users: ["archie-callout"]
}
}
+8
View File
@@ -225,11 +225,19 @@ docker compose up -d nats
[nats]
mode = "external"
url = "nats://172.19.0.1:4222"
token_env = "NATS_TOKEN"
callout_issuer_seed_env = "NATS_CALLOUT_SEED"
[containers]
network = "archie-core_default"
```
The broker defers every login to archied (NATS auth callout), so a worker logs
in with its run credential and reaches only its own task's subjects. Generate
an account nkey (`nk -gen account -pubout`), export its seed as
`NATS_CALLOUT_SEED` for archied and its public key as `NATS_CALLOUT_ISSUER` for
Compose, and set `NATS_TOKEN` to the same value in both.
The daemon hands workers the URL its own client connected to. The Compose
service name is not resolvable from the native host, and localhost inside a
worker is the worker itself; the pinned network gateway is reachable from both.
+7 -7
View File
@@ -46,19 +46,19 @@ services:
nats:
image: nats:2.14-alpine3.22
# The client port is published on every interface (agent containers reach
# it over the bridge), so it refuses to start without a token. Set
# NATS_TOKEN here and name the same value in [nats] token_env.
entrypoint:
- sh
- -c
- '[ -n "$$NATS_TOKEN" ] || { echo "NATS_TOKEN is required" >&2; exit 1; }; exec nats-server -js -m 8222 --auth "$$NATS_TOKEN"'
# it over the bridge), so it defers every login to archied. Set NATS_TOKEN
# (name it in [nats] token_env) and NATS_CALLOUT_ISSUER, the public key of
# the account nkey seed named in [nats] callout_issuer_seed_env.
command: ["-c", "/etc/nats/nats.conf"]
environment:
NATS_TOKEN: ${NATS_TOKEN:-}
NATS_TOKEN: ${NATS_TOKEN:?NATS_TOKEN is required}
NATS_CALLOUT_ISSUER: ${NATS_CALLOUT_ISSUER:?NATS_CALLOUT_ISSUER is required}
ports:
- "4222:4222" # client
- "127.0.0.1:8222:8222" # monitoring (http://localhost:8222)
volumes:
- nats_data:/data
- ./deployments/nats-callout.conf:/etc/nats/nats.conf:ro
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:8222/healthz"]
+3 -2
View File
@@ -38,8 +38,6 @@ require (
github.com/mfridman/interpolate v0.2.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/nats-io/jwt/v2 v2.8.3 // indirect
github.com/nats-io/nkeys v0.4.17 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/oklog/run v1.2.0 // indirect
github.com/philhofer/fwd v1.2.1 // indirect
@@ -86,12 +84,15 @@ require (
github.com/moby/moby/api v1.56.1
github.com/moby/moby/client v0.6.2
github.com/modelcontextprotocol/go-sdk v1.8.0
github.com/nats-io/jwt/v2 v2.8.3
github.com/nats-io/nats-server/v2 v2.15.3
github.com/nats-io/nats.go v1.55.0
github.com/nats-io/nkeys v0.4.17
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.1
github.com/pressly/goose/v3 v3.30.1
github.com/samcharles93/ai-sdk v0.6.0
github.com/synadia-io/callout.go v0.2.3
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
go.opentelemetry.io/otel v1.47.0
+12
View File
@@ -2,6 +2,8 @@ cel.dev/cel-go v0.33.2 h1:eAcEdBguc6AFrGbPOyE0g168Uey4kKFW9J6qsBLSeMU=
cel.dev/cel-go v0.33.2/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU=
cel.dev/expr v0.26.0 h1:oLXvOF0jiYA5A/tmzU91pLEWZkKynhLJlC9Gw7PrKvs=
cel.dev/expr v0.26.0/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/Microsoft/go-winio v0.6.4 h1:kJF2pamtYCWXf0xAT8sAkwKC0pMl2HYBrRPHAqyUc+g=
@@ -16,6 +18,8 @@ github.com/antithesishq/antithesis-sdk-go v0.9.0 h1:JQpWOgYSYBb9HqgPtaXBswHxZFmO
github.com/antithesishq/antithesis-sdk-go v0.9.0/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
github.com/aricart/nst.go v0.2.0 h1:vfC+upJofU7OFJZM5Ph+HNko1XfJtVuSE6Q7FCQmT3g=
github.com/aricart/nst.go v0.2.0/go.mod h1:13ztPOEw3Jz0/LT8yDi5wZB+T4Lsj3Uhkc17CyZthII=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw=
@@ -135,6 +139,8 @@ github.com/mfridman/interpolate v0.2.0 h1:HM9ID8p6CUxoMJX+fECOtGQ2Mu7TW/GIMPyu48
github.com/mfridman/interpolate v0.2.0/go.mod h1:8SfDy95IaPNnIokZh9mvIppmc3fGl8fx+Gu7uusuO9Q=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/moby/api v1.56.1 h1:PpWkvVPB7Fr/No8w+TfyJ/I6rWZ2YPZhT3JorIg+06c=
@@ -151,6 +157,8 @@ github.com/nats-io/nats.go v1.55.0 h1:Ng8QKHdvdHIWoaiq0+d/aH1BWkxvcJxWJ0Y/9BPKx3
github.com/nats-io/nats.go v1.55.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.17 h1:JAJrlDhOyQqOt0a1eg4cbIZ1Kc7+vmkCigAltpwQpVA=
github.com/nats-io/nkeys v0.4.17/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nsc/v2 v2.10.3-0.20250110165315-eeda721ecff6 h1:V1uh9L3rGIUeYoMd0/EZY2Tvy5+1CkgGpDRx8t3+PNY=
github.com/nats-io/nsc/v2 v2.10.3-0.20250110165315-eeda721ecff6/go.mod h1:ScomAvx1cgjiXzW3WpGo9x/lLENkwELhewCcok/GTU8=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk=
@@ -191,6 +199,10 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/synadia-io/callout.go v0.2.3 h1:22u8sIIdpkKuyySXqEWvGAkAQ4tMBnvLtpVPNBdbj6Y=
github.com/synadia-io/callout.go v0.2.3/go.mod h1:IDVofaME9xzbBO5nUvEL59Ml0jGd6gAsF1qC4Nm6Pn8=
github.com/synadia-io/jwt-auth-builder.go v0.0.9 h1:UmXQ0JsqiA7sboE0VC46YEqrGAFA4MYHnhw54uYKRi8=
github.com/synadia-io/jwt-auth-builder.go v0.0.9/go.mod h1:0XPmvG3TihNkiM3ZFVtcCAzv0oBgFtjnw/lb/h8NHs0=
github.com/tinylib/msgp v1.6.5 h1:iAH6XTP7BpzErjBZlujJQXxA+GmRi2YWs6M4KdLtNbE=
github.com/tinylib/msgp v1.6.5/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+4 -6
View File
@@ -35,9 +35,8 @@ func (e *StartupError) Unwrap() error { return e.Err }
// Settings are process-derived inputs for one worker runtime.
type Settings struct {
NATSURL string
NATSToken string
WorkDir string
NATSURL string
WorkDir string
// StateStoreTarget and StateStoreToken are the daemon-injected State Store
// gRPC endpoint and bearer token. StateStoreTarget is required: the legacy
@@ -128,8 +127,8 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies
}
dependencies.markSafe(ctx, workDir, log)
// The task is known before connecting: without a broker token the
// worker logs in as its task, with its run credential.
// The task is known before connecting: the worker logs in as its task,
// with its run credential.
taskID, err := dependencies.bootID(workDir)
if err != nil {
log.ErrorContext(ctx, "task boot failed", "err", err)
@@ -137,7 +136,6 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies
}
transport, err := dependencies.connect(ctx, agentnats.Config{
URL: settings.NATSURL,
Token: settings.NATSToken,
TaskID: taskID,
StateStoreURL: settings.StateStoreTarget,
StateStoreToken: settings.StateStoreToken,
+25 -5
View File
@@ -155,9 +155,6 @@ type boot struct {
// the embedded server's ClientURL(). Recorded so Daemon.ConnectedNATS
// carries the real endpoint rather than an empty config URL.
natsURL string
// natsToken is the credential the daemon's client connected with at
// startup. Embedded mode generates it; external mode resolves it once.
natsToken string
containerPool *container.Pool
// taskRuntime is what task agents run in: the container pool, or an
@@ -388,6 +385,9 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco
log.ErrorContext(ctx, "nats credentials", "err", err)
return err
}
if err := b.startCallout(ctx, url, natsToken); err != nil {
return err
}
} else {
endpoint, readErr := servicekit.ReadNATSEndpoint(cfg.StateDir)
if readErr == nil {
@@ -445,7 +445,6 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco
}
b.natsClient = natsClient
b.natsURL = url
b.natsToken = natsToken
b.addCleanup(func() { natsClient.Close() })
b.addCleanup(func() { reactionClient.Close() })
b.reactionClient = reactionClient
@@ -453,6 +452,27 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco
return nil
}
// startCallout makes the external broker defer every login to archie, so a task
// container logs in with its run credential and reaches only its own subjects.
// It runs before any other login, because those are decided here.
func (b *boot) startCallout(ctx context.Context, url, token string) error {
seed := os.Getenv(b.cfg.NATS.CalloutIssuerSeedEnv)
if seed == "" {
return fmt.Errorf("%s is required when nats.callout_issuer_seed_env is configured", b.cfg.NATS.CalloutIssuerSeedEnv)
}
opts := nats.CalloutOptions{URL: url, Token: token, IssuerSeed: seed}
if b.stateStoreGrants != nil {
opts.Tasks = taskBroker{runs: b.stateStoreGrants.Client}
}
svc, err := nats.StartCallout(ctx, opts, b.log)
if err != nil {
b.log.ErrorContext(ctx, "nats callout start failed", "err", err)
return err
}
b.addCleanup(svc.Close)
return nil
}
func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) {
cfg, log := b.cfg, b.log
if cfg.StateDir == "" {
@@ -627,7 +647,7 @@ func (b *boot) buildDaemon(ctx context.Context) {
d := &daemon.Daemon{
ResolveRepository: b.repositoryAccess,
Cfg: b.cfgHolder,
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken, Scoped: b.cfg.NATS.Mode != config.NATSModeExternal},
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL},
ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), CA: b.stateStoreCA(ctx)},
Store: b.stateStore,
Identity: daemon.NewIdentityGate(b.stateClient, b.stateClient, b.accessChain, b.stateClient, b.stateClient, b.orgPrincipals, workspaces, log),
-13
View File
@@ -53,9 +53,6 @@ func (b *boot) setupReadinessProbes() {
if b.accessLive != nil {
probes = append(probes, readiness.NewProblemProbe("access_policies", b.accessLive))
}
if cfg.NATS.Mode == config.NATSModeExternal {
probes = append(probes, unscopedBroker{})
}
b.healthRegistry = health.NewRegistry(probes...)
}
@@ -97,16 +94,6 @@ func diskProbePath(cfg config.Config) string {
}
}
// unscopedBroker reports that containers hold the external broker's own
// token: only the embedded broker can limit a container to its task.
type unscopedBroker struct{}
func (unscopedBroker) Name() string { return "nats_credential_scope" }
func (unscopedBroker) Check(context.Context) health.Result {
return health.Result{Status: health.StatusDegraded, Detail: "external NATS: containers get a broker token that reaches every subject; use the embedded broker"}
}
// stateCallDeadline bounds each State Store call the daemon makes without a
// deadline of its own.
const stateCallDeadline = 20 * time.Second
+9 -2
View File
@@ -647,9 +647,16 @@ type NATSConfig struct {
// URL is the NATS server address, e.g. "nats://localhost:4222".
// Required when Mode is "external"; must be empty otherwise.
URL string `toml:"url" yaml:"url"`
// TokenEnv optionally names an env var holding an external NATS auth token.
// Empty means no authentication. Embedded mode generates a per-start token.
// TokenEnv names an env var holding the external broker's instance token:
// the password of the "archie-callout" auth user, and the token archie's own
// services log in with. Required when Mode is "external". Embedded mode
// generates a per-start token.
TokenEnv string `toml:"token_env" yaml:"token_env"`
// CalloutIssuerSeedEnv names an env var holding the account nkey seed whose
// public key the broker lists as its auth_callout issuer. Required when
// Mode is "external": the broker defers every login to archied, so a task
// container holds only its run credential.
CalloutIssuerSeedEnv string `toml:"callout_issuer_seed_env" yaml:"callout_issuer_seed_env"`
}
// CaptureConfig configures the unbound webhook capture endpoint.
-1
View File
@@ -132,7 +132,6 @@ func (d *Daemon) acquireTaskContainer(
func (d *Daemon) workerEndpoints(stateStoreToken string) egress.Endpoints {
return egress.Endpoints{
NATS: d.ConnectedNATS.URL,
NATSToken: d.ConnectedNATS.ContainerToken(),
StateStore: d.ConnectedStateStore.URL,
StateStoreToken: stateStoreToken,
StateStoreCA: d.ConnectedStateStore.CA,
+4 -18
View File
@@ -41,25 +41,11 @@ type RunCredentialIssuer interface {
Issue(ctx context.Context, task *workflow.Task, limit time.Duration) (token string, revoke func(), err error)
}
// NATSEndpoint is the broker address and credential archied connected with at
// startup. It is runtime state rather than configuration: embedded mode
// generates both values, and external secret references are resolved once.
// NATSEndpoint is the broker address archied connected with at startup. It is
// runtime state rather than configuration: embedded mode generates it. A
// container logs in with its run credential, never the broker's token.
type NATSEndpoint struct {
URL string
Token string
// Scoped is true on the embedded broker, where a container logs in with
// its run credential and reaches only its own task's subjects.
Scoped bool
}
// ContainerToken is the broker token a container is handed: none on a
// scoped broker. An external broker cannot scope a container, so it still
// gets the broker's token (reported as a health issue by archied).
func (e NATSEndpoint) ContainerToken() string {
if e.Scoped {
return ""
}
return e.Token
URL string
}
// StateStoreEndpoint is the State Store target and the CA its certificate
@@ -35,10 +35,8 @@ import (
// deleted
type Config struct {
URL string
// Token is a broker-wide token, set only on an external broker. Without
// it the worker logs in as TaskID with StateStoreToken, its run
// credential, which reaches only its own task's subjects.
Token string
// The worker logs in as TaskID with StateStoreToken, its run credential,
// which reaches only its own task's subjects.
TaskID int64
StateStoreURL string
@@ -82,10 +80,7 @@ func Connect(ctx context.Context, config Config, log *slog.Logger) (*Transport,
return nil, err
}
options := []natsio.Option{natsio.Name("archie-agent")}
switch {
case config.Token != "":
options = append(options, natsio.Token(config.Token))
case config.StateStoreToken != "":
if config.StateStoreToken != "" {
options = append(options,
natsio.UserInfo(fmt.Sprintf("%s%d", eventbus.TaskUserPrefix, config.TaskID), config.StateStoreToken),
natsio.CustomInboxPrefix(eventbus.TaskInboxPrefix(config.TaskID)))
@@ -310,6 +310,9 @@ func validateNATS(cfg *config.Config) error {
if cfg.NATS.URL == "" {
return fmt.Errorf("%w: nats.url is required when nats.mode is %q", ErrInvalidInput, config.NATSModeExternal)
}
if cfg.NATS.TokenEnv == "" || cfg.NATS.CalloutIssuerSeedEnv == "" {
return fmt.Errorf("%w: nats.token_env and nats.callout_issuer_seed_env are required when nats.mode is %q", ErrInvalidInput, config.NATSModeExternal)
}
case config.NATSModeEmbedded:
if cfg.NATS.URL != "" {
return fmt.Errorf("%w: nats.url must be empty when nats.mode is %q", ErrInvalidInput, mode)
@@ -12,7 +12,6 @@ import (
// needs, with the task's own credentials for them.
type Endpoints struct {
NATS string
NATSToken string
StateStore string
StateStoreToken string
StateStoreCA []byte
@@ -34,9 +33,6 @@ func WorkerEnv(e Endpoints) ([]string, error) {
// credentials from e. The State Store is left out when e names none.
func DirectWorkerEnv(nats, stateStore string, e Endpoints) []string {
env := []string{"NATS_URL=" + nats}
if e.NATSToken != "" {
env = append(env, "NATS_TOKEN="+e.NATSToken)
}
if e.StateStore != "" {
env = append(env, "STATE_STORE_URL="+stateStore)
if e.StateStoreToken != "" {
+47 -19
View File
@@ -3,6 +3,7 @@ package nats
import (
"context"
"crypto/subtle"
"slices"
"strconv"
"strings"
"time"
@@ -38,34 +39,61 @@ const taskAuthTimeout = 2 * time.Second
// authenticator admits the instance token with every permission, and a task
// user only to its own task's subjects, so no container holds a credential
// that reaches past its run.
// that reaches past its run. The embedded broker and the auth callout of an
// external broker both decide through it.
type authenticator struct {
token string
tasks TaskAuthority
}
// grant is what a login is allowed. Full is the instance token's reach; a task
// login lists its subjects instead.
type grant struct {
Username string
Full bool
Publish []string
Subscribe []string
}
// authorize decides one login: the instance token, or "task-<id>" with that
// task's run credential.
func (a authenticator) authorize(ctx context.Context, token, username, password string) (grant, bool) {
if token != "" {
if subtle.ConstantTimeCompare([]byte(token), []byte(a.token)) != 1 {
return grant{}, false
}
return grant{Username: "archie", Full: true}, true
}
taskID, ok := taskUser(username)
if !ok || password == "" || a.tasks == nil {
return grant{}, false
}
ctx, cancel := context.WithTimeout(ctx, taskAuthTimeout)
defer cancel()
subjects, err := a.tasks.TaskSubjects(ctx, taskID, password)
if err != nil {
return grant{}, false
}
return grant{
Username: username,
Publish: slices.Clone(subjects.Publish),
Subscribe: append(slices.Clone(subjects.Subscribe), TaskInboxPrefix(taskID)+".>"),
}, true
}
func (a authenticator) Check(c server.ClientAuthentication) bool {
opts := c.GetOpts()
if opts.Token != "" {
if subtle.ConstantTimeCompare([]byte(opts.Token), []byte(a.token)) != 1 {
return false
}
c.RegisterUser(&server.User{Username: "archie"})
g, ok := a.authorize(context.Background(), opts.Token, opts.Username, opts.Password)
if !ok {
return false
}
if g.Full {
c.RegisterUser(&server.User{Username: g.Username})
return true
}
taskID, ok := taskUser(opts.Username)
if !ok || opts.Password == "" || a.tasks == nil {
return false
}
ctx, cancel := context.WithTimeout(context.Background(), taskAuthTimeout)
defer cancel()
subjects, err := a.tasks.TaskSubjects(ctx, taskID, opts.Password)
if err != nil {
return false
}
c.RegisterUser(&server.User{Username: opts.Username, Permissions: &server.Permissions{
Publish: &server.SubjectPermission{Allow: subjects.Publish},
Subscribe: &server.SubjectPermission{Allow: append(subjects.Subscribe, TaskInboxPrefix(taskID)+".>")},
c.RegisterUser(&server.User{Username: g.Username, Permissions: &server.Permissions{
Publish: &server.SubjectPermission{Allow: g.Publish},
Subscribe: &server.SubjectPermission{Allow: g.Subscribe},
// Lets the task answer the one request it is sent, on the requester's
// reply subject.
Response: &server.ResponsePermission{MaxMsgs: 1},
@@ -8,7 +8,9 @@ import (
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
natsio "github.com/nats-io/nats.go"
"github.com/nats-io/nkeys"
)
type taskSeven struct{}
@@ -22,14 +24,60 @@ func (taskSeven) TaskSubjects(_ context.Context, taskID int64, credential string
// TestBrokerScopesTaskCredentials pins that the instance token reaches
// everything and a task's run credential reaches only its own subjects, on a
// real embedded broker.
// real embedded broker and on an external one that defers logins to the
// callout. It guards a task container being handed more than its own subjects.
func TestBrokerScopesTaskCredentials(t *testing.T) {
srv, err := StartEmbedded(context.Background(), EmbeddedOptions{Token: "instance", StoreDir: t.TempDir(), Tasks: taskSeven{}}, nil)
embedded, err := StartEmbedded(context.Background(), EmbeddedOptions{Token: "instance", StoreDir: t.TempDir(), Tasks: taskSeven{}}, nil)
if err != nil {
t.Fatal(err)
}
defer srv.Shutdown()
defer embedded.Shutdown()
for name, url := range map[string]string{"embedded": embedded.ClientURL(), "callout": calloutBroker(t)} {
t.Run(name, func(t *testing.T) { checkScopes(t, url) })
}
}
// calloutBroker starts an external-style broker whose logins are decided by
// StartCallout, configured the way an operator would: one auth user for the
// callout itself, every other login deferred.
func calloutBroker(t *testing.T) string {
t.Helper()
issuer, err := nkeys.CreateAccount()
if err != nil {
t.Fatal(err)
}
pub, err := issuer.PublicKey()
if err != nil {
t.Fatal(err)
}
seed, err := issuer.Seed()
if err != nil {
t.Fatal(err)
}
srv, err := server.NewServer(&server.Options{
Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true,
Users: []*server.User{{Username: CalloutUser, Password: "instance"}},
AuthCallout: &server.AuthCallout{Issuer: pub, Account: "$G", AuthUsers: []string{CalloutUser}},
})
if err != nil {
t.Fatal(err)
}
srv.Start()
t.Cleanup(srv.Shutdown)
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("broker not ready")
}
svc, err := StartCallout(context.Background(), CalloutOptions{URL: srv.ClientURL(), Token: "instance", IssuerSeed: string(seed), Tasks: taskSeven{}}, nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(svc.Close)
return srv.ClientURL()
}
func checkScopes(t *testing.T, url string) {
t.Helper()
tests := []struct {
name string
opts []natsio.Option
@@ -61,7 +109,7 @@ func TestBrokerScopesTaskCredentials(t *testing.T) {
opts := append(slices.Clone(tt.opts), natsio.ErrorHandler(func(_ *natsio.Conn, _ *natsio.Subscription, err error) {
violations <- err.Error()
}))
nc, err := natsio.Connect(srv.ClientURL(), opts...)
nc, err := natsio.Connect(url, opts...)
if !tt.connects {
if err == nil {
nc.Close()
@@ -0,0 +1,101 @@
package nats
import (
"context"
"errors"
"fmt"
"log/slog"
"github.com/nats-io/jwt/v2"
natsio "github.com/nats-io/nats.go"
"github.com/nats-io/nkeys"
"github.com/synadia-io/callout.go"
)
// CalloutUser is the broker login archied's callout service connects with. The
// external broker lists it in auth_users, so its own login skips the callout;
// its password is the instance token.
const CalloutUser = "archie-callout"
// calloutAccount is the account a granted login is placed in: the broker's
// global account, which JetStream uses.
const calloutAccount = "$G"
// CalloutOptions configures StartCallout.
type CalloutOptions struct {
// URL is the external broker.
URL string
// Token is the instance token: the callout user's password, and the token
// that logs in with every permission.
Token string
// IssuerSeed is the account nkey seed whose public key the broker names as
// its auth_callout issuer.
IssuerSeed string
// Tasks admits task users to their own task's subjects.
Tasks TaskAuthority
}
// Callout is the authorization service of an external broker.
type Callout struct {
svc *callout.AuthorizationService
nc *natsio.Conn
}
// StartCallout answers an external broker's login requests with the decisions
// the embedded broker makes, so a task container holds only its run credential.
func StartCallout(ctx context.Context, opts CalloutOptions, log *slog.Logger) (*Callout, error) {
if log == nil {
log = slog.New(slog.DiscardHandler)
}
issuer, err := nkeys.FromSeed([]byte(opts.IssuerSeed))
if err != nil {
return nil, errors.New("nats callout: issuer seed is not an nkey seed")
}
nc, err := natsio.Connect(opts.URL, natsio.UserInfo(CalloutUser, opts.Token), natsio.Name("archie-callout"))
if err != nil {
return nil, fmt.Errorf("nats callout connect: %w", err)
}
svc, err := callout.NewAuthorizationService(nc,
callout.Authorizer(authenticator{token: opts.Token, tasks: opts.Tasks}.calloutAuthorizer(context.WithoutCancel(ctx), issuer)),
callout.ResponseSignerKey(issuer),
callout.Logger(brokerLogger{log: log.With("service", "nats", "component", "callout")}),
)
if err != nil {
nc.Close()
return nil, fmt.Errorf("nats callout service: %w", err)
}
log.InfoContext(ctx, "nats callout ready", "url", opts.URL)
return &Callout{svc: svc, nc: nc}, nil
}
// Close stops answering logins.
func (c *Callout) Close() {
_ = c.svc.Stop()
c.nc.Close()
}
// calloutAuthorizer turns an authenticator decision into a user JWT. An error
// refuses the login.
func (a authenticator) calloutAuthorizer(ctx context.Context, issuer nkeys.KeyPair) callout.AuthorizerFn {
return func(req *jwt.AuthorizationRequest) (string, error) {
g, ok := a.authorize(ctx, req.ConnectOptions.Token, req.ConnectOptions.Username, req.ConnectOptions.Password)
if !ok {
return "", errors.New("login refused")
}
return g.userJWT(req.UserNkey, issuer)
}
}
// userJWT is the grant as a broker user's permissions, signed by the issuer the
// broker trusts.
func (g grant) userJWT(userNkey string, issuer nkeys.KeyPair) (string, error) {
uc := jwt.NewUserClaims(userNkey)
uc.Name = g.Username
uc.Audience = calloutAccount
if !g.Full {
uc.Pub.Allow.Add(g.Publish...)
uc.Sub.Allow.Add(g.Subscribe...)
uc.Resp = &jwt.ResponsePermission{MaxMsgs: 1}
}
return uc.Encode(issuer)
}
@@ -30,14 +30,14 @@ func TestSandboxPolicyAdmitsOnlyWhatTheTaskNeeds(t *testing.T) {
name: "services by IP and loopback use the services host",
servicesHost: "archie.lan",
launch: Launch{Worker: egress.Endpoints{
NATS: "nats://172.17.0.1:4222", NATSToken: "t", StateStore: "dns:///127.0.0.1:50051", StateStoreToken: "run-1",
NATS: "nats://172.17.0.1:4222", StateStore: "dns:///127.0.0.1:50051", StateStoreToken: "run-1",
}},
wantRules: map[string]string{
"archie-nats": "archie.lan:4222 " + agentBinary,
"archie-state-store": "archie.lan:50051 " + agentBinary,
},
wantEnv: map[string]string{
"NATS_URL": "nats://archie.lan:4222", "NATS_TOKEN": "t",
"NATS_URL": "nats://archie.lan:4222",
"STATE_STORE_URL": "archie.lan:50051", "STATE_STORE_TOKEN": "run-1",
},
},