diff --git a/Dockerfile b/Dockerfile index 2069bf70b..f46e89020 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,5 +103,5 @@ ENV GOMODCACHE=/data/cache/go/mod \ COPY --from=builder /usr/local/bin/archie-agent /usr/local/bin/archie-agent -# The daemon injects NATS_URL and, when configured, NATS_TOKEN. +# The daemon injects NATS_URL. ENTRYPOINT ["archie-agent"] diff --git a/cmd/archie-agent/main.go b/cmd/archie-agent/main.go index 10fb807c5..676332f7b 100644 --- a/cmd/archie-agent/main.go +++ b/cmd/archie-agent/main.go @@ -52,7 +52,10 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run return 0 } - natsURL, natsToken := natsConnectionSettings(*natsURLFlag, getenv) + natsURL := *natsURLFlag + if natsURL == "" { + natsURL = getenv("NATS_URL") + } if natsURL == "" { _, _ = fmt.Fprintln(stderr, "error: -nats-url or NATS_URL is required") flags.Usage() @@ -79,7 +82,6 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run if err := runWorker(ctx, agentworker.Settings{ NATSURL: natsURL, - NATSToken: natsToken, StateStoreTarget: stateStoreURL, StateStoreToken: stateStoreToken, StateStoreCA: []byte(getenv("STATE_STORE_CA")), @@ -92,14 +94,6 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run return 0 } -func natsConnectionSettings(flagURL string, getenv func(string) string) (url, token string) { - url = flagURL - if url == "" { - url = getenv("NATS_URL") - } - return url, getenv("NATS_TOKEN") -} - type relayServer func(ctx context.Context, forwards map[string]string) error // runRelay is the sandbox egress relay: the container on both a sandbox's diff --git a/config.example.toml b/config.example.toml index d226f32c3..27265da02 100644 --- a/config.example.toml +++ b/config.example.toml @@ -74,7 +74,8 @@ target = "127.0.0.1:9090" # [nats] # mode = "embedded" # or "external" with url # url = "nats://localhost:4222" -# token_env = "NATS_TOKEN" +# token_env = "NATS_TOKEN" # external: the archie-callout user's password +# callout_issuer_seed_env = "NATS_CALLOUT_SEED" # external: account nkey seed; the broker's auth_callout issuer is its public key # [log] # file = "~/.local/share/archie/logs/archied.log" diff --git a/deployments/docker-nats-stack.toml b/deployments/docker-nats-stack.toml index 644a7dddf..c38c2db56 100644 --- a/deployments/docker-nats-stack.toml +++ b/deployments/docker-nats-stack.toml @@ -59,6 +59,8 @@ mode = "external" # The pinned Compose-network gateway is reachable from both the host daemon # and the task containers it launches on archie-core_default. url = "nats://172.19.0.1:4222" +token_env = "NATS_TOKEN" +callout_issuer_seed_env = "NATS_CALLOUT_SEED" [containers] network = "archie-core_default" diff --git a/deployments/nats-callout.conf b/deployments/nats-callout.conf new file mode 100644 index 000000000..d4d842681 --- /dev/null +++ b/deployments/nats-callout.conf @@ -0,0 +1,10 @@ +jetstream { store_dir: /data } +http_port: 8222 + +authorization { + users: [{ user: "archie-callout", password: $NATS_TOKEN }] + auth_callout { + issuer: $NATS_CALLOUT_ISSUER + auth_users: ["archie-callout"] + } +} diff --git a/deployments/systemd-user-service.md b/deployments/systemd-user-service.md index 5f9169d2c..d5779dbba 100644 --- a/deployments/systemd-user-service.md +++ b/deployments/systemd-user-service.md @@ -225,11 +225,19 @@ docker compose up -d nats [nats] mode = "external" url = "nats://172.19.0.1:4222" +token_env = "NATS_TOKEN" +callout_issuer_seed_env = "NATS_CALLOUT_SEED" [containers] network = "archie-core_default" ``` +The broker defers every login to archied (NATS auth callout), so a worker logs +in with its run credential and reaches only its own task's subjects. Generate +an account nkey (`nk -gen account -pubout`), export its seed as +`NATS_CALLOUT_SEED` for archied and its public key as `NATS_CALLOUT_ISSUER` for +Compose, and set `NATS_TOKEN` to the same value in both. + The daemon hands workers the URL its own client connected to. The Compose service name is not resolvable from the native host, and localhost inside a worker is the worker itself; the pinned network gateway is reachable from both. diff --git a/docker-compose.yml b/docker-compose.yml index 4bc5958ba..aab8017ee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,19 +46,19 @@ services: nats: image: nats:2.14-alpine3.22 # The client port is published on every interface (agent containers reach - # it over the bridge), so it refuses to start without a token. Set - # NATS_TOKEN here and name the same value in [nats] token_env. - entrypoint: - - sh - - -c - - '[ -n "$$NATS_TOKEN" ] || { echo "NATS_TOKEN is required" >&2; exit 1; }; exec nats-server -js -m 8222 --auth "$$NATS_TOKEN"' + # it over the bridge), so it defers every login to archied. Set NATS_TOKEN + # (name it in [nats] token_env) and NATS_CALLOUT_ISSUER, the public key of + # the account nkey seed named in [nats] callout_issuer_seed_env. + command: ["-c", "/etc/nats/nats.conf"] environment: - NATS_TOKEN: ${NATS_TOKEN:-} + NATS_TOKEN: ${NATS_TOKEN:?NATS_TOKEN is required} + NATS_CALLOUT_ISSUER: ${NATS_CALLOUT_ISSUER:?NATS_CALLOUT_ISSUER is required} ports: - "4222:4222" # client - "127.0.0.1:8222:8222" # monitoring (http://localhost:8222) volumes: - nats_data:/data + - ./deployments/nats-callout.conf:/etc/nats/nats.conf:ro restart: unless-stopped healthcheck: test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:8222/healthz"] diff --git a/go.mod b/go.mod index 345bd886e..1cf695e30 100644 --- a/go.mod +++ b/go.mod @@ -38,8 +38,6 @@ require ( github.com/mfridman/interpolate v0.2.0 // indirect github.com/minio/highwayhash v1.0.4 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/nats-io/jwt/v2 v2.8.3 // indirect - github.com/nats-io/nkeys v0.4.17 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/oklog/run v1.2.0 // indirect github.com/philhofer/fwd v1.2.1 // indirect @@ -86,12 +84,15 @@ require ( github.com/moby/moby/api v1.56.1 github.com/moby/moby/client v0.6.2 github.com/modelcontextprotocol/go-sdk v1.8.0 + github.com/nats-io/jwt/v2 v2.8.3 github.com/nats-io/nats-server/v2 v2.15.3 github.com/nats-io/nats.go v1.55.0 + github.com/nats-io/nkeys v0.4.17 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.1 github.com/pressly/goose/v3 v3.30.1 github.com/samcharles93/ai-sdk v0.6.0 + github.com/synadia-io/callout.go v0.2.3 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 go.opentelemetry.io/otel v1.47.0 diff --git a/go.sum b/go.sum index 3a79133a7..8507ef279 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cel.dev/cel-go v0.33.2 h1:eAcEdBguc6AFrGbPOyE0g168Uey4kKFW9J6qsBLSeMU= cel.dev/cel-go v0.33.2/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU= cel.dev/expr v0.26.0 h1:oLXvOF0jiYA5A/tmzU91pLEWZkKynhLJlC9Gw7PrKvs= cel.dev/expr v0.26.0/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Microsoft/go-winio v0.6.4 h1:kJF2pamtYCWXf0xAT8sAkwKC0pMl2HYBrRPHAqyUc+g= @@ -16,6 +18,8 @@ github.com/antithesishq/antithesis-sdk-go v0.9.0 h1:JQpWOgYSYBb9HqgPtaXBswHxZFmO github.com/antithesishq/antithesis-sdk-go v0.9.0/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/aricart/nst.go v0.2.0 h1:vfC+upJofU7OFJZM5Ph+HNko1XfJtVuSE6Q7FCQmT3g= +github.com/aricart/nst.go v0.2.0/go.mod h1:13ztPOEw3Jz0/LT8yDi5wZB+T4Lsj3Uhkc17CyZthII= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= @@ -135,6 +139,8 @@ github.com/mfridman/interpolate v0.2.0 h1:HM9ID8p6CUxoMJX+fECOtGQ2Mu7TW/GIMPyu48 github.com/mfridman/interpolate v0.2.0/go.mod h1:8SfDy95IaPNnIokZh9mvIppmc3fGl8fx+Gu7uusuO9Q= github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/moby/api v1.56.1 h1:PpWkvVPB7Fr/No8w+TfyJ/I6rWZ2YPZhT3JorIg+06c= @@ -151,6 +157,8 @@ github.com/nats-io/nats.go v1.55.0 h1:Ng8QKHdvdHIWoaiq0+d/aH1BWkxvcJxWJ0Y/9BPKx3 github.com/nats-io/nats.go v1.55.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= github.com/nats-io/nkeys v0.4.17 h1:JAJrlDhOyQqOt0a1eg4cbIZ1Kc7+vmkCigAltpwQpVA= github.com/nats-io/nkeys v0.4.17/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nsc/v2 v2.10.3-0.20250110165315-eeda721ecff6 h1:V1uh9L3rGIUeYoMd0/EZY2Tvy5+1CkgGpDRx8t3+PNY= +github.com/nats-io/nsc/v2 v2.10.3-0.20250110165315-eeda721ecff6/go.mod h1:ScomAvx1cgjiXzW3WpGo9x/lLENkwELhewCcok/GTU8= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk= @@ -191,6 +199,10 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/synadia-io/callout.go v0.2.3 h1:22u8sIIdpkKuyySXqEWvGAkAQ4tMBnvLtpVPNBdbj6Y= +github.com/synadia-io/callout.go v0.2.3/go.mod h1:IDVofaME9xzbBO5nUvEL59Ml0jGd6gAsF1qC4Nm6Pn8= +github.com/synadia-io/jwt-auth-builder.go v0.0.9 h1:UmXQ0JsqiA7sboE0VC46YEqrGAFA4MYHnhw54uYKRi8= +github.com/synadia-io/jwt-auth-builder.go v0.0.9/go.mod h1:0XPmvG3TihNkiM3ZFVtcCAzv0oBgFtjnw/lb/h8NHs0= github.com/tinylib/msgp v1.6.5 h1:iAH6XTP7BpzErjBZlujJQXxA+GmRi2YWs6M4KdLtNbE= github.com/tinylib/msgp v1.6.5/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/internal/app/agentworker/worker.go b/internal/app/agentworker/worker.go index 0a93a483e..06644db77 100644 --- a/internal/app/agentworker/worker.go +++ b/internal/app/agentworker/worker.go @@ -35,9 +35,8 @@ func (e *StartupError) Unwrap() error { return e.Err } // Settings are process-derived inputs for one worker runtime. type Settings struct { - NATSURL string - NATSToken string - WorkDir string + NATSURL string + WorkDir string // StateStoreTarget and StateStoreToken are the daemon-injected State Store // gRPC endpoint and bearer token. StateStoreTarget is required: the legacy @@ -128,8 +127,8 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies } dependencies.markSafe(ctx, workDir, log) - // The task is known before connecting: without a broker token the - // worker logs in as its task, with its run credential. + // The task is known before connecting: the worker logs in as its task, + // with its run credential. taskID, err := dependencies.bootID(workDir) if err != nil { log.ErrorContext(ctx, "task boot failed", "err", err) @@ -137,7 +136,6 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies } transport, err := dependencies.connect(ctx, agentnats.Config{ URL: settings.NATSURL, - Token: settings.NATSToken, TaskID: taskID, StateStoreURL: settings.StateStoreTarget, StateStoreToken: settings.StateStoreToken, diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 9e828725b..a2120b7e2 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -155,9 +155,6 @@ type boot struct { // the embedded server's ClientURL(). Recorded so Daemon.ConnectedNATS // carries the real endpoint rather than an empty config URL. natsURL string - // natsToken is the credential the daemon's client connected with at - // startup. Embedded mode generates it; external mode resolves it once. - natsToken string containerPool *container.Pool // taskRuntime is what task agents run in: the container pool, or an @@ -388,6 +385,9 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco log.ErrorContext(ctx, "nats credentials", "err", err) return err } + if err := b.startCallout(ctx, url, natsToken); err != nil { + return err + } } else { endpoint, readErr := servicekit.ReadNATSEndpoint(cfg.StateDir) if readErr == nil { @@ -445,7 +445,6 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco } b.natsClient = natsClient b.natsURL = url - b.natsToken = natsToken b.addCleanup(func() { natsClient.Close() }) b.addCleanup(func() { reactionClient.Close() }) b.reactionClient = reactionClient @@ -453,6 +452,27 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco return nil } +// startCallout makes the external broker defer every login to archie, so a task +// container logs in with its run credential and reaches only its own subjects. +// It runs before any other login, because those are decided here. +func (b *boot) startCallout(ctx context.Context, url, token string) error { + seed := os.Getenv(b.cfg.NATS.CalloutIssuerSeedEnv) + if seed == "" { + return fmt.Errorf("%s is required when nats.callout_issuer_seed_env is configured", b.cfg.NATS.CalloutIssuerSeedEnv) + } + opts := nats.CalloutOptions{URL: url, Token: token, IssuerSeed: seed} + if b.stateStoreGrants != nil { + opts.Tasks = taskBroker{runs: b.stateStoreGrants.Client} + } + svc, err := nats.StartCallout(ctx, opts, b.log) + if err != nil { + b.log.ErrorContext(ctx, "nats callout start failed", "err", err) + return err + } + b.addCleanup(svc.Close) + return nil +} + func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) { cfg, log := b.cfg, b.log if cfg.StateDir == "" { @@ -627,7 +647,7 @@ func (b *boot) buildDaemon(ctx context.Context) { d := &daemon.Daemon{ ResolveRepository: b.repositoryAccess, Cfg: b.cfgHolder, - ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken, Scoped: b.cfg.NATS.Mode != config.NATSModeExternal}, + ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL}, ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), CA: b.stateStoreCA(ctx)}, Store: b.stateStore, Identity: daemon.NewIdentityGate(b.stateClient, b.stateClient, b.accessChain, b.stateClient, b.stateClient, b.orgPrincipals, workspaces, log), diff --git a/internal/app/archied/readiness.go b/internal/app/archied/readiness.go index 22146f6e9..01cd7919d 100644 --- a/internal/app/archied/readiness.go +++ b/internal/app/archied/readiness.go @@ -53,9 +53,6 @@ func (b *boot) setupReadinessProbes() { if b.accessLive != nil { probes = append(probes, readiness.NewProblemProbe("access_policies", b.accessLive)) } - if cfg.NATS.Mode == config.NATSModeExternal { - probes = append(probes, unscopedBroker{}) - } b.healthRegistry = health.NewRegistry(probes...) } @@ -97,16 +94,6 @@ func diskProbePath(cfg config.Config) string { } } -// unscopedBroker reports that containers hold the external broker's own -// token: only the embedded broker can limit a container to its task. -type unscopedBroker struct{} - -func (unscopedBroker) Name() string { return "nats_credential_scope" } - -func (unscopedBroker) Check(context.Context) health.Result { - return health.Result{Status: health.StatusDegraded, Detail: "external NATS: containers get a broker token that reaches every subject; use the embedded broker"} -} - // stateCallDeadline bounds each State Store call the daemon makes without a // deadline of its own. const stateCallDeadline = 20 * time.Second diff --git a/internal/config/config.go b/internal/config/config.go index f9107f85d..80087ee68 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -647,9 +647,16 @@ type NATSConfig struct { // URL is the NATS server address, e.g. "nats://localhost:4222". // Required when Mode is "external"; must be empty otherwise. URL string `toml:"url" yaml:"url"` - // TokenEnv optionally names an env var holding an external NATS auth token. - // Empty means no authentication. Embedded mode generates a per-start token. + // TokenEnv names an env var holding the external broker's instance token: + // the password of the "archie-callout" auth user, and the token archie's own + // services log in with. Required when Mode is "external". Embedded mode + // generates a per-start token. TokenEnv string `toml:"token_env" yaml:"token_env"` + // CalloutIssuerSeedEnv names an env var holding the account nkey seed whose + // public key the broker lists as its auth_callout issuer. Required when + // Mode is "external": the broker defers every login to archied, so a task + // container holds only its run credential. + CalloutIssuerSeedEnv string `toml:"callout_issuer_seed_env" yaml:"callout_issuer_seed_env"` } // CaptureConfig configures the unbound webhook capture endpoint. diff --git a/internal/daemon/container_run.go b/internal/daemon/container_run.go index ab20eb78a..7c5542371 100644 --- a/internal/daemon/container_run.go +++ b/internal/daemon/container_run.go @@ -132,7 +132,6 @@ func (d *Daemon) acquireTaskContainer( func (d *Daemon) workerEndpoints(stateStoreToken string) egress.Endpoints { return egress.Endpoints{ NATS: d.ConnectedNATS.URL, - NATSToken: d.ConnectedNATS.ContainerToken(), StateStore: d.ConnectedStateStore.URL, StateStoreToken: stateStoreToken, StateStoreCA: d.ConnectedStateStore.CA, diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index da1c16894..cc7fa941b 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -41,25 +41,11 @@ type RunCredentialIssuer interface { Issue(ctx context.Context, task *workflow.Task, limit time.Duration) (token string, revoke func(), err error) } -// NATSEndpoint is the broker address and credential archied connected with at -// startup. It is runtime state rather than configuration: embedded mode -// generates both values, and external secret references are resolved once. +// NATSEndpoint is the broker address archied connected with at startup. It is +// runtime state rather than configuration: embedded mode generates it. A +// container logs in with its run credential, never the broker's token. type NATSEndpoint struct { - URL string - Token string - // Scoped is true on the embedded broker, where a container logs in with - // its run credential and reaches only its own task's subjects. - Scoped bool -} - -// ContainerToken is the broker token a container is handed: none on a -// scoped broker. An external broker cannot scope a container, so it still -// gets the broker's token (reported as a health issue by archied). -func (e NATSEndpoint) ContainerToken() string { - if e.Scoped { - return "" - } - return e.Token + URL string } // StateStoreEndpoint is the State Store target and the CA its certificate diff --git a/internal/infrastructure/agenttransport/nats/transport.go b/internal/infrastructure/agenttransport/nats/transport.go index 312a7fc3b..8af68d8f3 100644 --- a/internal/infrastructure/agenttransport/nats/transport.go +++ b/internal/infrastructure/agenttransport/nats/transport.go @@ -35,10 +35,8 @@ import ( // deleted type Config struct { URL string - // Token is a broker-wide token, set only on an external broker. Without - // it the worker logs in as TaskID with StateStoreToken, its run - // credential, which reaches only its own task's subjects. - Token string + // The worker logs in as TaskID with StateStoreToken, its run credential, + // which reaches only its own task's subjects. TaskID int64 StateStoreURL string @@ -82,10 +80,7 @@ func Connect(ctx context.Context, config Config, log *slog.Logger) (*Transport, return nil, err } options := []natsio.Option{natsio.Name("archie-agent")} - switch { - case config.Token != "": - options = append(options, natsio.Token(config.Token)) - case config.StateStoreToken != "": + if config.StateStoreToken != "" { options = append(options, natsio.UserInfo(fmt.Sprintf("%s%d", eventbus.TaskUserPrefix, config.TaskID), config.StateStoreToken), natsio.CustomInboxPrefix(eventbus.TaskInboxPrefix(config.TaskID))) diff --git a/internal/infrastructure/configuration/validate.go b/internal/infrastructure/configuration/validate.go index b9f294748..d1bb0e03a 100644 --- a/internal/infrastructure/configuration/validate.go +++ b/internal/infrastructure/configuration/validate.go @@ -310,6 +310,9 @@ func validateNATS(cfg *config.Config) error { if cfg.NATS.URL == "" { return fmt.Errorf("%w: nats.url is required when nats.mode is %q", ErrInvalidInput, config.NATSModeExternal) } + if cfg.NATS.TokenEnv == "" || cfg.NATS.CalloutIssuerSeedEnv == "" { + return fmt.Errorf("%w: nats.token_env and nats.callout_issuer_seed_env are required when nats.mode is %q", ErrInvalidInput, config.NATSModeExternal) + } case config.NATSModeEmbedded: if cfg.NATS.URL != "" { return fmt.Errorf("%w: nats.url must be empty when nats.mode is %q", ErrInvalidInput, mode) diff --git a/internal/infrastructure/egress/sandbox_env.go b/internal/infrastructure/egress/sandbox_env.go index 9dda57fbf..6992582fd 100644 --- a/internal/infrastructure/egress/sandbox_env.go +++ b/internal/infrastructure/egress/sandbox_env.go @@ -12,7 +12,6 @@ import ( // needs, with the task's own credentials for them. type Endpoints struct { NATS string - NATSToken string StateStore string StateStoreToken string StateStoreCA []byte @@ -34,9 +33,6 @@ func WorkerEnv(e Endpoints) ([]string, error) { // credentials from e. The State Store is left out when e names none. func DirectWorkerEnv(nats, stateStore string, e Endpoints) []string { env := []string{"NATS_URL=" + nats} - if e.NATSToken != "" { - env = append(env, "NATS_TOKEN="+e.NATSToken) - } if e.StateStore != "" { env = append(env, "STATE_STORE_URL="+stateStore) if e.StateStoreToken != "" { diff --git a/internal/infrastructure/eventbus/nats/auth.go b/internal/infrastructure/eventbus/nats/auth.go index 578225c65..3087d25ac 100644 --- a/internal/infrastructure/eventbus/nats/auth.go +++ b/internal/infrastructure/eventbus/nats/auth.go @@ -3,6 +3,7 @@ package nats import ( "context" "crypto/subtle" + "slices" "strconv" "strings" "time" @@ -38,34 +39,61 @@ const taskAuthTimeout = 2 * time.Second // authenticator admits the instance token with every permission, and a task // user only to its own task's subjects, so no container holds a credential -// that reaches past its run. +// that reaches past its run. The embedded broker and the auth callout of an +// external broker both decide through it. type authenticator struct { token string tasks TaskAuthority } +// grant is what a login is allowed. Full is the instance token's reach; a task +// login lists its subjects instead. +type grant struct { + Username string + Full bool + Publish []string + Subscribe []string +} + +// authorize decides one login: the instance token, or "task-" with that +// task's run credential. +func (a authenticator) authorize(ctx context.Context, token, username, password string) (grant, bool) { + if token != "" { + if subtle.ConstantTimeCompare([]byte(token), []byte(a.token)) != 1 { + return grant{}, false + } + return grant{Username: "archie", Full: true}, true + } + taskID, ok := taskUser(username) + if !ok || password == "" || a.tasks == nil { + return grant{}, false + } + ctx, cancel := context.WithTimeout(ctx, taskAuthTimeout) + defer cancel() + subjects, err := a.tasks.TaskSubjects(ctx, taskID, password) + if err != nil { + return grant{}, false + } + return grant{ + Username: username, + Publish: slices.Clone(subjects.Publish), + Subscribe: append(slices.Clone(subjects.Subscribe), TaskInboxPrefix(taskID)+".>"), + }, true +} + func (a authenticator) Check(c server.ClientAuthentication) bool { opts := c.GetOpts() - if opts.Token != "" { - if subtle.ConstantTimeCompare([]byte(opts.Token), []byte(a.token)) != 1 { - return false - } - c.RegisterUser(&server.User{Username: "archie"}) + g, ok := a.authorize(context.Background(), opts.Token, opts.Username, opts.Password) + if !ok { + return false + } + if g.Full { + c.RegisterUser(&server.User{Username: g.Username}) return true } - taskID, ok := taskUser(opts.Username) - if !ok || opts.Password == "" || a.tasks == nil { - return false - } - ctx, cancel := context.WithTimeout(context.Background(), taskAuthTimeout) - defer cancel() - subjects, err := a.tasks.TaskSubjects(ctx, taskID, opts.Password) - if err != nil { - return false - } - c.RegisterUser(&server.User{Username: opts.Username, Permissions: &server.Permissions{ - Publish: &server.SubjectPermission{Allow: subjects.Publish}, - Subscribe: &server.SubjectPermission{Allow: append(subjects.Subscribe, TaskInboxPrefix(taskID)+".>")}, + c.RegisterUser(&server.User{Username: g.Username, Permissions: &server.Permissions{ + Publish: &server.SubjectPermission{Allow: g.Publish}, + Subscribe: &server.SubjectPermission{Allow: g.Subscribe}, // Lets the task answer the one request it is sent, on the requester's // reply subject. Response: &server.ResponsePermission{MaxMsgs: 1}, diff --git a/internal/infrastructure/eventbus/nats/auth_test.go b/internal/infrastructure/eventbus/nats/auth_test.go index d84c19db7..7e67b8101 100644 --- a/internal/infrastructure/eventbus/nats/auth_test.go +++ b/internal/infrastructure/eventbus/nats/auth_test.go @@ -8,7 +8,9 @@ import ( "testing" "time" + "github.com/nats-io/nats-server/v2/server" natsio "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" ) type taskSeven struct{} @@ -22,14 +24,60 @@ func (taskSeven) TaskSubjects(_ context.Context, taskID int64, credential string // TestBrokerScopesTaskCredentials pins that the instance token reaches // everything and a task's run credential reaches only its own subjects, on a -// real embedded broker. +// real embedded broker and on an external one that defers logins to the +// callout. It guards a task container being handed more than its own subjects. func TestBrokerScopesTaskCredentials(t *testing.T) { - srv, err := StartEmbedded(context.Background(), EmbeddedOptions{Token: "instance", StoreDir: t.TempDir(), Tasks: taskSeven{}}, nil) + embedded, err := StartEmbedded(context.Background(), EmbeddedOptions{Token: "instance", StoreDir: t.TempDir(), Tasks: taskSeven{}}, nil) if err != nil { t.Fatal(err) } - defer srv.Shutdown() + defer embedded.Shutdown() + for name, url := range map[string]string{"embedded": embedded.ClientURL(), "callout": calloutBroker(t)} { + t.Run(name, func(t *testing.T) { checkScopes(t, url) }) + } +} + +// calloutBroker starts an external-style broker whose logins are decided by +// StartCallout, configured the way an operator would: one auth user for the +// callout itself, every other login deferred. +func calloutBroker(t *testing.T) string { + t.Helper() + issuer, err := nkeys.CreateAccount() + if err != nil { + t.Fatal(err) + } + pub, err := issuer.PublicKey() + if err != nil { + t.Fatal(err) + } + seed, err := issuer.Seed() + if err != nil { + t.Fatal(err) + } + srv, err := server.NewServer(&server.Options{ + Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true, + Users: []*server.User{{Username: CalloutUser, Password: "instance"}}, + AuthCallout: &server.AuthCallout{Issuer: pub, Account: "$G", AuthUsers: []string{CalloutUser}}, + }) + if err != nil { + t.Fatal(err) + } + srv.Start() + t.Cleanup(srv.Shutdown) + if !srv.ReadyForConnections(10 * time.Second) { + t.Fatal("broker not ready") + } + svc, err := StartCallout(context.Background(), CalloutOptions{URL: srv.ClientURL(), Token: "instance", IssuerSeed: string(seed), Tasks: taskSeven{}}, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(svc.Close) + return srv.ClientURL() +} + +func checkScopes(t *testing.T, url string) { + t.Helper() tests := []struct { name string opts []natsio.Option @@ -61,7 +109,7 @@ func TestBrokerScopesTaskCredentials(t *testing.T) { opts := append(slices.Clone(tt.opts), natsio.ErrorHandler(func(_ *natsio.Conn, _ *natsio.Subscription, err error) { violations <- err.Error() })) - nc, err := natsio.Connect(srv.ClientURL(), opts...) + nc, err := natsio.Connect(url, opts...) if !tt.connects { if err == nil { nc.Close() diff --git a/internal/infrastructure/eventbus/nats/callout.go b/internal/infrastructure/eventbus/nats/callout.go new file mode 100644 index 000000000..0cbde4b30 --- /dev/null +++ b/internal/infrastructure/eventbus/nats/callout.go @@ -0,0 +1,101 @@ +package nats + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/nats-io/jwt/v2" + natsio "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" + "github.com/synadia-io/callout.go" +) + +// CalloutUser is the broker login archied's callout service connects with. The +// external broker lists it in auth_users, so its own login skips the callout; +// its password is the instance token. +const CalloutUser = "archie-callout" + +// calloutAccount is the account a granted login is placed in: the broker's +// global account, which JetStream uses. +const calloutAccount = "$G" + +// CalloutOptions configures StartCallout. +type CalloutOptions struct { + // URL is the external broker. + URL string + // Token is the instance token: the callout user's password, and the token + // that logs in with every permission. + Token string + // IssuerSeed is the account nkey seed whose public key the broker names as + // its auth_callout issuer. + IssuerSeed string + // Tasks admits task users to their own task's subjects. + Tasks TaskAuthority +} + +// Callout is the authorization service of an external broker. +type Callout struct { + svc *callout.AuthorizationService + nc *natsio.Conn +} + +// StartCallout answers an external broker's login requests with the decisions +// the embedded broker makes, so a task container holds only its run credential. +func StartCallout(ctx context.Context, opts CalloutOptions, log *slog.Logger) (*Callout, error) { + if log == nil { + log = slog.New(slog.DiscardHandler) + } + issuer, err := nkeys.FromSeed([]byte(opts.IssuerSeed)) + if err != nil { + return nil, errors.New("nats callout: issuer seed is not an nkey seed") + } + nc, err := natsio.Connect(opts.URL, natsio.UserInfo(CalloutUser, opts.Token), natsio.Name("archie-callout")) + if err != nil { + return nil, fmt.Errorf("nats callout connect: %w", err) + } + svc, err := callout.NewAuthorizationService(nc, + callout.Authorizer(authenticator{token: opts.Token, tasks: opts.Tasks}.calloutAuthorizer(context.WithoutCancel(ctx), issuer)), + callout.ResponseSignerKey(issuer), + callout.Logger(brokerLogger{log: log.With("service", "nats", "component", "callout")}), + ) + if err != nil { + nc.Close() + return nil, fmt.Errorf("nats callout service: %w", err) + } + log.InfoContext(ctx, "nats callout ready", "url", opts.URL) + return &Callout{svc: svc, nc: nc}, nil +} + +// Close stops answering logins. +func (c *Callout) Close() { + _ = c.svc.Stop() + c.nc.Close() +} + +// calloutAuthorizer turns an authenticator decision into a user JWT. An error +// refuses the login. +func (a authenticator) calloutAuthorizer(ctx context.Context, issuer nkeys.KeyPair) callout.AuthorizerFn { + return func(req *jwt.AuthorizationRequest) (string, error) { + g, ok := a.authorize(ctx, req.ConnectOptions.Token, req.ConnectOptions.Username, req.ConnectOptions.Password) + if !ok { + return "", errors.New("login refused") + } + return g.userJWT(req.UserNkey, issuer) + } +} + +// userJWT is the grant as a broker user's permissions, signed by the issuer the +// broker trusts. +func (g grant) userJWT(userNkey string, issuer nkeys.KeyPair) (string, error) { + uc := jwt.NewUserClaims(userNkey) + uc.Name = g.Username + uc.Audience = calloutAccount + if !g.Full { + uc.Pub.Allow.Add(g.Publish...) + uc.Sub.Allow.Add(g.Subscribe...) + uc.Resp = &jwt.ResponsePermission{MaxMsgs: 1} + } + return uc.Encode(issuer) +} diff --git a/internal/infrastructure/openshell/policy_test.go b/internal/infrastructure/openshell/policy_test.go index d2507f39a..1ffe25c0d 100644 --- a/internal/infrastructure/openshell/policy_test.go +++ b/internal/infrastructure/openshell/policy_test.go @@ -30,14 +30,14 @@ func TestSandboxPolicyAdmitsOnlyWhatTheTaskNeeds(t *testing.T) { name: "services by IP and loopback use the services host", servicesHost: "archie.lan", launch: Launch{Worker: egress.Endpoints{ - NATS: "nats://172.17.0.1:4222", NATSToken: "t", StateStore: "dns:///127.0.0.1:50051", StateStoreToken: "run-1", + NATS: "nats://172.17.0.1:4222", StateStore: "dns:///127.0.0.1:50051", StateStoreToken: "run-1", }}, wantRules: map[string]string{ "archie-nats": "archie.lan:4222 " + agentBinary, "archie-state-store": "archie.lan:50051 " + agentBinary, }, wantEnv: map[string]string{ - "NATS_URL": "nats://archie.lan:4222", "NATS_TOKEN": "t", + "NATS_URL": "nats://archie.lan:4222", "STATE_STORE_URL": "archie.lan:50051", "STATE_STORE_TOKEN": "run-1", }, },