feat(nats): an external broker defers logins to archie through auth callout (#1866)
Task containers log in with their run credential on an external broker and reach only their own subjects, decided by the embedded broker's authenticator. The broker token is never put in a container environment.
This commit is contained in:
22 files changed
+299
-103
No files matched your search
+1
-1
@@ -103,5 +103,5 @@ ENV GOMODCACHE=/data/cache/go/mod \
|
||||
|
||||
COPY --from=builder /usr/local/bin/archie-agent /usr/local/bin/archie-agent
|
||||
|
||||
# The daemon injects NATS_URL and, when configured, NATS_TOKEN.
|
||||
# The daemon injects NATS_URL.
|
||||
ENTRYPOINT ["archie-agent"]
|
||||
@@ -52,7 +52,10 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run
|
||||
return 0
|
||||
}
|
||||
|
||||
natsURL, natsToken := natsConnectionSettings(*natsURLFlag, getenv)
|
||||
natsURL := *natsURLFlag
|
||||
if natsURL == "" {
|
||||
natsURL = getenv("NATS_URL")
|
||||
}
|
||||
if natsURL == "" {
|
||||
_, _ = fmt.Fprintln(stderr, "error: -nats-url or NATS_URL is required")
|
||||
flags.Usage()
|
||||
@@ -79,7 +82,6 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run
|
||||
|
||||
if err := runWorker(ctx, agentworker.Settings{
|
||||
NATSURL: natsURL,
|
||||
NATSToken: natsToken,
|
||||
StateStoreTarget: stateStoreURL,
|
||||
StateStoreToken: stateStoreToken,
|
||||
StateStoreCA: []byte(getenv("STATE_STORE_CA")),
|
||||
@@ -92,14 +94,6 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run
|
||||
return 0
|
||||
}
|
||||
|
||||
func natsConnectionSettings(flagURL string, getenv func(string) string) (url, token string) {
|
||||
url = flagURL
|
||||
if url == "" {
|
||||
url = getenv("NATS_URL")
|
||||
}
|
||||
return url, getenv("NATS_TOKEN")
|
||||
}
|
||||
|
||||
type relayServer func(ctx context.Context, forwards map[string]string) error
|
||||
|
||||
// runRelay is the sandbox egress relay: the container on both a sandbox's
|
||||
|
||||
+2
-1
@@ -74,7 +74,8 @@ target = "127.0.0.1:9090"
|
||||
# [nats]
|
||||
# mode = "embedded" # or "external" with url
|
||||
# url = "nats://localhost:4222"
|
||||
# token_env = "NATS_TOKEN"
|
||||
# token_env = "NATS_TOKEN" # external: the archie-callout user's password
|
||||
# callout_issuer_seed_env = "NATS_CALLOUT_SEED" # external: account nkey seed; the broker's auth_callout issuer is its public key
|
||||
|
||||
# [log]
|
||||
# file = "~/.local/share/archie/logs/archied.log"
|
||||
|
||||
@@ -59,6 +59,8 @@ mode = "external"
|
||||
# The pinned Compose-network gateway is reachable from both the host daemon
|
||||
# and the task containers it launches on archie-core_default.
|
||||
url = "nats://172.19.0.1:4222"
|
||||
token_env = "NATS_TOKEN"
|
||||
callout_issuer_seed_env = "NATS_CALLOUT_SEED"
|
||||
|
||||
[containers]
|
||||
network = "archie-core_default"
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
jetstream { store_dir: /data }
|
||||
http_port: 8222
|
||||
|
||||
authorization {
|
||||
users: [{ user: "archie-callout", password: $NATS_TOKEN }]
|
||||
auth_callout {
|
||||
issuer: $NATS_CALLOUT_ISSUER
|
||||
auth_users: ["archie-callout"]
|
||||
}
|
||||
}
|
||||
@@ -225,11 +225,19 @@ docker compose up -d nats
|
||||
[nats]
|
||||
mode = "external"
|
||||
url = "nats://172.19.0.1:4222"
|
||||
token_env = "NATS_TOKEN"
|
||||
callout_issuer_seed_env = "NATS_CALLOUT_SEED"
|
||||
|
||||
[containers]
|
||||
network = "archie-core_default"
|
||||
```
|
||||
|
||||
The broker defers every login to archied (NATS auth callout), so a worker logs
|
||||
in with its run credential and reaches only its own task's subjects. Generate
|
||||
an account nkey (`nk -gen account -pubout`), export its seed as
|
||||
`NATS_CALLOUT_SEED` for archied and its public key as `NATS_CALLOUT_ISSUER` for
|
||||
Compose, and set `NATS_TOKEN` to the same value in both.
|
||||
|
||||
The daemon hands workers the URL its own client connected to. The Compose
|
||||
service name is not resolvable from the native host, and localhost inside a
|
||||
worker is the worker itself; the pinned network gateway is reachable from both.
|
||||
|
||||
+7
-7
@@ -46,19 +46,19 @@ services:
|
||||
nats:
|
||||
image: nats:2.14-alpine3.22
|
||||
# The client port is published on every interface (agent containers reach
|
||||
# it over the bridge), so it refuses to start without a token. Set
|
||||
# NATS_TOKEN here and name the same value in [nats] token_env.
|
||||
entrypoint:
|
||||
- sh
|
||||
- -c
|
||||
- '[ -n "$$NATS_TOKEN" ] || { echo "NATS_TOKEN is required" >&2; exit 1; }; exec nats-server -js -m 8222 --auth "$$NATS_TOKEN"'
|
||||
# it over the bridge), so it defers every login to archied. Set NATS_TOKEN
|
||||
# (name it in [nats] token_env) and NATS_CALLOUT_ISSUER, the public key of
|
||||
# the account nkey seed named in [nats] callout_issuer_seed_env.
|
||||
command: ["-c", "/etc/nats/nats.conf"]
|
||||
environment:
|
||||
NATS_TOKEN: ${NATS_TOKEN:-}
|
||||
NATS_TOKEN: ${NATS_TOKEN:?NATS_TOKEN is required}
|
||||
NATS_CALLOUT_ISSUER: ${NATS_CALLOUT_ISSUER:?NATS_CALLOUT_ISSUER is required}
|
||||
ports:
|
||||
- "4222:4222" # client
|
||||
- "127.0.0.1:8222:8222" # monitoring (http://localhost:8222)
|
||||
volumes:
|
||||
- nats_data:/data
|
||||
- ./deployments/nats-callout.conf:/etc/nats/nats.conf:ro
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:8222/healthz"]
|
||||
|
||||
@@ -38,8 +38,6 @@ require (
|
||||
github.com/mfridman/interpolate v0.2.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.3 // indirect
|
||||
github.com/nats-io/nkeys v0.4.17 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/oklog/run v1.2.0 // indirect
|
||||
github.com/philhofer/fwd v1.2.1 // indirect
|
||||
@@ -86,12 +84,15 @@ require (
|
||||
github.com/moby/moby/api v1.56.1
|
||||
github.com/moby/moby/client v0.6.2
|
||||
github.com/modelcontextprotocol/go-sdk v1.8.0
|
||||
github.com/nats-io/jwt/v2 v2.8.3
|
||||
github.com/nats-io/nats-server/v2 v2.15.3
|
||||
github.com/nats-io/nats.go v1.55.0
|
||||
github.com/nats-io/nkeys v0.4.17
|
||||
github.com/opencontainers/go-digest v1.0.0
|
||||
github.com/opencontainers/image-spec v1.1.1
|
||||
github.com/pressly/goose/v3 v3.30.1
|
||||
github.com/samcharles93/ai-sdk v0.6.0
|
||||
github.com/synadia-io/callout.go v0.2.3
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
|
||||
go.opentelemetry.io/otel v1.47.0
|
||||
|
||||
@@ -2,6 +2,8 @@ cel.dev/cel-go v0.33.2 h1:eAcEdBguc6AFrGbPOyE0g168Uey4kKFW9J6qsBLSeMU=
|
||||
cel.dev/cel-go v0.33.2/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU=
|
||||
cel.dev/expr v0.26.0 h1:oLXvOF0jiYA5A/tmzU91pLEWZkKynhLJlC9Gw7PrKvs=
|
||||
cel.dev/expr v0.26.0/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
|
||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
|
||||
github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
|
||||
github.com/Microsoft/go-winio v0.6.4 h1:kJF2pamtYCWXf0xAT8sAkwKC0pMl2HYBrRPHAqyUc+g=
|
||||
@@ -16,6 +18,8 @@ github.com/antithesishq/antithesis-sdk-go v0.9.0 h1:JQpWOgYSYBb9HqgPtaXBswHxZFmO
|
||||
github.com/antithesishq/antithesis-sdk-go v0.9.0/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
|
||||
github.com/aricart/nst.go v0.2.0 h1:vfC+upJofU7OFJZM5Ph+HNko1XfJtVuSE6Q7FCQmT3g=
|
||||
github.com/aricart/nst.go v0.2.0/go.mod h1:13ztPOEw3Jz0/LT8yDi5wZB+T4Lsj3Uhkc17CyZthII=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw=
|
||||
@@ -135,6 +139,8 @@ github.com/mfridman/interpolate v0.2.0 h1:HM9ID8p6CUxoMJX+fECOtGQ2Mu7TW/GIMPyu48
|
||||
github.com/mfridman/interpolate v0.2.0/go.mod h1:8SfDy95IaPNnIokZh9mvIppmc3fGl8fx+Gu7uusuO9Q=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/moby/api v1.56.1 h1:PpWkvVPB7Fr/No8w+TfyJ/I6rWZ2YPZhT3JorIg+06c=
|
||||
@@ -151,6 +157,8 @@ github.com/nats-io/nats.go v1.55.0 h1:Ng8QKHdvdHIWoaiq0+d/aH1BWkxvcJxWJ0Y/9BPKx3
|
||||
github.com/nats-io/nats.go v1.55.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.17 h1:JAJrlDhOyQqOt0a1eg4cbIZ1Kc7+vmkCigAltpwQpVA=
|
||||
github.com/nats-io/nkeys v0.4.17/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nsc/v2 v2.10.3-0.20250110165315-eeda721ecff6 h1:V1uh9L3rGIUeYoMd0/EZY2Tvy5+1CkgGpDRx8t3+PNY=
|
||||
github.com/nats-io/nsc/v2 v2.10.3-0.20250110165315-eeda721ecff6/go.mod h1:ScomAvx1cgjiXzW3WpGo9x/lLENkwELhewCcok/GTU8=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk=
|
||||
@@ -191,6 +199,10 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/synadia-io/callout.go v0.2.3 h1:22u8sIIdpkKuyySXqEWvGAkAQ4tMBnvLtpVPNBdbj6Y=
|
||||
github.com/synadia-io/callout.go v0.2.3/go.mod h1:IDVofaME9xzbBO5nUvEL59Ml0jGd6gAsF1qC4Nm6Pn8=
|
||||
github.com/synadia-io/jwt-auth-builder.go v0.0.9 h1:UmXQ0JsqiA7sboE0VC46YEqrGAFA4MYHnhw54uYKRi8=
|
||||
github.com/synadia-io/jwt-auth-builder.go v0.0.9/go.mod h1:0XPmvG3TihNkiM3ZFVtcCAzv0oBgFtjnw/lb/h8NHs0=
|
||||
github.com/tinylib/msgp v1.6.5 h1:iAH6XTP7BpzErjBZlujJQXxA+GmRi2YWs6M4KdLtNbE=
|
||||
github.com/tinylib/msgp v1.6.5/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
|
||||
@@ -35,9 +35,8 @@ func (e *StartupError) Unwrap() error { return e.Err }
|
||||
|
||||
// Settings are process-derived inputs for one worker runtime.
|
||||
type Settings struct {
|
||||
NATSURL string
|
||||
NATSToken string
|
||||
WorkDir string
|
||||
NATSURL string
|
||||
WorkDir string
|
||||
|
||||
// StateStoreTarget and StateStoreToken are the daemon-injected State Store
|
||||
// gRPC endpoint and bearer token. StateStoreTarget is required: the legacy
|
||||
@@ -128,8 +127,8 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies
|
||||
}
|
||||
dependencies.markSafe(ctx, workDir, log)
|
||||
|
||||
// The task is known before connecting: without a broker token the
|
||||
// worker logs in as its task, with its run credential.
|
||||
// The task is known before connecting: the worker logs in as its task,
|
||||
// with its run credential.
|
||||
taskID, err := dependencies.bootID(workDir)
|
||||
if err != nil {
|
||||
log.ErrorContext(ctx, "task boot failed", "err", err)
|
||||
@@ -137,7 +136,6 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies
|
||||
}
|
||||
transport, err := dependencies.connect(ctx, agentnats.Config{
|
||||
URL: settings.NATSURL,
|
||||
Token: settings.NATSToken,
|
||||
TaskID: taskID,
|
||||
StateStoreURL: settings.StateStoreTarget,
|
||||
StateStoreToken: settings.StateStoreToken,
|
||||
|
||||
@@ -155,9 +155,6 @@ type boot struct {
|
||||
// the embedded server's ClientURL(). Recorded so Daemon.ConnectedNATS
|
||||
// carries the real endpoint rather than an empty config URL.
|
||||
natsURL string
|
||||
// natsToken is the credential the daemon's client connected with at
|
||||
// startup. Embedded mode generates it; external mode resolves it once.
|
||||
natsToken string
|
||||
|
||||
containerPool *container.Pool
|
||||
// taskRuntime is what task agents run in: the container pool, or an
|
||||
@@ -388,6 +385,9 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco
|
||||
log.ErrorContext(ctx, "nats credentials", "err", err)
|
||||
return err
|
||||
}
|
||||
if err := b.startCallout(ctx, url, natsToken); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
endpoint, readErr := servicekit.ReadNATSEndpoint(cfg.StateDir)
|
||||
if readErr == nil {
|
||||
@@ -445,7 +445,6 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco
|
||||
}
|
||||
b.natsClient = natsClient
|
||||
b.natsURL = url
|
||||
b.natsToken = natsToken
|
||||
b.addCleanup(func() { natsClient.Close() })
|
||||
b.addCleanup(func() { reactionClient.Close() })
|
||||
b.reactionClient = reactionClient
|
||||
@@ -453,6 +452,27 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco
|
||||
return nil
|
||||
}
|
||||
|
||||
// startCallout makes the external broker defer every login to archie, so a task
|
||||
// container logs in with its run credential and reaches only its own subjects.
|
||||
// It runs before any other login, because those are decided here.
|
||||
func (b *boot) startCallout(ctx context.Context, url, token string) error {
|
||||
seed := os.Getenv(b.cfg.NATS.CalloutIssuerSeedEnv)
|
||||
if seed == "" {
|
||||
return fmt.Errorf("%s is required when nats.callout_issuer_seed_env is configured", b.cfg.NATS.CalloutIssuerSeedEnv)
|
||||
}
|
||||
opts := nats.CalloutOptions{URL: url, Token: token, IssuerSeed: seed}
|
||||
if b.stateStoreGrants != nil {
|
||||
opts.Tasks = taskBroker{runs: b.stateStoreGrants.Client}
|
||||
}
|
||||
svc, err := nats.StartCallout(ctx, opts, b.log)
|
||||
if err != nil {
|
||||
b.log.ErrorContext(ctx, "nats callout start failed", "err", err)
|
||||
return err
|
||||
}
|
||||
b.addCleanup(svc.Close)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) {
|
||||
cfg, log := b.cfg, b.log
|
||||
if cfg.StateDir == "" {
|
||||
@@ -627,7 +647,7 @@ func (b *boot) buildDaemon(ctx context.Context) {
|
||||
d := &daemon.Daemon{
|
||||
ResolveRepository: b.repositoryAccess,
|
||||
Cfg: b.cfgHolder,
|
||||
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken, Scoped: b.cfg.NATS.Mode != config.NATSModeExternal},
|
||||
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL},
|
||||
ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), CA: b.stateStoreCA(ctx)},
|
||||
Store: b.stateStore,
|
||||
Identity: daemon.NewIdentityGate(b.stateClient, b.stateClient, b.accessChain, b.stateClient, b.stateClient, b.orgPrincipals, workspaces, log),
|
||||
|
||||
@@ -53,9 +53,6 @@ func (b *boot) setupReadinessProbes() {
|
||||
if b.accessLive != nil {
|
||||
probes = append(probes, readiness.NewProblemProbe("access_policies", b.accessLive))
|
||||
}
|
||||
if cfg.NATS.Mode == config.NATSModeExternal {
|
||||
probes = append(probes, unscopedBroker{})
|
||||
}
|
||||
b.healthRegistry = health.NewRegistry(probes...)
|
||||
}
|
||||
|
||||
@@ -97,16 +94,6 @@ func diskProbePath(cfg config.Config) string {
|
||||
}
|
||||
}
|
||||
|
||||
// unscopedBroker reports that containers hold the external broker's own
|
||||
// token: only the embedded broker can limit a container to its task.
|
||||
type unscopedBroker struct{}
|
||||
|
||||
func (unscopedBroker) Name() string { return "nats_credential_scope" }
|
||||
|
||||
func (unscopedBroker) Check(context.Context) health.Result {
|
||||
return health.Result{Status: health.StatusDegraded, Detail: "external NATS: containers get a broker token that reaches every subject; use the embedded broker"}
|
||||
}
|
||||
|
||||
// stateCallDeadline bounds each State Store call the daemon makes without a
|
||||
// deadline of its own.
|
||||
const stateCallDeadline = 20 * time.Second
|
||||
|
||||
@@ -647,9 +647,16 @@ type NATSConfig struct {
|
||||
// URL is the NATS server address, e.g. "nats://localhost:4222".
|
||||
// Required when Mode is "external"; must be empty otherwise.
|
||||
URL string `toml:"url" yaml:"url"`
|
||||
// TokenEnv optionally names an env var holding an external NATS auth token.
|
||||
// Empty means no authentication. Embedded mode generates a per-start token.
|
||||
// TokenEnv names an env var holding the external broker's instance token:
|
||||
// the password of the "archie-callout" auth user, and the token archie's own
|
||||
// services log in with. Required when Mode is "external". Embedded mode
|
||||
// generates a per-start token.
|
||||
TokenEnv string `toml:"token_env" yaml:"token_env"`
|
||||
// CalloutIssuerSeedEnv names an env var holding the account nkey seed whose
|
||||
// public key the broker lists as its auth_callout issuer. Required when
|
||||
// Mode is "external": the broker defers every login to archied, so a task
|
||||
// container holds only its run credential.
|
||||
CalloutIssuerSeedEnv string `toml:"callout_issuer_seed_env" yaml:"callout_issuer_seed_env"`
|
||||
}
|
||||
|
||||
// CaptureConfig configures the unbound webhook capture endpoint.
|
||||
|
||||
@@ -132,7 +132,6 @@ func (d *Daemon) acquireTaskContainer(
|
||||
func (d *Daemon) workerEndpoints(stateStoreToken string) egress.Endpoints {
|
||||
return egress.Endpoints{
|
||||
NATS: d.ConnectedNATS.URL,
|
||||
NATSToken: d.ConnectedNATS.ContainerToken(),
|
||||
StateStore: d.ConnectedStateStore.URL,
|
||||
StateStoreToken: stateStoreToken,
|
||||
StateStoreCA: d.ConnectedStateStore.CA,
|
||||
|
||||
@@ -41,25 +41,11 @@ type RunCredentialIssuer interface {
|
||||
Issue(ctx context.Context, task *workflow.Task, limit time.Duration) (token string, revoke func(), err error)
|
||||
}
|
||||
|
||||
// NATSEndpoint is the broker address and credential archied connected with at
|
||||
// startup. It is runtime state rather than configuration: embedded mode
|
||||
// generates both values, and external secret references are resolved once.
|
||||
// NATSEndpoint is the broker address archied connected with at startup. It is
|
||||
// runtime state rather than configuration: embedded mode generates it. A
|
||||
// container logs in with its run credential, never the broker's token.
|
||||
type NATSEndpoint struct {
|
||||
URL string
|
||||
Token string
|
||||
// Scoped is true on the embedded broker, where a container logs in with
|
||||
// its run credential and reaches only its own task's subjects.
|
||||
Scoped bool
|
||||
}
|
||||
|
||||
// ContainerToken is the broker token a container is handed: none on a
|
||||
// scoped broker. An external broker cannot scope a container, so it still
|
||||
// gets the broker's token (reported as a health issue by archied).
|
||||
func (e NATSEndpoint) ContainerToken() string {
|
||||
if e.Scoped {
|
||||
return ""
|
||||
}
|
||||
return e.Token
|
||||
URL string
|
||||
}
|
||||
|
||||
// StateStoreEndpoint is the State Store target and the CA its certificate
|
||||
|
||||
@@ -35,10 +35,8 @@ import (
|
||||
// deleted
|
||||
type Config struct {
|
||||
URL string
|
||||
// Token is a broker-wide token, set only on an external broker. Without
|
||||
// it the worker logs in as TaskID with StateStoreToken, its run
|
||||
// credential, which reaches only its own task's subjects.
|
||||
Token string
|
||||
// The worker logs in as TaskID with StateStoreToken, its run credential,
|
||||
// which reaches only its own task's subjects.
|
||||
TaskID int64
|
||||
|
||||
StateStoreURL string
|
||||
@@ -82,10 +80,7 @@ func Connect(ctx context.Context, config Config, log *slog.Logger) (*Transport,
|
||||
return nil, err
|
||||
}
|
||||
options := []natsio.Option{natsio.Name("archie-agent")}
|
||||
switch {
|
||||
case config.Token != "":
|
||||
options = append(options, natsio.Token(config.Token))
|
||||
case config.StateStoreToken != "":
|
||||
if config.StateStoreToken != "" {
|
||||
options = append(options,
|
||||
natsio.UserInfo(fmt.Sprintf("%s%d", eventbus.TaskUserPrefix, config.TaskID), config.StateStoreToken),
|
||||
natsio.CustomInboxPrefix(eventbus.TaskInboxPrefix(config.TaskID)))
|
||||
|
||||
@@ -310,6 +310,9 @@ func validateNATS(cfg *config.Config) error {
|
||||
if cfg.NATS.URL == "" {
|
||||
return fmt.Errorf("%w: nats.url is required when nats.mode is %q", ErrInvalidInput, config.NATSModeExternal)
|
||||
}
|
||||
if cfg.NATS.TokenEnv == "" || cfg.NATS.CalloutIssuerSeedEnv == "" {
|
||||
return fmt.Errorf("%w: nats.token_env and nats.callout_issuer_seed_env are required when nats.mode is %q", ErrInvalidInput, config.NATSModeExternal)
|
||||
}
|
||||
case config.NATSModeEmbedded:
|
||||
if cfg.NATS.URL != "" {
|
||||
return fmt.Errorf("%w: nats.url must be empty when nats.mode is %q", ErrInvalidInput, mode)
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
// needs, with the task's own credentials for them.
|
||||
type Endpoints struct {
|
||||
NATS string
|
||||
NATSToken string
|
||||
StateStore string
|
||||
StateStoreToken string
|
||||
StateStoreCA []byte
|
||||
@@ -34,9 +33,6 @@ func WorkerEnv(e Endpoints) ([]string, error) {
|
||||
// credentials from e. The State Store is left out when e names none.
|
||||
func DirectWorkerEnv(nats, stateStore string, e Endpoints) []string {
|
||||
env := []string{"NATS_URL=" + nats}
|
||||
if e.NATSToken != "" {
|
||||
env = append(env, "NATS_TOKEN="+e.NATSToken)
|
||||
}
|
||||
if e.StateStore != "" {
|
||||
env = append(env, "STATE_STORE_URL="+stateStore)
|
||||
if e.StateStoreToken != "" {
|
||||
|
||||
@@ -3,6 +3,7 @@ package nats
|
||||
import (
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -38,34 +39,61 @@ const taskAuthTimeout = 2 * time.Second
|
||||
|
||||
// authenticator admits the instance token with every permission, and a task
|
||||
// user only to its own task's subjects, so no container holds a credential
|
||||
// that reaches past its run.
|
||||
// that reaches past its run. The embedded broker and the auth callout of an
|
||||
// external broker both decide through it.
|
||||
type authenticator struct {
|
||||
token string
|
||||
tasks TaskAuthority
|
||||
}
|
||||
|
||||
// grant is what a login is allowed. Full is the instance token's reach; a task
|
||||
// login lists its subjects instead.
|
||||
type grant struct {
|
||||
Username string
|
||||
Full bool
|
||||
Publish []string
|
||||
Subscribe []string
|
||||
}
|
||||
|
||||
// authorize decides one login: the instance token, or "task-<id>" with that
|
||||
// task's run credential.
|
||||
func (a authenticator) authorize(ctx context.Context, token, username, password string) (grant, bool) {
|
||||
if token != "" {
|
||||
if subtle.ConstantTimeCompare([]byte(token), []byte(a.token)) != 1 {
|
||||
return grant{}, false
|
||||
}
|
||||
return grant{Username: "archie", Full: true}, true
|
||||
}
|
||||
taskID, ok := taskUser(username)
|
||||
if !ok || password == "" || a.tasks == nil {
|
||||
return grant{}, false
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, taskAuthTimeout)
|
||||
defer cancel()
|
||||
subjects, err := a.tasks.TaskSubjects(ctx, taskID, password)
|
||||
if err != nil {
|
||||
return grant{}, false
|
||||
}
|
||||
return grant{
|
||||
Username: username,
|
||||
Publish: slices.Clone(subjects.Publish),
|
||||
Subscribe: append(slices.Clone(subjects.Subscribe), TaskInboxPrefix(taskID)+".>"),
|
||||
}, true
|
||||
}
|
||||
|
||||
func (a authenticator) Check(c server.ClientAuthentication) bool {
|
||||
opts := c.GetOpts()
|
||||
if opts.Token != "" {
|
||||
if subtle.ConstantTimeCompare([]byte(opts.Token), []byte(a.token)) != 1 {
|
||||
return false
|
||||
}
|
||||
c.RegisterUser(&server.User{Username: "archie"})
|
||||
g, ok := a.authorize(context.Background(), opts.Token, opts.Username, opts.Password)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if g.Full {
|
||||
c.RegisterUser(&server.User{Username: g.Username})
|
||||
return true
|
||||
}
|
||||
taskID, ok := taskUser(opts.Username)
|
||||
if !ok || opts.Password == "" || a.tasks == nil {
|
||||
return false
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), taskAuthTimeout)
|
||||
defer cancel()
|
||||
subjects, err := a.tasks.TaskSubjects(ctx, taskID, opts.Password)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
c.RegisterUser(&server.User{Username: opts.Username, Permissions: &server.Permissions{
|
||||
Publish: &server.SubjectPermission{Allow: subjects.Publish},
|
||||
Subscribe: &server.SubjectPermission{Allow: append(subjects.Subscribe, TaskInboxPrefix(taskID)+".>")},
|
||||
c.RegisterUser(&server.User{Username: g.Username, Permissions: &server.Permissions{
|
||||
Publish: &server.SubjectPermission{Allow: g.Publish},
|
||||
Subscribe: &server.SubjectPermission{Allow: g.Subscribe},
|
||||
// Lets the task answer the one request it is sent, on the requester's
|
||||
// reply subject.
|
||||
Response: &server.ResponsePermission{MaxMsgs: 1},
|
||||
|
||||
@@ -8,7 +8,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
natsio "github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
type taskSeven struct{}
|
||||
@@ -22,14 +24,60 @@ func (taskSeven) TaskSubjects(_ context.Context, taskID int64, credential string
|
||||
|
||||
// TestBrokerScopesTaskCredentials pins that the instance token reaches
|
||||
// everything and a task's run credential reaches only its own subjects, on a
|
||||
// real embedded broker.
|
||||
// real embedded broker and on an external one that defers logins to the
|
||||
// callout. It guards a task container being handed more than its own subjects.
|
||||
func TestBrokerScopesTaskCredentials(t *testing.T) {
|
||||
srv, err := StartEmbedded(context.Background(), EmbeddedOptions{Token: "instance", StoreDir: t.TempDir(), Tasks: taskSeven{}}, nil)
|
||||
embedded, err := StartEmbedded(context.Background(), EmbeddedOptions{Token: "instance", StoreDir: t.TempDir(), Tasks: taskSeven{}}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer srv.Shutdown()
|
||||
defer embedded.Shutdown()
|
||||
|
||||
for name, url := range map[string]string{"embedded": embedded.ClientURL(), "callout": calloutBroker(t)} {
|
||||
t.Run(name, func(t *testing.T) { checkScopes(t, url) })
|
||||
}
|
||||
}
|
||||
|
||||
// calloutBroker starts an external-style broker whose logins are decided by
|
||||
// StartCallout, configured the way an operator would: one auth user for the
|
||||
// callout itself, every other login deferred.
|
||||
func calloutBroker(t *testing.T) string {
|
||||
t.Helper()
|
||||
issuer, err := nkeys.CreateAccount()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub, err := issuer.PublicKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed, err := issuer.Seed()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv, err := server.NewServer(&server.Options{
|
||||
Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true,
|
||||
Users: []*server.User{{Username: CalloutUser, Password: "instance"}},
|
||||
AuthCallout: &server.AuthCallout{Issuer: pub, Account: "$G", AuthUsers: []string{CalloutUser}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv.Start()
|
||||
t.Cleanup(srv.Shutdown)
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("broker not ready")
|
||||
}
|
||||
svc, err := StartCallout(context.Background(), CalloutOptions{URL: srv.ClientURL(), Token: "instance", IssuerSeed: string(seed), Tasks: taskSeven{}}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(svc.Close)
|
||||
return srv.ClientURL()
|
||||
}
|
||||
|
||||
func checkScopes(t *testing.T, url string) {
|
||||
t.Helper()
|
||||
tests := []struct {
|
||||
name string
|
||||
opts []natsio.Option
|
||||
@@ -61,7 +109,7 @@ func TestBrokerScopesTaskCredentials(t *testing.T) {
|
||||
opts := append(slices.Clone(tt.opts), natsio.ErrorHandler(func(_ *natsio.Conn, _ *natsio.Subscription, err error) {
|
||||
violations <- err.Error()
|
||||
}))
|
||||
nc, err := natsio.Connect(srv.ClientURL(), opts...)
|
||||
nc, err := natsio.Connect(url, opts...)
|
||||
if !tt.connects {
|
||||
if err == nil {
|
||||
nc.Close()
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
package nats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/nats-io/jwt/v2"
|
||||
natsio "github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nkeys"
|
||||
"github.com/synadia-io/callout.go"
|
||||
)
|
||||
|
||||
// CalloutUser is the broker login archied's callout service connects with. The
|
||||
// external broker lists it in auth_users, so its own login skips the callout;
|
||||
// its password is the instance token.
|
||||
const CalloutUser = "archie-callout"
|
||||
|
||||
// calloutAccount is the account a granted login is placed in: the broker's
|
||||
// global account, which JetStream uses.
|
||||
const calloutAccount = "$G"
|
||||
|
||||
// CalloutOptions configures StartCallout.
|
||||
type CalloutOptions struct {
|
||||
// URL is the external broker.
|
||||
URL string
|
||||
// Token is the instance token: the callout user's password, and the token
|
||||
// that logs in with every permission.
|
||||
Token string
|
||||
// IssuerSeed is the account nkey seed whose public key the broker names as
|
||||
// its auth_callout issuer.
|
||||
IssuerSeed string
|
||||
// Tasks admits task users to their own task's subjects.
|
||||
Tasks TaskAuthority
|
||||
}
|
||||
|
||||
// Callout is the authorization service of an external broker.
|
||||
type Callout struct {
|
||||
svc *callout.AuthorizationService
|
||||
nc *natsio.Conn
|
||||
}
|
||||
|
||||
// StartCallout answers an external broker's login requests with the decisions
|
||||
// the embedded broker makes, so a task container holds only its run credential.
|
||||
func StartCallout(ctx context.Context, opts CalloutOptions, log *slog.Logger) (*Callout, error) {
|
||||
if log == nil {
|
||||
log = slog.New(slog.DiscardHandler)
|
||||
}
|
||||
issuer, err := nkeys.FromSeed([]byte(opts.IssuerSeed))
|
||||
if err != nil {
|
||||
return nil, errors.New("nats callout: issuer seed is not an nkey seed")
|
||||
}
|
||||
nc, err := natsio.Connect(opts.URL, natsio.UserInfo(CalloutUser, opts.Token), natsio.Name("archie-callout"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nats callout connect: %w", err)
|
||||
}
|
||||
svc, err := callout.NewAuthorizationService(nc,
|
||||
callout.Authorizer(authenticator{token: opts.Token, tasks: opts.Tasks}.calloutAuthorizer(context.WithoutCancel(ctx), issuer)),
|
||||
callout.ResponseSignerKey(issuer),
|
||||
callout.Logger(brokerLogger{log: log.With("service", "nats", "component", "callout")}),
|
||||
)
|
||||
if err != nil {
|
||||
nc.Close()
|
||||
return nil, fmt.Errorf("nats callout service: %w", err)
|
||||
}
|
||||
log.InfoContext(ctx, "nats callout ready", "url", opts.URL)
|
||||
return &Callout{svc: svc, nc: nc}, nil
|
||||
}
|
||||
|
||||
// Close stops answering logins.
|
||||
func (c *Callout) Close() {
|
||||
_ = c.svc.Stop()
|
||||
c.nc.Close()
|
||||
}
|
||||
|
||||
// calloutAuthorizer turns an authenticator decision into a user JWT. An error
|
||||
// refuses the login.
|
||||
func (a authenticator) calloutAuthorizer(ctx context.Context, issuer nkeys.KeyPair) callout.AuthorizerFn {
|
||||
return func(req *jwt.AuthorizationRequest) (string, error) {
|
||||
g, ok := a.authorize(ctx, req.ConnectOptions.Token, req.ConnectOptions.Username, req.ConnectOptions.Password)
|
||||
if !ok {
|
||||
return "", errors.New("login refused")
|
||||
}
|
||||
return g.userJWT(req.UserNkey, issuer)
|
||||
}
|
||||
}
|
||||
|
||||
// userJWT is the grant as a broker user's permissions, signed by the issuer the
|
||||
// broker trusts.
|
||||
func (g grant) userJWT(userNkey string, issuer nkeys.KeyPair) (string, error) {
|
||||
uc := jwt.NewUserClaims(userNkey)
|
||||
uc.Name = g.Username
|
||||
uc.Audience = calloutAccount
|
||||
if !g.Full {
|
||||
uc.Pub.Allow.Add(g.Publish...)
|
||||
uc.Sub.Allow.Add(g.Subscribe...)
|
||||
uc.Resp = &jwt.ResponsePermission{MaxMsgs: 1}
|
||||
}
|
||||
return uc.Encode(issuer)
|
||||
}
|
||||
@@ -30,14 +30,14 @@ func TestSandboxPolicyAdmitsOnlyWhatTheTaskNeeds(t *testing.T) {
|
||||
name: "services by IP and loopback use the services host",
|
||||
servicesHost: "archie.lan",
|
||||
launch: Launch{Worker: egress.Endpoints{
|
||||
NATS: "nats://172.17.0.1:4222", NATSToken: "t", StateStore: "dns:///127.0.0.1:50051", StateStoreToken: "run-1",
|
||||
NATS: "nats://172.17.0.1:4222", StateStore: "dns:///127.0.0.1:50051", StateStoreToken: "run-1",
|
||||
}},
|
||||
wantRules: map[string]string{
|
||||
"archie-nats": "archie.lan:4222 " + agentBinary,
|
||||
"archie-state-store": "archie.lan:50051 " + agentBinary,
|
||||
},
|
||||
wantEnv: map[string]string{
|
||||
"NATS_URL": "nats://archie.lan:4222", "NATS_TOKEN": "t",
|
||||
"NATS_URL": "nats://archie.lan:4222",
|
||||
"STATE_STORE_URL": "archie.lan:50051", "STATE_STORE_TOKEN": "run-1",
|
||||
},
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user