feat(rpc): mTLS between services from operator-issued certificates

Replace the service bearer tokens with [tls] ca_file and cert_dir. Caller
identity comes from the verified certificate, not metadata. Without TLS,
services listen and dial on loopback only. Task containers get only the CA
and keep their grant.
This commit is contained in:
sam committed 2026-10-16 02:37:51 +11:00
1 parent 88ef6ca5b2
commit 53c95038de
51 files changed
+746 -689

No files matched your search

+1
View File
@@ -91,6 +91,7 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run
NATSToken: natsToken, NATSToken: natsToken,
StateStoreTarget: stateStoreURL, StateStoreTarget: stateStoreURL,
StateStoreToken: stateStoreToken, StateStoreToken: stateStoreToken,
StateStoreCA: []byte(getenv("STATE_STORE_CA")),
ModelProxy: getenv(egress.ModelProxyEnv), ModelProxy: getenv(egress.ModelProxyEnv),
ModelProxyHosts: egress.ParseHosts(getenv(egress.ModelProxyHostsEnv)), ModelProxyHosts: egress.ParseHosts(getenv(egress.ModelProxyHostsEnv)),
ModelProxyCA: getenv(egress.ModelProxyCAEnv), ModelProxyCA: getenv(egress.ModelProxyCAEnv),
-1
View File
@@ -22,7 +22,6 @@ func run() int {
flag.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory") flag.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory")
flag.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory") flag.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory")
flag.StringVar(&options.Listen, "listen", "", "gateway gRPC listen address (defaults to [services.gateway].listen, else 127.0.0.1:8585)") flag.StringVar(&options.Listen, "listen", "", "gateway gRPC listen address (defaults to [services.gateway].listen, else 127.0.0.1:8585)")
flag.StringVar(&options.Token, "token", "", "Bearer [REDACTED] required for a non-loopback listener (defaults to [services.gateway].target_token)")
flag.Parse() flag.Parse()
showVersion() showVersion()
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
-3
View File
@@ -23,11 +23,8 @@ func run() int {
flag.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory") flag.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory")
flag.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory") flag.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory")
flag.StringVar(&options.Gateway.Target, "gateway-target", "", "archie-gateway gRPC address (defaults to [services.gateway].target, else 127.0.0.1:8585)") flag.StringVar(&options.Gateway.Target, "gateway-target", "", "archie-gateway gRPC address (defaults to [services.gateway].target, else 127.0.0.1:8585)")
flag.StringVar(&options.Gateway.Token, "gateway-token", "", "bearer token presented to archie-gateway")
flag.StringVar(&options.StateStore.Target, "state-store-target", "", "archie-state-store gRPC address (defaults to [services.state].target)") flag.StringVar(&options.StateStore.Target, "state-store-target", "", "archie-state-store gRPC address (defaults to [services.state].target)")
flag.StringVar(&options.StateStore.Token, "state-store-token", "", "bearer token presented to archie-state-store")
flag.StringVar(&options.Listen, "listen", "", "MessagingService gRPC listen address (defaults to [services.messaging].listen)") flag.StringVar(&options.Listen, "listen", "", "MessagingService gRPC listen address (defaults to [services.messaging].listen)")
flag.StringVar(&options.Token, "token", "", "bearer token required from MessagingService callers on a non-loopback listener")
flag.DurationVar(&options.DependencyTimeout, "dependency-timeout", 0, "readiness probe timeout (default 5s)") flag.DurationVar(&options.DependencyTimeout, "dependency-timeout", 0, "readiness probe timeout (default 5s)")
flag.DurationVar(&options.ShutdownTimeout, "shutdown-timeout", 0, "shutdown timeout (default 5s)") flag.DurationVar(&options.ShutdownTimeout, "shutdown-timeout", 0, "shutdown timeout (default 5s)")
flag.Parse() flag.Parse()
-1
View File
@@ -34,7 +34,6 @@ func run() int {
flag.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory") flag.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory")
flag.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory") flag.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory")
flag.StringVar(&options.Listen, "listen", "", "state store gRPC listen address (defaults to [services.state].listen, else 127.0.0.1:9090)") flag.StringVar(&options.Listen, "listen", "", "state store gRPC listen address (defaults to [services.state].listen, else 127.0.0.1:9090)")
flag.StringVar(&options.Token, "token", "", "bearer token required for a non-loopback listener (defaults to [services.state].target_token)")
flag.StringVar(&options.ReadyAddr, "ready-addr", "", "optional readiness HTTP listen address (e.g. 127.0.0.1:9091)") flag.StringVar(&options.ReadyAddr, "ready-addr", "", "optional readiness HTTP listen address (e.g. 127.0.0.1:9091)")
flag.Usage = func() { flag.Usage = func() {
fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store <backup|restore|validate|rollback> -h | access reset -h\n\n") fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store <backup|restore|validate|rollback> -h | access reset -h\n\n")
-3
View File
@@ -35,11 +35,8 @@ func run() int {
flag.StringVar(&options.OidcClientSecretEnv, "oidc-client-secret-env", "", "name of the environment variable holding the sign-in client secret") flag.StringVar(&options.OidcClientSecretEnv, "oidc-client-secret-env", "", "name of the environment variable holding the sign-in client secret")
flag.StringVar(&options.OidcRedirectURL, "oidc-redirect-url", "", "callback URL registered at the provider (path must be /oauth2/callback)") flag.StringVar(&options.OidcRedirectURL, "oidc-redirect-url", "", "callback URL registered at the provider (path must be /oauth2/callback)")
flag.StringVar(&options.Gateway.Target, "gateway-target", "", "archie-gateway gRPC address (defaults to [services.gateway].target)") flag.StringVar(&options.Gateway.Target, "gateway-target", "", "archie-gateway gRPC address (defaults to [services.gateway].target)")
flag.StringVar(&options.Gateway.Token, "gateway-token", "", "bearer token presented to archie-gateway")
flag.StringVar(&options.State.Target, "state-target", "", "archie-state-store gRPC address (defaults to [services.state].target)") flag.StringVar(&options.State.Target, "state-target", "", "archie-state-store gRPC address (defaults to [services.state].target)")
flag.StringVar(&options.State.Token, "state-token", "", "bearer token presented to archie-state-store")
flag.StringVar(&options.Harness.Target, "harness-target", "", "archied harness session gRPC address (defaults to [services.harness].target); empty leaves the setup terminal unwired") flag.StringVar(&options.Harness.Target, "harness-target", "", "archied harness session gRPC address (defaults to [services.harness].target); empty leaves the setup terminal unwired")
flag.StringVar(&options.Harness.Token, "harness-token", "", "bearer token presented to the daemon's harness contract")
flag.DurationVar(&options.DependencyTimeout, "dependency-timeout", 0, "per-dependency readiness probe timeout (default 5s)") flag.DurationVar(&options.DependencyTimeout, "dependency-timeout", 0, "per-dependency readiness probe timeout (default 5s)")
flag.DurationVar(&options.EventPollInterval, "event-poll-interval", 0, "how often the activity feed polls the State Store for new events (default 1s)") flag.DurationVar(&options.EventPollInterval, "event-poll-interval", 0, "how often the activity feed polls the State Store for new events (default 1s)")
// BoolFunc rather than BoolVar so an unset flag stays distinguishable // BoolFunc rather than BoolVar so an unset flag stays distinguishable
+14 -17
View File
@@ -261,6 +261,18 @@ wall_clock = "30m" # one agent call
gate_max_failures = 5 gate_max_failures = 5
task_wall_clock = "4h" # a whole task run; keep containers.max_uptime at or above it task_wall_clock = "4h" # a whole task run; keep containers.max_uptime at or above it
# TLS: every gRPC hop between services uses certificates from your own CA
# (step-ca, cert-manager, Vault, openssl). Each process loads
# cert_dir/<service>.pem and cert_dir/<service>-key.pem, whose DNS SAN is its
# service name: archied, archie-gateway, archie-messaging, archie-state-store,
# archie-ui. Every peer must chain to ca_file. Certificates are reloaded when
# they change. Without [tls], services may only listen on and dial loopback
# addresses, task containers cannot reach the State Store, and the harness
# setup terminal is off.
# [tls]
# ca_file = "/etc/archie/tls/ca.pem"
# cert_dir = "/etc/archie/tls"
# Services selects contract adapters at composition time. # Services selects contract adapters at composition time.
[services.gateway] [services.gateway]
target = "127.0.0.1:8585" # archie-gateway's gRPC address (always remote once set) target = "127.0.0.1:8585" # archie-gateway's gRPC address (always remote once set)
@@ -269,11 +281,6 @@ target = "127.0.0.1:8585" # archie-gateway's gRPC address (always remot
# # to the target above, so a default deployment's client and # # to the target above, so a default deployment's client and
# # server cannot drift onto different ports. Set it when the # # server cannot drift onto different ports. Set it when the
# # host already owns that port. # # host already owns that port.
# target_token = "..." # Bearer [REDACTED] for a NON-loopback listener. A loopback listener
# # (127.0.0.1/::1/localhost) needs no token. A non-loopback
# # listener without a token fails closed, as does a
# # non-loopback dial without one. The same token is
# # presented as GATEWAY_TOKEN by remote consumers.
[services.state] [services.state]
target = "127.0.0.1:9090" # REQUIRED. The State Store gRPC address archied and target = "127.0.0.1:9090" # REQUIRED. The State Store gRPC address archied and
@@ -285,25 +292,15 @@ target = "127.0.0.1:9090" # REQUIRED. The State Store gRPC address arch
# # above, so a default deployment's client and server cannot # # above, so a default deployment's client and server cannot
# # drift onto different ports. Move it when a port is already # # drift onto different ports. Move it when a port is already
# # taken on the host. # # taken on the host.
# target_token = "..." # bearer token for a NON-loopback target. A loopback target
# # (127.0.0.1/::1/localhost) needs no token. A non-loopback
# # target without a token fails closed. The same token is injected as STATE_STORE_TOKEN into
# # archie-agent containers so the agent authenticates to the
# # same remote store. See deployments/README.md for how to
# # start archie-state-store.
# The daemon's harness session contract: the dashboard's setup terminal opens # The daemon's harness session contract: the dashboard's setup terminal opens
# a PTY into an ephemeral Kit container over it. Both ends are optional and # a PTY into an ephemeral Kit container over it. Both ends are optional and
# the dashboard's setup terminal degrades to 503 without them: the daemon # the dashboard's setup terminal degrades to 503 without them: the daemon
# serves the contract only when target_token (or HARNESS_TOKEN) is set, and # serves the contract only when [tls] is configured, and archie-ui dials it
# archie-ui dials it only when a target is set. # only when a target is set.
# [services.harness] # [services.harness]
# target = "127.0.0.1:8587" # The address archie-ui dials. Empty leaves the # target = "127.0.0.1:8587" # The address archie-ui dials. Empty leaves the
# # setup terminal unwired. # # setup terminal unwired.
# target_token = "..." # REQUIRED, even on loopback: a session is a
# # shell in a container that holds an org credential.
# # The daemon reads the same value to authenticate its
# # clients. Leave unset to disable the listener.
# NATS is the sole autonomous workflow handoff. mode changes only where the # NATS is the sole autonomous workflow handoff. mode changes only where the
# broker runs; every workflow still runs in a managed task container: # broker runs; every workflow still runs in a managed task container:
+1 -5
View File
@@ -58,11 +58,7 @@ as a host binary. Pull it from GHCR: `ghcr.io/samcharles93/archie-agent`.
Dashboard reachability comes from `[web].listen` (default Dashboard reachability comes from `[web].listen` (default
`127.0.0.1:8484`, which needs no dashboard token). For a non-loopback `127.0.0.1:8484`, which needs no dashboard token). For a non-loopback
bind pass `-listen` with `-token` or `-token-file` to mint and persist bind pass `-listen` with `-token` or `-token-file` to mint and persist
one. `archie-ui` reads `GATEWAY_TOKEN` / `STATE_STORE_TOKEN` from its one.
process environment only — it has no secret registry, so when those
tokens live in a secret manager, whatever starts `archie-ui` must export
them into its environment (the `[services.*].target_token` values in
`config.toml` are the other supported source).
4. For a persistent, 24/7 deployment instead of a foreground run, follow the 4. For a persistent, 24/7 deployment instead of a foreground run, follow the
systemd user-unit runbook: systemd user-unit runbook:
+8 -11
View File
@@ -52,10 +52,10 @@ PostgreSQL database `database_url` names and serves the `StateStore` gRPC
contract. BOTH `archied` and `archie-gateway` dial it via contract. BOTH `archied` and `archie-gateway` dial it via
`[services.state].target` and never open the task tables themselves. The templates set `[services.state].target` and never open the task tables themselves. The templates set
`[services.state]` accordingly: a host-only agent reaches a loopback bind `[services.state]` accordingly: a host-only agent reaches a loopback bind
(`127.0.0.1:9090`) with no token, while a container-mode agent needs the (`127.0.0.1:9090`), while a container-mode agent needs the Docker bridge
Docker bridge gateway address plus a matching bearer token (`target_token` / gateway address, because a container cannot reach the host's loopback. A
`--token` / `STATE_STORE_TOKEN`), because a container cannot reach the host's non-loopback address needs `[tls]` certificates from your own CA; see
loopback. `config.example.toml`.
## Running the dashboard ## Running the dashboard
@@ -84,17 +84,14 @@ archie-ui warns about exactly that at startup. Terminate TLS in front of it
warning; leaving it off while exposing the port directly is the case the warning; leaving it off while exposing the port directly is the case the
warning names. warning names.
Auth note: unlike `archied`, archie-ui has no secret registry -- Auth note: archie-ui authenticates to its dependencies with its `[tls]`
`GATEWAY_TOKEN`/`STATE_STORE_TOKEN` resolve from its process environment certificate (`archie-ui.pem`).
only (or from `[services.*].target_token` in `config.toml`), so whatever
starts archie-ui must export those tokens into its environment. See the UI
Service unit in `systemd-user-service.md` for the worked example.
```bash ```bash
# loopback (host-only agent) # loopback (host-only agent)
archie-state-store -config ~/.config/archie/config.toml -listen 127.0.0.1:9090 archie-state-store -config ~/.config/archie/config.toml -listen 127.0.0.1:9090
# or container-mode: bind the bridge gateway with a token # or container-mode: bind the bridge gateway ([tls] required)
archie-state-store -config ~/.config/archie/config.toml -listen 172.17.0.1:9090 -token <token> archie-state-store -config ~/.config/archie/config.toml -listen 172.17.0.1:9090
``` ```
## Verifying the daemon ## Verifying the daemon
+7 -5
View File
@@ -26,13 +26,15 @@ target = "127.0.0.1:8585"
# The State Store gRPC service owns the PostgreSQL task data; archied and # The State Store gRPC service owns the PostgreSQL task data; archied and
# archie-gateway dial it and never open those tables themselves. # archie-gateway dial it and never open those tables themselves.
# This deployment runs the agent in containers, so target is the Docker bridge # This deployment runs the agent in containers, so the target is the Docker
# gateway (172.17.0.1) the container can reach, with a matching target_token # bridge gateway (172.17.0.1) the container can reach. A non-loopback address
# that archie-state-store validates. Host-only agent: use 127.0.0.1:9090 with # needs [tls]; see config.example.toml.
# no token.
[services.state] [services.state]
target = "172.17.0.1:9090" target = "172.17.0.1:9090"
target_token = "change-me-state-store-token"
[tls]
ca_file = "/etc/archie/tls/ca.pem"
cert_dir = "/etc/archie/tls"
[forge] [forge]
type = "gitea" type = "gitea"
+3 -5
View File
@@ -18,13 +18,11 @@ target = "127.0.0.1:8585"
# The State Store gRPC service owns the PostgreSQL task data; archied and # The State Store gRPC service owns the PostgreSQL task data; archied and
# archie-gateway dial it and never open those tables themselves. # archie-gateway dial it and never open those tables themselves.
# This deployment runs the agent in containers, so target is the Docker bridge # This deployment runs the agent in containers, so the target is the Docker
# gateway (172.17.0.1) the container can reach, with a matching target_token # bridge gateway (172.17.0.1) the container can reach. A non-loopback address
# that archie-state-store validates. Host-only agent: use 127.0.0.1:9090 with # needs [tls]; see config.example.toml.
# no token.
[services.state] [services.state]
target = "172.17.0.1:9090" target = "172.17.0.1:9090"
target_token = "change-me-state-store-token"
[models] [models]
triage = "openai/gpt-6-luna" triage = "openai/gpt-6-luna"
+3 -8
View File
@@ -25,16 +25,11 @@ target = "127.0.0.1:8585"
# The State Store gRPC service owns the PostgreSQL task data. archied and # The State Store gRPC service owns the PostgreSQL task data. archied and
# archie-gateway dial it here and never open those tables themselves. # archie-gateway dial it here and never open those tables themselves.
# This deployment runs the # This deployment runs the agent in containers, so the target is the Docker
# agent in containers, so the target must be the Docker bridge gateway address # bridge gateway (172.17.0.1) the container can reach. A non-loopback address
# (172.17.0.1) that the container can reach, and a non-loopback target requires # needs [tls]; see config.example.toml.
# a matching target_token that archie-state-store validates -- start it with the
# same token (e.g. `archie-state-store ... -listen 172.17.0.1:9090 -token
# change-me-state-store-token`). For a host-only agent (no [containers]) use the
# loopback target 127.0.0.1:9090 with no token.
[services.state] [services.state]
target = "172.17.0.1:9090" target = "172.17.0.1:9090"
target_token = "change-me-state-store-token"
[forge] [forge]
type = "github" type = "github"
+8 -29
View File
@@ -20,10 +20,9 @@ The State Store unit is `archie-state-store.service`. It owns the task data in
PostgreSQL and serves the `StateStore` gRPC contract. BOTH `archied` and PostgreSQL and serves the `StateStore` gRPC contract. BOTH `archied` and
`archie-gateway` dial it via `[services.state].target` (see the `archied` `archie-gateway` dial it via `[services.state].target` (see the `archied`
config below) and never open the task tables themselves. A config below) and never open the task tables themselves. A
host-only agent uses the loopback bind with no token below; a container-mode host-only agent uses the loopback bind below; a container-mode agent needs the
agent needs the Docker bridge gateway address plus a bearer token via Docker bridge gateway address, and a non-loopback bind needs `[tls]` (see
`--token`/`STATE_STORE_TOKEN` or `[services.state].target_token`, and a `config.example.toml`):
non-loopback bind fails closed without one:
`install.sh` writes `archie-state-store.service` with that loopback bind. `install.sh` writes `archie-state-store.service` with that loopback bind.
@@ -41,9 +40,7 @@ update when a host is missing a unit and names what is missing, which now fires
for a host that drifted or predates this release rather than the normal path. for a host that drifted or predates this release rather than the normal path.
It dials the Gateway for the chat contract and the State Store for the stored It dials the Gateway for the chat contract and the State Store for the stored
channel settings, and reads the same `config.toml`. Like `archie-ui` it resolves channel settings, and reads the same `config.toml`, including `[tls]`.
`GATEWAY_TOKEN` and `STATE_STORE_TOKEN` from its process environment, hence the
`EnvironmentFile`:
`install.sh` writes `archie-messaging.service`. Its `After=` and `Wants=` `install.sh` writes `archie-messaging.service`. Its `After=` and `Wants=`
lines only tidy the start sequence: nothing depends on that order, because lines only tidy the start sequence: nothing depends on that order, because
@@ -56,34 +53,16 @@ the Gateway and State Store targets already in it, and binds `[web].listen`.
A non-loopback bind requires a dashboard token (`-token`, or `-token-file` to A non-loopback bind requires a dashboard token (`-token`, or `-token-file` to
mint and persist one); mint and persist one);
**Auth note:** unlike `archied`, **Auth note:** `archie-ui` authenticates to the Gateway, State Store and
`archie-ui` has no secret registry — it resolves `GATEWAY_TOKEN` and harness with its `[tls]` certificate (`archie-ui.pem`) from `config.toml`.
`STATE_STORE_TOKEN` from its **process environment only** (or from the
`target_token` values in `config.toml`). Whatever starts `archie-ui` is
therefore responsible for exporting those tokens into its environment: keep
them in `~/.config/archie/env` (loaded by the `EnvironmentFile` line below),
not only in a secret manager. A deployment that authenticates `archied`
through bws/Vault but exports nothing for `archie-ui` leaves the dashboard
unable to dial its dependencies.
For supervision, `GET /healthz` on `[web].listen` is the process's liveness For supervision, `GET /healthz` on `[web].listen` is the process's liveness
probe (token-free by design); the authenticated readiness surface lives at probe (token-free by design); the authenticated readiness surface lives at
the dashboard's health endpoint behind the token. A watchdog can curl the dashboard's health endpoint behind the token. A watchdog can curl
`/healthz` without credentials. `/healthz` without credentials.
`install.sh` writes `archie-ui.service`. It has no secret registry of its own, `install.sh` writes `archie-ui.service`. The dashboard token comes from
so its token handling is the `EnvironmentFile` note above. `-token-file` (there is no environment variable for it).
Concretely, for a deployment whose dependency tokens live outside
`config.toml`, `~/.config/archie/env` carries the two names and the unit
loads it; the dashboard token itself comes from `-token-file` (there is no
environment variable for it):
```bash
# ~/.config/archie/env
GATEWAY_TOKEN=... # presented to archie-gateway
STATE_STORE_TOKEN=... # presented to archie-state-store
```
`After=` orders the start, it does not wait for the store to bind. The UI `After=` orders the start, it does not wait for the store to bind. The UI
process closes that gap itself: its event pump retries priming with backoff process closes that gap itself: its event pump retries priming with backoff
+3
View File
@@ -50,6 +50,8 @@ type Settings struct {
// NATS storerpc path is deleted. // NATS storerpc path is deleted.
StateStoreTarget string StateStoreTarget string
StateStoreToken string StateStoreToken string
// StateStoreCA is the PEM CA the State Store's certificate chains to.
StateStoreCA []byte
// ModelProxy, ModelProxyHosts and ModelProxyCA route model provider // ModelProxy, ModelProxyHosts and ModelProxyCA route model provider
// requests through the daemon's egress proxy, which holds the keys. // requests through the daemon's egress proxy, which holds the keys.
@@ -148,6 +150,7 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies
TaskID: taskID, TaskID: taskID,
StateStoreURL: settings.StateStoreTarget, StateStoreURL: settings.StateStoreTarget,
StateStoreToken: settings.StateStoreToken, StateStoreToken: settings.StateStoreToken,
StateStoreCA: settings.StateStoreCA,
}, log) }, log)
if err != nil { if err != nil {
operation := "nats connect failed" operation := "nats connect failed"
+15 -6
View File
@@ -108,7 +108,6 @@ type boot struct {
// (daemon.RunCredentialIssuer), wrapping the same *staterpc.Client as // (daemon.RunCredentialIssuer), wrapping the same *staterpc.Client as
// stateStore, which also verifies a credential for a worktree push. // stateStore, which also verifies a credential for a worktree push.
stateStoreGrants *staterpc.GrantIssuer stateStoreGrants *staterpc.GrantIssuer
stateStoreToken string
controlPlane *controlplane.Client controlPlane *controlplane.Client
// controlPlaneRPC is the State Store's control-plane transport, kept so the // controlPlaneRPC is the State Store's control-plane transport, kept so the
// daemon root can build its workflow-definitions client from it // daemon root can build its workflow-definitions client from it
@@ -273,7 +272,7 @@ func (b *boot) openStateStoreAdapter(ctx context.Context) error {
if target == "" { if target == "" {
return fmt.Errorf("services.state.target is required: archied/archie-gateway no longer own archie.db; the standalone archie-state-store process owns it (docs/prds/state-store-contract.md §12 step 7)") return fmt.Errorf("services.state.target is required: archied/archie-gateway no longer own archie.db; the standalone archie-state-store process owns it (docs/prds/state-store-contract.md §12 step 7)")
} }
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, b.cfg.Services, b.secrets) client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, b.cfg)
if err != nil { if err != nil {
b.log.Error("state store adapter", "err", err) b.log.Error("state store adapter", "err", err)
return err return err
@@ -287,7 +286,6 @@ func (b *boot) openStateStoreAdapter(ctx context.Context) error {
b.controlPlane = controlplane.NewRPCClient(b.controlPlaneRPC) b.controlPlane = controlplane.NewRPCClient(b.controlPlaneRPC)
b.applyStatus = applystatus.New(b.processName, client, b.log) b.applyStatus = applystatus.New(b.processName, client, b.log)
b.stateStoreGrants = &staterpc.GrantIssuer{Client: client} b.stateStoreGrants = &staterpc.GrantIssuer{Client: client}
b.stateStoreToken = b.cfg.Services.ResolvedToken(config.ServiceNameState, b.secrets.Getenv)
b.addCleanup(cleanup) b.addCleanup(cleanup)
b.startSecretEngines(ctx, client) b.startSecretEngines(ctx, client)
return nil return nil
@@ -478,13 +476,13 @@ func (b *boot) setupContainers(ctx context.Context) func() {
// only enqueues the chat tasks scheduled workflows create. // only enqueues the chat tasks scheduled workflows create.
func (b *boot) setupGatewayClient(ctx context.Context) error { func (b *boot) setupGatewayClient(ctx context.Context) error {
b.setupChatTasks(b.cfg) b.setupChatTasks(b.cfg)
contract, cleanup, err := composeChatContract(b.cfg.Services, b.secrets) contract, cleanup, err := composeChatContract(b.cfg)
if err != nil { if err != nil {
return err return err
} }
b.addCleanup(cleanup) b.addCleanup(cleanup)
b.chat = &webui.ChatService{Contract: contract} b.chat = &webui.ChatService{Contract: contract}
messaging, closeMessaging, err := composeMessaging(b.cfg.Services, b.secrets) messaging, closeMessaging, err := composeMessaging(b.cfg)
if err != nil { if err != nil {
return err return err
} }
@@ -580,7 +578,7 @@ func (b *boot) buildDaemon() {
b.d = &daemon.Daemon{ b.d = &daemon.Daemon{
Cfg: b.cfgHolder, Cfg: b.cfgHolder,
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken, Scoped: b.cfg.NATS.Mode != config.NATSModeExternal}, ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken, Scoped: b.cfg.NATS.Mode != config.NATSModeExternal},
ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), Token: b.stateStoreToken}, ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), CA: b.stateStoreCA()},
Store: b.stateStore, Store: b.stateStore,
Bus: b.bus, Bus: b.bus,
Forge: b.forgeClient, Forge: b.forgeClient,
@@ -873,3 +871,14 @@ func (b *boot) livePlaybooks() *controlplane.LivePlaybooks {
// playbookRefreshInterval is how often archied re-reads the resources it // playbookRefreshInterval is how often archied re-reads the resources it
// would otherwise load only on first use, so their apply status stays reported. // would otherwise load only on first use, so their apply status stays reported.
const playbookRefreshInterval = 30 * time.Second const playbookRefreshInterval = 30 * time.Second
// stateStoreCA is the CA a task container verifies the State Store with;
// nil without [tls].
func (b *boot) stateStoreCA() []byte {
id, err := servicekit.Identity(b.cfg.TLS, presence.Daemon)
if err != nil {
b.log.Warn("state store CA for containers", "err", err)
return nil
}
return id.CAPEM()
}
+17 -11
View File
@@ -3,27 +3,29 @@ package archied
import ( import (
"google.golang.org/grpc" "google.golang.org/grpc"
"github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/gateway" "github.com/samcharles93/archie-core/internal/gateway"
"github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc" "github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc"
"github.com/samcharles93/archie-core/internal/infrastructure/messagingrpc" "github.com/samcharles93/archie-core/internal/infrastructure/messagingrpc"
"github.com/samcharles93/archie-core/internal/secret"
) )
// composeChatContract dials the standalone archie-gateway, the sole owner of // composeChatContract dials the standalone archie-gateway, the sole owner of
// ChatContract, from the daemon's own configuration key. Token resolution is // ChatContract, from the daemon's own configuration key.
// the daemon's (it owns the config keys and the secret registry); the dial func composeChatContract(cfg config.Config, options ...grpc.DialOption) (gateway.ChatContract, func(), error) {
// itself and the fail-closed non-loopback rule belong to the transport, so target, err := cfg.Services.RequireTarget(config.ServiceNameGateway)
// they come from gatewayrpc.Dial. if err != nil {
func composeChatContract(services config.Services, secrets *secret.Registry, options ...grpc.DialOption) (gateway.ChatContract, func(), error) { return nil, nil, err
target, err := services.RequireTarget(config.ServiceNameGateway) }
id, err := servicekit.Identity(cfg.TLS, presence.Daemon)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
// Assigned through a typed variable rather than returned directly: a // Assigned through a typed variable rather than returned directly: a
// direct forward would hand a failed dial's nil *Client back as a // direct forward would hand a failed dial's nil *Client back as a
// non-nil ChatContract interface. // non-nil ChatContract interface.
client, cleanup, err := gatewayrpc.Dial(target, services.ResolvedToken(config.ServiceNameGateway, secrets.Getenv), options...) client, cleanup, err := gatewayrpc.Dial(target, id, options...)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@@ -32,10 +34,14 @@ func composeChatContract(services config.Services, secrets *secret.Registry, opt
// composeMessaging builds the Messaging Service client. The connection is // composeMessaging builds the Messaging Service client. The connection is
// lazy, so an absent Messaging Service fails each delivery, not the daemon. // lazy, so an absent Messaging Service fails each delivery, not the daemon.
func composeMessaging(services config.Services, secrets *secret.Registry) (*messagingrpc.Client, func(), error) { func composeMessaging(cfg config.Config) (*messagingrpc.Client, func(), error) {
target, err := services.RequireTarget(config.ServiceNameMessaging) target, err := cfg.Services.RequireTarget(config.ServiceNameMessaging)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
return messagingrpc.Dial(target, services.ResolvedToken(config.ServiceNameMessaging, secrets.Getenv)) id, err := servicekit.Identity(cfg.TLS, presence.Daemon)
if err != nil {
return nil, nil, err
}
return messagingrpc.Dial(target, id)
} }
+14 -9
View File
@@ -6,7 +6,6 @@ import (
"io" "io"
"log/slog" "log/slog"
"net" "net"
"os"
"slices" "slices"
"sync" "sync"
"time" "time"
@@ -19,7 +18,7 @@ import (
"github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/container" "github.com/samcharles93/archie-core/internal/container"
pb "github.com/samcharles93/archie-core/internal/contracts/harness/v1" pb "github.com/samcharles93/archie-core/internal/contracts/harness/v1"
"github.com/samcharles93/archie-core/internal/infrastructure/harnessrpc" "github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/harnesssession" "github.com/samcharles93/archie-core/internal/infrastructure/harnesssession"
"github.com/samcharles93/archie-core/internal/infrastructure/kitrun" "github.com/samcharles93/archie-core/internal/infrastructure/kitrun"
) )
@@ -189,9 +188,18 @@ func (b *boot) setupHarness(ctx context.Context) {
b.log.Warn("harness sessions disabled", "err", err) b.log.Warn("harness sessions disabled", "err", err)
return return
} }
token := b.cfg.Services.ResolvedToken(config.ServiceNameHarness, os.Getenv) id, err := servicekit.Identity(b.cfg.TLS, presence.Daemon)
if token == "" { if err != nil {
b.log.Warn("harness sessions disabled: [services.harness].target_token or HARNESS_TOKEN is required, even on loopback") b.log.Warn("harness sessions disabled", "err", err)
return
}
if id == nil {
b.log.Warn("harness sessions disabled: a session is a shell holding an org's credential, so it needs [tls] ca_file and cert_dir")
return
}
creds, err := id.ServerOption(listen, false)
if err != nil {
b.log.Warn("harness sessions disabled", "err", err)
return return
} }
ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", listen) ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", listen)
@@ -205,10 +213,7 @@ func (b *boot) setupHarness(ctx context.Context) {
Profile: func(name string) (config.AgentProfile, error) { return b.cfgHolder.Get().Containers.Profile(name) }, Profile: func(name string) (config.AgentProfile, error) { return b.cfgHolder.Get().Containers.Profile(name) },
TTL: setupSessionTTL, TTL: setupSessionTTL,
} }
srv := grpc.NewServer( srv := grpc.NewServer(creds)
//nolint:contextcheck // grpc.StreamServerInterceptor has no context.Context parameter; the interceptor derives its context from stream.Context()
grpc.StreamInterceptor(harnessrpc.StreamServerInterceptor(token)),
)
pb.RegisterHarnessServiceServer(srv, &harnessServer{sessions: manager, runs: b.runView(), log: b.log}) pb.RegisterHarnessServiceServer(srv, &harnessServer{sessions: manager, runs: b.runView(), log: b.log})
go func() { go func() {
if err := srv.Serve(ln); err != nil && !errors.Is(err, grpc.ErrServerStopped) { if err := srv.Serve(ln); err != nil && !errors.Is(err, grpc.ErrServerStopped) {
+1 -2
View File
@@ -12,7 +12,6 @@ import (
"github.com/samcharles93/archie-core/internal/app/servicekit" "github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/domain/presence" "github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/secret"
) )
// RunStatus prints one line per service from the State Store's presence // RunStatus prints one line per service from the State Store's presence
@@ -31,7 +30,7 @@ func RunStatus(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "archied status: %v\n", err) fmt.Fprintf(stderr, "archied status: %v\n", err)
return 1 return 1
} }
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, doc.Config.Services, secret.NewRegistry()) client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, doc.Config)
if err != nil { if err != nil {
fmt.Fprintf(stderr, "archied status: %v\n", err) fmt.Fprintf(stderr, "archied status: %v\n", err)
return 1 return 1
+1 -2
View File
@@ -12,7 +12,6 @@ import (
"github.com/samcharles93/archie-core/internal/app/servicekit" "github.com/samcharles93/archie-core/internal/app/servicekit"
pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
"github.com/samcharles93/archie-core/internal/domain/presence" "github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/secret"
) )
// imagePolicy is the control plane's agent image, reached through the State // imagePolicy is the control plane's agent image, reached through the State
@@ -55,7 +54,7 @@ func (p imagePolicy) with(ctx context.Context, use func(pb.ControlPlaneServiceCl
if err != nil { if err != nil {
return err return err
} }
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, doc.Config.Services, secret.NewRegistry()) client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, doc.Config)
if err != nil { if err != nil {
return err return err
} }
+14 -25
View File
@@ -15,15 +15,16 @@ import (
"github.com/samcharles93/archie-core/internal/gateway" "github.com/samcharles93/archie-core/internal/gateway"
"github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc" "github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc"
"github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity" "github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/plugin" "github.com/samcharles93/archie-core/internal/plugin"
) )
// serveGatewayListener is the serving tail of Run: the listener is // serveGatewayListener is the serving tail of Run: the listener is
// bound, the process declares itself serving, and the chat contract answers // bound, the process declares itself serving, and the chat contract answers
// until ctx ends. // until ctx ends.
func (b *server) serveGatewayListener(ctx context.Context, listener net.Listener, loopback bool, contract gateway.ChatContract, opts []grpc.ServerOption) error { func (b *server) serveGatewayListener(ctx context.Context, listener net.Listener, contract gateway.ChatContract, opts []grpc.ServerOption) error {
defer listener.Close() defer listener.Close()
b.log.Info("archie-gateway running", "addr", listener.Addr().String(), "token_required", !loopback) b.log.Info("archie-gateway running", "addr", listener.Addr().String(), "tls", b.id != nil)
// Boot is over and this listener is about to accept: the same fact // Boot is over and this listener is about to accept: the same fact
// systemd's READY=1 asserts, so the announcement goes out here rather than // systemd's READY=1 asserts, so the announcement goes out here rather than
// from a second notion of "started". Not later: Serve // from a second notion of "started". Not later: Serve
@@ -90,37 +91,25 @@ func (b *server) startGatewayRuntime(ctx context.Context, actor gateway.ChatTask
// rather than rejected as an oversized gRPC message. // rather than rejected as an oversized gRPC message.
const inboundMessageHeadroomBytes = 1 << 20 const inboundMessageHeadroomBytes = 1 << 20
// gatewayServerOpts applies the transport security boundary: a loopback // gatewayServerOpts serves listen with id's credentials; every caller
// listener is served insecure with no token; any non-loopback address // presents a service certificate.
// requires a Bearer [REDACTED], else the process fails closed. It mirrors
// stateStoreServerOpts, minus the State Store's task-scoped grants: the
// Gateway has one administrative token for all callers.
// //
// The receive limit is raised above gRPC's 4 MB default because an inbound // The receive limit is raised above gRPC's 4 MB default because an inbound
// request carries a channel attachment's bytes: the channel frontend // request carries a channel attachment's bytes: the channel frontend
// downloads the file, and this process -- which runs the turn -- holds no // downloads the file, and this process -- which runs the turn -- holds no
// platform credential to fetch it again. // platform credential to fetch it again.
func gatewayServerOpts(listen, token string, principals access.PrincipalSource) (opts []grpc.ServerOption, loopback bool, err error) { func gatewayServerOpts(listen string, id *rpctls.Identity, principals access.PrincipalSource) ([]grpc.ServerOption, error) {
callers := rpcidentity.Callers{Principals: principals} creds, err := id.ServerOption(listen, false)
recv := grpc.MaxRecvMsgSize(messaging.MaxInboundAttachmentBytes + inboundMessageHeadroomBytes)
loopback, err = gatewayrpc.TargetIsLoopback(listen)
if err != nil { if err != nil {
return nil, false, err return nil, fmt.Errorf("gateway: %w", err)
}
if loopback {
return []grpc.ServerOption{recv, grpc.ChainUnaryInterceptor(callers.Unary()), grpc.ChainStreamInterceptor(callers.Stream())}, true, nil
}
if token == "" {
return nil, false, fmt.Errorf(
"gateway listen address %q is non-loopback; non-loopback exposure requires a Bearer [REDACTED] (--token / [services.gateway].target_token) or TLS",
listen,
)
} }
callers := rpcidentity.Callers{Principals: principals}
return []grpc.ServerOption{ return []grpc.ServerOption{
recv, creds,
grpc.ChainUnaryInterceptor(gatewayrpc.UnaryServerInterceptor(token), callers.Unary()), grpc.MaxRecvMsgSize(messaging.MaxInboundAttachmentBytes + inboundMessageHeadroomBytes),
grpc.ChainStreamInterceptor(gatewayrpc.StreamServerInterceptor(token), callers.Stream()), grpc.ChainUnaryInterceptor(callers.Unary()),
}, false, nil grpc.ChainStreamInterceptor(callers.Stream()),
}, nil
} }
func serveGateway(ctx context.Context, listener net.Listener, contract gateway.ChatContract, sessions gateway.SessionStore, catalog gatewayrpc.Catalog, opts []grpc.ServerOption) error { func serveGateway(ctx context.Context, listener net.Listener, contract gateway.ChatContract, sessions gateway.SessionStore, catalog gatewayrpc.Catalog, opts []grpc.ServerOption) error {
+7 -12
View File
@@ -30,6 +30,7 @@ import (
"github.com/samcharles93/archie-core/internal/infrastructure/controlplanerpc" "github.com/samcharles93/archie-core/internal/infrastructure/controlplanerpc"
"github.com/samcharles93/archie-core/internal/infrastructure/modelcatalog" "github.com/samcharles93/archie-core/internal/infrastructure/modelcatalog"
"github.com/samcharles93/archie-core/internal/infrastructure/readiness" "github.com/samcharles93/archie-core/internal/infrastructure/readiness"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/infrastructure/secretengine" "github.com/samcharles93/archie-core/internal/infrastructure/secretengine"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
"github.com/samcharles93/archie-core/internal/infrastructure/taskactions" "github.com/samcharles93/archie-core/internal/infrastructure/taskactions"
@@ -51,9 +52,6 @@ type Options struct {
// Listen is the gRPC listen address. Empty falls back to // Listen is the gRPC listen address. Empty falls back to
// [services.gateway].listen. // [services.gateway].listen.
Listen string Listen string
// Token is the bearer token a non-loopback listener requires. Empty falls
// back to [services.gateway].target_token.
Token string
} }
// server is the Gateway process: the owner of conversations and of every // server is the Gateway process: the owner of conversations and of every
@@ -65,6 +63,7 @@ type server struct {
log *slog.Logger log *slog.Logger
secrets *secret.Registry secrets *secret.Registry
id *rpctls.Identity
engines *secretengine.Runner engines *secretengine.Runner
stateStore *staterpc.Client stateStore *staterpc.Client
controlPlane *controlplane.Client controlPlane *controlplane.Client
@@ -130,12 +129,7 @@ func Run(ctx context.Context, options Options) error {
if err != nil { if err != nil {
return err return err
} }
token := options.Token opts, err := gatewayServerOpts(listen, b.id, b.stateStore)
if token == "" {
token = b.cfg.Services.ResolvedToken(config.ServiceNameGateway, b.secrets.Getenv)
}
//nolint:contextcheck // grpc.StreamServerInterceptor has no context.Context parameter; gatewayrpc.StreamServerInterceptor derives its context from stream.Context() instead
opts, loopback, err := gatewayServerOpts(listen, token, b.stateStore)
if err != nil { if err != nil {
return err return err
} }
@@ -143,7 +137,7 @@ func Run(ctx context.Context, options Options) error {
if err != nil { if err != nil {
return fmt.Errorf("listen for gateway: %w", err) return fmt.Errorf("listen for gateway: %w", err)
} }
return b.serveGatewayListener(ctx, listener, loopback, contract, opts) return b.serveGatewayListener(ctx, listener, contract, opts)
} }
func (b *server) addCleanup(fn func()) { b.cleanups = append(b.cleanups, fn) } func (b *server) addCleanup(fn func()) { b.cleanups = append(b.cleanups, fn) }
@@ -171,7 +165,8 @@ func (b *server) loadConfig(cfgPath, overlayPath string) error {
b.log, b.logFeed = logs.Log, logs.Feed b.log, b.logFeed = logs.Log, logs.Feed
b.addCleanup(func() { _ = logs.Closer.Close() }) b.addCleanup(func() { _ = logs.Closer.Close() })
b.secrets = secret.NewRegistry() b.secrets = secret.NewRegistry()
return nil b.id, err = servicekit.Identity(b.cfg.TLS, presence.Gateway)
return err
} }
// openState claims the conversation store, dials the State Store, starts the // openState claims the conversation store, dials the State Store, starts the
@@ -183,7 +178,7 @@ func (b *server) openState(ctx context.Context) error {
if err := b.claimGatewayOwnership(ctx); err != nil { if err := b.claimGatewayOwnership(ctx); err != nil {
return err return err
} }
client, cleanup, err := servicekit.StateStoreClient(presence.Gateway, b.cfg.Services, b.secrets) client, cleanup, err := servicekit.StateStoreClient(presence.Gateway, b.cfg)
if err != nil { if err != nil {
return fmt.Errorf("state store client: %w", err) return fmt.Errorf("state store client: %w", err)
} }
+14 -29
View File
@@ -8,13 +8,17 @@ import (
"os" "os"
"github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/secret" "github.com/samcharles93/archie-core/internal/secret"
) )
// ResolvedConfig contains the resolved inputs needed by the Messaging Service. // ResolvedConfig contains the resolved inputs needed by the Messaging Service.
type ResolvedConfig struct { type ResolvedConfig struct {
Options Options Options Options
// Identity is this process's certificate; nil is plaintext, loopback only.
Identity *rpctls.Identity
TelegramToken string TelegramToken string
Telegram config.TelegramConfig Telegram config.TelegramConfig
// WorkDir and BotUser are not channel transport settings. They locate // WorkDir and BotUser are not channel transport settings. They locate
@@ -28,6 +32,7 @@ type projection struct {
gateway ServiceTarget gateway ServiceTarget
stateStore ServiceTarget stateStore ServiceTarget
messaging config.ServiceConnection messaging config.ServiceConnection
tls config.TLS
telegram config.TelegramConfig telegram config.TelegramConfig
workDir string workDir string
botUser string botUser string
@@ -51,13 +56,17 @@ func Resolve(o Options, log *slog.Logger) (ResolvedConfig, error) {
} }
} }
o = withDefaults(o) o = withDefaults(o)
o = withEnvTokens(o)
if err := o.validate(); err != nil { if err := o.validate(); err != nil {
return ResolvedConfig{}, err return ResolvedConfig{}, err
} }
id, err := rpctls.Load(proj.tls.CAFile, proj.tls.CertDir, presence.Messaging)
if err != nil {
return ResolvedConfig{}, err
}
return ResolvedConfig{ return ResolvedConfig{
Options: o, Options: o,
Identity: id,
Telegram: proj.telegram, Telegram: proj.telegram,
WorkDir: proj.workDir, WorkDir: proj.workDir,
BotUser: proj.botUser, BotUser: proj.botUser,
@@ -78,12 +87,10 @@ func readProjection(path, overlay string, log *slog.Logger) (projection, bool, e
func project(cfg config.Config) projection { func project(cfg config.Config) projection {
return projection{ return projection{
gateway: ServiceTarget{ gateway: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameGateway).Target},
Target: cfg.Services.Get(config.ServiceNameGateway).Target, stateStore: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameState).Target},
Token: cfg.Services.Get(config.ServiceNameGateway).TargetToken,
},
stateStore: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameState).Target, Token: cfg.Services.Get(config.ServiceNameState).TargetToken},
messaging: cfg.Services.Get(config.ServiceNameMessaging), messaging: cfg.Services.Get(config.ServiceNameMessaging),
tls: cfg.TLS,
telegram: cfg.Chat.Telegram, telegram: cfg.Chat.Telegram,
workDir: cfg.WorkDir, workDir: cfg.WorkDir,
botUser: cfg.BotUser, botUser: cfg.BotUser,
@@ -91,38 +98,16 @@ func project(cfg config.Config) projection {
} }
} }
func withEnvTokens(o Options) Options {
if o.Gateway.Token == "" {
o.Gateway.Token = os.Getenv("GATEWAY_TOKEN")
}
if o.StateStore.Token == "" {
o.StateStore.Token = os.Getenv("STATE_STORE_TOKEN")
}
if o.Token == "" {
o.Token = os.Getenv("MESSAGING_TOKEN")
}
return o
}
func merge(o Options, p projection) Options { func merge(o Options, p projection) Options {
if o.Gateway.Target == "" { if o.Gateway.Target == "" {
o.Gateway.Target = p.gateway.Target o.Gateway.Target = p.gateway.Target
} }
if o.Gateway.Token == "" {
o.Gateway.Token = p.gateway.Token
}
if o.StateStore.Target == "" { if o.StateStore.Target == "" {
o.StateStore.Target = p.stateStore.Target o.StateStore.Target = p.stateStore.Target
} }
if o.StateStore.Token == "" {
o.StateStore.Token = p.stateStore.Token
}
if o.Listen == "" { if o.Listen == "" {
o.Listen = p.messaging.Listen o.Listen = p.messaging.Listen
} }
if o.Token == "" {
o.Token = p.messaging.TargetToken
}
return o return o
} }
+2 -39
View File
@@ -4,9 +4,6 @@ package archiemessaging
import ( import (
"errors" "errors"
"fmt"
"net"
"strings"
"time" "time"
) )
@@ -18,11 +15,9 @@ const (
defaultShutdownTimeout = 5 * time.Second defaultShutdownTimeout = 5 * time.Second
) )
// ServiceTarget is one remote contract's endpoint and the service-to-service // ServiceTarget is one remote contract's endpoint.
// bearer token this process presents to it.
type ServiceTarget struct { type ServiceTarget struct {
Target string Target string
Token string
} }
// Options contains the process inputs for the Messaging Service. // Options contains the process inputs for the Messaging Service.
@@ -32,10 +27,8 @@ type Options struct {
Gateway ServiceTarget Gateway ServiceTarget
StateStore ServiceTarget StateStore ServiceTarget
// Listen is where MessagingService is served; Token is the bearer token // Listen is where MessagingService is served.
// its callers present on a non-loopback address.
Listen string Listen string
Token string
DependencyTimeout time.Duration DependencyTimeout time.Duration
ShutdownTimeout time.Duration ShutdownTimeout time.Duration
} }
@@ -60,35 +53,5 @@ func (o Options) validate() error {
if o.Gateway.Target == "" { if o.Gateway.Target == "" {
return errors.New("gateway target is required (pass -gateway-target or configure [services.gateway].target)") return errors.New("gateway target is required (pass -gateway-target or configure [services.gateway].target)")
} }
if err := validateServiceTarget("gateway", o.Gateway); err != nil {
return err
}
if o.StateStore.Target != "" {
if err := validateServiceTarget("state store", o.StateStore); err != nil {
return err
}
}
return validateServiceTarget("messaging listen", ServiceTarget{Target: o.Listen, Token: o.Token})
}
func validateServiceTarget(serviceName string, target ServiceTarget) error {
host, _, err := net.SplitHostPort(target.Target)
if err != nil {
host = target.Target
}
if isLoopback(host) {
return nil
}
if target.Token == "" {
return fmt.Errorf("%s target %q is non-loopback and requires a bearer token", serviceName, target.Target)
}
return nil return nil
} }
func isLoopback(host string) bool {
if host == "localhost" {
return true
}
ip := net.ParseIP(strings.Trim(host, "[]"))
return ip != nil && ip.IsLoopback()
}
+9 -8
View File
@@ -19,6 +19,7 @@ import (
"github.com/samcharles93/archie-core/internal/infrastructure/extension" "github.com/samcharles93/archie-core/internal/infrastructure/extension"
"github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc" "github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc"
"github.com/samcharles93/archie-core/internal/infrastructure/messagingrpc" "github.com/samcharles93/archie-core/internal/infrastructure/messagingrpc"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/infrastructure/secretengine" "github.com/samcharles93/archie-core/internal/infrastructure/secretengine"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
"github.com/samcharles93/archie-core/internal/logging" "github.com/samcharles93/archie-core/internal/logging"
@@ -36,7 +37,7 @@ func Run(ctx context.Context, o Options) error {
return fmt.Errorf("resolve messaging options: %w", err) return fmt.Errorf("resolve messaging options: %w", err)
} }
chat, closeGateway, err := gatewayrpc.Dial(cfg.Options.Gateway.Target, cfg.Options.Gateway.Token) chat, closeGateway, err := gatewayrpc.Dial(cfg.Options.Gateway.Target, cfg.Identity)
if err != nil { if err != nil {
return fmt.Errorf("dial gateway (%s): %w", cfg.Options.Gateway.Target, err) return fmt.Errorf("dial gateway (%s): %w", cfg.Options.Gateway.Target, err)
} }
@@ -59,7 +60,7 @@ func Run(ctx context.Context, o Options) error {
var updates *releaseupdate.Service var updates *releaseupdate.Service
var extensionChannels []*channelInstance var extensionChannels []*channelInstance
if cfg.Options.StateStore.Target != "" { if cfg.Options.StateStore.Target != "" {
stateStore, closeClient, dialErr := staterpc.Dial(cfg.Options.StateStore.Target, presence.Messaging, cfg.Options.StateStore.Token, staterpc.WaitForPeer) stateStore, closeClient, dialErr := staterpc.Dial(cfg.Options.StateStore.Target, presence.Messaging, cfg.Identity, staterpc.WaitForPeer)
if dialErr != nil { if dialErr != nil {
return fmt.Errorf("dial state store control plane (%s): %w", cfg.Options.StateStore.Target, dialErr) return fmt.Errorf("dial state store control plane (%s): %w", cfg.Options.StateStore.Target, dialErr)
} }
@@ -112,18 +113,18 @@ func Run(ctx context.Context, o Options) error {
if err != nil { if err != nil {
return err return err
} }
return serve(ctx, cfg.Options, srv, log, feed) return serve(ctx, cfg.Options.Listen, cfg.Identity, srv, log, feed)
} }
// serve runs srv and serves MessagingService over it until ctx is cancelled. // serve runs srv and serves MessagingService over it until ctx is cancelled.
func serve(ctx context.Context, o Options, srv *Service, log *slog.Logger, feed *logging.Feed) error { func serve(ctx context.Context, listen string, id *rpctls.Identity, srv *Service, log *slog.Logger, feed *logging.Feed) error {
opts, err := messagingrpc.ServerOptions(o.Listen, o.Token) opts, err := messagingrpc.ServerOptions(listen, id)
if err != nil { if err != nil {
return err return err
} }
listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", o.Listen) listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", listen)
if err != nil { if err != nil {
return fmt.Errorf("listen messaging (%s): %w", o.Listen, err) return fmt.Errorf("listen messaging (%s): %w", listen, err)
} }
server := grpc.NewServer(opts...) server := grpc.NewServer(opts...)
messagingrpc.RegisterServer(server, srv, feed) messagingrpc.RegisterServer(server, srv, feed)
@@ -133,7 +134,7 @@ func serve(ctx context.Context, o Options, srv *Service, log *slog.Logger, feed
} }
}() }()
defer server.GracefulStop() defer server.GracefulStop()
log.Info("messaging service listening", "listen", o.Listen) log.Info("messaging service listening", "listen", listen)
return srv.Start(ctx) return srv.Start(ctx)
} }
+15 -25
View File
@@ -8,13 +8,16 @@ import (
"os" "os"
"github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/webui" "github.com/samcharles93/archie-core/internal/webui"
) )
// Resolve returns the effective Options: flags first, then the config file's // Resolve returns the effective Options: flags first, then the config file's
// projection, then defaults, and validates the result. // projection, then defaults, and validates the result.
func Resolve(o Options, log *slog.Logger) (Options, error) { func Resolve(o Options, log *slog.Logger) (Options, error) {
var tls config.TLS
if o.Config != "" { if o.Config != "" {
cfg, found, err := readProjection(o.Config, o.Overlay, log) cfg, found, err := readProjection(o.Config, o.Overlay, log)
if err != nil { if err != nil {
@@ -22,10 +25,15 @@ func Resolve(o Options, log *slog.Logger) (Options, error) {
} }
if found { if found {
o = merge(o, cfg) o = merge(o, cfg)
tls = cfg.tls
} }
} }
o = withDefaults(o) o = withDefaults(o)
o = withEnvTokens(o) id, err := rpctls.Load(tls.CAFile, tls.CertDir, presence.UI)
if err != nil {
return Options{}, err
}
o.identity = id
token, err := resolveToken(o) token, err := resolveToken(o)
if err != nil { if err != nil {
return Options{}, err return Options{}, err
@@ -60,6 +68,7 @@ type projection struct {
state ServiceTarget state ServiceTarget
messaging ServiceTarget messaging ServiceTarget
harness ServiceTarget harness ServiceTarget
tls config.TLS
capture CaptureOptions capture CaptureOptions
} }
@@ -67,10 +76,11 @@ func project(cfg config.Config) projection {
return projection{ return projection{
listen: cfg.Web.Listen, listen: cfg.Web.Listen,
trustForwardedHeaders: cfg.Web.TrustForwardedHeaders, trustForwardedHeaders: cfg.Web.TrustForwardedHeaders,
gateway: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameGateway).Target, Token: cfg.Services.Get(config.ServiceNameGateway).TargetToken}, gateway: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameGateway).Target},
messaging: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameMessaging).Target, Token: cfg.Services.Get(config.ServiceNameMessaging).TargetToken}, messaging: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameMessaging).Target},
state: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameState).Target, Token: cfg.Services.Get(config.ServiceNameState).TargetToken}, state: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameState).Target},
harness: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameHarness).Target, Token: cfg.Services.Get(config.ServiceNameHarness).TargetToken}, harness: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameHarness).Target},
tls: cfg.TLS,
capture: CaptureOptions{ capture: CaptureOptions{
Retention: cfg.Capture.Retention.Std(), Retention: cfg.Capture.Retention.Std(),
MaxEvents: cfg.Capture.MaxEvents, MaxEvents: cfg.Capture.MaxEvents,
@@ -81,23 +91,6 @@ func project(cfg config.Config) projection {
} }
} }
// withEnvTokens fills service tokens left unset from the environment.
func withEnvTokens(o Options) Options {
if o.Gateway.Token == "" {
o.Gateway.Token = os.Getenv("GATEWAY_TOKEN")
}
if o.State.Token == "" {
o.State.Token = os.Getenv("STATE_STORE_TOKEN")
}
if o.Messaging.Token == "" {
o.Messaging.Token = os.Getenv("MESSAGING_TOKEN")
}
if o.Harness.Token == "" {
o.Harness.Token = os.Getenv("HARNESS_TOKEN")
}
return o
}
// merge fills only the fields the flags left empty. // merge fills only the fields the flags left empty.
func merge(o Options, p projection) Options { func merge(o Options, p projection) Options {
if o.Listen == "" { if o.Listen == "" {
@@ -187,8 +180,5 @@ func mergeTarget(o, p ServiceTarget) ServiceTarget {
if o.Target == "" { if o.Target == "" {
o.Target = p.Target o.Target = p.Target
} }
if o.Token == "" {
o.Token = p.Token
}
return o return o
} }
+1 -1
View File
@@ -23,7 +23,7 @@ func wireServiceLogs(opts Options, srv *webui.Server, tasks *staterpc.Client, ch
return chat.RecentLogs(ctx, presence.Daemon, q) return chat.RecentLogs(ctx, presence.Daemon, q)
}, },
} }
messaging, closeMessaging, dialErr := messagingrpc.Dial(opts.Messaging.Target, opts.Messaging.Token) messaging, closeMessaging, dialErr := messagingrpc.Dial(opts.Messaging.Target, opts.identity)
if dialErr != nil { if dialErr != nil {
return nil, dialErr return nil, dialErr
} }
+6 -4
View File
@@ -6,6 +6,7 @@ import (
"time" "time"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
) )
// Process defaults. Listen matches the dashboard's historical bind // Process defaults. Listen matches the dashboard's historical bind
@@ -18,12 +19,9 @@ const (
defaultShutdownTimeout = 5 * time.Second defaultShutdownTimeout = 5 * time.Second
) )
// ServiceTarget is one remote contract's endpoint and the service-to-service // ServiceTarget is one remote contract's endpoint.
// bearer token this process presents to it. It is an endpoint reference, not
// a resolved daemon credential: the UI authenticates each client itself.
type ServiceTarget struct { type ServiceTarget struct {
Target string Target string
Token string
} }
// Options is the UI-owned input DTO. It carries process-local settings and // Options is the UI-owned input DTO. It carries process-local settings and
@@ -83,6 +81,10 @@ type Options struct {
// ReadHeaderTimeout and ShutdownTimeout bound the HTTP lifecycle. // ReadHeaderTimeout and ShutdownTimeout bound the HTTP lifecycle.
ReadHeaderTimeout time.Duration ReadHeaderTimeout time.Duration
ShutdownTimeout time.Duration ShutdownTimeout time.Duration
// identity is this process's certificate, loaded by Resolve from
// [tls]; nil is plaintext, loopback only.
identity *rpctls.Identity
} }
// CaptureOptions carries the [capture] settings the capture receiver needs. // CaptureOptions carries the [capture] settings the capture receiver needs.
+3 -3
View File
@@ -58,12 +58,12 @@ func Run(ctx context.Context, options Options) error {
cleanup() cleanup()
} }
}() }()
tasks, closeState, err := staterpc.Dial(opts.State.Target, presence.UI, opts.State.Token) tasks, closeState, err := staterpc.Dial(opts.State.Target, presence.UI, opts.identity)
if err != nil { if err != nil {
return err return err
} }
cleanups = append(cleanups, closeState) cleanups = append(cleanups, closeState)
chat, closeGateway, err := gatewayrpc.Dial(opts.Gateway.Target, opts.Gateway.Token) chat, closeGateway, err := gatewayrpc.Dial(opts.Gateway.Target, opts.identity)
if err != nil { if err != nil {
return err return err
} }
@@ -146,7 +146,7 @@ func wireHarness(opts Options, srv *webui.Server, cleanups *[]func()) error {
if opts.Harness.Target == "" { if opts.Harness.Target == "" {
return nil return nil
} }
harness, closeHarness, err := harnessrpc.Dial(opts.Harness.Target, opts.Harness.Token) harness, closeHarness, err := harnessrpc.Dial(opts.Harness.Target, opts.identity)
if err != nil { if err != nil {
return err return err
} }
+16 -9
View File
@@ -2,20 +2,27 @@ package servicekit
import ( import (
"github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
"github.com/samcharles93/archie-core/internal/secret"
) )
// Identity loads service's certificate from [tls]; nil means plaintext,
// loopback only.
func Identity(cfg config.TLS, service string) (*rpctls.Identity, error) {
return rpctls.Load(cfg.CAFile, cfg.CertDir, service)
}
// StateStoreClient dials the State Store gRPC service as the caller service. // StateStoreClient dials the State Store gRPC service as the caller service.
// Token resolution is the daemon's // Calls wait for the State Store rather than fail while it is down, so boot
// (it owns the config keys and the secret registry); the dial itself and the // order between services does not matter.
// fail-closed non-loopback rule belong to the transport, so they come func StateStoreClient(caller string, cfg config.Config) (*staterpc.Client, func(), error) {
// from staterpc.Dial. Calls wait for the State Store rather than fail while it target, err := cfg.Services.RequireTarget(config.ServiceNameState)
// is down, so boot order between services does not matter.
func StateStoreClient(caller string, services config.Services, secrets *secret.Registry) (*staterpc.Client, func(), error) {
target, err := services.RequireTarget(config.ServiceNameState)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
return staterpc.Dial(target, caller, services.ResolvedToken(config.ServiceNameState, secrets.Getenv), staterpc.WaitForPeer) id, err := Identity(cfg.TLS, caller)
if err != nil {
return nil, nil, err
}
return staterpc.Dial(target, caller, id, staterpc.WaitForPeer)
} }
+20 -55
View File
@@ -42,13 +42,8 @@ import (
type Options struct { type Options struct {
Config string Config string
Overlay string Overlay string
// Listen is the gRPC listen address. Loopback binds (127.0.0.1, ::1, // Listen is the gRPC listen address. Without [tls] it must be loopback.
// localhost) serve insecure with no token; any non-loopback address
// requires a non-empty token (or TLS), else the process fails closed
Listen string Listen string
// Token is the bearer token the State Store server validates for a
// non-loopback listener. Empty falls back to [services.state].target_token.
Token string
// ReadyAddr, when non-empty, starts an HTTP readiness surface on the // ReadyAddr, when non-empty, starts an HTTP readiness surface on the
// address: GET /healthz (liveness) and GET /health/detailed (the // address: GET /healthz (liveness) and GET /health/detailed (the
// state_db probe). Empty disables it. // state_db probe). Empty disables it.
@@ -114,13 +109,12 @@ func Run(ctx context.Context, options Options) error { //nolint:cyclop // the co
} }
b.reportUnseededResources(skipped) b.reportUnseededResources(skipped)
token := b.instanceToken(options)
grants, err := b.runCredentials(ctx) grants, err := b.runCredentials(ctx)
if err != nil { if err != nil {
return err return err
} }
//nolint:contextcheck // grpc.StreamServerInterceptor has no context.Context parameter; TaskGrants.StreamInterceptor derives its context from stream.Context() instead //nolint:contextcheck // grpc.StreamServerInterceptor has no context.Context parameter; TaskGrants.StreamInterceptor derives its context from stream.Context() instead
opts, loopback, err := stateStoreServerOpts(listen, token, grants, b.callers()) opts, err := stateStoreServerOpts(listen, b.cfg.TLS, grants, b.callers())
if err != nil { if err != nil {
return err return err
} }
@@ -129,7 +123,7 @@ func Run(ctx context.Context, options Options) error { //nolint:cyclop // the co
return fmt.Errorf("listen for state store: %w", err) return fmt.Errorf("listen for state store: %w", err)
} }
defer listener.Close() defer listener.Close()
b.log.Info("archie-state-store running", "addr", listener.Addr().String(), "token_required", !loopback) b.log.Info("archie-state-store running", "addr", listener.Addr().String(), "tls", b.cfg.TLS.CertDir != "")
if err := b.startOptionalSurfaces(ctx, options); err != nil { if err := b.startOptionalSurfaces(ctx, options); err != nil {
return err return err
@@ -383,15 +377,6 @@ func (b *server) executionDeps(deps *staterpc.Deps) {
} }
} }
// instanceToken is the bearer token a non-loopback listener requires: the
// flag, else [services.state].target_token.
func (b *server) instanceToken(options Options) string {
if options.Token != "" {
return options.Token
}
return b.cfg.Services.ResolvedToken(config.ServiceNameState, b.secrets.Getenv)
}
// runCredentials builds the run-credential check over this store and drops // runCredentials builds the run-credential check over this store and drops
// the credentials that expired while nothing revoked them. // the credentials that expired while nothing revoked them.
func (b *server) runCredentials(ctx context.Context) (*staterpc.TaskGrants, error) { func (b *server) runCredentials(ctx context.Context) (*staterpc.TaskGrants, error) {
@@ -507,48 +492,28 @@ func (b *server) startOptionalSurfaces(ctx context.Context, options Options) err
return nil return nil
} }
// stateStoreServerOpts applies the transport security boundary: a // stateStoreServerOpts serves listen with this process's [tls] certificate. A client
// loopback listener is served insecure with no token; any non-loopback // certificate is optional here alone: a task container has none and is
// address requires a bearer token, else the process fails closed. It cannot // confined by its grant. Authentication runs before attribution.
// enforce TLS/mTLS (an operator decision), so it narrows the non-loopback
// path to token auth -- the same confinement philosophy as the gateway's
// loopback-only --listen rule.
// //
// The keepalive enforcement policy is not part of that boundary and is // The keepalive enforcement policy matches staterpc.Dial's clients, whose
// installed on both listeners: a loopback State Store serves the same dialers // idle watches grpc's default policy would answer with GOAWAY
// as a remote one (staterpc.Dial installs the client keepalive on every target, // too_many_pings. See staterpc.ServerKeepaliveOption.
// loopback included), and grpc's default policy would answer their idle func stateStoreServerOpts(listen string, tls config.TLS, grants *staterpc.TaskGrants, callers rpcidentity.Callers) ([]grpc.ServerOption, error) {
// watches with GOAWAY too_many_pings. See staterpc.ServerKeepaliveOption. id, err := servicekit.Identity(tls, presence.StateStore)
func stateStoreServerOpts(listen, token string, grants *staterpc.TaskGrants, callers rpcidentity.Callers) (opts []grpc.ServerOption, loopback bool, err error) {
loopback, err = staterpc.TargetIsLoopback(listen)
if err != nil { if err != nil {
return nil, false, err return nil, err
} }
keepalive := staterpc.ServerKeepaliveOption() creds, err := id.ServerOption(listen, true)
if loopback { if err != nil {
return []grpc.ServerOption{ return nil, fmt.Errorf("state store: %w", err)
keepalive,
grpc.ChainUnaryInterceptor(callers.Unary()),
grpc.ChainStreamInterceptor(callers.Stream()),
}, true, nil
} }
if token == "" {
return nil, false, fmt.Errorf(
"state store listen address %q is non-loopback; non-loopback exposure requires a bearer token (--token / [services.state].target_token) or TLS, mirroring the gateway's --listen confinement (docs/prds/state-store-contract.md §9)",
listen,
)
}
// grants.UnaryInterceptor/StreamInterceptor separate the daemon's own
// administrative token (full access) from a container's task-scoped
// grant (Update/Transition/InsertEvent on its own task ID only), rather
// than the single all-or-nothing token check this replaced.
return []grpc.ServerOption{ return []grpc.ServerOption{
keepalive, creds,
// Authentication runs before attribution: callers trusts the metadata staterpc.ServerKeepaliveOption(),
// only of a call the grants check admitted as the instance. grpc.ChainUnaryInterceptor(grants.UnaryInterceptor(), callers.Unary()),
grpc.ChainUnaryInterceptor(grants.UnaryInterceptor(token), callers.Unary()), grpc.ChainStreamInterceptor(grants.StreamInterceptor(), callers.Stream()),
grpc.ChainStreamInterceptor(grants.StreamInterceptor(token), callers.Stream()), }, nil
}, false, nil
} }
// newPackageProjections builds the family projectors an installed package's // newPackageProjections builds the family projectors an installed package's
+3 -1
View File
@@ -311,7 +311,9 @@ type ToolsConfig struct {
// Config is the daemon configuration. // Config is the daemon configuration.
type Config struct { type Config struct {
Services Services `toml:"services" yaml:"services"` Services Services `toml:"services" yaml:"services"`
WorkDir string `toml:"work_dir" yaml:"work_dir"` // TLS is the certificate every gRPC hop uses. Restart-required.
TLS TLS `toml:"tls" yaml:"tls"`
WorkDir string `toml:"work_dir" yaml:"work_dir"`
// SkillsDir is an optional directory of */SKILL.md skills. Its workflows // SkillsDir is an optional directory of */SKILL.md skills. Its workflows
// override built-ins, and Kits mount it read-only. // override built-ins, and Kits mount it read-only.
SkillsDir string `toml:"skills_dir" yaml:"skills_dir"` SkillsDir string `toml:"skills_dir" yaml:"skills_dir"`
+10 -17
View File
@@ -16,19 +16,6 @@ func (s Services) Get(name string) ServiceConnection {
return s[name] return s[name]
} }
// ResolvedToken returns name's target_token, or the service's registered
// environment variable.
func (s Services) ResolvedToken(name string, getenv func(string) string) string {
if token := s[name].TargetToken; token != "" {
return token
}
spec, ok := LookupService(name)
if !ok || spec.TokenEnv == "" {
return ""
}
return getenv(spec.TokenEnv)
}
// RequireTarget returns name's dial address, or an error naming the config // RequireTarget returns name's dial address, or an error naming the config
// key. // key.
func (s Services) RequireTarget(name string) (string, error) { func (s Services) RequireTarget(name string) (string, error) {
@@ -39,13 +26,19 @@ func (s Services) RequireTarget(name string) (string, error) {
return target, nil return target, nil
} }
// ServiceConnection is a service's gRPC addresses and client token. // ServiceConnection is a service's gRPC addresses.
// Restart-required. // Restart-required.
type ServiceConnection struct { type ServiceConnection struct {
Target string `toml:"target" yaml:"target"` Target string `toml:"target" yaml:"target"`
// Listen is the address the owning process binds. Always defaulted. // Listen is the address the owning process binds. Always defaulted.
Listen string `toml:"listen" yaml:"listen"` Listen string `toml:"listen" yaml:"listen"`
// TargetToken is the bearer token clients present to a non-loopback target. }
// Empty means none.
TargetToken string `toml:"target_token" yaml:"target_token"` // TLS locates the operator-issued certificates. A process named <service>
// loads cert_dir/<service>.pem and cert_dir/<service>-key.pem, whose DNS name
// is <service>, and trusts ca_file for every peer. Both empty means plaintext,
// which is confined to loopback addresses.
type TLS struct {
CAFile string `toml:"ca_file" yaml:"ca_file"`
CertDir string `toml:"cert_dir" yaml:"cert_dir"`
} }
+11 -16
View File
@@ -19,8 +19,7 @@ const (
ServiceBoth = "both" ServiceBoth = "both"
) )
// ServiceSpec declares one service: name, defaults and token environment // ServiceSpec declares one service: name and default addresses.
// variable.
type ServiceSpec struct { type ServiceSpec struct {
// Context is ServiceClient, ServiceServer, or ServiceBoth. // Context is ServiceClient, ServiceServer, or ServiceBoth.
Context string Context string
@@ -35,9 +34,6 @@ type ServiceSpec struct {
// reaches net.Listen as "any free port", silently moving the service // reaches net.Listen as "any free port", silently moving the service
// off the address its clients dial. // off the address its clients dial.
Listen string Listen string
// TokenEnv names the environment variable a client falls back to when
// [services.<name>].target_token is empty.
TokenEnv string
} }
// Registered service names. They live beside the registrations below so the // Registered service names. They live beside the registrations below so the
@@ -57,7 +53,7 @@ var (
// RegisterService declares one service, replacing any existing spec of that // RegisterService declares one service, replacing any existing spec of that
// name. It panics on an invalid registration. // name. It panics on an invalid registration.
func RegisterService(context, name, target, listen, tokenEnv string) { func RegisterService(context, name, target, listen string) {
switch context { switch context {
case ServiceClient, ServiceServer, ServiceBoth: case ServiceClient, ServiceServer, ServiceBoth:
default: default:
@@ -76,11 +72,10 @@ func RegisterService(context, name, target, listen, tokenEnv string) {
serviceRegistryMu.Lock() serviceRegistryMu.Lock()
defer serviceRegistryMu.Unlock() defer serviceRegistryMu.Unlock()
serviceRegistry[name] = ServiceSpec{ serviceRegistry[name] = ServiceSpec{
Context: context, Context: context,
Name: name, Name: name,
Target: target, Target: target,
Listen: listen, Listen: listen,
TokenEnv: tokenEnv,
} }
} }
@@ -112,13 +107,13 @@ func init() {
// The services this repository ships. Everything that differs // The services this repository ships. Everything that differs
// between them is an argument here, not a branch somewhere else: the // between them is an argument here, not a branch somewhere else: the
// gateway's target is defaulted while the State Store's is supplied by // gateway's target is defaulted while the State Store's is supplied by
// the operator, and each falls back to its own token variable. // the operator.
RegisterService(ServiceBoth, ServiceNameGateway, "127.0.0.1:8585", "127.0.0.1:8585", "GATEWAY_TOKEN") RegisterService(ServiceBoth, ServiceNameGateway, "127.0.0.1:8585", "127.0.0.1:8585")
RegisterService(ServiceClient, ServiceNameState, "", "127.0.0.1:9090", "STATE_STORE_TOKEN") RegisterService(ServiceClient, ServiceNameState, "", "127.0.0.1:9090")
RegisterService(ServiceBoth, ServiceNameMessaging, "127.0.0.1:8586", "127.0.0.1:8586", "MESSAGING_TOKEN") RegisterService(ServiceBoth, ServiceNameMessaging, "127.0.0.1:8586", "127.0.0.1:8586")
// The daemon hosts the harness session contract in this repository; the // The daemon hosts the harness session contract in this repository; the
// dashboard dials it, and its target is the operator's to supply, so a // dashboard dials it, and its target is the operator's to supply, so a
// deployment with no [services.harness].target leaves the setup terminal // deployment with no [services.harness].target leaves the setup terminal
// unwired rather than dialling a default nothing owns. // unwired rather than dialling a default nothing owns.
RegisterService(ServiceBoth, ServiceNameHarness, "", "127.0.0.1:8587", "HARNESS_TOKEN") RegisterService(ServiceBoth, ServiceNameHarness, "", "127.0.0.1:8587")
} }
+7 -4
View File
@@ -77,11 +77,11 @@ func (e NATSEndpoint) ContainerToken() string {
return e.Token return e.Token
} }
// StateStoreEndpoint is the State Store target and token passed to agent // StateStoreEndpoint is the State Store target and the CA its certificate
// containers. // chains to, both passed to agent containers.
type StateStoreEndpoint struct { type StateStoreEndpoint struct {
URL string URL string
Token string CA []byte
} }
type Daemon struct { type Daemon struct {
@@ -2263,6 +2263,9 @@ func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken string) []str
// daemon's own. // daemon's own.
if d.ConnectedStateStore.URL != "" { if d.ConnectedStateStore.URL != "" {
env = append(env, "STATE_STORE_URL="+d.ConnectedStateStore.URL) env = append(env, "STATE_STORE_URL="+d.ConnectedStateStore.URL)
if len(d.ConnectedStateStore.CA) > 0 {
env = append(env, "STATE_STORE_CA="+string(d.ConnectedStateStore.CA))
}
if stateStoreToken != "" { if stateStoreToken != "" {
env = append(env, "STATE_STORE_TOKEN="+stateStoreToken) env = append(env, "STATE_STORE_TOKEN="+stateStoreToken)
} }
+1 -1
View File
@@ -47,7 +47,7 @@ func (d *Daemon) runKitTask(ctx context.Context, task *workflow.Task, repo confi
defer revokeCredential() defer revokeCredential()
env, err := kitrun.WorkerEnv(kitrun.Endpoints{ env, err := kitrun.WorkerEnv(kitrun.Endpoints{
NATS: d.ConnectedNATS.URL, NATSToken: d.ConnectedNATS.ContainerToken(), NATS: d.ConnectedNATS.URL, NATSToken: d.ConnectedNATS.ContainerToken(),
StateStore: d.ConnectedStateStore.URL, StateStoreToken: credential, StateStore: d.ConnectedStateStore.URL, StateStoreToken: credential, StateStoreCA: d.ConnectedStateStore.CA,
}, os.Getuid(), os.Getgid()) }, os.Getuid(), os.Getgid())
if err != nil { if err != nil {
park("kit worker environment", err) park("kit worker environment", err)
@@ -39,6 +39,9 @@ type Config struct {
StateStoreURL string StateStoreURL string
StateStoreToken string StateStoreToken string
// StateStoreCA is the PEM CA the State Store's certificate chains to.
// Empty dials plaintext, which only a loopback target allows.
StateStoreCA []byte
} }
const ( const (
@@ -96,7 +99,7 @@ func Connect(ctx context.Context, config Config, log *slog.Logger) (*Transport,
conn.Close() conn.Close()
return nil, fmt.Errorf("state store target is required: archie-agent no longer supports the legacy NATS storerpc path (docs/prds/state-store-contract.md §12 step 4)") return nil, fmt.Errorf("state store target is required: archie-agent no longer supports the legacy NATS storerpc path (docs/prds/state-store-contract.md §12 step 4)")
} }
state, closeState, err := staterpc.Dial(config.StateStoreURL, "archie-agent", config.StateStoreToken) state, closeState, err := staterpc.DialTask(config.StateStoreURL, config.StateStoreCA, config.StateStoreToken)
if err != nil { if err != nil {
conn.Close() conn.Close()
return nil, fmt.Errorf("connect state store: %w", err) return nil, fmt.Errorf("connect state store: %w", err)
@@ -114,6 +114,7 @@ func (h *Host) Start(ctx context.Context, spec Spec, surface string, impl goplug
Cmd: cmd, Cmd: cmd,
AllowedProtocols: []goplugin.Protocol{goplugin.ProtocolGRPC}, AllowedProtocols: []goplugin.Protocol{goplugin.ProtocolGRPC},
SkipHostEnv: true, SkipHostEnv: true,
AutoMTLS: true,
Logger: h.log.Named(spec.Name), Logger: h.log.Named(spec.Name),
})} })}
rpc, err := p.client.Client() rpc, err := p.client.Client()
+6 -82
View File
@@ -2,94 +2,18 @@ package gatewayrpc
import ( import (
"context" "context"
"crypto/subtle"
"net"
"strings"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity" "github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity"
) )
// tokenMetadataKey is the gRPC metadata key the Gateway Bearer [REDACTED] // unaryPrincipal forwards the context's principal on every unary call.
// travels in. Metadata, never a URL, mirroring the State Store's func unaryPrincipal(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
// token lifecycle. return invoker(rpcidentity.Outgoing(ctx, "gateway-client"), method, req, reply, cc, opts...)
const tokenMetadataKey = "gateway-token"
// TargetIsLoopback reports whether addr's host is a loopback IP or
// "localhost".
func TargetIsLoopback(addr string) (bool, error) {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return false, err
}
if strings.EqualFold(host, "localhost") {
return true, nil
}
ip := net.ParseIP(host)
if ip == nil {
return false, nil
}
return ip.IsLoopback(), nil
} }
// validToken compares candidate against the configured token in constant // streamPrincipal forwards the context's principal on every stream.
// time so a timing side channel cannot leak how many bytes matched. func streamPrincipal(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
func validToken(token, candidate string) bool { return streamer(rpcidentity.Outgoing(ctx, "gateway-client"), desc, cc, method, opts...)
if token == "" || candidate == "" {
return false
}
return subtle.ConstantTimeCompare([]byte(token), []byte(candidate)) == 1
}
// UnaryServerInterceptor validates the Bearer [REDACTED] carried in each unary
// call's metadata, returning codes.Unauthenticated on a missing or wrong
// token. It is installed only for a non-loopback listener; a loopback
// listener serves with no interceptors.
func UnaryServerInterceptor(token string) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return nil, status.Error(codes.Unauthenticated, "missing gateway token")
}
values := md.Get(tokenMetadataKey)
if len(values) == 0 || !validToken(token, values[0]) {
return nil, status.Error(codes.Unauthenticated, "missing or invalid gateway token")
}
return handler(ctx, req)
}
}
// StreamServerInterceptor is the streaming counterpart of
// UnaryServerInterceptor: the chat Stream RPC is server-streaming, so its
// credential must be checked on the stream context as well.
func StreamServerInterceptor(token string) grpc.StreamServerInterceptor {
return func(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
md, _ := metadata.FromIncomingContext(stream.Context())
values := md.Get(tokenMetadataKey)
if len(values) == 0 || !validToken(token, values[0]) {
return status.Error(codes.Unauthenticated, "missing or invalid gateway token")
}
return handler(srv, stream)
}
}
// UnaryClientTokenInterceptor attaches token to every outgoing unary call's
// metadata under tokenMetadataKey.
func UnaryClientTokenInterceptor(token string) grpc.UnaryClientInterceptor {
return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
return invoker(metadata.AppendToOutgoingContext(rpcidentity.Outgoing(ctx, "gateway-client"), tokenMetadataKey, token), method, req, reply, cc, opts...)
}
}
// StreamClientTokenInterceptor attaches token to the outgoing stream's
// context, covering the chat Stream RPC that the unary interceptor cannot
// see.
func StreamClientTokenInterceptor(token string) grpc.StreamClientInterceptor {
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
return streamer(metadata.AppendToOutgoingContext(rpcidentity.Outgoing(ctx, "gateway-client"), tokenMetadataKey, token), desc, cc, method, opts...)
}
} }
+12 -20
View File
@@ -5,34 +5,26 @@ import (
"strings" "strings"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
) )
// Dial returns a Gateway chat client for target. A loopback target dials // Dial returns a Gateway chat client for target, authenticated by id. The
// insecure; any other target requires a token. The cleanup closes the // cleanup closes the connection.
// connection. func Dial(target string, id *rpctls.Identity, options ...grpc.DialOption) (*Client, func(), error) {
func Dial(target, token string, options ...grpc.DialOption) (*Client, func(), error) {
target = strings.TrimSpace(target) target = strings.TrimSpace(target)
if target == "" { if target == "" {
return nil, nil, fmt.Errorf("gateway target is required") return nil, nil, fmt.Errorf("gateway target is required")
} }
loopback, err := TargetIsLoopback(target) creds, err := id.DialOption(target, presence.Gateway)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("gateway target must be host:port: %w", err) return nil, nil, fmt.Errorf("gateway: %w", err)
} }
if !loopback && token == "" { opts := []grpc.DialOption{
return nil, nil, fmt.Errorf( creds,
"gateway target %q is non-loopback; non-loopback exposure requires a Bearer [REDACTED] ([services.gateway].target_token / GATEWAY_TOKEN) or TLS", grpc.WithUnaryInterceptor(unaryPrincipal),
target, grpc.WithStreamInterceptor(streamPrincipal),
)
}
opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
if token != "" {
opts = append(
opts,
grpc.WithUnaryInterceptor(UnaryClientTokenInterceptor(token)),
grpc.WithStreamInterceptor(StreamClientTokenInterceptor(token)),
)
} }
conn, err := grpc.NewClient(target, append(opts, options...)...) conn, err := grpc.NewClient(target, append(opts, options...)...)
if err != nil { if err != nil {
@@ -6,7 +6,6 @@ package harnessrpc
import ( import (
"context" "context"
"crypto/subtle"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@@ -14,43 +13,33 @@ import (
"sync" "sync"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
pb "github.com/samcharles93/archie-core/internal/contracts/harness/v1" pb "github.com/samcharles93/archie-core/internal/contracts/harness/v1"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
) )
// tokenMetadataKey is the gRPC metadata key the harness service token travels
// in, mirroring the Gateway's and State Store's token keys.
const tokenMetadataKey = "harness-token"
// Client opens sessions against a daemon's HarnessService. // Client opens sessions against a daemon's HarnessService.
type Client struct { type Client struct {
svc pb.HarnessServiceClient svc pb.HarnessServiceClient
} }
// Dial returns a harness client for target. The token is required for every // Dial returns a harness client for target. A session is a shell in a
// target, loopback included: the daemon's listener enforces it because a // container that holds an org's credential, so it needs a service
// session is a shell in a container that holds an org's credential. The // certificate even on loopback. The cleanup closes the connection.
// cleanup closes the connection. func Dial(target string, id *rpctls.Identity, options ...grpc.DialOption) (*Client, func(), error) {
func Dial(target, token string, options ...grpc.DialOption) (*Client, func(), error) {
target = strings.TrimSpace(target) target = strings.TrimSpace(target)
if target == "" { if target == "" {
return nil, nil, errors.New("harness target is required") return nil, nil, errors.New("harness target is required")
} }
if token == "" { if id == nil {
return nil, nil, fmt.Errorf( return nil, nil, errors.New("harness sessions need [tls] ca_file and cert_dir")
"harness target %q reads a setup session, so it needs a bearer token ([services.harness].target_token / HARNESS_TOKEN)",
target,
)
} }
opts := []grpc.DialOption{ creds, err := id.DialOption(target, presence.Daemon)
grpc.WithTransportCredentials(insecure.NewCredentials()), if err != nil {
grpc.WithStreamInterceptor(streamClientToken(token)), return nil, nil, fmt.Errorf("harness: %w", err)
} }
conn, err := grpc.NewClient(target, append(opts, options...)...) conn, err := grpc.NewClient(target, append([]grpc.DialOption{creds}, options...)...)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("create harness client: %w", err) return nil, nil, fmt.Errorf("create harness client: %w", err)
} }
@@ -169,40 +158,3 @@ func (s *session) Close() error {
s.cancel() s.cancel()
return nil return nil
} }
// validToken compares candidate against the configured token in constant
// time.
func validToken(token, candidate string) bool {
if token == "" || candidate == "" {
return false
}
return subtle.ConstantTimeCompare([]byte(token), []byte(candidate)) == 1
}
// StreamServerInterceptor validates the harness token on a stream. The
// session RPC is bidi, so this is the interceptor that matters. Unlike the
// State Store, a loopback listener enforces it too: a session is a shell in a
// container that holds an org's credential.
func StreamServerInterceptor(token string) grpc.StreamServerInterceptor {
return func(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
if !validContextToken(stream.Context(), token) {
return status.Error(codes.Unauthenticated, "missing or invalid harness token")
}
return handler(srv, stream)
}
}
func validContextToken(ctx context.Context, token string) bool {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return false
}
values := md.Get(tokenMetadataKey)
return len(values) > 0 && validToken(token, values[0])
}
func streamClientToken(token string) grpc.StreamClientInterceptor {
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
return streamer(metadata.AppendToOutgoingContext(ctx, tokenMetadataKey, token), desc, cc, method, opts...)
}
}
+4
View File
@@ -17,6 +17,7 @@ type Endpoints struct {
NATSToken string NATSToken string
StateStore string StateStore string
StateStoreToken string StateStoreToken string
StateStoreCA []byte
} }
// WorkerEnv is the worker's entire environment in a Kit container: NATS and // WorkerEnv is the worker's entire environment in a Kit container: NATS and
@@ -36,6 +37,9 @@ func WorkerEnv(e Endpoints, uid, gid int) ([]string, error) {
if e.StateStoreToken != "" { if e.StateStoreToken != "" {
env = append(env, "STATE_STORE_TOKEN="+e.StateStoreToken) env = append(env, "STATE_STORE_TOKEN="+e.StateStoreToken)
} }
if len(e.StateStoreCA) > 0 {
env = append(env, "STATE_STORE_CA="+string(e.StateStoreCA))
}
} }
return append(env, "WORKTREE_UID="+strconv.Itoa(uid), "WORKTREE_GID="+strconv.Itoa(gid)), nil return append(env, "WORKTREE_UID="+strconv.Itoa(uid), "WORKTREE_GID="+strconv.Itoa(gid)), nil
} }
@@ -5,24 +5,20 @@ package messagingrpc
import ( import (
"context" "context"
"crypto/subtle"
"errors" "errors"
"fmt" "fmt"
"strings" "strings"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
pb "github.com/samcharles93/archie-core/internal/contracts/messaging/v1" pb "github.com/samcharles93/archie-core/internal/contracts/messaging/v1"
"github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc" "github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/logging" "github.com/samcharles93/archie-core/internal/logging"
) )
const tokenMetadataKey = "messaging-token"
// ErrUnavailable marks a delivery the channel could not make now: the channel // ErrUnavailable marks a delivery the channel could not make now: the channel
// is unknown, stopped or cannot send. The caller may retry later. // is unknown, stopped or cannot send. The caller may retry later.
var ErrUnavailable = errors.New("messaging: channel unavailable") var ErrUnavailable = errors.New("messaging: channel unavailable")
@@ -58,52 +54,26 @@ func (s *server) Deliver(ctx context.Context, req *pb.DeliverRequest) (*pb.Deliv
} }
} }
// ServerOptions serves a loopback listen address without credentials and // ServerOptions serves listen with id's credentials.
// requires token on any other, failing closed when it is empty. func ServerOptions(listen string, id *rpctls.Identity) ([]grpc.ServerOption, error) {
func ServerOptions(listen, token string) ([]grpc.ServerOption, error) { creds, err := id.ServerOption(listen, false)
loopback, err := gatewayrpc.TargetIsLoopback(listen)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("messaging: %w", err)
}
if loopback {
return nil, nil
}
if token == "" {
return nil, fmt.Errorf("messaging listen address %q is non-loopback and requires a bearer token", listen)
}
return []grpc.ServerOption{grpc.ChainUnaryInterceptor(requireToken(token))}, nil
}
func requireToken(token string) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
values := metadata.ValueFromIncomingContext(ctx, tokenMetadataKey)
if len(values) == 0 || subtle.ConstantTimeCompare([]byte(token), []byte(values[0])) != 1 {
return nil, status.Error(codes.Unauthenticated, "missing or invalid messaging token")
}
return handler(ctx, req)
} }
return []grpc.ServerOption{creds}, nil
} }
// Client delivers through a dialled MessagingService. // Client delivers through a dialled MessagingService.
type Client struct{ rpc pb.MessagingServiceClient } type Client struct{ rpc pb.MessagingServiceClient }
// Dial returns a client for target. A non-loopback target requires token. // Dial returns a client for target, authenticated by id.
func Dial(target, token string) (*Client, func(), error) { func Dial(target string, id *rpctls.Identity) (*Client, func(), error) {
target = strings.TrimSpace(target) target = strings.TrimSpace(target)
loopback, err := gatewayrpc.TargetIsLoopback(target) creds, err := id.DialOption(target, presence.Messaging)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("messaging target must be host:port: %w", err) return nil, nil, fmt.Errorf("messaging: %w", err)
} }
if !loopback && token == "" { conn, err := grpc.NewClient(target, creds)
return nil, nil, fmt.Errorf("messaging target %q is non-loopback and requires a bearer token ([services.messaging].target_token / MESSAGING_TOKEN)", target)
}
opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
if token != "" {
opts = append(opts, grpc.WithUnaryInterceptor(func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
return invoker(metadata.AppendToOutgoingContext(ctx, tokenMetadataKey, token), method, req, reply, cc, opts...)
}))
}
conn, err := grpc.NewClient(target, opts...)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("create messaging client: %w", err) return nil, nil, fmt.Errorf("create messaging client: %w", err)
} }
@@ -1,36 +0,0 @@
package messagingrpc
import (
"context"
"testing"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
)
func TestRequireToken(t *testing.T) {
ok := func(context.Context, any) (any, error) { return "served", nil }
for _, tc := range []struct {
name string
md metadata.MD
want codes.Code
}{
{"valid", metadata.Pairs(tokenMetadataKey, "secret"), codes.OK},
{"wrong", metadata.Pairs(tokenMetadataKey, "secrex"), codes.Unauthenticated},
{"missing", metadata.MD{}, codes.Unauthenticated},
{"other service's key", metadata.Pairs("gateway-token", "secret"), codes.Unauthenticated},
} {
t.Run(tc.name, func(t *testing.T) {
ctx := metadata.NewIncomingContext(context.Background(), tc.md)
_, err := requireToken("secret")(ctx, nil, &grpc.UnaryServerInfo{}, ok)
if got := status.Code(err); got != tc.want {
t.Fatalf("code = %v, want %v", got, tc.want)
}
})
}
if _, err := ServerOptions("0.0.0.0:8586", ""); err == nil {
t.Fatal("non-loopback listen with no token was accepted")
}
}
@@ -13,6 +13,7 @@ import (
"github.com/samcharles93/archie-core/internal/domain/identity" "github.com/samcharles93/archie-core/internal/domain/identity"
"github.com/samcharles93/archie-core/internal/domain/org" "github.com/samcharles93/archie-core/internal/domain/org"
"github.com/samcharles93/archie-core/internal/domain/presence" "github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
) )
// CallerKey names the service making the call. // CallerKey names the service making the call.
@@ -34,10 +35,10 @@ func Outgoing(ctx context.Context, caller string) context.Context {
return metadata.NewOutgoingContext(ctx, md) return metadata.NewOutgoingContext(ctx, md)
} }
// Callers resolves who a call is made by. It runs after authentication, // Callers resolves who a call is made by. It runs after authentication:
// which is what makes the metadata trustworthy: only holders of the instance // the caller is the verified certificate's service, else (a plaintext
// credential reach it with their metadata intact (a task grant's is // loopback peer) the metadata's claim, and a task grant's metadata is
// stripped). A call naming a principal acts in, and reads only, that principal's // stripped before it gets here. A call naming a principal acts in, and reads only, that principal's
// org; any other acts in the default org. // org; any other acts in the default org.
type Callers struct { type Callers struct {
Principals access.PrincipalSource Principals access.PrincipalSource
@@ -45,7 +46,10 @@ type Callers struct {
func (c Callers) resolve(ctx context.Context) (context.Context, error) { func (c Callers) resolve(ctx context.Context) (context.Context, error) {
md, _ := metadata.FromIncomingContext(ctx) md, _ := metadata.FromIncomingContext(ctx)
caller := first(md, CallerKey) caller := rpctls.PeerService(ctx)
if caller == "" {
caller = first(md, CallerKey)
}
if caller != "" { if caller != "" {
ctx = access.WithActor(access.WithSource(ctx, caller), caller) ctx = access.WithActor(access.WithSource(ctx, caller), caller)
} }
+204
View File
@@ -0,0 +1,204 @@
// Package rpctls builds the transport credentials every gRPC hop uses from
// certificate files the operator's PKI issues. With no identity configured a
// process speaks plaintext, and only over loopback.
package rpctls
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"strings"
"sync"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/peer"
)
// Identity is one process's certificate and the CA it trusts. A nil Identity
// is the plaintext loopback mode.
type Identity struct {
pool *x509.CertPool
caPEM []byte
cert *certFile
}
// Load reads the CA at caFile and service's key pair from
// certDir/<service>.pem and certDir/<service>-key.pem. Both paths empty means
// no TLS and returns nil.
func Load(caFile, certDir, service string) (*Identity, error) {
if caFile == "" && certDir == "" {
return nil, nil
}
if caFile == "" || certDir == "" {
return nil, errors.New("tls: ca_file and cert_dir are both required")
}
caPEM, err := os.ReadFile(caFile)
if err != nil {
return nil, fmt.Errorf("tls: read CA: %w", err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(caPEM) {
return nil, fmt.Errorf("tls: %s holds no PEM certificate", caFile)
}
cert := &certFile{
certPath: filepath.Join(certDir, service+".pem"),
keyPath: filepath.Join(certDir, service+"-key.pem"),
}
if _, err := cert.get(); err != nil {
return nil, err
}
return &Identity{pool: pool, caPEM: caPEM, cert: cert}, nil
}
// CAPEM is the CA certificate a container receives to verify the State
// Store. Nil in plaintext mode.
func (i *Identity) CAPEM() []byte {
if i == nil {
return nil
}
return i.caPEM
}
// ServerOption returns the credentials for a listener on listen. A listener
// that accepts calls without a client certificate (the State Store's task
// grants) passes optionalClient; every other listener requires one.
func (i *Identity) ServerOption(listen string, optionalClient bool) (grpc.ServerOption, error) {
if i == nil {
if err := requireLoopback(listen); err != nil {
return nil, err
}
return grpc.Creds(insecure.NewCredentials()), nil
}
auth := tls.RequireAndVerifyClientCert
if optionalClient {
auth = tls.VerifyClientCertIfGiven
}
return grpc.Creds(credentials.NewTLS(&tls.Config{
MinVersion: tls.VersionTLS13,
GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return i.cert.get() },
ClientCAs: i.pool,
ClientAuth: auth,
})), nil
}
// DialOption returns the credentials for dialling server (its certificate
// name) at target. The name, not target's host, is verified, so a service is
// reachable through any address, relay or proxy.
func (i *Identity) DialOption(target, server string) (grpc.DialOption, error) {
if i == nil {
if err := requireLoopback(target); err != nil {
return nil, err
}
return grpc.WithTransportCredentials(insecure.NewCredentials()), nil
}
return grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
MinVersion: tls.VersionTLS13,
RootCAs: i.pool,
ServerName: server,
GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { return i.cert.get() },
})), nil
}
// CAOnlyDialOption dials server trusting caPEM and presenting no
// certificate: a container's credentials are its task grant alone. Empty
// caPEM is plaintext, loopback only.
func CAOnlyDialOption(caPEM []byte, target, server string) (grpc.DialOption, error) {
if len(caPEM) == 0 {
var none *Identity
return none.DialOption(target, server)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(caPEM) {
return nil, errors.New("tls: CA holds no PEM certificate")
}
return grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
MinVersion: tls.VersionTLS13,
RootCAs: pool,
ServerName: server,
})), nil
}
// PeerService is the service named by the caller's verified certificate, or
// "" when it presented none.
func PeerService(ctx context.Context) string {
p, ok := peer.FromContext(ctx)
if !ok {
return ""
}
info, ok := p.AuthInfo.(credentials.TLSInfo)
if !ok || len(info.State.VerifiedChains) == 0 || len(info.State.VerifiedChains[0]) == 0 {
return ""
}
if names := info.State.VerifiedChains[0][0].DNSNames; len(names) > 0 {
return names[0]
}
return ""
}
// Trusted reports whether the caller is a service: it holds a verified
// certificate, or reached a plaintext listener, which is loopback only.
func Trusted(ctx context.Context) bool {
p, ok := peer.FromContext(ctx)
if !ok {
return false
}
if _, isTLS := p.AuthInfo.(credentials.TLSInfo); !isTLS {
return true
}
return PeerService(ctx) != ""
}
func requireLoopback(addr string) error {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return fmt.Errorf("address %q must be host:port: %w", addr, err)
}
if strings.EqualFold(host, "localhost") {
return nil
}
if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() {
return nil
}
return fmt.Errorf("%q is not loopback; configure [tls] ca_file and cert_dir to serve or dial it", addr)
}
// certFile reloads a key pair when its certificate file changes, so the
// operator's renewal tooling needs no restart.
type certFile struct {
certPath, keyPath string
mu sync.Mutex
modified time.Time
cert *tls.Certificate
}
func (c *certFile) get() (*tls.Certificate, error) {
info, err := os.Stat(c.certPath)
if err != nil {
return nil, fmt.Errorf("tls: %w", err)
}
c.mu.Lock()
defer c.mu.Unlock()
if c.cert != nil && info.ModTime().Equal(c.modified) {
return c.cert, nil
}
cert, err := tls.LoadX509KeyPair(c.certPath, c.keyPath)
if err != nil {
if c.cert != nil {
// A renewal caught between writing the two files keeps the
// previous pair until both match.
return c.cert, nil
}
return nil, fmt.Errorf("tls: load %s: %w", c.certPath, err)
}
c.cert, c.modified = &cert, info.ModTime()
return c.cert, nil
}
@@ -0,0 +1,171 @@
package rpctls
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"math/big"
"net"
"os"
"path/filepath"
"testing"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/health"
healthpb "google.golang.org/grpc/health/grpc_health_v1"
"google.golang.org/grpc/test/bufconn"
)
// TestHandshake pins the transport boundary: who may connect, and the
// identity the server sees.
func TestHandshake(t *testing.T) {
dir := t.TempDir()
ca := newCA(t, dir, "ca")
foreign := newCA(t, dir, "foreign")
ca.issue(t, dir, "archie-state-store")
ca.issue(t, dir, "archied")
foreignDir := filepath.Join(dir, "foreign-certs")
foreign.issue(t, foreignDir, "archied")
server := load(t, ca.path, dir, "archie-state-store")
daemon := load(t, ca.path, dir, "archied")
impostor := load(t, ca.path, foreignDir, "archied")
tests := []struct {
name string
optionalClient bool
dial func(target string) (grpc.DialOption, error)
wantService string
wantTrusted bool
wantErr bool
}{
{"service certificate names the caller", false, func(tg string) (grpc.DialOption, error) { return daemon.DialOption(tg, "archie-state-store") }, "archied", true, false},
{"foreign CA is refused", false, func(tg string) (grpc.DialOption, error) { return impostor.DialOption(tg, "archie-state-store") }, "", false, true},
{"no certificate where one is required", false, func(tg string) (grpc.DialOption, error) { return CAOnlyDialOption(ca.pem, tg, "archie-state-store") }, "", false, true},
{"no certificate where it is optional is untrusted", true, func(tg string) (grpc.DialOption, error) { return CAOnlyDialOption(ca.pem, tg, "archie-state-store") }, "", false, false},
{"wrong server name is refused", false, func(tg string) (grpc.DialOption, error) { return daemon.DialOption(tg, "archie-gateway") }, "", false, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
creds, err := server.ServerOption("127.0.0.1:0", tt.optionalClient)
if err != nil {
t.Fatal(err)
}
var gotService string
var gotTrusted bool
srv := grpc.NewServer(creds, grpc.UnaryInterceptor(func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, h grpc.UnaryHandler) (any, error) {
gotService, gotTrusted = PeerService(ctx), Trusted(ctx)
return h(ctx, req)
}))
healthpb.RegisterHealthServer(srv, health.NewServer())
lis := bufconn.Listen(1 << 16)
go func() { _ = srv.Serve(lis) }()
t.Cleanup(srv.Stop)
dialCreds, err := tt.dial("127.0.0.1:1")
if err != nil {
t.Fatal(err)
}
conn, err := grpc.NewClient("passthrough:///bufnet", dialCreds,
grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return lis.DialContext(ctx) }))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = conn.Close() })
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
_, err = healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{})
if (err != nil) != tt.wantErr {
t.Fatalf("err = %v, wantErr %v", err, tt.wantErr)
}
if gotService != tt.wantService || gotTrusted != tt.wantTrusted {
t.Fatalf("service %q trusted %v, want %q %v", gotService, gotTrusted, tt.wantService, tt.wantTrusted)
}
})
}
var plaintext *Identity
if _, err := plaintext.DialOption("10.0.0.1:9090", "archie-state-store"); err == nil {
t.Fatal("plaintext dial to a non-loopback target was allowed")
}
if _, err := plaintext.ServerOption("0.0.0.0:9090", false); err == nil {
t.Fatal("plaintext listener on a non-loopback address was allowed")
}
}
type testCA struct {
cert *x509.Certificate
key *ecdsa.PrivateKey
pem []byte
path string
}
func newCA(t *testing.T, dir, name string) testCA {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: name},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour),
IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign,
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
cert, _ := x509.ParseCertificate(der)
p := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
path := filepath.Join(dir, name+"-ca.pem")
write(t, path, p)
return testCA{cert: cert, key: key, pem: p, path: path}
}
func (c testCA) issue(t *testing.T, dir, service string) {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: service}, DNSNames: []string{service},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, c.cert, &key.PublicKey, c.key)
if err != nil {
t.Fatal(err)
}
keyDER, err := x509.MarshalECPrivateKey(key)
if err != nil {
t.Fatal(err)
}
write(t, filepath.Join(dir, service+".pem"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
write(t, filepath.Join(dir, service+"-key.pem"), pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}))
}
func load(t *testing.T, caFile, certDir, service string) *Identity {
t.Helper()
id, err := Load(caFile, certDir, service)
if err != nil {
t.Fatal(err)
}
return id
}
func write(t *testing.T, path string, data []byte) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, data, 0o600); err != nil {
t.Fatal(err)
}
}
+5 -5
View File
@@ -9,7 +9,7 @@ import (
"github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity" "github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity"
) )
// tokenMetadataKey is the gRPC metadata key the bearer token travels in. // tokenMetadataKey is the gRPC metadata key a task grant travels in.
// Metadata, never a URL. // Metadata, never a URL.
const tokenMetadataKey = "state-store-token" const tokenMetadataKey = "state-store-token"
@@ -21,16 +21,16 @@ func outgoing(ctx context.Context, caller, token string) context.Context {
return ctx return ctx
} }
// UnaryClientCredentialInterceptor attaches caller and token to every // UnaryClientCredentialInterceptor attaches caller and any task grant to
// outgoing call. // every outgoing call.
func UnaryClientCredentialInterceptor(caller, token string) grpc.UnaryClientInterceptor { func UnaryClientCredentialInterceptor(caller, token string) grpc.UnaryClientInterceptor {
return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
return invoker(outgoing(ctx, caller, token), method, req, reply, cc, opts...) return invoker(outgoing(ctx, caller, token), method, req, reply, cc, opts...)
} }
} }
// StreamClientCredentialInterceptor attaches caller and token to every // StreamClientCredentialInterceptor attaches caller and any task grant to
// outgoing stream. // every outgoing stream.
func StreamClientCredentialInterceptor(caller, token string) grpc.StreamClientInterceptor { func StreamClientCredentialInterceptor(caller, token string) grpc.StreamClientInterceptor {
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
return streamer(outgoing(ctx, caller, token), desc, cc, method, opts...) return streamer(outgoing(ctx, caller, token), desc, cc, method, opts...)
@@ -7,7 +7,9 @@ import (
"time" "time"
"google.golang.org/grpc" "google.golang.org/grpc"
grpccreds "google.golang.org/grpc/credentials"
"google.golang.org/grpc/metadata" "google.golang.org/grpc/metadata"
"google.golang.org/grpc/peer"
pb "github.com/samcharles93/archie-core/internal/contracts/state/v1" pb "github.com/samcharles93/archie-core/internal/contracts/state/v1"
"github.com/samcharles93/archie-core/internal/domain/access" "github.com/samcharles93/archie-core/internal/domain/access"
@@ -45,10 +47,10 @@ func (p principalsByID) PrincipalFor(_ context.Context, id identity.IdentityID)
return access.Principal{IdentityID: id, Org: p[id]}, nil return access.Principal{IdentityID: id, Org: p[id]}, nil
} }
// TestCallerAttribution pins who a call acts as: the instance credential's // TestCallerAttribution pins who a call acts as: a trusted service's
// metadata is trusted, a task grant's is not. // metadata is trusted, a task grant's is not, and a caller with neither a
// certificate nor a grant is refused.
func TestCallerAttribution(t *testing.T) { func TestCallerAttribution(t *testing.T) {
const admin = "admin-token"
taskToken := strings.Repeat("ab", 32) taskToken := strings.Repeat("ab", 32)
grants := &TaskGrants{Store: credentials{}} grants := &TaskGrants{Store: credentials{}}
if err := grants.register(context.Background(), 7, taskToken, time.Hour); err != nil { if err := grants.register(context.Background(), 7, taskToken, time.Hour); err != nil {
@@ -56,20 +58,27 @@ func TestCallerAttribution(t *testing.T) {
} }
callers := rpcidentity.Callers{Principals: principalsByID{"alice": "acme"}} callers := rpcidentity.Callers{Principals: principalsByID{"alice": "acme"}}
update := &grpc.UnaryServerInfo{FullMethod: pb.StateStoreService_Update_FullMethodName} update := &grpc.UnaryServerInfo{FullMethod: pb.StateStoreService_Update_FullMethodName}
// A plaintext peer reached a loopback listener; a TLS peer without a
// verified chain presented no client certificate.
plaintext := &peer.Peer{}
noCert := &peer.Peer{AuthInfo: grpccreds.TLSInfo{}}
tests := []struct { tests := []struct {
name string name string
peer *peer.Peer
md []string md []string
wantActor string wantActor string
wantOrg org.OrgID wantOrg org.OrgID
wantErr bool
}{ }{
{"service alone acts in the default org", []string{tokenMetadataKey, admin, rpcidentity.CallerKey, "archied"}, "archied", org.DefaultOrgID}, {"service alone acts in the default org", plaintext, []string{rpcidentity.CallerKey, "archied"}, "archied", org.DefaultOrgID, false},
{"forwarded principal sets actor and org", []string{tokenMetadataKey, admin, rpcidentity.CallerKey, "archie-ui", rpcidentity.PrincipalKey, "alice"}, "alice", "acme"}, {"forwarded principal sets actor and org", plaintext, []string{rpcidentity.CallerKey, "archie-ui", rpcidentity.PrincipalKey, "alice"}, "alice", "acme", false},
{"task grant cannot claim a principal", []string{tokenMetadataKey, taskToken, rpcidentity.CallerKey, "archie-ui", rpcidentity.PrincipalKey, "alice"}, "task/7", org.DefaultOrgID}, {"task grant cannot claim a principal", noCert, []string{tokenMetadataKey, taskToken, rpcidentity.CallerKey, "archie-ui", rpcidentity.PrincipalKey, "alice"}, "task/7", org.DefaultOrgID, false},
{"no certificate and no grant is refused", noCert, []string{rpcidentity.CallerKey, "archied"}, "", "", true},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs(tt.md...)) ctx := peer.NewContext(metadata.NewIncomingContext(context.Background(), metadata.Pairs(tt.md...)), tt.peer)
var gotActor string var gotActor string
var gotOrg org.OrgID var gotOrg org.OrgID
handler := func(ctx context.Context, _ any) (any, error) { handler := func(ctx context.Context, _ any) (any, error) {
@@ -80,8 +89,9 @@ func TestCallerAttribution(t *testing.T) {
return callers.Unary()(ctx, req, update, handler) return callers.Unary()(ctx, req, update, handler)
} }
req := &pb.UpdateRequest{Task: &pb.Task{Id: 7}} req := &pb.UpdateRequest{Task: &pb.Task{Id: 7}}
if _, err := grants.UnaryInterceptor(admin)(ctx, req, update, chained); err != nil { _, err := grants.UnaryInterceptor()(ctx, req, update, chained)
t.Fatal(err) if (err != nil) != tt.wantErr {
t.Fatalf("err = %v, wantErr %v", err, tt.wantErr)
} }
if gotActor != tt.wantActor || gotOrg != tt.wantOrg { if gotActor != tt.wantActor || gotOrg != tt.wantOrg {
t.Fatalf("got actor %q org %q, want %q %q", gotActor, gotOrg, tt.wantActor, tt.wantOrg) t.Fatalf("got actor %q org %q, want %q %q", gotActor, gotOrg, tt.wantActor, tt.wantOrg)
+23 -35
View File
@@ -2,45 +2,50 @@ package staterpc
import ( import (
"fmt" "fmt"
"net"
"strings" "strings"
"time" "time"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/keepalive" "google.golang.org/grpc/keepalive"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
) )
// Dial returns a State Store client for target, attributed to the caller // Dial returns a State Store client for target, attributed to the caller
// service. A loopback target dials insecure; any other target requires a // service and authenticated by id. The cleanup closes the connection.
// token. The cleanup closes the connection. func Dial(target, caller string, id *rpctls.Identity, options ...grpc.DialOption) (*Client, func(), error) {
func Dial(target, caller, token string, options ...grpc.DialOption) (*Client, func(), error) {
target = strings.TrimSpace(target) target = strings.TrimSpace(target)
if target == "" { if target == "" {
return nil, nil, fmt.Errorf("state store target is required") return nil, nil, fmt.Errorf("state store target is required")
} }
loopback, err := TargetIsLoopback(target) creds, err := id.DialOption(target, presence.StateStore)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("state store target must be host:port: %w", err) return nil, nil, fmt.Errorf("state store: %w", err)
} }
if !loopback && token == "" { return dial(target, creds, caller, "", options)
return nil, nil, fmt.Errorf( }
"state store target %q is non-loopback; non-loopback exposure requires a bearer token ([services.state].target_token / STATE_STORE_TOKEN) or TLS, mirroring the gateway's --listen confinement (docs/prds/state-store-contract.md §9)",
target, // DialTask returns the client a task container uses: it trusts caPEM and
) // presents only the task grant, never a service certificate.
func DialTask(target string, caPEM []byte, grant string) (*Client, func(), error) {
creds, err := rpctls.CAOnlyDialOption(caPEM, target, presence.StateStore)
if err != nil {
return nil, nil, fmt.Errorf("state store: %w", err)
} }
return dial(target, creds, presence.Agent, grant, nil)
}
func dial(target string, creds grpc.DialOption, caller, grant string, options []grpc.DialOption) (*Client, func(), error) {
opts := []grpc.DialOption{ opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()), creds,
// Keepalive detects a dead peer on an otherwise silent watch stream. // Keepalive detects a dead peer on an otherwise silent watch stream.
grpc.WithKeepaliveParams(ClientKeepaliveParams()), grpc.WithKeepaliveParams(ClientKeepaliveParams()),
// GetInstalledPackage returns the whole layer, which for an extension is // GetInstalledPackage returns the whole layer, which for an extension is
// a plugin binary well past gRPC's default 4 MiB receive limit. // a plugin binary well past gRPC's default 4 MiB receive limit.
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxPackageResponseBytes)), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxPackageResponseBytes)),
// Both call shapes need the credential: the server's interceptors grpc.WithUnaryInterceptor(UnaryClientCredentialInterceptor(caller, grant)),
// guard unary RPCs and the streaming capture reads separately, so a grpc.WithStreamInterceptor(StreamClientCredentialInterceptor(caller, grant)),
// unary-only interceptor leaves the streams unauthenticated.
grpc.WithUnaryInterceptor(UnaryClientCredentialInterceptor(caller, token)),
grpc.WithStreamInterceptor(StreamClientCredentialInterceptor(caller, token)),
} }
conn, err := grpc.NewClient(target, append(opts, options...)...) conn, err := grpc.NewClient(target, append(opts, options...)...)
if err != nil { if err != nil {
@@ -82,23 +87,6 @@ func ServerKeepalivePolicy() keepalive.EnforcementPolicy {
} }
} }
// TargetIsLoopback reports whether addr's host is a loopback IP or
// "localhost".
func TargetIsLoopback(addr string) (bool, error) {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return false, fmt.Errorf("address must be host:port: %w", err)
}
if strings.EqualFold(host, "localhost") {
return true, nil
}
ip := net.ParseIP(host)
if ip == nil {
return false, nil
}
return ip.IsLoopback(), nil
}
// WaitForPeer makes calls to the State Store wait for it to come up // WaitForPeer makes calls to the State Store wait for it to come up
// instead of failing while it is down or restarting. A call that must not // instead of failing while it is down or restarting. A call that must not
// wait carries its own deadline. // wait carries its own deadline.
@@ -8,13 +8,15 @@ import (
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
grpccreds "google.golang.org/grpc/credentials"
"google.golang.org/grpc/metadata" "google.golang.org/grpc/metadata"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
pb "github.com/samcharles93/archie-core/internal/contracts/state/v1" pb "github.com/samcharles93/archie-core/internal/contracts/state/v1"
) )
func TestServiceLogsRequireInstanceCredential(t *testing.T) { func TestServiceLogsRequireService(t *testing.T) {
run := strings.Repeat("ab", 32) run := strings.Repeat("ab", 32)
grants := &TaskGrants{Store: credentials{}} grants := &TaskGrants{Store: credentials{}}
if err := grants.register(t.Context(), 7, run, time.Hour); err != nil { if err := grants.register(t.Context(), 7, run, time.Hour); err != nil {
@@ -22,15 +24,16 @@ func TestServiceLogsRequireInstanceCredential(t *testing.T) {
} }
for _, tt := range []struct { for _, tt := range []struct {
name, token string name, token string
peer *peer.Peer
code codes.Code code codes.Code
}{ }{
{"instance", "admin", codes.OK}, {"service", "", &peer.Peer{}, codes.OK},
{"task cannot read other services", run, codes.PermissionDenied}, {"task cannot read other services", run, &peer.Peer{AuthInfo: grpccreds.TLSInfo{}}, codes.PermissionDenied},
{"anonymous", "", codes.Unauthenticated}, {"anonymous", "", &peer.Peer{AuthInfo: grpccreds.TLSInfo{}}, codes.Unauthenticated},
} { } {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
ctx := metadata.NewIncomingContext(t.Context(), metadata.Pairs(tokenMetadataKey, tt.token)) ctx := peer.NewContext(metadata.NewIncomingContext(t.Context(), metadata.Pairs(tokenMetadataKey, tt.token)), tt.peer)
_, err := grants.UnaryInterceptor("admin")(ctx, &pb.RecentLogsRequest{}, &grpc.UnaryServerInfo{FullMethod: pb.StateStoreService_RecentLogs_FullMethodName}, func(context.Context, any) (any, error) { return &pb.RecentLogsResponse{}, nil }) _, err := grants.UnaryInterceptor()(ctx, &pb.RecentLogsRequest{}, &grpc.UnaryServerInfo{FullMethod: pb.StateStoreService_RecentLogs_FullMethodName}, func(context.Context, any) (any, error) { return &pb.RecentLogsResponse{}, nil })
if status.Code(err) != tt.code { if status.Code(err) != tt.code {
t.Fatalf("code %s want %s", status.Code(err), tt.code) t.Fatalf("code %s want %s", status.Code(err), tt.code)
} }
+13 -17
View File
@@ -4,7 +4,6 @@ import (
"context" "context"
"crypto/rand" "crypto/rand"
"crypto/sha256" "crypto/sha256"
"crypto/subtle"
"encoding/hex" "encoding/hex"
"errors" "errors"
"fmt" "fmt"
@@ -23,6 +22,7 @@ import (
"github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow/task" "github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
) )
// TaskGrants verifies run credentials: the one token a task's container // TaskGrants verifies run credentials: the one token a task's container
@@ -158,24 +158,25 @@ var taskScopedTargets = map[string]func(any) int64{
}, },
} }
// UnaryInterceptor separates administrator access from task-scoped worker // UnaryInterceptor separates services from task-scoped workers. A trusted
// access. A tokenless listener is permitted only on loopback by composition. // caller (a verified service certificate, or a plaintext peer, which only a
// Supplied worker tokens are still validated on loopback. // loopback listener admits) has full access; any other call needs a task
func (g *TaskGrants) UnaryInterceptor(adminToken string) grpc.UnaryServerInterceptor { // grant and is confined to its task.
func (g *TaskGrants) UnaryInterceptor() grpc.UnaryServerInterceptor {
return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
token := requestToken(ctx) if rpctls.Trusted(ctx) {
if (adminToken == "" && token == "") || (adminToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(adminToken)) == 1) {
return handler(ctx, req) return handler(ctx, req)
} }
token := requestToken(ctx)
if token == "" { if token == "" {
return nil, status.Error(codes.Unauthenticated, "missing state store token") return nil, status.Error(codes.Unauthenticated, "missing client certificate or task grant")
} }
taskID, err := g.taskFor(ctx, token) taskID, err := g.taskFor(ctx, token)
if err != nil { if err != nil {
return nil, status.Error(codes.Unavailable, "run credentials unavailable") return nil, status.Error(codes.Unavailable, "run credentials unavailable")
} }
if taskID == 0 { if taskID == 0 {
return nil, status.Error(codes.Unauthenticated, "invalid state store token") return nil, status.Error(codes.Unauthenticated, "invalid task grant")
} }
if !authorizesTaskScopedCall(info.FullMethod, req, taskID) { if !authorizesTaskScopedCall(info.FullMethod, req, taskID) {
return nil, status.Error(codes.PermissionDenied, "task grant does not authorize this operation") return nil, status.Error(codes.PermissionDenied, "task grant does not authorize this operation")
@@ -202,15 +203,10 @@ func dropDefinitionPin(req any) {
r.Task.WorkflowDefinitionYaml = "" r.Task.WorkflowDefinitionYaml = ""
} }
// StreamInterceptor admits only administrators to store read streams. // StreamInterceptor admits only services to store read streams.
func (g *TaskGrants) StreamInterceptor(adminToken string) grpc.StreamServerInterceptor { func (g *TaskGrants) StreamInterceptor() grpc.StreamServerInterceptor {
return func(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { return func(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
md, _ := metadata.FromIncomingContext(stream.Context()) if rpctls.Trusted(stream.Context()) {
tokens := md.Get(tokenMetadataKey)
if adminToken == "" && len(tokens) == 0 {
return handler(srv, stream)
}
if len(tokens) > 0 && adminToken != "" && subtle.ConstantTimeCompare([]byte(tokens[0]), []byte(adminToken)) == 1 {
return handler(srv, stream) return handler(srv, stream)
} }
return status.Error(codes.PermissionDenied, "task grant does not authorize streaming reads") return status.Error(codes.PermissionDenied, "task grant does not authorize streaming reads")
@@ -7,7 +7,9 @@ import (
"testing" "testing"
"google.golang.org/grpc" "google.golang.org/grpc"
grpccreds "google.golang.org/grpc/credentials"
"google.golang.org/grpc/metadata" "google.golang.org/grpc/metadata"
"google.golang.org/grpc/peer"
pb "github.com/samcharles93/archie-core/internal/contracts/state/v1" pb "github.com/samcharles93/archie-core/internal/contracts/state/v1"
) )
@@ -19,16 +21,17 @@ import (
func TestTaskCredentialCannotReplaceItsPin(t *testing.T) { func TestTaskCredentialCannotReplaceItsPin(t *testing.T) {
taskToken := strings.Repeat("ab", 32) taskToken := strings.Repeat("ab", 32)
grants := &TaskGrants{Store: credentials{sha256.Sum256([]byte(taskToken)): 7}} grants := &TaskGrants{Store: credentials{sha256.Sum256([]byte(taskToken)): 7}}
intercept := grants.UnaryInterceptor("admin") intercept := grants.UnaryInterceptor()
tests := []struct { tests := []struct {
name, token, wantYAML string name, token, wantYAML string
peer *peer.Peer
}{ }{
{"task credential", taskToken, ""}, {"task credential", taskToken, "", &peer.Peer{AuthInfo: grpccreds.TLSInfo{}}},
{"daemon credential", "admin", "yaml-b"}, {"daemon", "", "yaml-b", &peer.Peer{}},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs(tokenMetadataKey, tt.token)) ctx := peer.NewContext(metadata.NewIncomingContext(context.Background(), metadata.Pairs(tokenMetadataKey, tt.token)), tt.peer)
req := &pb.UpdateRequest{Task: &pb.Task{ req := &pb.UpdateRequest{Task: &pb.Task{
Id: 7, Workflow: "handoff-target", Id: 7, Workflow: "handoff-target",
WorkflowDefinitionVersion: 4, WorkflowDefinitionDigest: "digest-b", WorkflowDefinitionYaml: "yaml-b", WorkflowDefinitionVersion: 4, WorkflowDefinitionDigest: "digest-b", WorkflowDefinitionYaml: "yaml-b",