feat(rpc): mTLS between services from operator-issued certificates

Replace the service bearer tokens with [tls] ca_file and cert_dir. Caller
identity comes from the verified certificate, not metadata. Without TLS,
services listen and dial on loopback only. Task containers get only the CA
and keep their grant.
This commit is contained in:
sam committed 2026-10-16 02:37:51 +11:00
1 parent 88ef6ca5b2
commit 53c95038de
51 files changed
+746 -689

No files matched your search

+1
View File
@@ -91,6 +91,7 @@ func runCommand(args []string, getenv func(string) string, stderr io.Writer, run
NATSToken: natsToken,
StateStoreTarget: stateStoreURL,
StateStoreToken: stateStoreToken,
StateStoreCA: []byte(getenv("STATE_STORE_CA")),
ModelProxy: getenv(egress.ModelProxyEnv),
ModelProxyHosts: egress.ParseHosts(getenv(egress.ModelProxyHostsEnv)),
ModelProxyCA: getenv(egress.ModelProxyCAEnv),
-1
View File
@@ -22,7 +22,6 @@ func run() int {
flag.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory")
flag.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory")
flag.StringVar(&options.Listen, "listen", "", "gateway gRPC listen address (defaults to [services.gateway].listen, else 127.0.0.1:8585)")
flag.StringVar(&options.Token, "token", "", "Bearer [REDACTED] required for a non-loopback listener (defaults to [services.gateway].target_token)")
flag.Parse()
showVersion()
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
-3
View File
@@ -23,11 +23,8 @@ func run() int {
flag.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory")
flag.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory")
flag.StringVar(&options.Gateway.Target, "gateway-target", "", "archie-gateway gRPC address (defaults to [services.gateway].target, else 127.0.0.1:8585)")
flag.StringVar(&options.Gateway.Token, "gateway-token", "", "bearer token presented to archie-gateway")
flag.StringVar(&options.StateStore.Target, "state-store-target", "", "archie-state-store gRPC address (defaults to [services.state].target)")
flag.StringVar(&options.StateStore.Token, "state-store-token", "", "bearer token presented to archie-state-store")
flag.StringVar(&options.Listen, "listen", "", "MessagingService gRPC listen address (defaults to [services.messaging].listen)")
flag.StringVar(&options.Token, "token", "", "bearer token required from MessagingService callers on a non-loopback listener")
flag.DurationVar(&options.DependencyTimeout, "dependency-timeout", 0, "readiness probe timeout (default 5s)")
flag.DurationVar(&options.ShutdownTimeout, "shutdown-timeout", 0, "shutdown timeout (default 5s)")
flag.Parse()
-1
View File
@@ -34,7 +34,6 @@ func run() int {
flag.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory")
flag.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory")
flag.StringVar(&options.Listen, "listen", "", "state store gRPC listen address (defaults to [services.state].listen, else 127.0.0.1:9090)")
flag.StringVar(&options.Token, "token", "", "bearer token required for a non-loopback listener (defaults to [services.state].target_token)")
flag.StringVar(&options.ReadyAddr, "ready-addr", "", "optional readiness HTTP listen address (e.g. 127.0.0.1:9091)")
flag.Usage = func() {
fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store <backup|restore|validate|rollback> -h | access reset -h\n\n")
-3
View File
@@ -35,11 +35,8 @@ func run() int {
flag.StringVar(&options.OidcClientSecretEnv, "oidc-client-secret-env", "", "name of the environment variable holding the sign-in client secret")
flag.StringVar(&options.OidcRedirectURL, "oidc-redirect-url", "", "callback URL registered at the provider (path must be /oauth2/callback)")
flag.StringVar(&options.Gateway.Target, "gateway-target", "", "archie-gateway gRPC address (defaults to [services.gateway].target)")
flag.StringVar(&options.Gateway.Token, "gateway-token", "", "bearer token presented to archie-gateway")
flag.StringVar(&options.State.Target, "state-target", "", "archie-state-store gRPC address (defaults to [services.state].target)")
flag.StringVar(&options.State.Token, "state-token", "", "bearer token presented to archie-state-store")
flag.StringVar(&options.Harness.Target, "harness-target", "", "archied harness session gRPC address (defaults to [services.harness].target); empty leaves the setup terminal unwired")
flag.StringVar(&options.Harness.Token, "harness-token", "", "bearer token presented to the daemon's harness contract")
flag.DurationVar(&options.DependencyTimeout, "dependency-timeout", 0, "per-dependency readiness probe timeout (default 5s)")
flag.DurationVar(&options.EventPollInterval, "event-poll-interval", 0, "how often the activity feed polls the State Store for new events (default 1s)")
// BoolFunc rather than BoolVar so an unset flag stays distinguishable
+14 -17
View File
@@ -261,6 +261,18 @@ wall_clock = "30m" # one agent call
gate_max_failures = 5
task_wall_clock = "4h" # a whole task run; keep containers.max_uptime at or above it
# TLS: every gRPC hop between services uses certificates from your own CA
# (step-ca, cert-manager, Vault, openssl). Each process loads
# cert_dir/<service>.pem and cert_dir/<service>-key.pem, whose DNS SAN is its
# service name: archied, archie-gateway, archie-messaging, archie-state-store,
# archie-ui. Every peer must chain to ca_file. Certificates are reloaded when
# they change. Without [tls], services may only listen on and dial loopback
# addresses, task containers cannot reach the State Store, and the harness
# setup terminal is off.
# [tls]
# ca_file = "/etc/archie/tls/ca.pem"
# cert_dir = "/etc/archie/tls"
# Services selects contract adapters at composition time.
[services.gateway]
target = "127.0.0.1:8585" # archie-gateway's gRPC address (always remote once set)
@@ -269,11 +281,6 @@ target = "127.0.0.1:8585" # archie-gateway's gRPC address (always remot
# # to the target above, so a default deployment's client and
# # server cannot drift onto different ports. Set it when the
# # host already owns that port.
# target_token = "..." # Bearer [REDACTED] for a NON-loopback listener. A loopback listener
# # (127.0.0.1/::1/localhost) needs no token. A non-loopback
# # listener without a token fails closed, as does a
# # non-loopback dial without one. The same token is
# # presented as GATEWAY_TOKEN by remote consumers.
[services.state]
target = "127.0.0.1:9090" # REQUIRED. The State Store gRPC address archied and
@@ -285,25 +292,15 @@ target = "127.0.0.1:9090" # REQUIRED. The State Store gRPC address arch
# # above, so a default deployment's client and server cannot
# # drift onto different ports. Move it when a port is already
# # taken on the host.
# target_token = "..." # bearer token for a NON-loopback target. A loopback target
# # (127.0.0.1/::1/localhost) needs no token. A non-loopback
# # target without a token fails closed. The same token is injected as STATE_STORE_TOKEN into
# # archie-agent containers so the agent authenticates to the
# # same remote store. See deployments/README.md for how to
# # start archie-state-store.
# The daemon's harness session contract: the dashboard's setup terminal opens
# a PTY into an ephemeral Kit container over it. Both ends are optional and
# the dashboard's setup terminal degrades to 503 without them: the daemon
# serves the contract only when target_token (or HARNESS_TOKEN) is set, and
# archie-ui dials it only when a target is set.
# serves the contract only when [tls] is configured, and archie-ui dials it
# only when a target is set.
# [services.harness]
# target = "127.0.0.1:8587" # The address archie-ui dials. Empty leaves the
# # setup terminal unwired.
# target_token = "..." # REQUIRED, even on loopback: a session is a
# # shell in a container that holds an org credential.
# # The daemon reads the same value to authenticate its
# # clients. Leave unset to disable the listener.
# NATS is the sole autonomous workflow handoff. mode changes only where the
# broker runs; every workflow still runs in a managed task container:
+1 -5
View File
@@ -58,11 +58,7 @@ as a host binary. Pull it from GHCR: `ghcr.io/samcharles93/archie-agent`.
Dashboard reachability comes from `[web].listen` (default
`127.0.0.1:8484`, which needs no dashboard token). For a non-loopback
bind pass `-listen` with `-token` or `-token-file` to mint and persist
one. `archie-ui` reads `GATEWAY_TOKEN` / `STATE_STORE_TOKEN` from its
process environment only — it has no secret registry, so when those
tokens live in a secret manager, whatever starts `archie-ui` must export
them into its environment (the `[services.*].target_token` values in
`config.toml` are the other supported source).
one.
4. For a persistent, 24/7 deployment instead of a foreground run, follow the
systemd user-unit runbook:
+8 -11
View File
@@ -52,10 +52,10 @@ PostgreSQL database `database_url` names and serves the `StateStore` gRPC
contract. BOTH `archied` and `archie-gateway` dial it via
`[services.state].target` and never open the task tables themselves. The templates set
`[services.state]` accordingly: a host-only agent reaches a loopback bind
(`127.0.0.1:9090`) with no token, while a container-mode agent needs the
Docker bridge gateway address plus a matching bearer token (`target_token` /
`--token` / `STATE_STORE_TOKEN`), because a container cannot reach the host's
loopback.
(`127.0.0.1:9090`), while a container-mode agent needs the Docker bridge
gateway address, because a container cannot reach the host's loopback. A
non-loopback address needs `[tls]` certificates from your own CA; see
`config.example.toml`.
## Running the dashboard
@@ -84,17 +84,14 @@ archie-ui warns about exactly that at startup. Terminate TLS in front of it
warning; leaving it off while exposing the port directly is the case the
warning names.
Auth note: unlike `archied`, archie-ui has no secret registry --
`GATEWAY_TOKEN`/`STATE_STORE_TOKEN` resolve from its process environment
only (or from `[services.*].target_token` in `config.toml`), so whatever
starts archie-ui must export those tokens into its environment. See the UI
Service unit in `systemd-user-service.md` for the worked example.
Auth note: archie-ui authenticates to its dependencies with its `[tls]`
certificate (`archie-ui.pem`).
```bash
# loopback (host-only agent)
archie-state-store -config ~/.config/archie/config.toml -listen 127.0.0.1:9090
# or container-mode: bind the bridge gateway with a token
archie-state-store -config ~/.config/archie/config.toml -listen 172.17.0.1:9090 -token <token>
# or container-mode: bind the bridge gateway ([tls] required)
archie-state-store -config ~/.config/archie/config.toml -listen 172.17.0.1:9090
```
## Verifying the daemon
+7 -5
View File
@@ -26,13 +26,15 @@ target = "127.0.0.1:8585"
# The State Store gRPC service owns the PostgreSQL task data; archied and
# archie-gateway dial it and never open those tables themselves.
# This deployment runs the agent in containers, so target is the Docker bridge
# gateway (172.17.0.1) the container can reach, with a matching target_token
# that archie-state-store validates. Host-only agent: use 127.0.0.1:9090 with
# no token.
# This deployment runs the agent in containers, so the target is the Docker
# bridge gateway (172.17.0.1) the container can reach. A non-loopback address
# needs [tls]; see config.example.toml.
[services.state]
target = "172.17.0.1:9090"
target_token = "change-me-state-store-token"
[tls]
ca_file = "/etc/archie/tls/ca.pem"
cert_dir = "/etc/archie/tls"
[forge]
type = "gitea"
+3 -5
View File
@@ -18,13 +18,11 @@ target = "127.0.0.1:8585"
# The State Store gRPC service owns the PostgreSQL task data; archied and
# archie-gateway dial it and never open those tables themselves.
# This deployment runs the agent in containers, so target is the Docker bridge
# gateway (172.17.0.1) the container can reach, with a matching target_token
# that archie-state-store validates. Host-only agent: use 127.0.0.1:9090 with
# no token.
# This deployment runs the agent in containers, so the target is the Docker
# bridge gateway (172.17.0.1) the container can reach. A non-loopback address
# needs [tls]; see config.example.toml.
[services.state]
target = "172.17.0.1:9090"
target_token = "change-me-state-store-token"
[models]
triage = "openai/gpt-6-luna"
+3 -8
View File
@@ -25,16 +25,11 @@ target = "127.0.0.1:8585"
# The State Store gRPC service owns the PostgreSQL task data. archied and
# archie-gateway dial it here and never open those tables themselves.
# This deployment runs the
# agent in containers, so the target must be the Docker bridge gateway address
# (172.17.0.1) that the container can reach, and a non-loopback target requires
# a matching target_token that archie-state-store validates -- start it with the
# same token (e.g. `archie-state-store ... -listen 172.17.0.1:9090 -token
# change-me-state-store-token`). For a host-only agent (no [containers]) use the
# loopback target 127.0.0.1:9090 with no token.
# This deployment runs the agent in containers, so the target is the Docker
# bridge gateway (172.17.0.1) the container can reach. A non-loopback address
# needs [tls]; see config.example.toml.
[services.state]
target = "172.17.0.1:9090"
target_token = "change-me-state-store-token"
[forge]
type = "github"
+8 -29
View File
@@ -20,10 +20,9 @@ The State Store unit is `archie-state-store.service`. It owns the task data in
PostgreSQL and serves the `StateStore` gRPC contract. BOTH `archied` and
`archie-gateway` dial it via `[services.state].target` (see the `archied`
config below) and never open the task tables themselves. A
host-only agent uses the loopback bind with no token below; a container-mode
agent needs the Docker bridge gateway address plus a bearer token via
`--token`/`STATE_STORE_TOKEN` or `[services.state].target_token`, and a
non-loopback bind fails closed without one:
host-only agent uses the loopback bind below; a container-mode agent needs the
Docker bridge gateway address, and a non-loopback bind needs `[tls]` (see
`config.example.toml`):
`install.sh` writes `archie-state-store.service` with that loopback bind.
@@ -41,9 +40,7 @@ update when a host is missing a unit and names what is missing, which now fires
for a host that drifted or predates this release rather than the normal path.
It dials the Gateway for the chat contract and the State Store for the stored
channel settings, and reads the same `config.toml`. Like `archie-ui` it resolves
`GATEWAY_TOKEN` and `STATE_STORE_TOKEN` from its process environment, hence the
`EnvironmentFile`:
channel settings, and reads the same `config.toml`, including `[tls]`.
`install.sh` writes `archie-messaging.service`. Its `After=` and `Wants=`
lines only tidy the start sequence: nothing depends on that order, because
@@ -56,34 +53,16 @@ the Gateway and State Store targets already in it, and binds `[web].listen`.
A non-loopback bind requires a dashboard token (`-token`, or `-token-file` to
mint and persist one);
**Auth note:** unlike `archied`,
`archie-ui` has no secret registry — it resolves `GATEWAY_TOKEN` and
`STATE_STORE_TOKEN` from its **process environment only** (or from the
`target_token` values in `config.toml`). Whatever starts `archie-ui` is
therefore responsible for exporting those tokens into its environment: keep
them in `~/.config/archie/env` (loaded by the `EnvironmentFile` line below),
not only in a secret manager. A deployment that authenticates `archied`
through bws/Vault but exports nothing for `archie-ui` leaves the dashboard
unable to dial its dependencies.
**Auth note:** `archie-ui` authenticates to the Gateway, State Store and
harness with its `[tls]` certificate (`archie-ui.pem`) from `config.toml`.
For supervision, `GET /healthz` on `[web].listen` is the process's liveness
probe (token-free by design); the authenticated readiness surface lives at
the dashboard's health endpoint behind the token. A watchdog can curl
`/healthz` without credentials.
`install.sh` writes `archie-ui.service`. It has no secret registry of its own,
so its token handling is the `EnvironmentFile` note above.
Concretely, for a deployment whose dependency tokens live outside
`config.toml`, `~/.config/archie/env` carries the two names and the unit
loads it; the dashboard token itself comes from `-token-file` (there is no
environment variable for it):
```bash
# ~/.config/archie/env
GATEWAY_TOKEN=... # presented to archie-gateway
STATE_STORE_TOKEN=... # presented to archie-state-store
```
`install.sh` writes `archie-ui.service`. The dashboard token comes from
`-token-file` (there is no environment variable for it).
`After=` orders the start, it does not wait for the store to bind. The UI
process closes that gap itself: its event pump retries priming with backoff
+3
View File
@@ -50,6 +50,8 @@ type Settings struct {
// NATS storerpc path is deleted.
StateStoreTarget string
StateStoreToken string
// StateStoreCA is the PEM CA the State Store's certificate chains to.
StateStoreCA []byte
// ModelProxy, ModelProxyHosts and ModelProxyCA route model provider
// requests through the daemon's egress proxy, which holds the keys.
@@ -148,6 +150,7 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies
TaskID: taskID,
StateStoreURL: settings.StateStoreTarget,
StateStoreToken: settings.StateStoreToken,
StateStoreCA: settings.StateStoreCA,
}, log)
if err != nil {
operation := "nats connect failed"
+15 -6
View File
@@ -108,7 +108,6 @@ type boot struct {
// (daemon.RunCredentialIssuer), wrapping the same *staterpc.Client as
// stateStore, which also verifies a credential for a worktree push.
stateStoreGrants *staterpc.GrantIssuer
stateStoreToken string
controlPlane *controlplane.Client
// controlPlaneRPC is the State Store's control-plane transport, kept so the
// daemon root can build its workflow-definitions client from it
@@ -273,7 +272,7 @@ func (b *boot) openStateStoreAdapter(ctx context.Context) error {
if target == "" {
return fmt.Errorf("services.state.target is required: archied/archie-gateway no longer own archie.db; the standalone archie-state-store process owns it (docs/prds/state-store-contract.md §12 step 7)")
}
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, b.cfg.Services, b.secrets)
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, b.cfg)
if err != nil {
b.log.Error("state store adapter", "err", err)
return err
@@ -287,7 +286,6 @@ func (b *boot) openStateStoreAdapter(ctx context.Context) error {
b.controlPlane = controlplane.NewRPCClient(b.controlPlaneRPC)
b.applyStatus = applystatus.New(b.processName, client, b.log)
b.stateStoreGrants = &staterpc.GrantIssuer{Client: client}
b.stateStoreToken = b.cfg.Services.ResolvedToken(config.ServiceNameState, b.secrets.Getenv)
b.addCleanup(cleanup)
b.startSecretEngines(ctx, client)
return nil
@@ -478,13 +476,13 @@ func (b *boot) setupContainers(ctx context.Context) func() {
// only enqueues the chat tasks scheduled workflows create.
func (b *boot) setupGatewayClient(ctx context.Context) error {
b.setupChatTasks(b.cfg)
contract, cleanup, err := composeChatContract(b.cfg.Services, b.secrets)
contract, cleanup, err := composeChatContract(b.cfg)
if err != nil {
return err
}
b.addCleanup(cleanup)
b.chat = &webui.ChatService{Contract: contract}
messaging, closeMessaging, err := composeMessaging(b.cfg.Services, b.secrets)
messaging, closeMessaging, err := composeMessaging(b.cfg)
if err != nil {
return err
}
@@ -580,7 +578,7 @@ func (b *boot) buildDaemon() {
b.d = &daemon.Daemon{
Cfg: b.cfgHolder,
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken, Scoped: b.cfg.NATS.Mode != config.NATSModeExternal},
ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), Token: b.stateStoreToken},
ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), CA: b.stateStoreCA()},
Store: b.stateStore,
Bus: b.bus,
Forge: b.forgeClient,
@@ -873,3 +871,14 @@ func (b *boot) livePlaybooks() *controlplane.LivePlaybooks {
// playbookRefreshInterval is how often archied re-reads the resources it
// would otherwise load only on first use, so their apply status stays reported.
const playbookRefreshInterval = 30 * time.Second
// stateStoreCA is the CA a task container verifies the State Store with;
// nil without [tls].
func (b *boot) stateStoreCA() []byte {
id, err := servicekit.Identity(b.cfg.TLS, presence.Daemon)
if err != nil {
b.log.Warn("state store CA for containers", "err", err)
return nil
}
return id.CAPEM()
}
+17 -11
View File
@@ -3,27 +3,29 @@ package archied
import (
"google.golang.org/grpc"
"github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/gateway"
"github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc"
"github.com/samcharles93/archie-core/internal/infrastructure/messagingrpc"
"github.com/samcharles93/archie-core/internal/secret"
)
// composeChatContract dials the standalone archie-gateway, the sole owner of
// ChatContract, from the daemon's own configuration key. Token resolution is
// the daemon's (it owns the config keys and the secret registry); the dial
// itself and the fail-closed non-loopback rule belong to the transport, so
// they come from gatewayrpc.Dial.
func composeChatContract(services config.Services, secrets *secret.Registry, options ...grpc.DialOption) (gateway.ChatContract, func(), error) {
target, err := services.RequireTarget(config.ServiceNameGateway)
// ChatContract, from the daemon's own configuration key.
func composeChatContract(cfg config.Config, options ...grpc.DialOption) (gateway.ChatContract, func(), error) {
target, err := cfg.Services.RequireTarget(config.ServiceNameGateway)
if err != nil {
return nil, nil, err
}
id, err := servicekit.Identity(cfg.TLS, presence.Daemon)
if err != nil {
return nil, nil, err
}
// Assigned through a typed variable rather than returned directly: a
// direct forward would hand a failed dial's nil *Client back as a
// non-nil ChatContract interface.
client, cleanup, err := gatewayrpc.Dial(target, services.ResolvedToken(config.ServiceNameGateway, secrets.Getenv), options...)
client, cleanup, err := gatewayrpc.Dial(target, id, options...)
if err != nil {
return nil, nil, err
}
@@ -32,10 +34,14 @@ func composeChatContract(services config.Services, secrets *secret.Registry, opt
// composeMessaging builds the Messaging Service client. The connection is
// lazy, so an absent Messaging Service fails each delivery, not the daemon.
func composeMessaging(services config.Services, secrets *secret.Registry) (*messagingrpc.Client, func(), error) {
target, err := services.RequireTarget(config.ServiceNameMessaging)
func composeMessaging(cfg config.Config) (*messagingrpc.Client, func(), error) {
target, err := cfg.Services.RequireTarget(config.ServiceNameMessaging)
if err != nil {
return nil, nil, err
}
return messagingrpc.Dial(target, services.ResolvedToken(config.ServiceNameMessaging, secrets.Getenv))
id, err := servicekit.Identity(cfg.TLS, presence.Daemon)
if err != nil {
return nil, nil, err
}
return messagingrpc.Dial(target, id)
}
+14 -9
View File
@@ -6,7 +6,6 @@ import (
"io"
"log/slog"
"net"
"os"
"slices"
"sync"
"time"
@@ -19,7 +18,7 @@ import (
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/container"
pb "github.com/samcharles93/archie-core/internal/contracts/harness/v1"
"github.com/samcharles93/archie-core/internal/infrastructure/harnessrpc"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/harnesssession"
"github.com/samcharles93/archie-core/internal/infrastructure/kitrun"
)
@@ -189,9 +188,18 @@ func (b *boot) setupHarness(ctx context.Context) {
b.log.Warn("harness sessions disabled", "err", err)
return
}
token := b.cfg.Services.ResolvedToken(config.ServiceNameHarness, os.Getenv)
if token == "" {
b.log.Warn("harness sessions disabled: [services.harness].target_token or HARNESS_TOKEN is required, even on loopback")
id, err := servicekit.Identity(b.cfg.TLS, presence.Daemon)
if err != nil {
b.log.Warn("harness sessions disabled", "err", err)
return
}
if id == nil {
b.log.Warn("harness sessions disabled: a session is a shell holding an org's credential, so it needs [tls] ca_file and cert_dir")
return
}
creds, err := id.ServerOption(listen, false)
if err != nil {
b.log.Warn("harness sessions disabled", "err", err)
return
}
ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", listen)
@@ -205,10 +213,7 @@ func (b *boot) setupHarness(ctx context.Context) {
Profile: func(name string) (config.AgentProfile, error) { return b.cfgHolder.Get().Containers.Profile(name) },
TTL: setupSessionTTL,
}
srv := grpc.NewServer(
//nolint:contextcheck // grpc.StreamServerInterceptor has no context.Context parameter; the interceptor derives its context from stream.Context()
grpc.StreamInterceptor(harnessrpc.StreamServerInterceptor(token)),
)
srv := grpc.NewServer(creds)
pb.RegisterHarnessServiceServer(srv, &harnessServer{sessions: manager, runs: b.runView(), log: b.log})
go func() {
if err := srv.Serve(ln); err != nil && !errors.Is(err, grpc.ErrServerStopped) {
+1 -2
View File
@@ -12,7 +12,6 @@ import (
"github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/secret"
)
// RunStatus prints one line per service from the State Store's presence
@@ -31,7 +30,7 @@ func RunStatus(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "archied status: %v\n", err)
return 1
}
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, doc.Config.Services, secret.NewRegistry())
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, doc.Config)
if err != nil {
fmt.Fprintf(stderr, "archied status: %v\n", err)
return 1
+1 -2
View File
@@ -12,7 +12,6 @@ import (
"github.com/samcharles93/archie-core/internal/app/servicekit"
pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/secret"
)
// imagePolicy is the control plane's agent image, reached through the State
@@ -55,7 +54,7 @@ func (p imagePolicy) with(ctx context.Context, use func(pb.ControlPlaneServiceCl
if err != nil {
return err
}
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, doc.Config.Services, secret.NewRegistry())
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, doc.Config)
if err != nil {
return err
}
+14 -25
View File
@@ -15,15 +15,16 @@ import (
"github.com/samcharles93/archie-core/internal/gateway"
"github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc"
"github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/plugin"
)
// serveGatewayListener is the serving tail of Run: the listener is
// bound, the process declares itself serving, and the chat contract answers
// until ctx ends.
func (b *server) serveGatewayListener(ctx context.Context, listener net.Listener, loopback bool, contract gateway.ChatContract, opts []grpc.ServerOption) error {
func (b *server) serveGatewayListener(ctx context.Context, listener net.Listener, contract gateway.ChatContract, opts []grpc.ServerOption) error {
defer listener.Close()
b.log.Info("archie-gateway running", "addr", listener.Addr().String(), "token_required", !loopback)
b.log.Info("archie-gateway running", "addr", listener.Addr().String(), "tls", b.id != nil)
// Boot is over and this listener is about to accept: the same fact
// systemd's READY=1 asserts, so the announcement goes out here rather than
// from a second notion of "started". Not later: Serve
@@ -90,37 +91,25 @@ func (b *server) startGatewayRuntime(ctx context.Context, actor gateway.ChatTask
// rather than rejected as an oversized gRPC message.
const inboundMessageHeadroomBytes = 1 << 20
// gatewayServerOpts applies the transport security boundary: a loopback
// listener is served insecure with no token; any non-loopback address
// requires a Bearer [REDACTED], else the process fails closed. It mirrors
// stateStoreServerOpts, minus the State Store's task-scoped grants: the
// Gateway has one administrative token for all callers.
// gatewayServerOpts serves listen with id's credentials; every caller
// presents a service certificate.
//
// The receive limit is raised above gRPC's 4 MB default because an inbound
// request carries a channel attachment's bytes: the channel frontend
// downloads the file, and this process -- which runs the turn -- holds no
// platform credential to fetch it again.
func gatewayServerOpts(listen, token string, principals access.PrincipalSource) (opts []grpc.ServerOption, loopback bool, err error) {
callers := rpcidentity.Callers{Principals: principals}
recv := grpc.MaxRecvMsgSize(messaging.MaxInboundAttachmentBytes + inboundMessageHeadroomBytes)
loopback, err = gatewayrpc.TargetIsLoopback(listen)
func gatewayServerOpts(listen string, id *rpctls.Identity, principals access.PrincipalSource) ([]grpc.ServerOption, error) {
creds, err := id.ServerOption(listen, false)
if err != nil {
return nil, false, err
}
if loopback {
return []grpc.ServerOption{recv, grpc.ChainUnaryInterceptor(callers.Unary()), grpc.ChainStreamInterceptor(callers.Stream())}, true, nil
}
if token == "" {
return nil, false, fmt.Errorf(
"gateway listen address %q is non-loopback; non-loopback exposure requires a Bearer [REDACTED] (--token / [services.gateway].target_token) or TLS",
listen,
)
return nil, fmt.Errorf("gateway: %w", err)
}
callers := rpcidentity.Callers{Principals: principals}
return []grpc.ServerOption{
recv,
grpc.ChainUnaryInterceptor(gatewayrpc.UnaryServerInterceptor(token), callers.Unary()),
grpc.ChainStreamInterceptor(gatewayrpc.StreamServerInterceptor(token), callers.Stream()),
}, false, nil
creds,
grpc.MaxRecvMsgSize(messaging.MaxInboundAttachmentBytes + inboundMessageHeadroomBytes),
grpc.ChainUnaryInterceptor(callers.Unary()),
grpc.ChainStreamInterceptor(callers.Stream()),
}, nil
}
func serveGateway(ctx context.Context, listener net.Listener, contract gateway.ChatContract, sessions gateway.SessionStore, catalog gatewayrpc.Catalog, opts []grpc.ServerOption) error {
+7 -12
View File
@@ -30,6 +30,7 @@ import (
"github.com/samcharles93/archie-core/internal/infrastructure/controlplanerpc"
"github.com/samcharles93/archie-core/internal/infrastructure/modelcatalog"
"github.com/samcharles93/archie-core/internal/infrastructure/readiness"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/infrastructure/secretengine"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
"github.com/samcharles93/archie-core/internal/infrastructure/taskactions"
@@ -51,9 +52,6 @@ type Options struct {
// Listen is the gRPC listen address. Empty falls back to
// [services.gateway].listen.
Listen string
// Token is the bearer token a non-loopback listener requires. Empty falls
// back to [services.gateway].target_token.
Token string
}
// server is the Gateway process: the owner of conversations and of every
@@ -65,6 +63,7 @@ type server struct {
log *slog.Logger
secrets *secret.Registry
id *rpctls.Identity
engines *secretengine.Runner
stateStore *staterpc.Client
controlPlane *controlplane.Client
@@ -130,12 +129,7 @@ func Run(ctx context.Context, options Options) error {
if err != nil {
return err
}
token := options.Token
if token == "" {
token = b.cfg.Services.ResolvedToken(config.ServiceNameGateway, b.secrets.Getenv)
}
//nolint:contextcheck // grpc.StreamServerInterceptor has no context.Context parameter; gatewayrpc.StreamServerInterceptor derives its context from stream.Context() instead
opts, loopback, err := gatewayServerOpts(listen, token, b.stateStore)
opts, err := gatewayServerOpts(listen, b.id, b.stateStore)
if err != nil {
return err
}
@@ -143,7 +137,7 @@ func Run(ctx context.Context, options Options) error {
if err != nil {
return fmt.Errorf("listen for gateway: %w", err)
}
return b.serveGatewayListener(ctx, listener, loopback, contract, opts)
return b.serveGatewayListener(ctx, listener, contract, opts)
}
func (b *server) addCleanup(fn func()) { b.cleanups = append(b.cleanups, fn) }
@@ -171,7 +165,8 @@ func (b *server) loadConfig(cfgPath, overlayPath string) error {
b.log, b.logFeed = logs.Log, logs.Feed
b.addCleanup(func() { _ = logs.Closer.Close() })
b.secrets = secret.NewRegistry()
return nil
b.id, err = servicekit.Identity(b.cfg.TLS, presence.Gateway)
return err
}
// openState claims the conversation store, dials the State Store, starts the
@@ -183,7 +178,7 @@ func (b *server) openState(ctx context.Context) error {
if err := b.claimGatewayOwnership(ctx); err != nil {
return err
}
client, cleanup, err := servicekit.StateStoreClient(presence.Gateway, b.cfg.Services, b.secrets)
client, cleanup, err := servicekit.StateStoreClient(presence.Gateway, b.cfg)
if err != nil {
return fmt.Errorf("state store client: %w", err)
}
+14 -29
View File
@@ -8,13 +8,17 @@ import (
"os"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/secret"
)
// ResolvedConfig contains the resolved inputs needed by the Messaging Service.
type ResolvedConfig struct {
Options Options
Options Options
// Identity is this process's certificate; nil is plaintext, loopback only.
Identity *rpctls.Identity
TelegramToken string
Telegram config.TelegramConfig
// WorkDir and BotUser are not channel transport settings. They locate
@@ -28,6 +32,7 @@ type projection struct {
gateway ServiceTarget
stateStore ServiceTarget
messaging config.ServiceConnection
tls config.TLS
telegram config.TelegramConfig
workDir string
botUser string
@@ -51,13 +56,17 @@ func Resolve(o Options, log *slog.Logger) (ResolvedConfig, error) {
}
}
o = withDefaults(o)
o = withEnvTokens(o)
if err := o.validate(); err != nil {
return ResolvedConfig{}, err
}
id, err := rpctls.Load(proj.tls.CAFile, proj.tls.CertDir, presence.Messaging)
if err != nil {
return ResolvedConfig{}, err
}
return ResolvedConfig{
Options: o,
Identity: id,
Telegram: proj.telegram,
WorkDir: proj.workDir,
BotUser: proj.botUser,
@@ -78,12 +87,10 @@ func readProjection(path, overlay string, log *slog.Logger) (projection, bool, e
func project(cfg config.Config) projection {
return projection{
gateway: ServiceTarget{
Target: cfg.Services.Get(config.ServiceNameGateway).Target,
Token: cfg.Services.Get(config.ServiceNameGateway).TargetToken,
},
stateStore: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameState).Target, Token: cfg.Services.Get(config.ServiceNameState).TargetToken},
gateway: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameGateway).Target},
stateStore: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameState).Target},
messaging: cfg.Services.Get(config.ServiceNameMessaging),
tls: cfg.TLS,
telegram: cfg.Chat.Telegram,
workDir: cfg.WorkDir,
botUser: cfg.BotUser,
@@ -91,38 +98,16 @@ func project(cfg config.Config) projection {
}
}
func withEnvTokens(o Options) Options {
if o.Gateway.Token == "" {
o.Gateway.Token = os.Getenv("GATEWAY_TOKEN")
}
if o.StateStore.Token == "" {
o.StateStore.Token = os.Getenv("STATE_STORE_TOKEN")
}
if o.Token == "" {
o.Token = os.Getenv("MESSAGING_TOKEN")
}
return o
}
func merge(o Options, p projection) Options {
if o.Gateway.Target == "" {
o.Gateway.Target = p.gateway.Target
}
if o.Gateway.Token == "" {
o.Gateway.Token = p.gateway.Token
}
if o.StateStore.Target == "" {
o.StateStore.Target = p.stateStore.Target
}
if o.StateStore.Token == "" {
o.StateStore.Token = p.stateStore.Token
}
if o.Listen == "" {
o.Listen = p.messaging.Listen
}
if o.Token == "" {
o.Token = p.messaging.TargetToken
}
return o
}
+2 -39
View File
@@ -4,9 +4,6 @@ package archiemessaging
import (
"errors"
"fmt"
"net"
"strings"
"time"
)
@@ -18,11 +15,9 @@ const (
defaultShutdownTimeout = 5 * time.Second
)
// ServiceTarget is one remote contract's endpoint and the service-to-service
// bearer token this process presents to it.
// ServiceTarget is one remote contract's endpoint.
type ServiceTarget struct {
Target string
Token string
}
// Options contains the process inputs for the Messaging Service.
@@ -32,10 +27,8 @@ type Options struct {
Gateway ServiceTarget
StateStore ServiceTarget
// Listen is where MessagingService is served; Token is the bearer token
// its callers present on a non-loopback address.
// Listen is where MessagingService is served.
Listen string
Token string
DependencyTimeout time.Duration
ShutdownTimeout time.Duration
}
@@ -60,35 +53,5 @@ func (o Options) validate() error {
if o.Gateway.Target == "" {
return errors.New("gateway target is required (pass -gateway-target or configure [services.gateway].target)")
}
if err := validateServiceTarget("gateway", o.Gateway); err != nil {
return err
}
if o.StateStore.Target != "" {
if err := validateServiceTarget("state store", o.StateStore); err != nil {
return err
}
}
return validateServiceTarget("messaging listen", ServiceTarget{Target: o.Listen, Token: o.Token})
}
func validateServiceTarget(serviceName string, target ServiceTarget) error {
host, _, err := net.SplitHostPort(target.Target)
if err != nil {
host = target.Target
}
if isLoopback(host) {
return nil
}
if target.Token == "" {
return fmt.Errorf("%s target %q is non-loopback and requires a bearer token", serviceName, target.Target)
}
return nil
}
func isLoopback(host string) bool {
if host == "localhost" {
return true
}
ip := net.ParseIP(strings.Trim(host, "[]"))
return ip != nil && ip.IsLoopback()
}
+9 -8
View File
@@ -19,6 +19,7 @@ import (
"github.com/samcharles93/archie-core/internal/infrastructure/extension"
"github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc"
"github.com/samcharles93/archie-core/internal/infrastructure/messagingrpc"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/infrastructure/secretengine"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
"github.com/samcharles93/archie-core/internal/logging"
@@ -36,7 +37,7 @@ func Run(ctx context.Context, o Options) error {
return fmt.Errorf("resolve messaging options: %w", err)
}
chat, closeGateway, err := gatewayrpc.Dial(cfg.Options.Gateway.Target, cfg.Options.Gateway.Token)
chat, closeGateway, err := gatewayrpc.Dial(cfg.Options.Gateway.Target, cfg.Identity)
if err != nil {
return fmt.Errorf("dial gateway (%s): %w", cfg.Options.Gateway.Target, err)
}
@@ -59,7 +60,7 @@ func Run(ctx context.Context, o Options) error {
var updates *releaseupdate.Service
var extensionChannels []*channelInstance
if cfg.Options.StateStore.Target != "" {
stateStore, closeClient, dialErr := staterpc.Dial(cfg.Options.StateStore.Target, presence.Messaging, cfg.Options.StateStore.Token, staterpc.WaitForPeer)
stateStore, closeClient, dialErr := staterpc.Dial(cfg.Options.StateStore.Target, presence.Messaging, cfg.Identity, staterpc.WaitForPeer)
if dialErr != nil {
return fmt.Errorf("dial state store control plane (%s): %w", cfg.Options.StateStore.Target, dialErr)
}
@@ -112,18 +113,18 @@ func Run(ctx context.Context, o Options) error {
if err != nil {
return err
}
return serve(ctx, cfg.Options, srv, log, feed)
return serve(ctx, cfg.Options.Listen, cfg.Identity, srv, log, feed)
}
// serve runs srv and serves MessagingService over it until ctx is cancelled.
func serve(ctx context.Context, o Options, srv *Service, log *slog.Logger, feed *logging.Feed) error {
opts, err := messagingrpc.ServerOptions(o.Listen, o.Token)
func serve(ctx context.Context, listen string, id *rpctls.Identity, srv *Service, log *slog.Logger, feed *logging.Feed) error {
opts, err := messagingrpc.ServerOptions(listen, id)
if err != nil {
return err
}
listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", o.Listen)
listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", listen)
if err != nil {
return fmt.Errorf("listen messaging (%s): %w", o.Listen, err)
return fmt.Errorf("listen messaging (%s): %w", listen, err)
}
server := grpc.NewServer(opts...)
messagingrpc.RegisterServer(server, srv, feed)
@@ -133,7 +134,7 @@ func serve(ctx context.Context, o Options, srv *Service, log *slog.Logger, feed
}
}()
defer server.GracefulStop()
log.Info("messaging service listening", "listen", o.Listen)
log.Info("messaging service listening", "listen", listen)
return srv.Start(ctx)
}
+15 -25
View File
@@ -8,13 +8,16 @@ import (
"os"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/webui"
)
// Resolve returns the effective Options: flags first, then the config file's
// projection, then defaults, and validates the result.
func Resolve(o Options, log *slog.Logger) (Options, error) {
var tls config.TLS
if o.Config != "" {
cfg, found, err := readProjection(o.Config, o.Overlay, log)
if err != nil {
@@ -22,10 +25,15 @@ func Resolve(o Options, log *slog.Logger) (Options, error) {
}
if found {
o = merge(o, cfg)
tls = cfg.tls
}
}
o = withDefaults(o)
o = withEnvTokens(o)
id, err := rpctls.Load(tls.CAFile, tls.CertDir, presence.UI)
if err != nil {
return Options{}, err
}
o.identity = id
token, err := resolveToken(o)
if err != nil {
return Options{}, err
@@ -60,6 +68,7 @@ type projection struct {
state ServiceTarget
messaging ServiceTarget
harness ServiceTarget
tls config.TLS
capture CaptureOptions
}
@@ -67,10 +76,11 @@ func project(cfg config.Config) projection {
return projection{
listen: cfg.Web.Listen,
trustForwardedHeaders: cfg.Web.TrustForwardedHeaders,
gateway: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameGateway).Target, Token: cfg.Services.Get(config.ServiceNameGateway).TargetToken},
messaging: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameMessaging).Target, Token: cfg.Services.Get(config.ServiceNameMessaging).TargetToken},
state: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameState).Target, Token: cfg.Services.Get(config.ServiceNameState).TargetToken},
harness: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameHarness).Target, Token: cfg.Services.Get(config.ServiceNameHarness).TargetToken},
gateway: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameGateway).Target},
messaging: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameMessaging).Target},
state: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameState).Target},
harness: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameHarness).Target},
tls: cfg.TLS,
capture: CaptureOptions{
Retention: cfg.Capture.Retention.Std(),
MaxEvents: cfg.Capture.MaxEvents,
@@ -81,23 +91,6 @@ func project(cfg config.Config) projection {
}
}
// withEnvTokens fills service tokens left unset from the environment.
func withEnvTokens(o Options) Options {
if o.Gateway.Token == "" {
o.Gateway.Token = os.Getenv("GATEWAY_TOKEN")
}
if o.State.Token == "" {
o.State.Token = os.Getenv("STATE_STORE_TOKEN")
}
if o.Messaging.Token == "" {
o.Messaging.Token = os.Getenv("MESSAGING_TOKEN")
}
if o.Harness.Token == "" {
o.Harness.Token = os.Getenv("HARNESS_TOKEN")
}
return o
}
// merge fills only the fields the flags left empty.
func merge(o Options, p projection) Options {
if o.Listen == "" {
@@ -187,8 +180,5 @@ func mergeTarget(o, p ServiceTarget) ServiceTarget {
if o.Target == "" {
o.Target = p.Target
}
if o.Token == "" {
o.Token = p.Token
}
return o
}
+1 -1
View File
@@ -23,7 +23,7 @@ func wireServiceLogs(opts Options, srv *webui.Server, tasks *staterpc.Client, ch
return chat.RecentLogs(ctx, presence.Daemon, q)
},
}
messaging, closeMessaging, dialErr := messagingrpc.Dial(opts.Messaging.Target, opts.Messaging.Token)
messaging, closeMessaging, dialErr := messagingrpc.Dial(opts.Messaging.Target, opts.identity)
if dialErr != nil {
return nil, dialErr
}
+6 -4
View File
@@ -6,6 +6,7 @@ import (
"time"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
)
// Process defaults. Listen matches the dashboard's historical bind
@@ -18,12 +19,9 @@ const (
defaultShutdownTimeout = 5 * time.Second
)
// ServiceTarget is one remote contract's endpoint and the service-to-service
// bearer token this process presents to it. It is an endpoint reference, not
// a resolved daemon credential: the UI authenticates each client itself.
// ServiceTarget is one remote contract's endpoint.
type ServiceTarget struct {
Target string
Token string
}
// Options is the UI-owned input DTO. It carries process-local settings and
@@ -83,6 +81,10 @@ type Options struct {
// ReadHeaderTimeout and ShutdownTimeout bound the HTTP lifecycle.
ReadHeaderTimeout time.Duration
ShutdownTimeout time.Duration
// identity is this process's certificate, loaded by Resolve from
// [tls]; nil is plaintext, loopback only.
identity *rpctls.Identity
}
// CaptureOptions carries the [capture] settings the capture receiver needs.
+3 -3
View File
@@ -58,12 +58,12 @@ func Run(ctx context.Context, options Options) error {
cleanup()
}
}()
tasks, closeState, err := staterpc.Dial(opts.State.Target, presence.UI, opts.State.Token)
tasks, closeState, err := staterpc.Dial(opts.State.Target, presence.UI, opts.identity)
if err != nil {
return err
}
cleanups = append(cleanups, closeState)
chat, closeGateway, err := gatewayrpc.Dial(opts.Gateway.Target, opts.Gateway.Token)
chat, closeGateway, err := gatewayrpc.Dial(opts.Gateway.Target, opts.identity)
if err != nil {
return err
}
@@ -146,7 +146,7 @@ func wireHarness(opts Options, srv *webui.Server, cleanups *[]func()) error {
if opts.Harness.Target == "" {
return nil
}
harness, closeHarness, err := harnessrpc.Dial(opts.Harness.Target, opts.Harness.Token)
harness, closeHarness, err := harnessrpc.Dial(opts.Harness.Target, opts.identity)
if err != nil {
return err
}
+16 -9
View File
@@ -2,20 +2,27 @@ package servicekit
import (
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
"github.com/samcharles93/archie-core/internal/secret"
)
// Identity loads service's certificate from [tls]; nil means plaintext,
// loopback only.
func Identity(cfg config.TLS, service string) (*rpctls.Identity, error) {
return rpctls.Load(cfg.CAFile, cfg.CertDir, service)
}
// StateStoreClient dials the State Store gRPC service as the caller service.
// Token resolution is the daemon's
// (it owns the config keys and the secret registry); the dial itself and the
// fail-closed non-loopback rule belong to the transport, so they come
// from staterpc.Dial. Calls wait for the State Store rather than fail while it
// is down, so boot order between services does not matter.
func StateStoreClient(caller string, services config.Services, secrets *secret.Registry) (*staterpc.Client, func(), error) {
target, err := services.RequireTarget(config.ServiceNameState)
// Calls wait for the State Store rather than fail while it is down, so boot
// order between services does not matter.
func StateStoreClient(caller string, cfg config.Config) (*staterpc.Client, func(), error) {
target, err := cfg.Services.RequireTarget(config.ServiceNameState)
if err != nil {
return nil, nil, err
}
return staterpc.Dial(target, caller, services.ResolvedToken(config.ServiceNameState, secrets.Getenv), staterpc.WaitForPeer)
id, err := Identity(cfg.TLS, caller)
if err != nil {
return nil, nil, err
}
return staterpc.Dial(target, caller, id, staterpc.WaitForPeer)
}
+20 -55
View File
@@ -42,13 +42,8 @@ import (
type Options struct {
Config string
Overlay string
// Listen is the gRPC listen address. Loopback binds (127.0.0.1, ::1,
// localhost) serve insecure with no token; any non-loopback address
// requires a non-empty token (or TLS), else the process fails closed
// Listen is the gRPC listen address. Without [tls] it must be loopback.
Listen string
// Token is the bearer token the State Store server validates for a
// non-loopback listener. Empty falls back to [services.state].target_token.
Token string
// ReadyAddr, when non-empty, starts an HTTP readiness surface on the
// address: GET /healthz (liveness) and GET /health/detailed (the
// state_db probe). Empty disables it.
@@ -114,13 +109,12 @@ func Run(ctx context.Context, options Options) error { //nolint:cyclop // the co
}
b.reportUnseededResources(skipped)
token := b.instanceToken(options)
grants, err := b.runCredentials(ctx)
if err != nil {
return err
}
//nolint:contextcheck // grpc.StreamServerInterceptor has no context.Context parameter; TaskGrants.StreamInterceptor derives its context from stream.Context() instead
opts, loopback, err := stateStoreServerOpts(listen, token, grants, b.callers())
opts, err := stateStoreServerOpts(listen, b.cfg.TLS, grants, b.callers())
if err != nil {
return err
}
@@ -129,7 +123,7 @@ func Run(ctx context.Context, options Options) error { //nolint:cyclop // the co
return fmt.Errorf("listen for state store: %w", err)
}
defer listener.Close()
b.log.Info("archie-state-store running", "addr", listener.Addr().String(), "token_required", !loopback)
b.log.Info("archie-state-store running", "addr", listener.Addr().String(), "tls", b.cfg.TLS.CertDir != "")
if err := b.startOptionalSurfaces(ctx, options); err != nil {
return err
@@ -383,15 +377,6 @@ func (b *server) executionDeps(deps *staterpc.Deps) {
}
}
// instanceToken is the bearer token a non-loopback listener requires: the
// flag, else [services.state].target_token.
func (b *server) instanceToken(options Options) string {
if options.Token != "" {
return options.Token
}
return b.cfg.Services.ResolvedToken(config.ServiceNameState, b.secrets.Getenv)
}
// runCredentials builds the run-credential check over this store and drops
// the credentials that expired while nothing revoked them.
func (b *server) runCredentials(ctx context.Context) (*staterpc.TaskGrants, error) {
@@ -507,48 +492,28 @@ func (b *server) startOptionalSurfaces(ctx context.Context, options Options) err
return nil
}
// stateStoreServerOpts applies the transport security boundary: a
// loopback listener is served insecure with no token; any non-loopback
// address requires a bearer token, else the process fails closed. It cannot
// enforce TLS/mTLS (an operator decision), so it narrows the non-loopback
// path to token auth -- the same confinement philosophy as the gateway's
// loopback-only --listen rule.
// stateStoreServerOpts serves listen with this process's [tls] certificate. A client
// certificate is optional here alone: a task container has none and is
// confined by its grant. Authentication runs before attribution.
//
// The keepalive enforcement policy is not part of that boundary and is
// installed on both listeners: a loopback State Store serves the same dialers
// as a remote one (staterpc.Dial installs the client keepalive on every target,
// loopback included), and grpc's default policy would answer their idle
// watches with GOAWAY too_many_pings. See staterpc.ServerKeepaliveOption.
func stateStoreServerOpts(listen, token string, grants *staterpc.TaskGrants, callers rpcidentity.Callers) (opts []grpc.ServerOption, loopback bool, err error) {
loopback, err = staterpc.TargetIsLoopback(listen)
// The keepalive enforcement policy matches staterpc.Dial's clients, whose
// idle watches grpc's default policy would answer with GOAWAY
// too_many_pings. See staterpc.ServerKeepaliveOption.
func stateStoreServerOpts(listen string, tls config.TLS, grants *staterpc.TaskGrants, callers rpcidentity.Callers) ([]grpc.ServerOption, error) {
id, err := servicekit.Identity(tls, presence.StateStore)
if err != nil {
return nil, false, err
return nil, err
}
keepalive := staterpc.ServerKeepaliveOption()
if loopback {
return []grpc.ServerOption{
keepalive,
grpc.ChainUnaryInterceptor(callers.Unary()),
grpc.ChainStreamInterceptor(callers.Stream()),
}, true, nil
creds, err := id.ServerOption(listen, true)
if err != nil {
return nil, fmt.Errorf("state store: %w", err)
}
if token == "" {
return nil, false, fmt.Errorf(
"state store listen address %q is non-loopback; non-loopback exposure requires a bearer token (--token / [services.state].target_token) or TLS, mirroring the gateway's --listen confinement (docs/prds/state-store-contract.md §9)",
listen,
)
}
// grants.UnaryInterceptor/StreamInterceptor separate the daemon's own
// administrative token (full access) from a container's task-scoped
// grant (Update/Transition/InsertEvent on its own task ID only), rather
// than the single all-or-nothing token check this replaced.
return []grpc.ServerOption{
keepalive,
// Authentication runs before attribution: callers trusts the metadata
// only of a call the grants check admitted as the instance.
grpc.ChainUnaryInterceptor(grants.UnaryInterceptor(token), callers.Unary()),
grpc.ChainStreamInterceptor(grants.StreamInterceptor(token), callers.Stream()),
}, false, nil
creds,
staterpc.ServerKeepaliveOption(),
grpc.ChainUnaryInterceptor(grants.UnaryInterceptor(), callers.Unary()),
grpc.ChainStreamInterceptor(grants.StreamInterceptor(), callers.Stream()),
}, nil
}
// newPackageProjections builds the family projectors an installed package's
+3 -1
View File
@@ -311,7 +311,9 @@ type ToolsConfig struct {
// Config is the daemon configuration.
type Config struct {
Services Services `toml:"services" yaml:"services"`
WorkDir string `toml:"work_dir" yaml:"work_dir"`
// TLS is the certificate every gRPC hop uses. Restart-required.
TLS TLS `toml:"tls" yaml:"tls"`
WorkDir string `toml:"work_dir" yaml:"work_dir"`
// SkillsDir is an optional directory of */SKILL.md skills. Its workflows
// override built-ins, and Kits mount it read-only.
SkillsDir string `toml:"skills_dir" yaml:"skills_dir"`
+10 -17
View File
@@ -16,19 +16,6 @@ func (s Services) Get(name string) ServiceConnection {
return s[name]
}
// ResolvedToken returns name's target_token, or the service's registered
// environment variable.
func (s Services) ResolvedToken(name string, getenv func(string) string) string {
if token := s[name].TargetToken; token != "" {
return token
}
spec, ok := LookupService(name)
if !ok || spec.TokenEnv == "" {
return ""
}
return getenv(spec.TokenEnv)
}
// RequireTarget returns name's dial address, or an error naming the config
// key.
func (s Services) RequireTarget(name string) (string, error) {
@@ -39,13 +26,19 @@ func (s Services) RequireTarget(name string) (string, error) {
return target, nil
}
// ServiceConnection is a service's gRPC addresses and client token.
// ServiceConnection is a service's gRPC addresses.
// Restart-required.
type ServiceConnection struct {
Target string `toml:"target" yaml:"target"`
// Listen is the address the owning process binds. Always defaulted.
Listen string `toml:"listen" yaml:"listen"`
// TargetToken is the bearer token clients present to a non-loopback target.
// Empty means none.
TargetToken string `toml:"target_token" yaml:"target_token"`
}
// TLS locates the operator-issued certificates. A process named <service>
// loads cert_dir/<service>.pem and cert_dir/<service>-key.pem, whose DNS name
// is <service>, and trusts ca_file for every peer. Both empty means plaintext,
// which is confined to loopback addresses.
type TLS struct {
CAFile string `toml:"ca_file" yaml:"ca_file"`
CertDir string `toml:"cert_dir" yaml:"cert_dir"`
}
+11 -16
View File
@@ -19,8 +19,7 @@ const (
ServiceBoth = "both"
)
// ServiceSpec declares one service: name, defaults and token environment
// variable.
// ServiceSpec declares one service: name and default addresses.
type ServiceSpec struct {
// Context is ServiceClient, ServiceServer, or ServiceBoth.
Context string
@@ -35,9 +34,6 @@ type ServiceSpec struct {
// reaches net.Listen as "any free port", silently moving the service
// off the address its clients dial.
Listen string
// TokenEnv names the environment variable a client falls back to when
// [services.<name>].target_token is empty.
TokenEnv string
}
// Registered service names. They live beside the registrations below so the
@@ -57,7 +53,7 @@ var (
// RegisterService declares one service, replacing any existing spec of that
// name. It panics on an invalid registration.
func RegisterService(context, name, target, listen, tokenEnv string) {
func RegisterService(context, name, target, listen string) {
switch context {
case ServiceClient, ServiceServer, ServiceBoth:
default:
@@ -76,11 +72,10 @@ func RegisterService(context, name, target, listen, tokenEnv string) {
serviceRegistryMu.Lock()
defer serviceRegistryMu.Unlock()
serviceRegistry[name] = ServiceSpec{
Context: context,
Name: name,
Target: target,
Listen: listen,
TokenEnv: tokenEnv,
Context: context,
Name: name,
Target: target,
Listen: listen,
}
}
@@ -112,13 +107,13 @@ func init() {
// The services this repository ships. Everything that differs
// between them is an argument here, not a branch somewhere else: the
// gateway's target is defaulted while the State Store's is supplied by
// the operator, and each falls back to its own token variable.
RegisterService(ServiceBoth, ServiceNameGateway, "127.0.0.1:8585", "127.0.0.1:8585", "GATEWAY_TOKEN")
RegisterService(ServiceClient, ServiceNameState, "", "127.0.0.1:9090", "STATE_STORE_TOKEN")
RegisterService(ServiceBoth, ServiceNameMessaging, "127.0.0.1:8586", "127.0.0.1:8586", "MESSAGING_TOKEN")
// the operator.
RegisterService(ServiceBoth, ServiceNameGateway, "127.0.0.1:8585", "127.0.0.1:8585")
RegisterService(ServiceClient, ServiceNameState, "", "127.0.0.1:9090")
RegisterService(ServiceBoth, ServiceNameMessaging, "127.0.0.1:8586", "127.0.0.1:8586")
// The daemon hosts the harness session contract in this repository; the
// dashboard dials it, and its target is the operator's to supply, so a
// deployment with no [services.harness].target leaves the setup terminal
// unwired rather than dialling a default nothing owns.
RegisterService(ServiceBoth, ServiceNameHarness, "", "127.0.0.1:8587", "HARNESS_TOKEN")
RegisterService(ServiceBoth, ServiceNameHarness, "", "127.0.0.1:8587")
}
+7 -4
View File
@@ -77,11 +77,11 @@ func (e NATSEndpoint) ContainerToken() string {
return e.Token
}
// StateStoreEndpoint is the State Store target and token passed to agent
// containers.
// StateStoreEndpoint is the State Store target and the CA its certificate
// chains to, both passed to agent containers.
type StateStoreEndpoint struct {
URL string
Token string
URL string
CA []byte
}
type Daemon struct {
@@ -2263,6 +2263,9 @@ func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken string) []str
// daemon's own.
if d.ConnectedStateStore.URL != "" {
env = append(env, "STATE_STORE_URL="+d.ConnectedStateStore.URL)
if len(d.ConnectedStateStore.CA) > 0 {
env = append(env, "STATE_STORE_CA="+string(d.ConnectedStateStore.CA))
}
if stateStoreToken != "" {
env = append(env, "STATE_STORE_TOKEN="+stateStoreToken)
}
+1 -1
View File
@@ -47,7 +47,7 @@ func (d *Daemon) runKitTask(ctx context.Context, task *workflow.Task, repo confi
defer revokeCredential()
env, err := kitrun.WorkerEnv(kitrun.Endpoints{
NATS: d.ConnectedNATS.URL, NATSToken: d.ConnectedNATS.ContainerToken(),
StateStore: d.ConnectedStateStore.URL, StateStoreToken: credential,
StateStore: d.ConnectedStateStore.URL, StateStoreToken: credential, StateStoreCA: d.ConnectedStateStore.CA,
}, os.Getuid(), os.Getgid())
if err != nil {
park("kit worker environment", err)
@@ -39,6 +39,9 @@ type Config struct {
StateStoreURL string
StateStoreToken string
// StateStoreCA is the PEM CA the State Store's certificate chains to.
// Empty dials plaintext, which only a loopback target allows.
StateStoreCA []byte
}
const (
@@ -96,7 +99,7 @@ func Connect(ctx context.Context, config Config, log *slog.Logger) (*Transport,
conn.Close()
return nil, fmt.Errorf("state store target is required: archie-agent no longer supports the legacy NATS storerpc path (docs/prds/state-store-contract.md §12 step 4)")
}
state, closeState, err := staterpc.Dial(config.StateStoreURL, "archie-agent", config.StateStoreToken)
state, closeState, err := staterpc.DialTask(config.StateStoreURL, config.StateStoreCA, config.StateStoreToken)
if err != nil {
conn.Close()
return nil, fmt.Errorf("connect state store: %w", err)
@@ -114,6 +114,7 @@ func (h *Host) Start(ctx context.Context, spec Spec, surface string, impl goplug
Cmd: cmd,
AllowedProtocols: []goplugin.Protocol{goplugin.ProtocolGRPC},
SkipHostEnv: true,
AutoMTLS: true,
Logger: h.log.Named(spec.Name),
})}
rpc, err := p.client.Client()
+6 -82
View File
@@ -2,94 +2,18 @@ package gatewayrpc
import (
"context"
"crypto/subtle"
"net"
"strings"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity"
)
// tokenMetadataKey is the gRPC metadata key the Gateway Bearer [REDACTED]
// travels in. Metadata, never a URL, mirroring the State Store's
// token lifecycle.
const tokenMetadataKey = "gateway-token"
// TargetIsLoopback reports whether addr's host is a loopback IP or
// "localhost".
func TargetIsLoopback(addr string) (bool, error) {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return false, err
}
if strings.EqualFold(host, "localhost") {
return true, nil
}
ip := net.ParseIP(host)
if ip == nil {
return false, nil
}
return ip.IsLoopback(), nil
// unaryPrincipal forwards the context's principal on every unary call.
func unaryPrincipal(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
return invoker(rpcidentity.Outgoing(ctx, "gateway-client"), method, req, reply, cc, opts...)
}
// validToken compares candidate against the configured token in constant
// time so a timing side channel cannot leak how many bytes matched.
func validToken(token, candidate string) bool {
if token == "" || candidate == "" {
return false
}
return subtle.ConstantTimeCompare([]byte(token), []byte(candidate)) == 1
}
// UnaryServerInterceptor validates the Bearer [REDACTED] carried in each unary
// call's metadata, returning codes.Unauthenticated on a missing or wrong
// token. It is installed only for a non-loopback listener; a loopback
// listener serves with no interceptors.
func UnaryServerInterceptor(token string) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return nil, status.Error(codes.Unauthenticated, "missing gateway token")
}
values := md.Get(tokenMetadataKey)
if len(values) == 0 || !validToken(token, values[0]) {
return nil, status.Error(codes.Unauthenticated, "missing or invalid gateway token")
}
return handler(ctx, req)
}
}
// StreamServerInterceptor is the streaming counterpart of
// UnaryServerInterceptor: the chat Stream RPC is server-streaming, so its
// credential must be checked on the stream context as well.
func StreamServerInterceptor(token string) grpc.StreamServerInterceptor {
return func(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
md, _ := metadata.FromIncomingContext(stream.Context())
values := md.Get(tokenMetadataKey)
if len(values) == 0 || !validToken(token, values[0]) {
return status.Error(codes.Unauthenticated, "missing or invalid gateway token")
}
return handler(srv, stream)
}
}
// UnaryClientTokenInterceptor attaches token to every outgoing unary call's
// metadata under tokenMetadataKey.
func UnaryClientTokenInterceptor(token string) grpc.UnaryClientInterceptor {
return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
return invoker(metadata.AppendToOutgoingContext(rpcidentity.Outgoing(ctx, "gateway-client"), tokenMetadataKey, token), method, req, reply, cc, opts...)
}
}
// StreamClientTokenInterceptor attaches token to the outgoing stream's
// context, covering the chat Stream RPC that the unary interceptor cannot
// see.
func StreamClientTokenInterceptor(token string) grpc.StreamClientInterceptor {
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
return streamer(metadata.AppendToOutgoingContext(rpcidentity.Outgoing(ctx, "gateway-client"), tokenMetadataKey, token), desc, cc, method, opts...)
}
// streamPrincipal forwards the context's principal on every stream.
func streamPrincipal(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
return streamer(rpcidentity.Outgoing(ctx, "gateway-client"), desc, cc, method, opts...)
}
+12 -20
View File
@@ -5,34 +5,26 @@ import (
"strings"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
)
// Dial returns a Gateway chat client for target. A loopback target dials
// insecure; any other target requires a token. The cleanup closes the
// connection.
func Dial(target, token string, options ...grpc.DialOption) (*Client, func(), error) {
// Dial returns a Gateway chat client for target, authenticated by id. The
// cleanup closes the connection.
func Dial(target string, id *rpctls.Identity, options ...grpc.DialOption) (*Client, func(), error) {
target = strings.TrimSpace(target)
if target == "" {
return nil, nil, fmt.Errorf("gateway target is required")
}
loopback, err := TargetIsLoopback(target)
creds, err := id.DialOption(target, presence.Gateway)
if err != nil {
return nil, nil, fmt.Errorf("gateway target must be host:port: %w", err)
return nil, nil, fmt.Errorf("gateway: %w", err)
}
if !loopback && token == "" {
return nil, nil, fmt.Errorf(
"gateway target %q is non-loopback; non-loopback exposure requires a Bearer [REDACTED] ([services.gateway].target_token / GATEWAY_TOKEN) or TLS",
target,
)
}
opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
if token != "" {
opts = append(
opts,
grpc.WithUnaryInterceptor(UnaryClientTokenInterceptor(token)),
grpc.WithStreamInterceptor(StreamClientTokenInterceptor(token)),
)
opts := []grpc.DialOption{
creds,
grpc.WithUnaryInterceptor(unaryPrincipal),
grpc.WithStreamInterceptor(streamPrincipal),
}
conn, err := grpc.NewClient(target, append(opts, options...)...)
if err != nil {
@@ -6,7 +6,6 @@ package harnessrpc
import (
"context"
"crypto/subtle"
"errors"
"fmt"
"io"
@@ -14,43 +13,33 @@ import (
"sync"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
pb "github.com/samcharles93/archie-core/internal/contracts/harness/v1"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
)
// tokenMetadataKey is the gRPC metadata key the harness service token travels
// in, mirroring the Gateway's and State Store's token keys.
const tokenMetadataKey = "harness-token"
// Client opens sessions against a daemon's HarnessService.
type Client struct {
svc pb.HarnessServiceClient
}
// Dial returns a harness client for target. The token is required for every
// target, loopback included: the daemon's listener enforces it because a
// session is a shell in a container that holds an org's credential. The
// cleanup closes the connection.
func Dial(target, token string, options ...grpc.DialOption) (*Client, func(), error) {
// Dial returns a harness client for target. A session is a shell in a
// container that holds an org's credential, so it needs a service
// certificate even on loopback. The cleanup closes the connection.
func Dial(target string, id *rpctls.Identity, options ...grpc.DialOption) (*Client, func(), error) {
target = strings.TrimSpace(target)
if target == "" {
return nil, nil, errors.New("harness target is required")
}
if token == "" {
return nil, nil, fmt.Errorf(
"harness target %q reads a setup session, so it needs a bearer token ([services.harness].target_token / HARNESS_TOKEN)",
target,
)
if id == nil {
return nil, nil, errors.New("harness sessions need [tls] ca_file and cert_dir")
}
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithStreamInterceptor(streamClientToken(token)),
creds, err := id.DialOption(target, presence.Daemon)
if err != nil {
return nil, nil, fmt.Errorf("harness: %w", err)
}
conn, err := grpc.NewClient(target, append(opts, options...)...)
conn, err := grpc.NewClient(target, append([]grpc.DialOption{creds}, options...)...)
if err != nil {
return nil, nil, fmt.Errorf("create harness client: %w", err)
}
@@ -169,40 +158,3 @@ func (s *session) Close() error {
s.cancel()
return nil
}
// validToken compares candidate against the configured token in constant
// time.
func validToken(token, candidate string) bool {
if token == "" || candidate == "" {
return false
}
return subtle.ConstantTimeCompare([]byte(token), []byte(candidate)) == 1
}
// StreamServerInterceptor validates the harness token on a stream. The
// session RPC is bidi, so this is the interceptor that matters. Unlike the
// State Store, a loopback listener enforces it too: a session is a shell in a
// container that holds an org's credential.
func StreamServerInterceptor(token string) grpc.StreamServerInterceptor {
return func(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
if !validContextToken(stream.Context(), token) {
return status.Error(codes.Unauthenticated, "missing or invalid harness token")
}
return handler(srv, stream)
}
}
func validContextToken(ctx context.Context, token string) bool {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return false
}
values := md.Get(tokenMetadataKey)
return len(values) > 0 && validToken(token, values[0])
}
func streamClientToken(token string) grpc.StreamClientInterceptor {
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
return streamer(metadata.AppendToOutgoingContext(ctx, tokenMetadataKey, token), desc, cc, method, opts...)
}
}
+4
View File
@@ -17,6 +17,7 @@ type Endpoints struct {
NATSToken string
StateStore string
StateStoreToken string
StateStoreCA []byte
}
// WorkerEnv is the worker's entire environment in a Kit container: NATS and
@@ -36,6 +37,9 @@ func WorkerEnv(e Endpoints, uid, gid int) ([]string, error) {
if e.StateStoreToken != "" {
env = append(env, "STATE_STORE_TOKEN="+e.StateStoreToken)
}
if len(e.StateStoreCA) > 0 {
env = append(env, "STATE_STORE_CA="+string(e.StateStoreCA))
}
}
return append(env, "WORKTREE_UID="+strconv.Itoa(uid), "WORKTREE_GID="+strconv.Itoa(gid)), nil
}
@@ -5,24 +5,20 @@ package messagingrpc
import (
"context"
"crypto/subtle"
"errors"
"fmt"
"strings"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
pb "github.com/samcharles93/archie-core/internal/contracts/messaging/v1"
"github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/logging"
)
const tokenMetadataKey = "messaging-token"
// ErrUnavailable marks a delivery the channel could not make now: the channel
// is unknown, stopped or cannot send. The caller may retry later.
var ErrUnavailable = errors.New("messaging: channel unavailable")
@@ -58,52 +54,26 @@ func (s *server) Deliver(ctx context.Context, req *pb.DeliverRequest) (*pb.Deliv
}
}
// ServerOptions serves a loopback listen address without credentials and
// requires token on any other, failing closed when it is empty.
func ServerOptions(listen, token string) ([]grpc.ServerOption, error) {
loopback, err := gatewayrpc.TargetIsLoopback(listen)
// ServerOptions serves listen with id's credentials.
func ServerOptions(listen string, id *rpctls.Identity) ([]grpc.ServerOption, error) {
creds, err := id.ServerOption(listen, false)
if err != nil {
return nil, err
}
if loopback {
return nil, nil
}
if token == "" {
return nil, fmt.Errorf("messaging listen address %q is non-loopback and requires a bearer token", listen)
}
return []grpc.ServerOption{grpc.ChainUnaryInterceptor(requireToken(token))}, nil
}
func requireToken(token string) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
values := metadata.ValueFromIncomingContext(ctx, tokenMetadataKey)
if len(values) == 0 || subtle.ConstantTimeCompare([]byte(token), []byte(values[0])) != 1 {
return nil, status.Error(codes.Unauthenticated, "missing or invalid messaging token")
}
return handler(ctx, req)
return nil, fmt.Errorf("messaging: %w", err)
}
return []grpc.ServerOption{creds}, nil
}
// Client delivers through a dialled MessagingService.
type Client struct{ rpc pb.MessagingServiceClient }
// Dial returns a client for target. A non-loopback target requires token.
func Dial(target, token string) (*Client, func(), error) {
// Dial returns a client for target, authenticated by id.
func Dial(target string, id *rpctls.Identity) (*Client, func(), error) {
target = strings.TrimSpace(target)
loopback, err := gatewayrpc.TargetIsLoopback(target)
creds, err := id.DialOption(target, presence.Messaging)
if err != nil {
return nil, nil, fmt.Errorf("messaging target must be host:port: %w", err)
return nil, nil, fmt.Errorf("messaging: %w", err)
}
if !loopback && token == "" {
return nil, nil, fmt.Errorf("messaging target %q is non-loopback and requires a bearer token ([services.messaging].target_token / MESSAGING_TOKEN)", target)
}
opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
if token != "" {
opts = append(opts, grpc.WithUnaryInterceptor(func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
return invoker(metadata.AppendToOutgoingContext(ctx, tokenMetadataKey, token), method, req, reply, cc, opts...)
}))
}
conn, err := grpc.NewClient(target, opts...)
conn, err := grpc.NewClient(target, creds)
if err != nil {
return nil, nil, fmt.Errorf("create messaging client: %w", err)
}
@@ -1,36 +0,0 @@
package messagingrpc
import (
"context"
"testing"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
)
func TestRequireToken(t *testing.T) {
ok := func(context.Context, any) (any, error) { return "served", nil }
for _, tc := range []struct {
name string
md metadata.MD
want codes.Code
}{
{"valid", metadata.Pairs(tokenMetadataKey, "secret"), codes.OK},
{"wrong", metadata.Pairs(tokenMetadataKey, "secrex"), codes.Unauthenticated},
{"missing", metadata.MD{}, codes.Unauthenticated},
{"other service's key", metadata.Pairs("gateway-token", "secret"), codes.Unauthenticated},
} {
t.Run(tc.name, func(t *testing.T) {
ctx := metadata.NewIncomingContext(context.Background(), tc.md)
_, err := requireToken("secret")(ctx, nil, &grpc.UnaryServerInfo{}, ok)
if got := status.Code(err); got != tc.want {
t.Fatalf("code = %v, want %v", got, tc.want)
}
})
}
if _, err := ServerOptions("0.0.0.0:8586", ""); err == nil {
t.Fatal("non-loopback listen with no token was accepted")
}
}
@@ -13,6 +13,7 @@ import (
"github.com/samcharles93/archie-core/internal/domain/identity"
"github.com/samcharles93/archie-core/internal/domain/org"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
)
// CallerKey names the service making the call.
@@ -34,10 +35,10 @@ func Outgoing(ctx context.Context, caller string) context.Context {
return metadata.NewOutgoingContext(ctx, md)
}
// Callers resolves who a call is made by. It runs after authentication,
// which is what makes the metadata trustworthy: only holders of the instance
// credential reach it with their metadata intact (a task grant's is
// stripped). A call naming a principal acts in, and reads only, that principal's
// Callers resolves who a call is made by. It runs after authentication:
// the caller is the verified certificate's service, else (a plaintext
// loopback peer) the metadata's claim, and a task grant's metadata is
// stripped before it gets here. A call naming a principal acts in, and reads only, that principal's
// org; any other acts in the default org.
type Callers struct {
Principals access.PrincipalSource
@@ -45,7 +46,10 @@ type Callers struct {
func (c Callers) resolve(ctx context.Context) (context.Context, error) {
md, _ := metadata.FromIncomingContext(ctx)
caller := first(md, CallerKey)
caller := rpctls.PeerService(ctx)
if caller == "" {
caller = first(md, CallerKey)
}
if caller != "" {
ctx = access.WithActor(access.WithSource(ctx, caller), caller)
}
+204
View File
@@ -0,0 +1,204 @@
// Package rpctls builds the transport credentials every gRPC hop uses from
// certificate files the operator's PKI issues. With no identity configured a
// process speaks plaintext, and only over loopback.
package rpctls
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"strings"
"sync"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/peer"
)
// Identity is one process's certificate and the CA it trusts. A nil Identity
// is the plaintext loopback mode.
type Identity struct {
pool *x509.CertPool
caPEM []byte
cert *certFile
}
// Load reads the CA at caFile and service's key pair from
// certDir/<service>.pem and certDir/<service>-key.pem. Both paths empty means
// no TLS and returns nil.
func Load(caFile, certDir, service string) (*Identity, error) {
if caFile == "" && certDir == "" {
return nil, nil
}
if caFile == "" || certDir == "" {
return nil, errors.New("tls: ca_file and cert_dir are both required")
}
caPEM, err := os.ReadFile(caFile)
if err != nil {
return nil, fmt.Errorf("tls: read CA: %w", err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(caPEM) {
return nil, fmt.Errorf("tls: %s holds no PEM certificate", caFile)
}
cert := &certFile{
certPath: filepath.Join(certDir, service+".pem"),
keyPath: filepath.Join(certDir, service+"-key.pem"),
}
if _, err := cert.get(); err != nil {
return nil, err
}
return &Identity{pool: pool, caPEM: caPEM, cert: cert}, nil
}
// CAPEM is the CA certificate a container receives to verify the State
// Store. Nil in plaintext mode.
func (i *Identity) CAPEM() []byte {
if i == nil {
return nil
}
return i.caPEM
}
// ServerOption returns the credentials for a listener on listen. A listener
// that accepts calls without a client certificate (the State Store's task
// grants) passes optionalClient; every other listener requires one.
func (i *Identity) ServerOption(listen string, optionalClient bool) (grpc.ServerOption, error) {
if i == nil {
if err := requireLoopback(listen); err != nil {
return nil, err
}
return grpc.Creds(insecure.NewCredentials()), nil
}
auth := tls.RequireAndVerifyClientCert
if optionalClient {
auth = tls.VerifyClientCertIfGiven
}
return grpc.Creds(credentials.NewTLS(&tls.Config{
MinVersion: tls.VersionTLS13,
GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return i.cert.get() },
ClientCAs: i.pool,
ClientAuth: auth,
})), nil
}
// DialOption returns the credentials for dialling server (its certificate
// name) at target. The name, not target's host, is verified, so a service is
// reachable through any address, relay or proxy.
func (i *Identity) DialOption(target, server string) (grpc.DialOption, error) {
if i == nil {
if err := requireLoopback(target); err != nil {
return nil, err
}
return grpc.WithTransportCredentials(insecure.NewCredentials()), nil
}
return grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
MinVersion: tls.VersionTLS13,
RootCAs: i.pool,
ServerName: server,
GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { return i.cert.get() },
})), nil
}
// CAOnlyDialOption dials server trusting caPEM and presenting no
// certificate: a container's credentials are its task grant alone. Empty
// caPEM is plaintext, loopback only.
func CAOnlyDialOption(caPEM []byte, target, server string) (grpc.DialOption, error) {
if len(caPEM) == 0 {
var none *Identity
return none.DialOption(target, server)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(caPEM) {
return nil, errors.New("tls: CA holds no PEM certificate")
}
return grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
MinVersion: tls.VersionTLS13,
RootCAs: pool,
ServerName: server,
})), nil
}
// PeerService is the service named by the caller's verified certificate, or
// "" when it presented none.
func PeerService(ctx context.Context) string {
p, ok := peer.FromContext(ctx)
if !ok {
return ""
}
info, ok := p.AuthInfo.(credentials.TLSInfo)
if !ok || len(info.State.VerifiedChains) == 0 || len(info.State.VerifiedChains[0]) == 0 {
return ""
}
if names := info.State.VerifiedChains[0][0].DNSNames; len(names) > 0 {
return names[0]
}
return ""
}
// Trusted reports whether the caller is a service: it holds a verified
// certificate, or reached a plaintext listener, which is loopback only.
func Trusted(ctx context.Context) bool {
p, ok := peer.FromContext(ctx)
if !ok {
return false
}
if _, isTLS := p.AuthInfo.(credentials.TLSInfo); !isTLS {
return true
}
return PeerService(ctx) != ""
}
func requireLoopback(addr string) error {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return fmt.Errorf("address %q must be host:port: %w", addr, err)
}
if strings.EqualFold(host, "localhost") {
return nil
}
if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() {
return nil
}
return fmt.Errorf("%q is not loopback; configure [tls] ca_file and cert_dir to serve or dial it", addr)
}
// certFile reloads a key pair when its certificate file changes, so the
// operator's renewal tooling needs no restart.
type certFile struct {
certPath, keyPath string
mu sync.Mutex
modified time.Time
cert *tls.Certificate
}
func (c *certFile) get() (*tls.Certificate, error) {
info, err := os.Stat(c.certPath)
if err != nil {
return nil, fmt.Errorf("tls: %w", err)
}
c.mu.Lock()
defer c.mu.Unlock()
if c.cert != nil && info.ModTime().Equal(c.modified) {
return c.cert, nil
}
cert, err := tls.LoadX509KeyPair(c.certPath, c.keyPath)
if err != nil {
if c.cert != nil {
// A renewal caught between writing the two files keeps the
// previous pair until both match.
return c.cert, nil
}
return nil, fmt.Errorf("tls: load %s: %w", c.certPath, err)
}
c.cert, c.modified = &cert, info.ModTime()
return c.cert, nil
}
@@ -0,0 +1,171 @@
package rpctls
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"math/big"
"net"
"os"
"path/filepath"
"testing"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/health"
healthpb "google.golang.org/grpc/health/grpc_health_v1"
"google.golang.org/grpc/test/bufconn"
)
// TestHandshake pins the transport boundary: who may connect, and the
// identity the server sees.
func TestHandshake(t *testing.T) {
dir := t.TempDir()
ca := newCA(t, dir, "ca")
foreign := newCA(t, dir, "foreign")
ca.issue(t, dir, "archie-state-store")
ca.issue(t, dir, "archied")
foreignDir := filepath.Join(dir, "foreign-certs")
foreign.issue(t, foreignDir, "archied")
server := load(t, ca.path, dir, "archie-state-store")
daemon := load(t, ca.path, dir, "archied")
impostor := load(t, ca.path, foreignDir, "archied")
tests := []struct {
name string
optionalClient bool
dial func(target string) (grpc.DialOption, error)
wantService string
wantTrusted bool
wantErr bool
}{
{"service certificate names the caller", false, func(tg string) (grpc.DialOption, error) { return daemon.DialOption(tg, "archie-state-store") }, "archied", true, false},
{"foreign CA is refused", false, func(tg string) (grpc.DialOption, error) { return impostor.DialOption(tg, "archie-state-store") }, "", false, true},
{"no certificate where one is required", false, func(tg string) (grpc.DialOption, error) { return CAOnlyDialOption(ca.pem, tg, "archie-state-store") }, "", false, true},
{"no certificate where it is optional is untrusted", true, func(tg string) (grpc.DialOption, error) { return CAOnlyDialOption(ca.pem, tg, "archie-state-store") }, "", false, false},
{"wrong server name is refused", false, func(tg string) (grpc.DialOption, error) { return daemon.DialOption(tg, "archie-gateway") }, "", false, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
creds, err := server.ServerOption("127.0.0.1:0", tt.optionalClient)
if err != nil {
t.Fatal(err)
}
var gotService string
var gotTrusted bool
srv := grpc.NewServer(creds, grpc.UnaryInterceptor(func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, h grpc.UnaryHandler) (any, error) {
gotService, gotTrusted = PeerService(ctx), Trusted(ctx)
return h(ctx, req)
}))
healthpb.RegisterHealthServer(srv, health.NewServer())
lis := bufconn.Listen(1 << 16)
go func() { _ = srv.Serve(lis) }()
t.Cleanup(srv.Stop)
dialCreds, err := tt.dial("127.0.0.1:1")
if err != nil {
t.Fatal(err)
}
conn, err := grpc.NewClient("passthrough:///bufnet", dialCreds,
grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return lis.DialContext(ctx) }))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = conn.Close() })
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
_, err = healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{})
if (err != nil) != tt.wantErr {
t.Fatalf("err = %v, wantErr %v", err, tt.wantErr)
}
if gotService != tt.wantService || gotTrusted != tt.wantTrusted {
t.Fatalf("service %q trusted %v, want %q %v", gotService, gotTrusted, tt.wantService, tt.wantTrusted)
}
})
}
var plaintext *Identity
if _, err := plaintext.DialOption("10.0.0.1:9090", "archie-state-store"); err == nil {
t.Fatal("plaintext dial to a non-loopback target was allowed")
}
if _, err := plaintext.ServerOption("0.0.0.0:9090", false); err == nil {
t.Fatal("plaintext listener on a non-loopback address was allowed")
}
}
type testCA struct {
cert *x509.Certificate
key *ecdsa.PrivateKey
pem []byte
path string
}
func newCA(t *testing.T, dir, name string) testCA {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: name},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour),
IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign,
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
cert, _ := x509.ParseCertificate(der)
p := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
path := filepath.Join(dir, name+"-ca.pem")
write(t, path, p)
return testCA{cert: cert, key: key, pem: p, path: path}
}
func (c testCA) issue(t *testing.T, dir, service string) {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: service}, DNSNames: []string{service},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, c.cert, &key.PublicKey, c.key)
if err != nil {
t.Fatal(err)
}
keyDER, err := x509.MarshalECPrivateKey(key)
if err != nil {
t.Fatal(err)
}
write(t, filepath.Join(dir, service+".pem"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
write(t, filepath.Join(dir, service+"-key.pem"), pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}))
}
func load(t *testing.T, caFile, certDir, service string) *Identity {
t.Helper()
id, err := Load(caFile, certDir, service)
if err != nil {
t.Fatal(err)
}
return id
}
func write(t *testing.T, path string, data []byte) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, data, 0o600); err != nil {
t.Fatal(err)
}
}
+5 -5
View File
@@ -9,7 +9,7 @@ import (
"github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity"
)
// tokenMetadataKey is the gRPC metadata key the bearer token travels in.
// tokenMetadataKey is the gRPC metadata key a task grant travels in.
// Metadata, never a URL.
const tokenMetadataKey = "state-store-token"
@@ -21,16 +21,16 @@ func outgoing(ctx context.Context, caller, token string) context.Context {
return ctx
}
// UnaryClientCredentialInterceptor attaches caller and token to every
// outgoing call.
// UnaryClientCredentialInterceptor attaches caller and any task grant to
// every outgoing call.
func UnaryClientCredentialInterceptor(caller, token string) grpc.UnaryClientInterceptor {
return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
return invoker(outgoing(ctx, caller, token), method, req, reply, cc, opts...)
}
}
// StreamClientCredentialInterceptor attaches caller and token to every
// outgoing stream.
// StreamClientCredentialInterceptor attaches caller and any task grant to
// every outgoing stream.
func StreamClientCredentialInterceptor(caller, token string) grpc.StreamClientInterceptor {
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
return streamer(outgoing(ctx, caller, token), desc, cc, method, opts...)
@@ -7,7 +7,9 @@ import (
"time"
"google.golang.org/grpc"
grpccreds "google.golang.org/grpc/credentials"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/peer"
pb "github.com/samcharles93/archie-core/internal/contracts/state/v1"
"github.com/samcharles93/archie-core/internal/domain/access"
@@ -45,10 +47,10 @@ func (p principalsByID) PrincipalFor(_ context.Context, id identity.IdentityID)
return access.Principal{IdentityID: id, Org: p[id]}, nil
}
// TestCallerAttribution pins who a call acts as: the instance credential's
// metadata is trusted, a task grant's is not.
// TestCallerAttribution pins who a call acts as: a trusted service's
// metadata is trusted, a task grant's is not, and a caller with neither a
// certificate nor a grant is refused.
func TestCallerAttribution(t *testing.T) {
const admin = "admin-token"
taskToken := strings.Repeat("ab", 32)
grants := &TaskGrants{Store: credentials{}}
if err := grants.register(context.Background(), 7, taskToken, time.Hour); err != nil {
@@ -56,20 +58,27 @@ func TestCallerAttribution(t *testing.T) {
}
callers := rpcidentity.Callers{Principals: principalsByID{"alice": "acme"}}
update := &grpc.UnaryServerInfo{FullMethod: pb.StateStoreService_Update_FullMethodName}
// A plaintext peer reached a loopback listener; a TLS peer without a
// verified chain presented no client certificate.
plaintext := &peer.Peer{}
noCert := &peer.Peer{AuthInfo: grpccreds.TLSInfo{}}
tests := []struct {
name string
peer *peer.Peer
md []string
wantActor string
wantOrg org.OrgID
wantErr bool
}{
{"service alone acts in the default org", []string{tokenMetadataKey, admin, rpcidentity.CallerKey, "archied"}, "archied", org.DefaultOrgID},
{"forwarded principal sets actor and org", []string{tokenMetadataKey, admin, rpcidentity.CallerKey, "archie-ui", rpcidentity.PrincipalKey, "alice"}, "alice", "acme"},
{"task grant cannot claim a principal", []string{tokenMetadataKey, taskToken, rpcidentity.CallerKey, "archie-ui", rpcidentity.PrincipalKey, "alice"}, "task/7", org.DefaultOrgID},
{"service alone acts in the default org", plaintext, []string{rpcidentity.CallerKey, "archied"}, "archied", org.DefaultOrgID, false},
{"forwarded principal sets actor and org", plaintext, []string{rpcidentity.CallerKey, "archie-ui", rpcidentity.PrincipalKey, "alice"}, "alice", "acme", false},
{"task grant cannot claim a principal", noCert, []string{tokenMetadataKey, taskToken, rpcidentity.CallerKey, "archie-ui", rpcidentity.PrincipalKey, "alice"}, "task/7", org.DefaultOrgID, false},
{"no certificate and no grant is refused", noCert, []string{rpcidentity.CallerKey, "archied"}, "", "", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs(tt.md...))
ctx := peer.NewContext(metadata.NewIncomingContext(context.Background(), metadata.Pairs(tt.md...)), tt.peer)
var gotActor string
var gotOrg org.OrgID
handler := func(ctx context.Context, _ any) (any, error) {
@@ -80,8 +89,9 @@ func TestCallerAttribution(t *testing.T) {
return callers.Unary()(ctx, req, update, handler)
}
req := &pb.UpdateRequest{Task: &pb.Task{Id: 7}}
if _, err := grants.UnaryInterceptor(admin)(ctx, req, update, chained); err != nil {
t.Fatal(err)
_, err := grants.UnaryInterceptor()(ctx, req, update, chained)
if (err != nil) != tt.wantErr {
t.Fatalf("err = %v, wantErr %v", err, tt.wantErr)
}
if gotActor != tt.wantActor || gotOrg != tt.wantOrg {
t.Fatalf("got actor %q org %q, want %q %q", gotActor, gotOrg, tt.wantActor, tt.wantOrg)
+23 -35
View File
@@ -2,45 +2,50 @@ package staterpc
import (
"fmt"
"net"
"strings"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/keepalive"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
)
// Dial returns a State Store client for target, attributed to the caller
// service. A loopback target dials insecure; any other target requires a
// token. The cleanup closes the connection.
func Dial(target, caller, token string, options ...grpc.DialOption) (*Client, func(), error) {
// service and authenticated by id. The cleanup closes the connection.
func Dial(target, caller string, id *rpctls.Identity, options ...grpc.DialOption) (*Client, func(), error) {
target = strings.TrimSpace(target)
if target == "" {
return nil, nil, fmt.Errorf("state store target is required")
}
loopback, err := TargetIsLoopback(target)
creds, err := id.DialOption(target, presence.StateStore)
if err != nil {
return nil, nil, fmt.Errorf("state store target must be host:port: %w", err)
return nil, nil, fmt.Errorf("state store: %w", err)
}
if !loopback && token == "" {
return nil, nil, fmt.Errorf(
"state store target %q is non-loopback; non-loopback exposure requires a bearer token ([services.state].target_token / STATE_STORE_TOKEN) or TLS, mirroring the gateway's --listen confinement (docs/prds/state-store-contract.md §9)",
target,
)
return dial(target, creds, caller, "", options)
}
// DialTask returns the client a task container uses: it trusts caPEM and
// presents only the task grant, never a service certificate.
func DialTask(target string, caPEM []byte, grant string) (*Client, func(), error) {
creds, err := rpctls.CAOnlyDialOption(caPEM, target, presence.StateStore)
if err != nil {
return nil, nil, fmt.Errorf("state store: %w", err)
}
return dial(target, creds, presence.Agent, grant, nil)
}
func dial(target string, creds grpc.DialOption, caller, grant string, options []grpc.DialOption) (*Client, func(), error) {
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()),
creds,
// Keepalive detects a dead peer on an otherwise silent watch stream.
grpc.WithKeepaliveParams(ClientKeepaliveParams()),
// GetInstalledPackage returns the whole layer, which for an extension is
// a plugin binary well past gRPC's default 4 MiB receive limit.
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxPackageResponseBytes)),
// Both call shapes need the credential: the server's interceptors
// guard unary RPCs and the streaming capture reads separately, so a
// unary-only interceptor leaves the streams unauthenticated.
grpc.WithUnaryInterceptor(UnaryClientCredentialInterceptor(caller, token)),
grpc.WithStreamInterceptor(StreamClientCredentialInterceptor(caller, token)),
grpc.WithUnaryInterceptor(UnaryClientCredentialInterceptor(caller, grant)),
grpc.WithStreamInterceptor(StreamClientCredentialInterceptor(caller, grant)),
}
conn, err := grpc.NewClient(target, append(opts, options...)...)
if err != nil {
@@ -82,23 +87,6 @@ func ServerKeepalivePolicy() keepalive.EnforcementPolicy {
}
}
// TargetIsLoopback reports whether addr's host is a loopback IP or
// "localhost".
func TargetIsLoopback(addr string) (bool, error) {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return false, fmt.Errorf("address must be host:port: %w", err)
}
if strings.EqualFold(host, "localhost") {
return true, nil
}
ip := net.ParseIP(host)
if ip == nil {
return false, nil
}
return ip.IsLoopback(), nil
}
// WaitForPeer makes calls to the State Store wait for it to come up
// instead of failing while it is down or restarting. A call that must not
// wait carries its own deadline.
@@ -8,13 +8,15 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
grpccreds "google.golang.org/grpc/credentials"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
pb "github.com/samcharles93/archie-core/internal/contracts/state/v1"
)
func TestServiceLogsRequireInstanceCredential(t *testing.T) {
func TestServiceLogsRequireService(t *testing.T) {
run := strings.Repeat("ab", 32)
grants := &TaskGrants{Store: credentials{}}
if err := grants.register(t.Context(), 7, run, time.Hour); err != nil {
@@ -22,15 +24,16 @@ func TestServiceLogsRequireInstanceCredential(t *testing.T) {
}
for _, tt := range []struct {
name, token string
peer *peer.Peer
code codes.Code
}{
{"instance", "admin", codes.OK},
{"task cannot read other services", run, codes.PermissionDenied},
{"anonymous", "", codes.Unauthenticated},
{"service", "", &peer.Peer{}, codes.OK},
{"task cannot read other services", run, &peer.Peer{AuthInfo: grpccreds.TLSInfo{}}, codes.PermissionDenied},
{"anonymous", "", &peer.Peer{AuthInfo: grpccreds.TLSInfo{}}, codes.Unauthenticated},
} {
t.Run(tt.name, func(t *testing.T) {
ctx := metadata.NewIncomingContext(t.Context(), metadata.Pairs(tokenMetadataKey, tt.token))
_, err := grants.UnaryInterceptor("admin")(ctx, &pb.RecentLogsRequest{}, &grpc.UnaryServerInfo{FullMethod: pb.StateStoreService_RecentLogs_FullMethodName}, func(context.Context, any) (any, error) { return &pb.RecentLogsResponse{}, nil })
ctx := peer.NewContext(metadata.NewIncomingContext(t.Context(), metadata.Pairs(tokenMetadataKey, tt.token)), tt.peer)
_, err := grants.UnaryInterceptor()(ctx, &pb.RecentLogsRequest{}, &grpc.UnaryServerInfo{FullMethod: pb.StateStoreService_RecentLogs_FullMethodName}, func(context.Context, any) (any, error) { return &pb.RecentLogsResponse{}, nil })
if status.Code(err) != tt.code {
t.Fatalf("code %s want %s", status.Code(err), tt.code)
}
+13 -17
View File
@@ -4,7 +4,6 @@ import (
"context"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/hex"
"errors"
"fmt"
@@ -23,6 +22,7 @@ import (
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
)
// TaskGrants verifies run credentials: the one token a task's container
@@ -158,24 +158,25 @@ var taskScopedTargets = map[string]func(any) int64{
},
}
// UnaryInterceptor separates administrator access from task-scoped worker
// access. A tokenless listener is permitted only on loopback by composition.
// Supplied worker tokens are still validated on loopback.
func (g *TaskGrants) UnaryInterceptor(adminToken string) grpc.UnaryServerInterceptor {
// UnaryInterceptor separates services from task-scoped workers. A trusted
// caller (a verified service certificate, or a plaintext peer, which only a
// loopback listener admits) has full access; any other call needs a task
// grant and is confined to its task.
func (g *TaskGrants) UnaryInterceptor() grpc.UnaryServerInterceptor {
return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
token := requestToken(ctx)
if (adminToken == "" && token == "") || (adminToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(adminToken)) == 1) {
if rpctls.Trusted(ctx) {
return handler(ctx, req)
}
token := requestToken(ctx)
if token == "" {
return nil, status.Error(codes.Unauthenticated, "missing state store token")
return nil, status.Error(codes.Unauthenticated, "missing client certificate or task grant")
}
taskID, err := g.taskFor(ctx, token)
if err != nil {
return nil, status.Error(codes.Unavailable, "run credentials unavailable")
}
if taskID == 0 {
return nil, status.Error(codes.Unauthenticated, "invalid state store token")
return nil, status.Error(codes.Unauthenticated, "invalid task grant")
}
if !authorizesTaskScopedCall(info.FullMethod, req, taskID) {
return nil, status.Error(codes.PermissionDenied, "task grant does not authorize this operation")
@@ -202,15 +203,10 @@ func dropDefinitionPin(req any) {
r.Task.WorkflowDefinitionYaml = ""
}
// StreamInterceptor admits only administrators to store read streams.
func (g *TaskGrants) StreamInterceptor(adminToken string) grpc.StreamServerInterceptor {
// StreamInterceptor admits only services to store read streams.
func (g *TaskGrants) StreamInterceptor() grpc.StreamServerInterceptor {
return func(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
md, _ := metadata.FromIncomingContext(stream.Context())
tokens := md.Get(tokenMetadataKey)
if adminToken == "" && len(tokens) == 0 {
return handler(srv, stream)
}
if len(tokens) > 0 && adminToken != "" && subtle.ConstantTimeCompare([]byte(tokens[0]), []byte(adminToken)) == 1 {
if rpctls.Trusted(stream.Context()) {
return handler(srv, stream)
}
return status.Error(codes.PermissionDenied, "task grant does not authorize streaming reads")
@@ -7,7 +7,9 @@ import (
"testing"
"google.golang.org/grpc"
grpccreds "google.golang.org/grpc/credentials"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/peer"
pb "github.com/samcharles93/archie-core/internal/contracts/state/v1"
)
@@ -19,16 +21,17 @@ import (
func TestTaskCredentialCannotReplaceItsPin(t *testing.T) {
taskToken := strings.Repeat("ab", 32)
grants := &TaskGrants{Store: credentials{sha256.Sum256([]byte(taskToken)): 7}}
intercept := grants.UnaryInterceptor("admin")
intercept := grants.UnaryInterceptor()
tests := []struct {
name, token, wantYAML string
peer *peer.Peer
}{
{"task credential", taskToken, ""},
{"daemon credential", "admin", "yaml-b"},
{"task credential", taskToken, "", &peer.Peer{AuthInfo: grpccreds.TLSInfo{}}},
{"daemon", "", "yaml-b", &peer.Peer{}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs(tokenMetadataKey, tt.token))
ctx := peer.NewContext(metadata.NewIncomingContext(context.Background(), metadata.Pairs(tokenMetadataKey, tt.token)), tt.peer)
req := &pb.UpdateRequest{Task: &pb.Task{
Id: 7, Workflow: "handoff-target",
WorkflowDefinitionVersion: 4, WorkflowDefinitionDigest: "digest-b", WorkflowDefinitionYaml: "yaml-b",