feat(nats): expose embedded broker to managed workers

This commit is contained in:
sam committed 2026-08-24 14:56:57 +10:00
1 parent 7b5dca3c42
commit 1b7bc004b6
16 files changed
+678 -178

No files matched your search

+21 -15
View File
@@ -100,6 +100,9 @@ type boot struct {
// the embedded server's ClientURL(). Recorded so Daemon.ConnectedNATS
// carries the real endpoint rather than an empty config URL.
natsURL string
// natsToken is the credential the daemon's client connected with at
// startup. Embedded mode generates it; external mode resolves it once.
natsToken string
containerPool *container.Pool
storeBackend storage.Backend
@@ -376,7 +379,12 @@ func (b *boot) connectNATS(ctx context.Context) error {
// Embedded. Start the in-process server before dialing, and register
// its shutdown BEFORE the client close below so the client closes
// first (cleanups run LIFO).
host := ""
if b.containerPool != nil {
host = b.containerPool.HostGateway()
}
srv, err := nats.StartEmbedded(ctx, nats.EmbeddedOptions{
Host: host,
StoreDir: filepath.Join(filepath.Dir(cfg.DBPath), "nats"),
}, log)
if err != nil {
@@ -385,6 +393,7 @@ func (b *boot) connectNATS(ctx context.Context) error {
}
b.addCleanup(func() { srv.Shutdown() })
url = srv.ClientURL()
natsToken = srv.Token()
log.Info("embedded nats started", "url", url)
}
@@ -402,6 +411,7 @@ func (b *boot) connectNATS(ctx context.Context) error {
}
b.natsClient = natsClient
b.natsURL = url
b.natsToken = natsToken
b.addCleanup(func() { natsClient.Close() })
log.Info("nats connected", "url", url)
return nil
@@ -414,15 +424,15 @@ func (b *boot) connectNATS(ctx context.Context) error {
// feature they may not be exercising right now. Under a systemd unit
// with Restart=on-failure it also turns a recoverable problem into a
// crash loop with the real error scrolling past unread.
func (b *boot) setupContainers(ctx context.Context) {
func (b *boot) setupContainers(ctx context.Context) func() {
containerPool, storeBackend, closeDocker := startContainers(ctx, b.cfg, b.log)
b.containerPool = containerPool
b.storeBackend = storeBackend
b.addCleanup(closeDocker)
// Sandboxing that was asked for and could not start must not silently
// become no sandboxing: the daemon stays up, but tasks park rather than
// running the agent loop on the host.
b.sandboxRequired = b.cfg.Containers.Enabled && containerPool == nil
return closeDocker
}
// setupLLMAndChat wires the runtime, tool registry, model management,
@@ -995,16 +1005,9 @@ func (b *boot) registerMinimaxTool(cfg config.Config, log *slog.Logger) {
// it shares with it.
func (b *boot) buildDaemon() {
cfg, log := b.cfg, b.log
// ConnectedNATS must carry the endpoint the daemon's own client connected
// with at startup, not the raw config: for embedded mode the config URL is
// empty, but container env must never see an empty NATS_URL. (Containers
// are rejected in embedded mode by validation, but the field should mean
// the same thing regardless of mode.)
connectedNATS := cfg.NATS
connectedNATS.URL = b.natsURL
b.d = &daemon.Daemon{
Cfg: config.NewHolder(cfg),
ConnectedNATS: connectedNATS,
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken},
Store: b.st,
Bus: b.bus,
Forge: b.forgeClient,
@@ -1323,11 +1326,14 @@ func (b *boot) startServices(ctx context.Context) error {
// setupBackends wires the optional runtime backends: the NATS client
// and the container pool.
func (b *boot) setupBackends(ctx context.Context) error {
if err := b.connectNATS(ctx); err != nil {
return err
}
b.setupContainers(ctx)
return nil
closeContainers := b.setupContainers(ctx)
err := b.connectNATS(ctx)
// Container discovery must precede embedded NATS so it can bind the
// worker bridge gateway, but cleanup registration comes afterwards.
// cleanups run LIFO: workers stop first, then the daemon client closes,
// then the embedded server shuts down.
b.addCleanup(closeContainers)
return err
}
// buildAgentAndWorkflows builds the agent runner and the workflow
+152
View File
@@ -0,0 +1,152 @@
package main
import (
"encoding/json"
"fmt"
"go/ast"
"go/parser"
"go/token"
"log/slog"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"github.com/moby/moby/client"
natsio "github.com/nats-io/nats.go"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/container"
)
// setupContainers must discover the bridge before embedded NATS starts, but
// its cleanup registration comes afterwards. boot.cleanup is LIFO, so this
// ordering stops worker containers before closing the daemon client and the
// embedded server they are connected to.
func TestSetupBackendsRegistersContainerCleanupAfterNATS(t *testing.T) {
fileset := token.NewFileSet()
file, err := parser.ParseFile(fileset, "bootstrap.go", nil, 0)
if err != nil {
t.Fatal(err)
}
setupContainers := methodBody(t, file, "setupContainers")
if methodCallPosition(setupContainers, "addCleanup") != token.NoPos {
t.Fatal("setupContainers registers cleanup before NATS; LIFO would stop the broker before workers")
}
setupBackends := methodBody(t, file, "setupBackends")
connect := methodCallPosition(setupBackends, "connectNATS")
register := methodCallPosition(setupBackends, "addCleanup")
if connect == token.NoPos || register == token.NoPos || register < connect {
t.Fatalf("setupBackends call positions: connectNATS=%d addCleanup=%d, want container cleanup registered after connect", connect, register)
}
}
func methodBody(t *testing.T, file *ast.File, name string) *ast.BlockStmt {
t.Helper()
for _, decl := range file.Decls {
fn, ok := decl.(*ast.FuncDecl)
if ok && fn.Recv != nil && fn.Name.Name == name {
return fn.Body
}
}
t.Fatalf("method %s not found", name)
return nil
}
func methodCallPosition(body *ast.BlockStmt, name string) token.Pos {
position := token.NoPos
ast.Inspect(body, func(node ast.Node) bool {
call, ok := node.(*ast.CallExpr)
if !ok {
return true
}
selector, ok := call.Fun.(*ast.SelectorExpr)
if ok && selector.Sel.Name == name && position == token.NoPos {
position = call.Pos()
}
return true
})
return position
}
// This is the native-daemon deployment seam: Docker supplies the host-side
// gateway of the worker bridge, embedded NATS binds that address, and the
// runtime credential shared with workers is required by the same listener.
func TestConnectEmbeddedNATSUsesManagedWorkerBridge(t *testing.T) {
const gateway = "127.0.0.1"
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/networks/workers"):
writeDockerResponse(t, w, map[string]any{
"Name": "workers",
"Driver": "bridge",
"Scope": "local",
"IPAM": map[string]any{
"Config": []map[string]string{{
"Subnet": "127.0.0.0/8",
"Gateway": gateway,
}},
},
})
case strings.HasSuffix(r.URL.Path, "/containers/json"):
writeDockerResponse(t, w, []any{})
default:
http.Error(w, fmt.Sprintf("unexpected Docker API path %s", r.URL.Path), http.StatusNotFound)
}
}))
t.Cleanup(dockerAPI.Close)
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = dockerClient.Close() })
pool, err := container.NewPool(t.Context(), container.Config{
DockerClient: dockerClient,
Network: "workers",
RequireHostGateway: true,
}, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatalf("container.NewPool = %v", err)
}
t.Cleanup(func() { _ = pool.Close() })
b := &boot{
cfg: config.Config{
DBPath: filepath.Join(t.TempDir(), "archie.db"),
NATS: config.NATSConfig{Mode: config.NATSModeEmbedded},
},
log: slog.New(slog.DiscardHandler),
containerPool: pool,
}
if err := b.connectNATS(t.Context()); err != nil {
t.Fatalf("connectNATS = %v", err)
}
t.Cleanup(b.cleanup)
if !strings.HasPrefix(b.natsURL, "nats://"+gateway+":") {
t.Errorf("connected NATS URL = %q, want bridge gateway %s", b.natsURL, gateway)
}
if b.natsToken == "" {
t.Fatal("connected NATS token is empty")
}
if unauthenticated, connectErr := natsio.Connect(b.natsURL); connectErr == nil {
unauthenticated.Close()
t.Fatal("unauthenticated worker connection succeeded")
}
authenticated, err := natsio.Connect(b.natsURL, natsio.Token(b.natsToken))
if err != nil {
t.Fatalf("authenticated worker connection = %v", err)
}
authenticated.Close()
}
func writeDockerResponse(t *testing.T, w http.ResponseWriter, value any) {
t.Helper()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(value); err != nil {
t.Errorf("encode Docker response: %v", err)
}
}
+8 -7
View File
@@ -979,13 +979,14 @@ func startContainers(
}
pool, err := container.NewPool(ctx, container.Config{
Image: cfg.Containers.Image,
MaxConcurrency: cfg.Containers.MaxConcurrency,
MaxUptime: cfg.Containers.MaxUptime.Std(),
PullPolicy: cfg.Containers.PullPolicy,
Network: cfg.Containers.Network,
DockerClient: dockerCli,
}, cfg.NATS.URL, log)
Image: cfg.Containers.Image,
MaxConcurrency: cfg.Containers.MaxConcurrency,
MaxUptime: cfg.Containers.MaxUptime.Std(),
PullPolicy: cfg.Containers.PullPolicy,
Network: cfg.Containers.Network,
DockerClient: dockerCli,
RequireHostGateway: cfg.NATS.Mode == config.NATSModeEmbedded,
}, log)
if err != nil {
// A missing image is recoverable by hand. The daemon sends no registry
// credentials on pull (internal/container/pool.go), so a private
+4 -3
View File
@@ -231,14 +231,14 @@ gate_max_failures = 5
# NATS JetStream for task distribution and reaction delivery. mode selects
# how NATS is provided:
# "embedded" – in-process nats-server (the default; no external server)
# "external" – connect to url (required for agent.mode="nats" / [containers])
# "external" – connect to url
# "off" – no NATS; the SQLite ClaimNext flow is used for tasks
# Leave the section absent (or set no url) to run embedded; set mode="off"
# only to opt out of NATS entirely.
# [nats]
# mode = "embedded" # "embedded" | "external" | "off"
# url = "nats://localhost:4222" # required when mode = "external"
# token_env = "NATS_TOKEN" # optional; named variable must be set
# token_env = "NATS_TOKEN" # optional external-server credential; embedded generates one per start
# Unbound webhook capture: POST /webhooks/capture/{source} accepts and
# stores any inbound payload archie doesn't yet have a binding for, so it can
@@ -255,7 +255,8 @@ gate_max_failures = 5
# rate_per_second = 1.0 # per-remote-address token bucket
# rate_burst = 5
# Docker sandbox execution of archie-agent. Requires [nats] and agent.mode = "nats".
# Docker sandbox execution of archie-agent. Requires NATS and agent.mode = "nats";
# embedded and external broker modes are both supported.
# [containers]
# enabled = true
# image = "archie-agent:latest" # Docker image to run
+61 -71
View File
@@ -1,86 +1,76 @@
# Embedded NATS -- decision
# Embedded NATS execution -- decision
**Status:** Decided, not yet implemented
**Date:** 2026-08-22
**Beads issue:** `archie-core-7d5u.2`
**Status:** Amended; implementation in progress
`event-sources-and-reactions.md` settled the reaction-delivery mechanism (reuse
`internal/infrastructure/eventbus/nats`, a second stream under a fan-out
retention policy) and left the deployment shape to this issue. This document
decides that shape. `github.com/nats-io/nats-server/v2` is already a direct
dependency, used today only in `_test.go` files; the gap is that production
startup only knows "external URL configured" vs "no NATS, use SQLite
`ClaimNext`".
**Date:** 2026-08-24
## 1. Embedded runs alongside SQLite, not instead of it
**Beads epic:** `archie-core-q15y`
The SQLite store remains authoritative for task state. NATS -- external or
embedded -- is the *distribution* path, and `drainNATS` already falls back to
`Store.ClaimNext` for requeued tasks (waiting_human approval, retry-parked)
that never went through a NATS publish. So wiring embedded NATS does not
remove the SQLite flow; it makes `Daemon.Tasks != nil` true in single-process
deployments, which routes poll→enqueue through `pollNATS` and drain through
`drainNATS` exactly as an external-NATS deployment already does. `ClaimNext`
stays as the requeue fallback inside `drainNATS`. No new drain path, no second
eventbus type.
This decision supersedes the 2026-08-22 position that embedded NATS was only a
distribution mechanism and could not be an agent-execution topology. That was
true only because the embedded listener was unauthenticated loopback. It left a
second production architecture in which `archied` ran autonomous workflows in
its own process, weakening the isolation boundary that managed task containers
exist to provide.
## 2. Config: `[nats] mode = "embedded" | "external" | "off"`
## One execution architecture, two broker deployments
`NATSConfig` gains a `mode` field. `url` no longer doubles as the on/off
switch -- that coupling is what made "no NATS" a permanent stance instead of a
deployment shape.
Every autonomous repository workflow is handed over as one complete task via
NATS and runs in a task-scoped `archie-agent` container. `archied` remains a
native, long-lived orchestration and interactive-chat process; its operator
configured host filesystem access is unchanged. It owns task admission,
state, forge and worktree operations, container lifecycle, and the broker when
the embedded deployment is selected. It does not run an autonomous model loop.
| mode | meaning |
|------------|----------------------------------------------------------------|
| `external` | connect to `url` (today's behaviour). `url` is required. |
| `embedded` | start an in-process nats-server; the client connects to it. |
| `off` | no NATS at all; the old SQLite-only flow (`ClaimNext`). |
NATS has two supported deployment shapes:
Default resolution: `url` set → `external`; `url` empty and `mode` unset or
`embedded` → `embedded`; `mode = "off"` → `off`. `mode = "external"` with no
`url` is a validation error; `mode = "off"`/`"embedded"` with a `url` set is a
validation error. This is a meaning change and is stated as one: `url` empty
*used to mean* "no NATS, SQLite only"; it *now means* "embedded NATS", and
getting the old behaviour requires the explicit opt-out `mode = "off"`.
| mode | meaning |
| --- | --- |
| `embedded` | `archied` owns an in-process JetStream server (default). |
| `external` | `archied` and workers connect to the configured NATS URL. |
`agent.mode = "nats"` and `[containers]` require `mode = "external"` (a
container cannot reach an in-process loopback server, and embedded NATS is not
an agent-execution topology).
Both shapes have identical task semantics: one container and one full-task
request per workflow. Broker placement must not select a different executor.
The legacy `nats.mode = "off"`, `agent.mode = "inprocess"`, per-stage NATS, and
subprocess execution paths are migration inputs only and will be removed rather
than retained as production fallbacks.
## 3. The daemon owns the embedded server's lifecycle
SQLite remains authoritative for task state. NATS is the sole autonomous work
handoff. SQLite recovery may identify work that needs republishing, but it must
not cause `archied` to call `workflow.Run` locally.
`internal/infrastructure/eventbus/nats` gains an `EmbeddedServer` type that
wraps `nats-server/v2/server.Server`, exposing only `ClientURL()` and
`Shutdown()`. The composition root (`cmd/archied/bootstrap.go`) starts it in
`connectNATS` (before dialing the client), binds it to `127.0.0.1` on a random
port, enables JetStream with a store dir under the daemon's data directory
(`<dir of DBPath>/nats`), and registers a cleanup that shuts the server down
*after* the client closes. Cleanup ordering is LIFO: the server shutdown is
registered before the client close, so the client closes first.
## Embedded reachability and authentication
## Retention is a Config field, not a constant
An embedded listener uses a cryptographically random bearer token generated for
each daemon start. The token exists only in process memory and the environment
of managed worker containers; it is never written to config or logs.
`nats.Config` gains `Retention *jetstream.RetentionPolicy`, defaulting to
`jetstream.WorkQueuePolicy` in `withDefaults`, replacing the hardcoded value in
`Connect` (`client.go:65`). `WorkQueuePolicy` is correct for `ARCHIE_TASKS`
(one consumer claims each message) and wrong for reactions (fan-out). This is
the enabler for the second reaction stream: `event-sources-and-reactions.md`
requires it, and nothing here creates the reaction stream itself -- that is
provisioned by the beads that build reaction producers and consumers, which do
not exist yet. A test proves the parameterization: two overlapping `Subscribe`
consumers both receive a message under `LimitsPolicy` (fan-out) and starve
under `WorkQueuePolicy`, the exact trap `CLAUDE.md` documents.
For a native daemon with managed containers, startup resolves the Docker bridge
the workers will join. The listener binds only to that bridge's host gateway
address on a random port. Both the native daemon and workers use that same
reachable endpoint, and every client supplies the generated token. Binding a
Docker bridge address keeps the listener off ordinary LAN interfaces without
publishing a host port. A configured user bridge is respected; otherwise the
Docker default bridge is used. A network without a usable local IPv4 gateway is
a startup error for autonomous execution.
## Packages this touches
If Docker is unavailable, interactive chat may still start and embedded NATS
may remain loopback-only, but autonomous tasks park with an explicit container
capability failure. They never fall back to host execution.
- `internal/infrastructure/eventbus/nats`: `Retention` on `Config`, an
`EmbeddedServer` type.
- `internal/config`: `NATSConfig.Mode`.
- `internal/infrastructure/configuration`: `mode` defaulting and validation.
- `cmd/archied/bootstrap.go`: start/stop the embedded server; set
`Daemon.ConnectedNATS.URL` to the embedded client URL so container env (if
ever enabled) never receives an empty endpoint.
- `config.example.toml`: document `mode`. The `deployments/*` profiles are
unchanged on purpose: their absent `[nats]` section now selects embedded by
default, which is the intended single-process behaviour this issue exists to
provide, not a regression to paper over with `mode = "off"`.
## Ownership and lifecycle
- `internal/infrastructure/eventbus/nats` owns the authenticated embedded
server and exposes its runtime endpoint, not the server implementation.
- `internal/container` resolves the worker bridge and gateway and owns task
container lifecycle.
- `internal/app` composition selects embedded or external broker deployment,
freezes the connected runtime endpoint for worker environments, and wires the
full-task handoff.
- `cmd/archied` remains process input only as the domain migration reaches this
composition path.
The daemon starts the embedded server before its client, closes clients before
the server, and keeps JetStream data beneath the daemon data directory. NATS URL
and token are startup-built values: a reload cannot send new workers to a broker
different from the one the daemon is using.
+8 -11
View File
@@ -623,8 +623,7 @@ const (
// delivery. The mode selects how NATS is provided:
//
// "embedded" – an in-process nats-server (default; no external server)
// "external" – connect to URL (required; the only mode agent.mode="nats"
// and [containers] can use)
// "external" – connect to URL (required)
// "off" – no NATS; the SQLite ClaimNext flow is used unchanged
//
// An empty Mode resolves from URL: URL set means "external", URL empty means
@@ -637,8 +636,8 @@ type NATSConfig struct {
// URL is the NATS server address, e.g. "nats://localhost:4222".
// Required when Mode is "external"; must be empty otherwise.
URL string `toml:"url" yaml:"url"`
// TokenEnv optionally names an env var holding a NATS auth token.
// Empty means no authentication. Ignored when Mode is "embedded".
// TokenEnv optionally names an env var holding an external NATS auth token.
// Empty means no authentication. Embedded mode generates a per-start token.
TokenEnv string `toml:"token_env" yaml:"token_env"`
}
@@ -685,13 +684,11 @@ type ContainerConfig struct {
VolumeTTL Duration `toml:"volume_ttl" yaml:"volume_ttl"`
// PullPolicy controls image pulling: "missing" (default) or "always".
PullPolicy string `toml:"pull_policy" yaml:"pull_policy"`
// Network is the Docker network spawned agent containers join, so they
// can resolve sibling compose services (nats, etc.) by name. Empty
// falls back to auto-detecting the daemon's own network, which is
// best-effort and silently yields the default bridge network (no
// hostname resolution) if detection fails — set this explicitly in
// compose/production deployments (e.g. "archie-core_default") rather
// than relying on auto-detection.
// Network is the Docker network spawned agent containers join. Empty first
// tries the daemon's own network (for containerized composition), then uses
// Docker's default bridge. Embedded NATS binds the selected bridge's host
// gateway; external deployments can set a Compose network explicitly when
// workers must resolve broker service names (e.g. "archie-core_default").
Network string `toml:"network" yaml:"network"`
}
+44
View File
@@ -0,0 +1,44 @@
package container
import (
"context"
"fmt"
"github.com/moby/moby/client"
)
// NetworkName is the Docker network managed worker containers join.
func (p *Pool) NetworkName() string { return p.network }
// HostGateway is the host-side IPv4 gateway of NetworkName. It is non-empty
// only when Config.RequireHostGateway requested embedded broker reachability.
func (p *Pool) HostGateway() string { return p.hostGateway }
func resolveHostGateway(
ctx context.Context,
cli *client.Client,
networkName string,
required bool,
) (string, error) {
if !required {
return "", nil
}
result, err := cli.NetworkInspect(ctx, networkName, client.NetworkInspectOptions{})
if err != nil {
return "", fmt.Errorf("inspect Docker network %q for embedded NATS: %w", networkName, err)
}
network := result.Network.Network
if network.Driver != "bridge" {
return "", fmt.Errorf("docker network %q uses driver %q, want bridge for embedded NATS", networkName, network.Driver)
}
if network.Scope != "" && network.Scope != "local" {
return "", fmt.Errorf("docker network %q has scope %q, want local for embedded NATS", networkName, network.Scope)
}
for _, ipam := range network.IPAM.Config {
gateway := ipam.Gateway
if gateway.IsValid() && gateway.Is4() && !gateway.IsUnspecified() {
return gateway.String(), nil
}
}
return "", fmt.Errorf("docker network %q has no usable IPv4 host gateway for embedded NATS", networkName)
}
+160
View File
@@ -0,0 +1,160 @@
package container
import (
"encoding/json"
"fmt"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"github.com/moby/moby/client"
)
// A native archied process is not itself attached to a Docker network. When
// embedded NATS must be reachable by managed workers, the pool therefore uses
// Docker's default bridge and exposes its host-side gateway to composition.
func TestNewPoolResolvesDefaultBridgeGatewayForNativeDaemon(t *testing.T) {
hostname, err := os.Hostname()
if err != nil {
t.Fatal(err)
}
const gateway = "172.31.0.1"
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/containers/"+hostname+"/json"):
http.Error(w, `{"message":"no such container"}`, http.StatusNotFound)
case strings.HasSuffix(r.URL.Path, "/networks/bridge"):
writeDockerJSON(t, w, map[string]any{
"Name": "bridge",
"Driver": "bridge",
"Scope": "local",
"IPAM": map[string]any{
"Config": []map[string]string{{
"Subnet": "172.31.0.0/16",
"Gateway": gateway,
}},
},
})
case strings.HasSuffix(r.URL.Path, "/containers/json"):
writeDockerJSON(t, w, []any{})
default:
http.Error(w, fmt.Sprintf("unexpected Docker API path %s", r.URL.Path), http.StatusNotFound)
}
}))
t.Cleanup(dockerAPI.Close)
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = dockerClient.Close() })
pool, err := NewPool(t.Context(), Config{
DockerClient: dockerClient,
RequireHostGateway: true,
}, discardLogger())
if err != nil {
t.Fatalf("NewPool = %v", err)
}
t.Cleanup(func() { _ = pool.Close() })
access, ok := any(pool).(interface {
NetworkName() string
HostGateway() string
})
if !ok {
t.Fatal("Pool does not expose its resolved worker network and host gateway")
}
if got := access.NetworkName(); got != "bridge" {
t.Errorf("NetworkName() = %q, want bridge", got)
}
if got := access.HostGateway(); got != gateway {
t.Errorf("HostGateway() = %q, want %q", got, gateway)
}
}
func TestNewPoolRejectsUnreachableEmbeddedNetwork(t *testing.T) {
tests := []struct {
name string
statusCode int
network map[string]any
}{
{
name: "inspect failure",
statusCode: http.StatusInternalServerError,
},
{
name: "non bridge driver",
network: map[string]any{
"Name": "workers", "Driver": "overlay", "Scope": "local",
"IPAM": map[string]any{"Config": []map[string]string{{"Gateway": "172.31.0.1"}}},
},
},
{
name: "non local scope",
network: map[string]any{
"Name": "workers", "Driver": "bridge", "Scope": "swarm",
"IPAM": map[string]any{"Config": []map[string]string{{"Gateway": "172.31.0.1"}}},
},
},
{
name: "no IPv4 gateway",
network: map[string]any{
"Name": "workers", "Driver": "bridge", "Scope": "local",
"IPAM": map[string]any{"Config": []map[string]string{{"Gateway": "fd00::1"}}},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/networks/workers"):
if tt.statusCode != 0 {
http.Error(w, `{"message":"network inspect failed"}`, tt.statusCode)
return
}
writeDockerJSON(t, w, tt.network)
case strings.HasSuffix(r.URL.Path, "/containers/json"):
writeDockerJSON(t, w, []any{})
default:
http.Error(w, fmt.Sprintf("unexpected Docker API path %s", r.URL.Path), http.StatusNotFound)
}
}))
t.Cleanup(dockerAPI.Close)
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = dockerClient.Close() })
pool, err := NewPool(t.Context(), Config{
DockerClient: dockerClient,
Network: "workers",
RequireHostGateway: true,
}, discardLogger())
if err == nil {
_ = pool.Close()
t.Fatal("NewPool = nil error, want embedded reachability validation failure")
}
})
}
}
func writeDockerJSON(t *testing.T, w http.ResponseWriter, value any) {
t.Helper()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(value); err != nil {
t.Errorf("encode Docker response: %v", err)
}
}
func discardLogger() *slog.Logger {
return slog.New(slog.DiscardHandler)
}
+35 -17
View File
@@ -72,17 +72,19 @@ func WriteTaskJSON(workspace string, payload TaskPayload) error {
// Pool manages a set of Docker containers running archie-agent.
type Pool struct {
cli *client.Client
ownCli bool // true when the pool created the client (owns Close)
cfg Config
log *slog.Logger
natsURL string
// network is the user-defined Docker network spawned containers join,
// detected from the daemon's own container so they can resolve
// sibling services (nats, etc.) by compose service name. Empty when
// the daemon isn't running in a container on such a network, or when
// detection fails -- Docker then falls back to its default bridge.
cli *client.Client
ownCli bool // true when the pool created the client (owns Close)
cfg Config
log *slog.Logger
// network is the Docker network spawned containers join. External broker
// deployments may leave it empty to use Docker's implicit default bridge;
// embedded deployments resolve that default explicitly as "bridge" so its
// host gateway can be inspected.
network string
// hostGateway is the local IPv4 gateway of network. It is populated when
// embedded broker reachability is required, so native composition can bind
// the listener to an address visible only to workers on this bridge.
hostGateway string
mu sync.Mutex
active int
@@ -107,6 +109,9 @@ type Config struct {
// NewPool creates its own via client.New(client.FromEnv). Pass a
// shared client to avoid multiple independent connections.
DockerClient *client.Client
// RequireHostGateway resolves and validates a local Docker bridge gateway.
// Embedded NATS needs this address; external brokers do not.
RequireHostGateway bool
}
// NewPool connects to the Docker daemon, optionally pulls the image,
@@ -114,7 +119,7 @@ type Config struct {
//
// If cfg.DockerClient is set, it is reused (caller owns Close).
// Otherwise a new client is created via client.FromEnv (pool owns Close).
func NewPool(ctx context.Context, cfg Config, natsURL string, log *slog.Logger) (*Pool, error) {
func NewPool(ctx context.Context, cfg Config, log *slog.Logger) (*Pool, error) {
cli := cfg.DockerClient
if cli == nil {
var err error
@@ -128,14 +133,27 @@ func NewPool(ctx context.Context, cfg Config, natsURL string, log *slog.Logger)
if network == "" {
network = selfNetwork(ctx, cli, log)
}
if cfg.RequireHostGateway && network == "" {
// A native daemon is not itself attached to a Docker network. Its
// managed workers use Docker's default bridge unless configured
// otherwise, so that is the bridge whose host gateway must be bound.
network = "bridge"
}
hostGateway, err := resolveHostGateway(ctx, cli, network, cfg.RequireHostGateway)
if err != nil {
if cfg.DockerClient == nil {
_ = cli.Close()
}
return nil, err
}
p := &Pool{
cli: cli,
cfg: cfg,
log: log,
natsURL: natsURL,
ownCli: cfg.DockerClient == nil,
network: network,
cli: cli,
cfg: cfg,
log: log,
ownCli: cfg.DockerClient == nil,
network: network,
hostGateway: hostGateway,
}
// Pull image if needed.
+107
View File
@@ -0,0 +1,107 @@
package daemon
import (
"encoding/json"
"fmt"
"log/slog"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
dockercontainer "github.com/moby/moby/api/types/container"
"github.com/moby/moby/client"
"github.com/samcharles93/archie-core/internal/config"
archiecontainer "github.com/samcharles93/archie-core/internal/container"
)
// This pins the last handoff seam, not just its two halves: the runtime NATS
// endpoint frozen in Daemon becomes the environment on a container created on
// the same Docker network whose gateway embedded NATS binds.
func TestConnectedNATSEndpointReachesManagedContainerCreate(t *testing.T) {
const (
networkName = "workers"
gateway = "172.31.0.1"
containerID = "12345678901234567890123456789012"
token = "runtime-only-token"
)
var createRequest dockercontainer.CreateRequest
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/networks/"+networkName):
writeContainerDockerJSON(t, w, map[string]any{
"Name": networkName,
"Driver": "bridge",
"Scope": "local",
"IPAM": map[string]any{
"Config": []map[string]string{{
"Subnet": "172.31.0.0/16",
"Gateway": gateway,
}},
},
})
case strings.HasSuffix(r.URL.Path, "/containers/create"):
if err := json.NewDecoder(r.Body).Decode(&createRequest); err != nil {
t.Errorf("decode container create request: %v", err)
}
writeContainerDockerJSON(t, w, dockercontainer.CreateResponse{ID: containerID})
case strings.HasSuffix(r.URL.Path, "/containers/"+containerID+"/start"):
w.WriteHeader(http.StatusNoContent)
case strings.HasSuffix(r.URL.Path, "/containers/json"):
writeContainerDockerJSON(t, w, []any{})
default:
http.Error(w, fmt.Sprintf("unexpected Docker API path %s", r.URL.Path), http.StatusNotFound)
}
}))
t.Cleanup(dockerAPI.Close)
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = dockerClient.Close() })
pool, err := archiecontainer.NewPool(t.Context(), archiecontainer.Config{
Image: "archie-agent:test",
Network: networkName,
DockerClient: dockerClient,
RequireHostGateway: true,
}, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatalf("container.NewPool = %v", err)
}
t.Cleanup(func() { _ = pool.Close() })
d := &Daemon{
Cfg: config.NewHolder(config.Config{}),
ConnectedNATS: NATSEndpoint{
URL: "nats://" + gateway + ":4222",
Token: token,
},
}
if _, err := pool.Acquire(t.Context(), nil, d.containerEnv(nil)); err != nil {
t.Fatalf("Pool.Acquire = %v", err)
}
if createRequest.HostConfig == nil || string(createRequest.HostConfig.NetworkMode) != networkName {
t.Fatalf("container network = %v, want %q", createRequest.HostConfig, networkName)
}
for _, want := range []string{
"NATS_URL=nats://" + gateway + ":4222",
"NATS_TOKEN=" + token,
} {
if !slices.Contains(createRequest.Env, want) {
t.Errorf("container env = %q, want %q", createRequest.Env, want)
}
}
}
func writeContainerDockerJSON(t *testing.T, w http.ResponseWriter, value any) {
t.Helper()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(value); err != nil {
t.Errorf("encode Docker response: %v", err)
}
}
+15 -11
View File
@@ -54,17 +54,23 @@ type WorktreeGrantIssuer interface {
Issue(task *store.Task) (token string, revoke func(), err error)
}
// NATSEndpoint is the broker address and credential archied connected with at
// startup. It is runtime state rather than configuration: embedded mode
// generates both values, and external secret references are resolved once.
type NATSEndpoint struct {
URL string
Token string
}
type Daemon struct {
// Cfg publishes the running configuration. Read through Cfg() so a
// reload swaps the published snapshot atomically. See config.Holder.
Cfg *config.Holder
// ConnectedNATS is the [nats] configuration the daemon's own client
// connected with at startup (the endpoint it publishes on). Container
// env is built from this, not from the live config, so a reload of
// nats.url/token_env cannot point new containers at a server the
// daemon is not publishing on. Zero value means NATS is not
// configured.
ConnectedNATS config.NATSConfig
// ConnectedNATS is the endpoint and credential the daemon's own client
// connected with at startup. Container env is built from this, not from
// live config or environment, so reload cannot point new containers at a
// broker the daemon is not publishing on. Zero value means no NATS.
ConnectedNATS NATSEndpoint
Store store.TaskStore
Forge forge.Forge
Trees *worktree.Manager
@@ -1168,10 +1174,8 @@ func (d *Daemon) containerEnv(task *store.Task) []string {
// the live config: a reloaded [nats] section must not point new
// containers at a server the daemon is not publishing on.
env = append(env, "NATS_URL="+d.ConnectedNATS.URL)
if tokenEnv := d.ConnectedNATS.TokenEnv; tokenEnv != "" {
if token := os.Getenv(tokenEnv); token != "" {
env = append(env, "NATS_TOKEN="+token)
}
if token := d.ConnectedNATS.Token; token != "" {
env = append(env, "NATS_TOKEN="+token)
}
// The agent process runs as root inside the container (no USER in the
// Dockerfile, no userns-remap), so a commit it writes to the bind-mounted
+7 -10
View File
@@ -305,20 +305,17 @@ func TestAllowConcurrentForTaskPrefersOwningIdentityRepo(t *testing.T) {
}
func TestContainerEnvIncludesConfiguredNATSCredentials(t *testing.T) {
t.Setenv("ARCHIE_NATS_SECRET", "test-nats-token")
// ConnectedNATS is the endpoint the daemon's own client connected with
// at startup; containerEnv deliberately reads this, not the live Cfg,
// so a reload of nats.url cannot point new containers at a server the
// daemon is not publishing on.
natsCfg := config.NATSConfig{
URL: "nats://nats.example:4222",
TokenEnv: "ARCHIE_NATS_SECRET",
}
// at startup; containerEnv deliberately reads this frozen runtime value,
// not live config or environment.
d := &Daemon{
Cfg: config.NewHolder(config.Config{}),
ConnectedNATS: natsCfg,
ConnectedNATS: NATSEndpoint{URL: "nats://nats.example:4222", Token: "test-nats-token"},
}
// Runtime credentials are frozen when the daemon connects. Mutating the
// source environment later must not send new workers to the same endpoint
// with a credential the broker never adopted.
t.Setenv("ARCHIE_NATS_SECRET", "changed-after-startup")
got := d.containerEnv(nil)
for _, want := range []string{
@@ -278,10 +278,10 @@ func effectiveNATSMode(cfg *config.Config) string {
return config.NATSModeEmbedded
}
// validateNATS checks the nats mode and its consistency with url and the
// agent execution mode. It resolves an unset mode from url without mutating
// cfg, so Validate (which documents it does not apply defaults) accepts a
// hand-built config the same way Loader.File accepts its on-disk form.
// validateNATS checks the nats mode and its consistency with url. It resolves
// an unset mode from url without mutating cfg, so Validate (which documents it
// does not apply defaults) accepts a hand-built config the same way Loader.File
// accepts its on-disk form.
func validateNATS(cfg *config.Config) error {
mode := effectiveNATSMode(cfg)
if !oneOf(mode, natsModes) {
@@ -297,17 +297,15 @@ func validateNATS(cfg *config.Config) error {
return fmt.Errorf("%w: nats.url must be empty when nats.mode is %q", ErrInvalidInput, mode)
}
}
// Agent execution over NATS needs workers that can reach the server. An
// embedded server binds loopback on a random port, which workers cannot
// discover, so agent.mode="nats" requires an external URL.
if cfg.Agent.Mode == agentModeNATS && mode != config.NATSModeExternal {
return fmt.Errorf("%w: nats.mode must be %q when agent.mode is %q", ErrInvalidInput, config.NATSModeExternal, agentModeNATS)
if cfg.Agent.Mode == agentModeNATS && mode == config.NATSModeEmbedded && !cfg.Containers.Enabled {
return fmt.Errorf("%w: containers.enabled must be true when agent.mode is %q and nats.mode is %q", ErrInvalidInput, agentModeNATS, config.NATSModeEmbedded)
}
return nil
}
// validateContainers checks the combination that container execution
// requires: a NATS transport and somewhere to reach it.
// validateContainers checks the combination container execution requires.
// Embedded and external NATS are both worker-reachable deployment shapes;
// composition resolves the Docker bridge gateway for embedded mode.
func validateContainers(cfg *config.Config) error {
if !cfg.Containers.Enabled {
return nil
@@ -318,8 +316,8 @@ func validateContainers(cfg *config.Config) error {
if cfg.Agent.Mode != agentModeNATS {
return fmt.Errorf("%w: agent.mode must be %q when containers.enabled is true", ErrInvalidInput, agentModeNATS)
}
if effectiveNATSMode(cfg) != config.NATSModeExternal {
return fmt.Errorf("%w: nats.mode must be %q when containers.enabled is true (containers cannot reach an embedded loopback server)", ErrInvalidInput, config.NATSModeExternal)
if effectiveNATSMode(cfg) == config.NATSModeOff {
return fmt.Errorf("%w: nats.mode must not be %q when containers.enabled is true", ErrInvalidInput, config.NATSModeOff)
}
if cfg.Containers.VolumeTTL < 0 {
return fmt.Errorf("%w: containers.volume_ttl must not be negative", ErrInvalidInput)
@@ -192,14 +192,16 @@ func TestValidate_RejectsTheSameProblemsAsLoaderLoad(t *testing.T) {
// TestValidateNATS pins the nats mode contract from docs/prds/embedded-nats.md:
// an unset mode resolves from url without mutating cfg, external requires url,
// embedded/off forbid a url, and agent.mode="nats" requires external.
// embedded/off forbid a url, and NATS agent execution works with either
// embedded or external broker deployment.
func TestValidateNATS(t *testing.T) {
tests := []struct {
name string
mode string
url string
agent string
wantErr bool
name string
mode string
url string
agent string
containers bool
wantErr bool
}{
{name: "unset mode with url resolves external", url: "nats://localhost:4222"},
{name: "unset mode without url resolves embedded"},
@@ -210,7 +212,8 @@ func TestValidateNATS(t *testing.T) {
{name: "embedded with url", mode: config.NATSModeEmbedded, url: "nats://localhost:4222", wantErr: true},
{name: "off with url", mode: config.NATSModeOff, url: "nats://localhost:4222", wantErr: true},
{name: "unknown mode", mode: "not-a-mode", wantErr: true},
{name: "agent nats with embedded", mode: config.NATSModeEmbedded, agent: "nats", wantErr: true},
{name: "agent nats with embedded requires containers", mode: config.NATSModeEmbedded, agent: "nats", wantErr: true},
{name: "agent nats with embedded containers", mode: config.NATSModeEmbedded, agent: "nats", containers: true},
{name: "agent nats with external", mode: config.NATSModeExternal, url: "nats://localhost:4222", agent: "nats"},
}
for _, tc := range tests {
@@ -220,6 +223,9 @@ func TestValidateNATS(t *testing.T) {
if tc.agent != "" {
cfg.Agent.Mode = tc.agent
}
if tc.containers {
cfg.Containers.Enabled = true
}
err := validateNATS(&cfg)
if tc.wantErr && err == nil {
t.Fatal("validateNATS() = nil, want an error")
@@ -252,11 +258,11 @@ func TestValidateContainersResolvesNATSMode(t *testing.T) {
}
})
t.Run("embedded mode rejects containers", func(t *testing.T) {
t.Run("embedded mode supports containers", func(t *testing.T) {
cfg := base()
cfg.NATS.Mode = config.NATSModeEmbedded
if err := Validate(&cfg); err == nil {
t.Fatal("Validate() = nil, want an error for containers with embedded nats")
if err := Validate(&cfg); err != nil {
t.Fatalf("Validate() = %v, want nil for managed containers with embedded nats", err)
}
})
}
@@ -2,6 +2,7 @@ package nats
import (
"context"
"crypto/rand"
"fmt"
"log/slog"
"time"
@@ -19,14 +20,16 @@ import (
// owns the lifecycle: StartEmbedded before dialing the client, Shutdown after
// the client closes.
type EmbeddedServer struct {
srv *server.Server
log *slog.Logger
srv *server.Server
log *slog.Logger
token string
}
// EmbeddedOptions configures StartEmbedded.
type EmbeddedOptions struct {
// Host is the bind address. Empty means "127.0.0.1" (loopback only,
// which is correct: an embedded server carries no authentication).
// Host is the bind address. Empty means "127.0.0.1" (loopback only).
// Container-backed deployments pass the host gateway address of the
// Docker bridge their workers join.
Host string
// Port is the listen port. Zero means a random port.
Port int
@@ -50,13 +53,15 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger)
if log == nil {
log = slog.New(slog.DiscardHandler)
}
token := rand.Text()
srv, err := server.NewServer(&server.Options{
Host: opts.Host,
Port: opts.Port,
JetStream: true,
StoreDir: opts.StoreDir,
NoSigs: true, // the daemon owns signals, not the embedded server
Host: opts.Host,
Port: opts.Port,
JetStream: true,
StoreDir: opts.StoreDir,
Authorization: token,
NoSigs: true, // the daemon owns signals, not the embedded server
// NoLog leaves the server's logger nil (ConfigureLogger is only ever
// called on reload). A nil logger makes the server's Fatalf a no-op
// rather than an os.Exit, so a JetStream startup failure cannot crash
@@ -83,7 +88,7 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger)
}
log.Info("embedded nats ready", "url", srv.ClientURL())
return &EmbeddedServer{srv: srv, log: log}, nil
return &EmbeddedServer{srv: srv, log: log, token: token}, nil
}
// ClientURL returns the address the server is listening on. It is the URL the
@@ -91,6 +96,11 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger)
// connection used.
func (e *EmbeddedServer) ClientURL() string { return e.srv.ClientURL() }
// Token returns the per-start credential required by every embedded NATS
// client. Composition keeps it in memory and passes it only to the daemon's
// client and managed worker environments; it must never be logged or persisted.
func (e *EmbeddedServer) Token() string { return e.token }
// Shutdown stops the server and waits for it to finish. It is safe to call on
// a nil server so callers can defer it unconditionally.
func (e *EmbeddedServer) Shutdown() {
@@ -21,11 +21,20 @@ func TestStartEmbeddedLifecycle(t *testing.T) {
if url == "" {
t.Fatal("ClientURL() = empty, want the bound address")
}
if unauthenticated, connectErr := nats.Connect(url); connectErr == nil {
unauthenticated.Close()
t.Fatal("unauthenticated connection succeeded, want embedded NATS to require its per-start token")
}
token := srv.Token()
if token == "" {
t.Fatal("Token() = empty, want a generated runtime token")
}
// Connecting and provisioning a stream proves JetStream is live, not just
// the core NATS socket.
c, err := Connect(t.Context(), Config{
URL: url,
Token: token,
Subjects: []string{"embedded.test.>"},
FilterSubject: "embedded.test.>",
}, discardLogger())