feat(nats): expose embedded broker to managed workers
This commit is contained in:
1 parent
7b5dca3c42
commit
1b7bc004b6
16 files changed
+678
-178
No files matched your search
+21
-15
@@ -100,6 +100,9 @@ type boot struct {
|
||||
// the embedded server's ClientURL(). Recorded so Daemon.ConnectedNATS
|
||||
// carries the real endpoint rather than an empty config URL.
|
||||
natsURL string
|
||||
// natsToken is the credential the daemon's client connected with at
|
||||
// startup. Embedded mode generates it; external mode resolves it once.
|
||||
natsToken string
|
||||
|
||||
containerPool *container.Pool
|
||||
storeBackend storage.Backend
|
||||
@@ -376,7 +379,12 @@ func (b *boot) connectNATS(ctx context.Context) error {
|
||||
// Embedded. Start the in-process server before dialing, and register
|
||||
// its shutdown BEFORE the client close below so the client closes
|
||||
// first (cleanups run LIFO).
|
||||
host := ""
|
||||
if b.containerPool != nil {
|
||||
host = b.containerPool.HostGateway()
|
||||
}
|
||||
srv, err := nats.StartEmbedded(ctx, nats.EmbeddedOptions{
|
||||
Host: host,
|
||||
StoreDir: filepath.Join(filepath.Dir(cfg.DBPath), "nats"),
|
||||
}, log)
|
||||
if err != nil {
|
||||
@@ -385,6 +393,7 @@ func (b *boot) connectNATS(ctx context.Context) error {
|
||||
}
|
||||
b.addCleanup(func() { srv.Shutdown() })
|
||||
url = srv.ClientURL()
|
||||
natsToken = srv.Token()
|
||||
log.Info("embedded nats started", "url", url)
|
||||
}
|
||||
|
||||
@@ -402,6 +411,7 @@ func (b *boot) connectNATS(ctx context.Context) error {
|
||||
}
|
||||
b.natsClient = natsClient
|
||||
b.natsURL = url
|
||||
b.natsToken = natsToken
|
||||
b.addCleanup(func() { natsClient.Close() })
|
||||
log.Info("nats connected", "url", url)
|
||||
return nil
|
||||
@@ -414,15 +424,15 @@ func (b *boot) connectNATS(ctx context.Context) error {
|
||||
// feature they may not be exercising right now. Under a systemd unit
|
||||
// with Restart=on-failure it also turns a recoverable problem into a
|
||||
// crash loop with the real error scrolling past unread.
|
||||
func (b *boot) setupContainers(ctx context.Context) {
|
||||
func (b *boot) setupContainers(ctx context.Context) func() {
|
||||
containerPool, storeBackend, closeDocker := startContainers(ctx, b.cfg, b.log)
|
||||
b.containerPool = containerPool
|
||||
b.storeBackend = storeBackend
|
||||
b.addCleanup(closeDocker)
|
||||
// Sandboxing that was asked for and could not start must not silently
|
||||
// become no sandboxing: the daemon stays up, but tasks park rather than
|
||||
// running the agent loop on the host.
|
||||
b.sandboxRequired = b.cfg.Containers.Enabled && containerPool == nil
|
||||
return closeDocker
|
||||
}
|
||||
|
||||
// setupLLMAndChat wires the runtime, tool registry, model management,
|
||||
@@ -995,16 +1005,9 @@ func (b *boot) registerMinimaxTool(cfg config.Config, log *slog.Logger) {
|
||||
// it shares with it.
|
||||
func (b *boot) buildDaemon() {
|
||||
cfg, log := b.cfg, b.log
|
||||
// ConnectedNATS must carry the endpoint the daemon's own client connected
|
||||
// with at startup, not the raw config: for embedded mode the config URL is
|
||||
// empty, but container env must never see an empty NATS_URL. (Containers
|
||||
// are rejected in embedded mode by validation, but the field should mean
|
||||
// the same thing regardless of mode.)
|
||||
connectedNATS := cfg.NATS
|
||||
connectedNATS.URL = b.natsURL
|
||||
b.d = &daemon.Daemon{
|
||||
Cfg: config.NewHolder(cfg),
|
||||
ConnectedNATS: connectedNATS,
|
||||
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken},
|
||||
Store: b.st,
|
||||
Bus: b.bus,
|
||||
Forge: b.forgeClient,
|
||||
@@ -1323,11 +1326,14 @@ func (b *boot) startServices(ctx context.Context) error {
|
||||
// setupBackends wires the optional runtime backends: the NATS client
|
||||
// and the container pool.
|
||||
func (b *boot) setupBackends(ctx context.Context) error {
|
||||
if err := b.connectNATS(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
b.setupContainers(ctx)
|
||||
return nil
|
||||
closeContainers := b.setupContainers(ctx)
|
||||
err := b.connectNATS(ctx)
|
||||
// Container discovery must precede embedded NATS so it can bind the
|
||||
// worker bridge gateway, but cleanup registration comes afterwards.
|
||||
// cleanups run LIFO: workers stop first, then the daemon client closes,
|
||||
// then the embedded server shuts down.
|
||||
b.addCleanup(closeContainers)
|
||||
return err
|
||||
}
|
||||
|
||||
// buildAgentAndWorkflows builds the agent runner and the workflow
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/moby/moby/client"
|
||||
natsio "github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/container"
|
||||
)
|
||||
|
||||
// setupContainers must discover the bridge before embedded NATS starts, but
|
||||
// its cleanup registration comes afterwards. boot.cleanup is LIFO, so this
|
||||
// ordering stops worker containers before closing the daemon client and the
|
||||
// embedded server they are connected to.
|
||||
func TestSetupBackendsRegistersContainerCleanupAfterNATS(t *testing.T) {
|
||||
fileset := token.NewFileSet()
|
||||
file, err := parser.ParseFile(fileset, "bootstrap.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
setupContainers := methodBody(t, file, "setupContainers")
|
||||
if methodCallPosition(setupContainers, "addCleanup") != token.NoPos {
|
||||
t.Fatal("setupContainers registers cleanup before NATS; LIFO would stop the broker before workers")
|
||||
}
|
||||
setupBackends := methodBody(t, file, "setupBackends")
|
||||
connect := methodCallPosition(setupBackends, "connectNATS")
|
||||
register := methodCallPosition(setupBackends, "addCleanup")
|
||||
if connect == token.NoPos || register == token.NoPos || register < connect {
|
||||
t.Fatalf("setupBackends call positions: connectNATS=%d addCleanup=%d, want container cleanup registered after connect", connect, register)
|
||||
}
|
||||
}
|
||||
|
||||
func methodBody(t *testing.T, file *ast.File, name string) *ast.BlockStmt {
|
||||
t.Helper()
|
||||
for _, decl := range file.Decls {
|
||||
fn, ok := decl.(*ast.FuncDecl)
|
||||
if ok && fn.Recv != nil && fn.Name.Name == name {
|
||||
return fn.Body
|
||||
}
|
||||
}
|
||||
t.Fatalf("method %s not found", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func methodCallPosition(body *ast.BlockStmt, name string) token.Pos {
|
||||
position := token.NoPos
|
||||
ast.Inspect(body, func(node ast.Node) bool {
|
||||
call, ok := node.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
selector, ok := call.Fun.(*ast.SelectorExpr)
|
||||
if ok && selector.Sel.Name == name && position == token.NoPos {
|
||||
position = call.Pos()
|
||||
}
|
||||
return true
|
||||
})
|
||||
return position
|
||||
}
|
||||
|
||||
// This is the native-daemon deployment seam: Docker supplies the host-side
|
||||
// gateway of the worker bridge, embedded NATS binds that address, and the
|
||||
// runtime credential shared with workers is required by the same listener.
|
||||
func TestConnectEmbeddedNATSUsesManagedWorkerBridge(t *testing.T) {
|
||||
const gateway = "127.0.0.1"
|
||||
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/networks/workers"):
|
||||
writeDockerResponse(t, w, map[string]any{
|
||||
"Name": "workers",
|
||||
"Driver": "bridge",
|
||||
"Scope": "local",
|
||||
"IPAM": map[string]any{
|
||||
"Config": []map[string]string{{
|
||||
"Subnet": "127.0.0.0/8",
|
||||
"Gateway": gateway,
|
||||
}},
|
||||
},
|
||||
})
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/json"):
|
||||
writeDockerResponse(t, w, []any{})
|
||||
default:
|
||||
http.Error(w, fmt.Sprintf("unexpected Docker API path %s", r.URL.Path), http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(dockerAPI.Close)
|
||||
|
||||
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = dockerClient.Close() })
|
||||
|
||||
pool, err := container.NewPool(t.Context(), container.Config{
|
||||
DockerClient: dockerClient,
|
||||
Network: "workers",
|
||||
RequireHostGateway: true,
|
||||
}, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatalf("container.NewPool = %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = pool.Close() })
|
||||
|
||||
b := &boot{
|
||||
cfg: config.Config{
|
||||
DBPath: filepath.Join(t.TempDir(), "archie.db"),
|
||||
NATS: config.NATSConfig{Mode: config.NATSModeEmbedded},
|
||||
},
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
containerPool: pool,
|
||||
}
|
||||
if err := b.connectNATS(t.Context()); err != nil {
|
||||
t.Fatalf("connectNATS = %v", err)
|
||||
}
|
||||
t.Cleanup(b.cleanup)
|
||||
|
||||
if !strings.HasPrefix(b.natsURL, "nats://"+gateway+":") {
|
||||
t.Errorf("connected NATS URL = %q, want bridge gateway %s", b.natsURL, gateway)
|
||||
}
|
||||
if b.natsToken == "" {
|
||||
t.Fatal("connected NATS token is empty")
|
||||
}
|
||||
if unauthenticated, connectErr := natsio.Connect(b.natsURL); connectErr == nil {
|
||||
unauthenticated.Close()
|
||||
t.Fatal("unauthenticated worker connection succeeded")
|
||||
}
|
||||
authenticated, err := natsio.Connect(b.natsURL, natsio.Token(b.natsToken))
|
||||
if err != nil {
|
||||
t.Fatalf("authenticated worker connection = %v", err)
|
||||
}
|
||||
authenticated.Close()
|
||||
}
|
||||
|
||||
func writeDockerResponse(t *testing.T, w http.ResponseWriter, value any) {
|
||||
t.Helper()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(value); err != nil {
|
||||
t.Errorf("encode Docker response: %v", err)
|
||||
}
|
||||
}
|
||||
+8
-7
@@ -979,13 +979,14 @@ func startContainers(
|
||||
}
|
||||
|
||||
pool, err := container.NewPool(ctx, container.Config{
|
||||
Image: cfg.Containers.Image,
|
||||
MaxConcurrency: cfg.Containers.MaxConcurrency,
|
||||
MaxUptime: cfg.Containers.MaxUptime.Std(),
|
||||
PullPolicy: cfg.Containers.PullPolicy,
|
||||
Network: cfg.Containers.Network,
|
||||
DockerClient: dockerCli,
|
||||
}, cfg.NATS.URL, log)
|
||||
Image: cfg.Containers.Image,
|
||||
MaxConcurrency: cfg.Containers.MaxConcurrency,
|
||||
MaxUptime: cfg.Containers.MaxUptime.Std(),
|
||||
PullPolicy: cfg.Containers.PullPolicy,
|
||||
Network: cfg.Containers.Network,
|
||||
DockerClient: dockerCli,
|
||||
RequireHostGateway: cfg.NATS.Mode == config.NATSModeEmbedded,
|
||||
}, log)
|
||||
if err != nil {
|
||||
// A missing image is recoverable by hand. The daemon sends no registry
|
||||
// credentials on pull (internal/container/pool.go), so a private
|
||||
|
||||
+4
-3
@@ -231,14 +231,14 @@ gate_max_failures = 5
|
||||
# NATS JetStream for task distribution and reaction delivery. mode selects
|
||||
# how NATS is provided:
|
||||
# "embedded" – in-process nats-server (the default; no external server)
|
||||
# "external" – connect to url (required for agent.mode="nats" / [containers])
|
||||
# "external" – connect to url
|
||||
# "off" – no NATS; the SQLite ClaimNext flow is used for tasks
|
||||
# Leave the section absent (or set no url) to run embedded; set mode="off"
|
||||
# only to opt out of NATS entirely.
|
||||
# [nats]
|
||||
# mode = "embedded" # "embedded" | "external" | "off"
|
||||
# url = "nats://localhost:4222" # required when mode = "external"
|
||||
# token_env = "NATS_TOKEN" # optional; named variable must be set
|
||||
# token_env = "NATS_TOKEN" # optional external-server credential; embedded generates one per start
|
||||
|
||||
# Unbound webhook capture: POST /webhooks/capture/{source} accepts and
|
||||
# stores any inbound payload archie doesn't yet have a binding for, so it can
|
||||
@@ -255,7 +255,8 @@ gate_max_failures = 5
|
||||
# rate_per_second = 1.0 # per-remote-address token bucket
|
||||
# rate_burst = 5
|
||||
|
||||
# Docker sandbox execution of archie-agent. Requires [nats] and agent.mode = "nats".
|
||||
# Docker sandbox execution of archie-agent. Requires NATS and agent.mode = "nats";
|
||||
# embedded and external broker modes are both supported.
|
||||
# [containers]
|
||||
# enabled = true
|
||||
# image = "archie-agent:latest" # Docker image to run
|
||||
|
||||
+61
-71
@@ -1,86 +1,76 @@
|
||||
# Embedded NATS -- decision
|
||||
# Embedded NATS execution -- decision
|
||||
|
||||
**Status:** Decided, not yet implemented
|
||||
**Date:** 2026-08-22
|
||||
**Beads issue:** `archie-core-7d5u.2`
|
||||
**Status:** Amended; implementation in progress
|
||||
|
||||
`event-sources-and-reactions.md` settled the reaction-delivery mechanism (reuse
|
||||
`internal/infrastructure/eventbus/nats`, a second stream under a fan-out
|
||||
retention policy) and left the deployment shape to this issue. This document
|
||||
decides that shape. `github.com/nats-io/nats-server/v2` is already a direct
|
||||
dependency, used today only in `_test.go` files; the gap is that production
|
||||
startup only knows "external URL configured" vs "no NATS, use SQLite
|
||||
`ClaimNext`".
|
||||
**Date:** 2026-08-24
|
||||
|
||||
## 1. Embedded runs alongside SQLite, not instead of it
|
||||
**Beads epic:** `archie-core-q15y`
|
||||
|
||||
The SQLite store remains authoritative for task state. NATS -- external or
|
||||
embedded -- is the *distribution* path, and `drainNATS` already falls back to
|
||||
`Store.ClaimNext` for requeued tasks (waiting_human approval, retry-parked)
|
||||
that never went through a NATS publish. So wiring embedded NATS does not
|
||||
remove the SQLite flow; it makes `Daemon.Tasks != nil` true in single-process
|
||||
deployments, which routes poll→enqueue through `pollNATS` and drain through
|
||||
`drainNATS` exactly as an external-NATS deployment already does. `ClaimNext`
|
||||
stays as the requeue fallback inside `drainNATS`. No new drain path, no second
|
||||
eventbus type.
|
||||
This decision supersedes the 2026-08-22 position that embedded NATS was only a
|
||||
distribution mechanism and could not be an agent-execution topology. That was
|
||||
true only because the embedded listener was unauthenticated loopback. It left a
|
||||
second production architecture in which `archied` ran autonomous workflows in
|
||||
its own process, weakening the isolation boundary that managed task containers
|
||||
exist to provide.
|
||||
|
||||
## 2. Config: `[nats] mode = "embedded" | "external" | "off"`
|
||||
## One execution architecture, two broker deployments
|
||||
|
||||
`NATSConfig` gains a `mode` field. `url` no longer doubles as the on/off
|
||||
switch -- that coupling is what made "no NATS" a permanent stance instead of a
|
||||
deployment shape.
|
||||
Every autonomous repository workflow is handed over as one complete task via
|
||||
NATS and runs in a task-scoped `archie-agent` container. `archied` remains a
|
||||
native, long-lived orchestration and interactive-chat process; its operator
|
||||
configured host filesystem access is unchanged. It owns task admission,
|
||||
state, forge and worktree operations, container lifecycle, and the broker when
|
||||
the embedded deployment is selected. It does not run an autonomous model loop.
|
||||
|
||||
| mode | meaning |
|
||||
|------------|----------------------------------------------------------------|
|
||||
| `external` | connect to `url` (today's behaviour). `url` is required. |
|
||||
| `embedded` | start an in-process nats-server; the client connects to it. |
|
||||
| `off` | no NATS at all; the old SQLite-only flow (`ClaimNext`). |
|
||||
NATS has two supported deployment shapes:
|
||||
|
||||
Default resolution: `url` set → `external`; `url` empty and `mode` unset or
|
||||
`embedded` → `embedded`; `mode = "off"` → `off`. `mode = "external"` with no
|
||||
`url` is a validation error; `mode = "off"`/`"embedded"` with a `url` set is a
|
||||
validation error. This is a meaning change and is stated as one: `url` empty
|
||||
*used to mean* "no NATS, SQLite only"; it *now means* "embedded NATS", and
|
||||
getting the old behaviour requires the explicit opt-out `mode = "off"`.
|
||||
| mode | meaning |
|
||||
| --- | --- |
|
||||
| `embedded` | `archied` owns an in-process JetStream server (default). |
|
||||
| `external` | `archied` and workers connect to the configured NATS URL. |
|
||||
|
||||
`agent.mode = "nats"` and `[containers]` require `mode = "external"` (a
|
||||
container cannot reach an in-process loopback server, and embedded NATS is not
|
||||
an agent-execution topology).
|
||||
Both shapes have identical task semantics: one container and one full-task
|
||||
request per workflow. Broker placement must not select a different executor.
|
||||
The legacy `nats.mode = "off"`, `agent.mode = "inprocess"`, per-stage NATS, and
|
||||
subprocess execution paths are migration inputs only and will be removed rather
|
||||
than retained as production fallbacks.
|
||||
|
||||
## 3. The daemon owns the embedded server's lifecycle
|
||||
SQLite remains authoritative for task state. NATS is the sole autonomous work
|
||||
handoff. SQLite recovery may identify work that needs republishing, but it must
|
||||
not cause `archied` to call `workflow.Run` locally.
|
||||
|
||||
`internal/infrastructure/eventbus/nats` gains an `EmbeddedServer` type that
|
||||
wraps `nats-server/v2/server.Server`, exposing only `ClientURL()` and
|
||||
`Shutdown()`. The composition root (`cmd/archied/bootstrap.go`) starts it in
|
||||
`connectNATS` (before dialing the client), binds it to `127.0.0.1` on a random
|
||||
port, enables JetStream with a store dir under the daemon's data directory
|
||||
(`<dir of DBPath>/nats`), and registers a cleanup that shuts the server down
|
||||
*after* the client closes. Cleanup ordering is LIFO: the server shutdown is
|
||||
registered before the client close, so the client closes first.
|
||||
## Embedded reachability and authentication
|
||||
|
||||
## Retention is a Config field, not a constant
|
||||
An embedded listener uses a cryptographically random bearer token generated for
|
||||
each daemon start. The token exists only in process memory and the environment
|
||||
of managed worker containers; it is never written to config or logs.
|
||||
|
||||
`nats.Config` gains `Retention *jetstream.RetentionPolicy`, defaulting to
|
||||
`jetstream.WorkQueuePolicy` in `withDefaults`, replacing the hardcoded value in
|
||||
`Connect` (`client.go:65`). `WorkQueuePolicy` is correct for `ARCHIE_TASKS`
|
||||
(one consumer claims each message) and wrong for reactions (fan-out). This is
|
||||
the enabler for the second reaction stream: `event-sources-and-reactions.md`
|
||||
requires it, and nothing here creates the reaction stream itself -- that is
|
||||
provisioned by the beads that build reaction producers and consumers, which do
|
||||
not exist yet. A test proves the parameterization: two overlapping `Subscribe`
|
||||
consumers both receive a message under `LimitsPolicy` (fan-out) and starve
|
||||
under `WorkQueuePolicy`, the exact trap `CLAUDE.md` documents.
|
||||
For a native daemon with managed containers, startup resolves the Docker bridge
|
||||
the workers will join. The listener binds only to that bridge's host gateway
|
||||
address on a random port. Both the native daemon and workers use that same
|
||||
reachable endpoint, and every client supplies the generated token. Binding a
|
||||
Docker bridge address keeps the listener off ordinary LAN interfaces without
|
||||
publishing a host port. A configured user bridge is respected; otherwise the
|
||||
Docker default bridge is used. A network without a usable local IPv4 gateway is
|
||||
a startup error for autonomous execution.
|
||||
|
||||
## Packages this touches
|
||||
If Docker is unavailable, interactive chat may still start and embedded NATS
|
||||
may remain loopback-only, but autonomous tasks park with an explicit container
|
||||
capability failure. They never fall back to host execution.
|
||||
|
||||
- `internal/infrastructure/eventbus/nats`: `Retention` on `Config`, an
|
||||
`EmbeddedServer` type.
|
||||
- `internal/config`: `NATSConfig.Mode`.
|
||||
- `internal/infrastructure/configuration`: `mode` defaulting and validation.
|
||||
- `cmd/archied/bootstrap.go`: start/stop the embedded server; set
|
||||
`Daemon.ConnectedNATS.URL` to the embedded client URL so container env (if
|
||||
ever enabled) never receives an empty endpoint.
|
||||
- `config.example.toml`: document `mode`. The `deployments/*` profiles are
|
||||
unchanged on purpose: their absent `[nats]` section now selects embedded by
|
||||
default, which is the intended single-process behaviour this issue exists to
|
||||
provide, not a regression to paper over with `mode = "off"`.
|
||||
## Ownership and lifecycle
|
||||
|
||||
- `internal/infrastructure/eventbus/nats` owns the authenticated embedded
|
||||
server and exposes its runtime endpoint, not the server implementation.
|
||||
- `internal/container` resolves the worker bridge and gateway and owns task
|
||||
container lifecycle.
|
||||
- `internal/app` composition selects embedded or external broker deployment,
|
||||
freezes the connected runtime endpoint for worker environments, and wires the
|
||||
full-task handoff.
|
||||
- `cmd/archied` remains process input only as the domain migration reaches this
|
||||
composition path.
|
||||
|
||||
The daemon starts the embedded server before its client, closes clients before
|
||||
the server, and keeps JetStream data beneath the daemon data directory. NATS URL
|
||||
and token are startup-built values: a reload cannot send new workers to a broker
|
||||
different from the one the daemon is using.
|
||||
@@ -623,8 +623,7 @@ const (
|
||||
// delivery. The mode selects how NATS is provided:
|
||||
//
|
||||
// "embedded" – an in-process nats-server (default; no external server)
|
||||
// "external" – connect to URL (required; the only mode agent.mode="nats"
|
||||
// and [containers] can use)
|
||||
// "external" – connect to URL (required)
|
||||
// "off" – no NATS; the SQLite ClaimNext flow is used unchanged
|
||||
//
|
||||
// An empty Mode resolves from URL: URL set means "external", URL empty means
|
||||
@@ -637,8 +636,8 @@ type NATSConfig struct {
|
||||
// URL is the NATS server address, e.g. "nats://localhost:4222".
|
||||
// Required when Mode is "external"; must be empty otherwise.
|
||||
URL string `toml:"url" yaml:"url"`
|
||||
// TokenEnv optionally names an env var holding a NATS auth token.
|
||||
// Empty means no authentication. Ignored when Mode is "embedded".
|
||||
// TokenEnv optionally names an env var holding an external NATS auth token.
|
||||
// Empty means no authentication. Embedded mode generates a per-start token.
|
||||
TokenEnv string `toml:"token_env" yaml:"token_env"`
|
||||
}
|
||||
|
||||
@@ -685,13 +684,11 @@ type ContainerConfig struct {
|
||||
VolumeTTL Duration `toml:"volume_ttl" yaml:"volume_ttl"`
|
||||
// PullPolicy controls image pulling: "missing" (default) or "always".
|
||||
PullPolicy string `toml:"pull_policy" yaml:"pull_policy"`
|
||||
// Network is the Docker network spawned agent containers join, so they
|
||||
// can resolve sibling compose services (nats, etc.) by name. Empty
|
||||
// falls back to auto-detecting the daemon's own network, which is
|
||||
// best-effort and silently yields the default bridge network (no
|
||||
// hostname resolution) if detection fails — set this explicitly in
|
||||
// compose/production deployments (e.g. "archie-core_default") rather
|
||||
// than relying on auto-detection.
|
||||
// Network is the Docker network spawned agent containers join. Empty first
|
||||
// tries the daemon's own network (for containerized composition), then uses
|
||||
// Docker's default bridge. Embedded NATS binds the selected bridge's host
|
||||
// gateway; external deployments can set a Compose network explicitly when
|
||||
// workers must resolve broker service names (e.g. "archie-core_default").
|
||||
Network string `toml:"network" yaml:"network"`
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/moby/moby/client"
|
||||
)
|
||||
|
||||
// NetworkName is the Docker network managed worker containers join.
|
||||
func (p *Pool) NetworkName() string { return p.network }
|
||||
|
||||
// HostGateway is the host-side IPv4 gateway of NetworkName. It is non-empty
|
||||
// only when Config.RequireHostGateway requested embedded broker reachability.
|
||||
func (p *Pool) HostGateway() string { return p.hostGateway }
|
||||
|
||||
func resolveHostGateway(
|
||||
ctx context.Context,
|
||||
cli *client.Client,
|
||||
networkName string,
|
||||
required bool,
|
||||
) (string, error) {
|
||||
if !required {
|
||||
return "", nil
|
||||
}
|
||||
result, err := cli.NetworkInspect(ctx, networkName, client.NetworkInspectOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("inspect Docker network %q for embedded NATS: %w", networkName, err)
|
||||
}
|
||||
network := result.Network.Network
|
||||
if network.Driver != "bridge" {
|
||||
return "", fmt.Errorf("docker network %q uses driver %q, want bridge for embedded NATS", networkName, network.Driver)
|
||||
}
|
||||
if network.Scope != "" && network.Scope != "local" {
|
||||
return "", fmt.Errorf("docker network %q has scope %q, want local for embedded NATS", networkName, network.Scope)
|
||||
}
|
||||
for _, ipam := range network.IPAM.Config {
|
||||
gateway := ipam.Gateway
|
||||
if gateway.IsValid() && gateway.Is4() && !gateway.IsUnspecified() {
|
||||
return gateway.String(), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("docker network %q has no usable IPv4 host gateway for embedded NATS", networkName)
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/moby/moby/client"
|
||||
)
|
||||
|
||||
// A native archied process is not itself attached to a Docker network. When
|
||||
// embedded NATS must be reachable by managed workers, the pool therefore uses
|
||||
// Docker's default bridge and exposes its host-side gateway to composition.
|
||||
func TestNewPoolResolvesDefaultBridgeGatewayForNativeDaemon(t *testing.T) {
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const gateway = "172.31.0.1"
|
||||
|
||||
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/"+hostname+"/json"):
|
||||
http.Error(w, `{"message":"no such container"}`, http.StatusNotFound)
|
||||
case strings.HasSuffix(r.URL.Path, "/networks/bridge"):
|
||||
writeDockerJSON(t, w, map[string]any{
|
||||
"Name": "bridge",
|
||||
"Driver": "bridge",
|
||||
"Scope": "local",
|
||||
"IPAM": map[string]any{
|
||||
"Config": []map[string]string{{
|
||||
"Subnet": "172.31.0.0/16",
|
||||
"Gateway": gateway,
|
||||
}},
|
||||
},
|
||||
})
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/json"):
|
||||
writeDockerJSON(t, w, []any{})
|
||||
default:
|
||||
http.Error(w, fmt.Sprintf("unexpected Docker API path %s", r.URL.Path), http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(dockerAPI.Close)
|
||||
|
||||
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = dockerClient.Close() })
|
||||
|
||||
pool, err := NewPool(t.Context(), Config{
|
||||
DockerClient: dockerClient,
|
||||
RequireHostGateway: true,
|
||||
}, discardLogger())
|
||||
if err != nil {
|
||||
t.Fatalf("NewPool = %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = pool.Close() })
|
||||
|
||||
access, ok := any(pool).(interface {
|
||||
NetworkName() string
|
||||
HostGateway() string
|
||||
})
|
||||
if !ok {
|
||||
t.Fatal("Pool does not expose its resolved worker network and host gateway")
|
||||
}
|
||||
if got := access.NetworkName(); got != "bridge" {
|
||||
t.Errorf("NetworkName() = %q, want bridge", got)
|
||||
}
|
||||
if got := access.HostGateway(); got != gateway {
|
||||
t.Errorf("HostGateway() = %q, want %q", got, gateway)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewPoolRejectsUnreachableEmbeddedNetwork(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
statusCode int
|
||||
network map[string]any
|
||||
}{
|
||||
{
|
||||
name: "inspect failure",
|
||||
statusCode: http.StatusInternalServerError,
|
||||
},
|
||||
{
|
||||
name: "non bridge driver",
|
||||
network: map[string]any{
|
||||
"Name": "workers", "Driver": "overlay", "Scope": "local",
|
||||
"IPAM": map[string]any{"Config": []map[string]string{{"Gateway": "172.31.0.1"}}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "non local scope",
|
||||
network: map[string]any{
|
||||
"Name": "workers", "Driver": "bridge", "Scope": "swarm",
|
||||
"IPAM": map[string]any{"Config": []map[string]string{{"Gateway": "172.31.0.1"}}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "no IPv4 gateway",
|
||||
network: map[string]any{
|
||||
"Name": "workers", "Driver": "bridge", "Scope": "local",
|
||||
"IPAM": map[string]any{"Config": []map[string]string{{"Gateway": "fd00::1"}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/networks/workers"):
|
||||
if tt.statusCode != 0 {
|
||||
http.Error(w, `{"message":"network inspect failed"}`, tt.statusCode)
|
||||
return
|
||||
}
|
||||
writeDockerJSON(t, w, tt.network)
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/json"):
|
||||
writeDockerJSON(t, w, []any{})
|
||||
default:
|
||||
http.Error(w, fmt.Sprintf("unexpected Docker API path %s", r.URL.Path), http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(dockerAPI.Close)
|
||||
|
||||
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = dockerClient.Close() })
|
||||
|
||||
pool, err := NewPool(t.Context(), Config{
|
||||
DockerClient: dockerClient,
|
||||
Network: "workers",
|
||||
RequireHostGateway: true,
|
||||
}, discardLogger())
|
||||
if err == nil {
|
||||
_ = pool.Close()
|
||||
t.Fatal("NewPool = nil error, want embedded reachability validation failure")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func writeDockerJSON(t *testing.T, w http.ResponseWriter, value any) {
|
||||
t.Helper()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(value); err != nil {
|
||||
t.Errorf("encode Docker response: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func discardLogger() *slog.Logger {
|
||||
return slog.New(slog.DiscardHandler)
|
||||
}
|
||||
+35
-17
@@ -72,17 +72,19 @@ func WriteTaskJSON(workspace string, payload TaskPayload) error {
|
||||
|
||||
// Pool manages a set of Docker containers running archie-agent.
|
||||
type Pool struct {
|
||||
cli *client.Client
|
||||
ownCli bool // true when the pool created the client (owns Close)
|
||||
cfg Config
|
||||
log *slog.Logger
|
||||
natsURL string
|
||||
// network is the user-defined Docker network spawned containers join,
|
||||
// detected from the daemon's own container so they can resolve
|
||||
// sibling services (nats, etc.) by compose service name. Empty when
|
||||
// the daemon isn't running in a container on such a network, or when
|
||||
// detection fails -- Docker then falls back to its default bridge.
|
||||
cli *client.Client
|
||||
ownCli bool // true when the pool created the client (owns Close)
|
||||
cfg Config
|
||||
log *slog.Logger
|
||||
// network is the Docker network spawned containers join. External broker
|
||||
// deployments may leave it empty to use Docker's implicit default bridge;
|
||||
// embedded deployments resolve that default explicitly as "bridge" so its
|
||||
// host gateway can be inspected.
|
||||
network string
|
||||
// hostGateway is the local IPv4 gateway of network. It is populated when
|
||||
// embedded broker reachability is required, so native composition can bind
|
||||
// the listener to an address visible only to workers on this bridge.
|
||||
hostGateway string
|
||||
|
||||
mu sync.Mutex
|
||||
active int
|
||||
@@ -107,6 +109,9 @@ type Config struct {
|
||||
// NewPool creates its own via client.New(client.FromEnv). Pass a
|
||||
// shared client to avoid multiple independent connections.
|
||||
DockerClient *client.Client
|
||||
// RequireHostGateway resolves and validates a local Docker bridge gateway.
|
||||
// Embedded NATS needs this address; external brokers do not.
|
||||
RequireHostGateway bool
|
||||
}
|
||||
|
||||
// NewPool connects to the Docker daemon, optionally pulls the image,
|
||||
@@ -114,7 +119,7 @@ type Config struct {
|
||||
//
|
||||
// If cfg.DockerClient is set, it is reused (caller owns Close).
|
||||
// Otherwise a new client is created via client.FromEnv (pool owns Close).
|
||||
func NewPool(ctx context.Context, cfg Config, natsURL string, log *slog.Logger) (*Pool, error) {
|
||||
func NewPool(ctx context.Context, cfg Config, log *slog.Logger) (*Pool, error) {
|
||||
cli := cfg.DockerClient
|
||||
if cli == nil {
|
||||
var err error
|
||||
@@ -128,14 +133,27 @@ func NewPool(ctx context.Context, cfg Config, natsURL string, log *slog.Logger)
|
||||
if network == "" {
|
||||
network = selfNetwork(ctx, cli, log)
|
||||
}
|
||||
if cfg.RequireHostGateway && network == "" {
|
||||
// A native daemon is not itself attached to a Docker network. Its
|
||||
// managed workers use Docker's default bridge unless configured
|
||||
// otherwise, so that is the bridge whose host gateway must be bound.
|
||||
network = "bridge"
|
||||
}
|
||||
hostGateway, err := resolveHostGateway(ctx, cli, network, cfg.RequireHostGateway)
|
||||
if err != nil {
|
||||
if cfg.DockerClient == nil {
|
||||
_ = cli.Close()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
p := &Pool{
|
||||
cli: cli,
|
||||
cfg: cfg,
|
||||
log: log,
|
||||
natsURL: natsURL,
|
||||
ownCli: cfg.DockerClient == nil,
|
||||
network: network,
|
||||
cli: cli,
|
||||
cfg: cfg,
|
||||
log: log,
|
||||
ownCli: cfg.DockerClient == nil,
|
||||
network: network,
|
||||
hostGateway: hostGateway,
|
||||
}
|
||||
|
||||
// Pull image if needed.
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
dockercontainer "github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/client"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
archiecontainer "github.com/samcharles93/archie-core/internal/container"
|
||||
)
|
||||
|
||||
// This pins the last handoff seam, not just its two halves: the runtime NATS
|
||||
// endpoint frozen in Daemon becomes the environment on a container created on
|
||||
// the same Docker network whose gateway embedded NATS binds.
|
||||
func TestConnectedNATSEndpointReachesManagedContainerCreate(t *testing.T) {
|
||||
const (
|
||||
networkName = "workers"
|
||||
gateway = "172.31.0.1"
|
||||
containerID = "12345678901234567890123456789012"
|
||||
token = "runtime-only-token"
|
||||
)
|
||||
var createRequest dockercontainer.CreateRequest
|
||||
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/networks/"+networkName):
|
||||
writeContainerDockerJSON(t, w, map[string]any{
|
||||
"Name": networkName,
|
||||
"Driver": "bridge",
|
||||
"Scope": "local",
|
||||
"IPAM": map[string]any{
|
||||
"Config": []map[string]string{{
|
||||
"Subnet": "172.31.0.0/16",
|
||||
"Gateway": gateway,
|
||||
}},
|
||||
},
|
||||
})
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/create"):
|
||||
if err := json.NewDecoder(r.Body).Decode(&createRequest); err != nil {
|
||||
t.Errorf("decode container create request: %v", err)
|
||||
}
|
||||
writeContainerDockerJSON(t, w, dockercontainer.CreateResponse{ID: containerID})
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/"+containerID+"/start"):
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/json"):
|
||||
writeContainerDockerJSON(t, w, []any{})
|
||||
default:
|
||||
http.Error(w, fmt.Sprintf("unexpected Docker API path %s", r.URL.Path), http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(dockerAPI.Close)
|
||||
|
||||
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = dockerClient.Close() })
|
||||
|
||||
pool, err := archiecontainer.NewPool(t.Context(), archiecontainer.Config{
|
||||
Image: "archie-agent:test",
|
||||
Network: networkName,
|
||||
DockerClient: dockerClient,
|
||||
RequireHostGateway: true,
|
||||
}, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatalf("container.NewPool = %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = pool.Close() })
|
||||
|
||||
d := &Daemon{
|
||||
Cfg: config.NewHolder(config.Config{}),
|
||||
ConnectedNATS: NATSEndpoint{
|
||||
URL: "nats://" + gateway + ":4222",
|
||||
Token: token,
|
||||
},
|
||||
}
|
||||
if _, err := pool.Acquire(t.Context(), nil, d.containerEnv(nil)); err != nil {
|
||||
t.Fatalf("Pool.Acquire = %v", err)
|
||||
}
|
||||
|
||||
if createRequest.HostConfig == nil || string(createRequest.HostConfig.NetworkMode) != networkName {
|
||||
t.Fatalf("container network = %v, want %q", createRequest.HostConfig, networkName)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"NATS_URL=nats://" + gateway + ":4222",
|
||||
"NATS_TOKEN=" + token,
|
||||
} {
|
||||
if !slices.Contains(createRequest.Env, want) {
|
||||
t.Errorf("container env = %q, want %q", createRequest.Env, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeContainerDockerJSON(t *testing.T, w http.ResponseWriter, value any) {
|
||||
t.Helper()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(value); err != nil {
|
||||
t.Errorf("encode Docker response: %v", err)
|
||||
}
|
||||
}
|
||||
+15
-11
@@ -54,17 +54,23 @@ type WorktreeGrantIssuer interface {
|
||||
Issue(task *store.Task) (token string, revoke func(), err error)
|
||||
}
|
||||
|
||||
// NATSEndpoint is the broker address and credential archied connected with at
|
||||
// startup. It is runtime state rather than configuration: embedded mode
|
||||
// generates both values, and external secret references are resolved once.
|
||||
type NATSEndpoint struct {
|
||||
URL string
|
||||
Token string
|
||||
}
|
||||
|
||||
type Daemon struct {
|
||||
// Cfg publishes the running configuration. Read through Cfg() so a
|
||||
// reload swaps the published snapshot atomically. See config.Holder.
|
||||
Cfg *config.Holder
|
||||
// ConnectedNATS is the [nats] configuration the daemon's own client
|
||||
// connected with at startup (the endpoint it publishes on). Container
|
||||
// env is built from this, not from the live config, so a reload of
|
||||
// nats.url/token_env cannot point new containers at a server the
|
||||
// daemon is not publishing on. Zero value means NATS is not
|
||||
// configured.
|
||||
ConnectedNATS config.NATSConfig
|
||||
// ConnectedNATS is the endpoint and credential the daemon's own client
|
||||
// connected with at startup. Container env is built from this, not from
|
||||
// live config or environment, so reload cannot point new containers at a
|
||||
// broker the daemon is not publishing on. Zero value means no NATS.
|
||||
ConnectedNATS NATSEndpoint
|
||||
Store store.TaskStore
|
||||
Forge forge.Forge
|
||||
Trees *worktree.Manager
|
||||
@@ -1168,10 +1174,8 @@ func (d *Daemon) containerEnv(task *store.Task) []string {
|
||||
// the live config: a reloaded [nats] section must not point new
|
||||
// containers at a server the daemon is not publishing on.
|
||||
env = append(env, "NATS_URL="+d.ConnectedNATS.URL)
|
||||
if tokenEnv := d.ConnectedNATS.TokenEnv; tokenEnv != "" {
|
||||
if token := os.Getenv(tokenEnv); token != "" {
|
||||
env = append(env, "NATS_TOKEN="+token)
|
||||
}
|
||||
if token := d.ConnectedNATS.Token; token != "" {
|
||||
env = append(env, "NATS_TOKEN="+token)
|
||||
}
|
||||
// The agent process runs as root inside the container (no USER in the
|
||||
// Dockerfile, no userns-remap), so a commit it writes to the bind-mounted
|
||||
|
||||
@@ -305,20 +305,17 @@ func TestAllowConcurrentForTaskPrefersOwningIdentityRepo(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestContainerEnvIncludesConfiguredNATSCredentials(t *testing.T) {
|
||||
t.Setenv("ARCHIE_NATS_SECRET", "test-nats-token")
|
||||
|
||||
// ConnectedNATS is the endpoint the daemon's own client connected with
|
||||
// at startup; containerEnv deliberately reads this, not the live Cfg,
|
||||
// so a reload of nats.url cannot point new containers at a server the
|
||||
// daemon is not publishing on.
|
||||
natsCfg := config.NATSConfig{
|
||||
URL: "nats://nats.example:4222",
|
||||
TokenEnv: "ARCHIE_NATS_SECRET",
|
||||
}
|
||||
// at startup; containerEnv deliberately reads this frozen runtime value,
|
||||
// not live config or environment.
|
||||
d := &Daemon{
|
||||
Cfg: config.NewHolder(config.Config{}),
|
||||
ConnectedNATS: natsCfg,
|
||||
ConnectedNATS: NATSEndpoint{URL: "nats://nats.example:4222", Token: "test-nats-token"},
|
||||
}
|
||||
// Runtime credentials are frozen when the daemon connects. Mutating the
|
||||
// source environment later must not send new workers to the same endpoint
|
||||
// with a credential the broker never adopted.
|
||||
t.Setenv("ARCHIE_NATS_SECRET", "changed-after-startup")
|
||||
|
||||
got := d.containerEnv(nil)
|
||||
for _, want := range []string{
|
||||
|
||||
@@ -278,10 +278,10 @@ func effectiveNATSMode(cfg *config.Config) string {
|
||||
return config.NATSModeEmbedded
|
||||
}
|
||||
|
||||
// validateNATS checks the nats mode and its consistency with url and the
|
||||
// agent execution mode. It resolves an unset mode from url without mutating
|
||||
// cfg, so Validate (which documents it does not apply defaults) accepts a
|
||||
// hand-built config the same way Loader.File accepts its on-disk form.
|
||||
// validateNATS checks the nats mode and its consistency with url. It resolves
|
||||
// an unset mode from url without mutating cfg, so Validate (which documents it
|
||||
// does not apply defaults) accepts a hand-built config the same way Loader.File
|
||||
// accepts its on-disk form.
|
||||
func validateNATS(cfg *config.Config) error {
|
||||
mode := effectiveNATSMode(cfg)
|
||||
if !oneOf(mode, natsModes) {
|
||||
@@ -297,17 +297,15 @@ func validateNATS(cfg *config.Config) error {
|
||||
return fmt.Errorf("%w: nats.url must be empty when nats.mode is %q", ErrInvalidInput, mode)
|
||||
}
|
||||
}
|
||||
// Agent execution over NATS needs workers that can reach the server. An
|
||||
// embedded server binds loopback on a random port, which workers cannot
|
||||
// discover, so agent.mode="nats" requires an external URL.
|
||||
if cfg.Agent.Mode == agentModeNATS && mode != config.NATSModeExternal {
|
||||
return fmt.Errorf("%w: nats.mode must be %q when agent.mode is %q", ErrInvalidInput, config.NATSModeExternal, agentModeNATS)
|
||||
if cfg.Agent.Mode == agentModeNATS && mode == config.NATSModeEmbedded && !cfg.Containers.Enabled {
|
||||
return fmt.Errorf("%w: containers.enabled must be true when agent.mode is %q and nats.mode is %q", ErrInvalidInput, agentModeNATS, config.NATSModeEmbedded)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateContainers checks the combination that container execution
|
||||
// requires: a NATS transport and somewhere to reach it.
|
||||
// validateContainers checks the combination container execution requires.
|
||||
// Embedded and external NATS are both worker-reachable deployment shapes;
|
||||
// composition resolves the Docker bridge gateway for embedded mode.
|
||||
func validateContainers(cfg *config.Config) error {
|
||||
if !cfg.Containers.Enabled {
|
||||
return nil
|
||||
@@ -318,8 +316,8 @@ func validateContainers(cfg *config.Config) error {
|
||||
if cfg.Agent.Mode != agentModeNATS {
|
||||
return fmt.Errorf("%w: agent.mode must be %q when containers.enabled is true", ErrInvalidInput, agentModeNATS)
|
||||
}
|
||||
if effectiveNATSMode(cfg) != config.NATSModeExternal {
|
||||
return fmt.Errorf("%w: nats.mode must be %q when containers.enabled is true (containers cannot reach an embedded loopback server)", ErrInvalidInput, config.NATSModeExternal)
|
||||
if effectiveNATSMode(cfg) == config.NATSModeOff {
|
||||
return fmt.Errorf("%w: nats.mode must not be %q when containers.enabled is true", ErrInvalidInput, config.NATSModeOff)
|
||||
}
|
||||
if cfg.Containers.VolumeTTL < 0 {
|
||||
return fmt.Errorf("%w: containers.volume_ttl must not be negative", ErrInvalidInput)
|
||||
|
||||
@@ -192,14 +192,16 @@ func TestValidate_RejectsTheSameProblemsAsLoaderLoad(t *testing.T) {
|
||||
|
||||
// TestValidateNATS pins the nats mode contract from docs/prds/embedded-nats.md:
|
||||
// an unset mode resolves from url without mutating cfg, external requires url,
|
||||
// embedded/off forbid a url, and agent.mode="nats" requires external.
|
||||
// embedded/off forbid a url, and NATS agent execution works with either
|
||||
// embedded or external broker deployment.
|
||||
func TestValidateNATS(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mode string
|
||||
url string
|
||||
agent string
|
||||
wantErr bool
|
||||
name string
|
||||
mode string
|
||||
url string
|
||||
agent string
|
||||
containers bool
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "unset mode with url resolves external", url: "nats://localhost:4222"},
|
||||
{name: "unset mode without url resolves embedded"},
|
||||
@@ -210,7 +212,8 @@ func TestValidateNATS(t *testing.T) {
|
||||
{name: "embedded with url", mode: config.NATSModeEmbedded, url: "nats://localhost:4222", wantErr: true},
|
||||
{name: "off with url", mode: config.NATSModeOff, url: "nats://localhost:4222", wantErr: true},
|
||||
{name: "unknown mode", mode: "not-a-mode", wantErr: true},
|
||||
{name: "agent nats with embedded", mode: config.NATSModeEmbedded, agent: "nats", wantErr: true},
|
||||
{name: "agent nats with embedded requires containers", mode: config.NATSModeEmbedded, agent: "nats", wantErr: true},
|
||||
{name: "agent nats with embedded containers", mode: config.NATSModeEmbedded, agent: "nats", containers: true},
|
||||
{name: "agent nats with external", mode: config.NATSModeExternal, url: "nats://localhost:4222", agent: "nats"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
@@ -220,6 +223,9 @@ func TestValidateNATS(t *testing.T) {
|
||||
if tc.agent != "" {
|
||||
cfg.Agent.Mode = tc.agent
|
||||
}
|
||||
if tc.containers {
|
||||
cfg.Containers.Enabled = true
|
||||
}
|
||||
err := validateNATS(&cfg)
|
||||
if tc.wantErr && err == nil {
|
||||
t.Fatal("validateNATS() = nil, want an error")
|
||||
@@ -252,11 +258,11 @@ func TestValidateContainersResolvesNATSMode(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("embedded mode rejects containers", func(t *testing.T) {
|
||||
t.Run("embedded mode supports containers", func(t *testing.T) {
|
||||
cfg := base()
|
||||
cfg.NATS.Mode = config.NATSModeEmbedded
|
||||
if err := Validate(&cfg); err == nil {
|
||||
t.Fatal("Validate() = nil, want an error for containers with embedded nats")
|
||||
if err := Validate(&cfg); err != nil {
|
||||
t.Fatalf("Validate() = %v, want nil for managed containers with embedded nats", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package nats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
@@ -19,14 +20,16 @@ import (
|
||||
// owns the lifecycle: StartEmbedded before dialing the client, Shutdown after
|
||||
// the client closes.
|
||||
type EmbeddedServer struct {
|
||||
srv *server.Server
|
||||
log *slog.Logger
|
||||
srv *server.Server
|
||||
log *slog.Logger
|
||||
token string
|
||||
}
|
||||
|
||||
// EmbeddedOptions configures StartEmbedded.
|
||||
type EmbeddedOptions struct {
|
||||
// Host is the bind address. Empty means "127.0.0.1" (loopback only,
|
||||
// which is correct: an embedded server carries no authentication).
|
||||
// Host is the bind address. Empty means "127.0.0.1" (loopback only).
|
||||
// Container-backed deployments pass the host gateway address of the
|
||||
// Docker bridge their workers join.
|
||||
Host string
|
||||
// Port is the listen port. Zero means a random port.
|
||||
Port int
|
||||
@@ -50,13 +53,15 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger)
|
||||
if log == nil {
|
||||
log = slog.New(slog.DiscardHandler)
|
||||
}
|
||||
token := rand.Text()
|
||||
|
||||
srv, err := server.NewServer(&server.Options{
|
||||
Host: opts.Host,
|
||||
Port: opts.Port,
|
||||
JetStream: true,
|
||||
StoreDir: opts.StoreDir,
|
||||
NoSigs: true, // the daemon owns signals, not the embedded server
|
||||
Host: opts.Host,
|
||||
Port: opts.Port,
|
||||
JetStream: true,
|
||||
StoreDir: opts.StoreDir,
|
||||
Authorization: token,
|
||||
NoSigs: true, // the daemon owns signals, not the embedded server
|
||||
// NoLog leaves the server's logger nil (ConfigureLogger is only ever
|
||||
// called on reload). A nil logger makes the server's Fatalf a no-op
|
||||
// rather than an os.Exit, so a JetStream startup failure cannot crash
|
||||
@@ -83,7 +88,7 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger)
|
||||
}
|
||||
|
||||
log.Info("embedded nats ready", "url", srv.ClientURL())
|
||||
return &EmbeddedServer{srv: srv, log: log}, nil
|
||||
return &EmbeddedServer{srv: srv, log: log, token: token}, nil
|
||||
}
|
||||
|
||||
// ClientURL returns the address the server is listening on. It is the URL the
|
||||
@@ -91,6 +96,11 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger)
|
||||
// connection used.
|
||||
func (e *EmbeddedServer) ClientURL() string { return e.srv.ClientURL() }
|
||||
|
||||
// Token returns the per-start credential required by every embedded NATS
|
||||
// client. Composition keeps it in memory and passes it only to the daemon's
|
||||
// client and managed worker environments; it must never be logged or persisted.
|
||||
func (e *EmbeddedServer) Token() string { return e.token }
|
||||
|
||||
// Shutdown stops the server and waits for it to finish. It is safe to call on
|
||||
// a nil server so callers can defer it unconditionally.
|
||||
func (e *EmbeddedServer) Shutdown() {
|
||||
|
||||
@@ -21,11 +21,20 @@ func TestStartEmbeddedLifecycle(t *testing.T) {
|
||||
if url == "" {
|
||||
t.Fatal("ClientURL() = empty, want the bound address")
|
||||
}
|
||||
if unauthenticated, connectErr := nats.Connect(url); connectErr == nil {
|
||||
unauthenticated.Close()
|
||||
t.Fatal("unauthenticated connection succeeded, want embedded NATS to require its per-start token")
|
||||
}
|
||||
token := srv.Token()
|
||||
if token == "" {
|
||||
t.Fatal("Token() = empty, want a generated runtime token")
|
||||
}
|
||||
|
||||
// Connecting and provisioning a stream proves JetStream is live, not just
|
||||
// the core NATS socket.
|
||||
c, err := Connect(t.Context(), Config{
|
||||
URL: url,
|
||||
Token: token,
|
||||
Subjects: []string{"embedded.test.>"},
|
||||
FilterSubject: "embedded.test.>",
|
||||
}, discardLogger())
|
||||
|
||||
Reference in new issue
Block a user