feat(nats): expose embedded broker to managed workers
This commit is contained in:
1 parent
7b5dca3c42
commit
1b7bc004b6
16 files changed
+678
-178
No files matched your search
+21
-15
@@ -100,6 +100,9 @@ type boot struct {
|
|||||||
// the embedded server's ClientURL(). Recorded so Daemon.ConnectedNATS
|
// the embedded server's ClientURL(). Recorded so Daemon.ConnectedNATS
|
||||||
// carries the real endpoint rather than an empty config URL.
|
// carries the real endpoint rather than an empty config URL.
|
||||||
natsURL string
|
natsURL string
|
||||||
|
// natsToken is the credential the daemon's client connected with at
|
||||||
|
// startup. Embedded mode generates it; external mode resolves it once.
|
||||||
|
natsToken string
|
||||||
|
|
||||||
containerPool *container.Pool
|
containerPool *container.Pool
|
||||||
storeBackend storage.Backend
|
storeBackend storage.Backend
|
||||||
@@ -376,7 +379,12 @@ func (b *boot) connectNATS(ctx context.Context) error {
|
|||||||
// Embedded. Start the in-process server before dialing, and register
|
// Embedded. Start the in-process server before dialing, and register
|
||||||
// its shutdown BEFORE the client close below so the client closes
|
// its shutdown BEFORE the client close below so the client closes
|
||||||
// first (cleanups run LIFO).
|
// first (cleanups run LIFO).
|
||||||
|
host := ""
|
||||||
|
if b.containerPool != nil {
|
||||||
|
host = b.containerPool.HostGateway()
|
||||||
|
}
|
||||||
srv, err := nats.StartEmbedded(ctx, nats.EmbeddedOptions{
|
srv, err := nats.StartEmbedded(ctx, nats.EmbeddedOptions{
|
||||||
|
Host: host,
|
||||||
StoreDir: filepath.Join(filepath.Dir(cfg.DBPath), "nats"),
|
StoreDir: filepath.Join(filepath.Dir(cfg.DBPath), "nats"),
|
||||||
}, log)
|
}, log)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -385,6 +393,7 @@ func (b *boot) connectNATS(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
b.addCleanup(func() { srv.Shutdown() })
|
b.addCleanup(func() { srv.Shutdown() })
|
||||||
url = srv.ClientURL()
|
url = srv.ClientURL()
|
||||||
|
natsToken = srv.Token()
|
||||||
log.Info("embedded nats started", "url", url)
|
log.Info("embedded nats started", "url", url)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -402,6 +411,7 @@ func (b *boot) connectNATS(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
b.natsClient = natsClient
|
b.natsClient = natsClient
|
||||||
b.natsURL = url
|
b.natsURL = url
|
||||||
|
b.natsToken = natsToken
|
||||||
b.addCleanup(func() { natsClient.Close() })
|
b.addCleanup(func() { natsClient.Close() })
|
||||||
log.Info("nats connected", "url", url)
|
log.Info("nats connected", "url", url)
|
||||||
return nil
|
return nil
|
||||||
@@ -414,15 +424,15 @@ func (b *boot) connectNATS(ctx context.Context) error {
|
|||||||
// feature they may not be exercising right now. Under a systemd unit
|
// feature they may not be exercising right now. Under a systemd unit
|
||||||
// with Restart=on-failure it also turns a recoverable problem into a
|
// with Restart=on-failure it also turns a recoverable problem into a
|
||||||
// crash loop with the real error scrolling past unread.
|
// crash loop with the real error scrolling past unread.
|
||||||
func (b *boot) setupContainers(ctx context.Context) {
|
func (b *boot) setupContainers(ctx context.Context) func() {
|
||||||
containerPool, storeBackend, closeDocker := startContainers(ctx, b.cfg, b.log)
|
containerPool, storeBackend, closeDocker := startContainers(ctx, b.cfg, b.log)
|
||||||
b.containerPool = containerPool
|
b.containerPool = containerPool
|
||||||
b.storeBackend = storeBackend
|
b.storeBackend = storeBackend
|
||||||
b.addCleanup(closeDocker)
|
|
||||||
// Sandboxing that was asked for and could not start must not silently
|
// Sandboxing that was asked for and could not start must not silently
|
||||||
// become no sandboxing: the daemon stays up, but tasks park rather than
|
// become no sandboxing: the daemon stays up, but tasks park rather than
|
||||||
// running the agent loop on the host.
|
// running the agent loop on the host.
|
||||||
b.sandboxRequired = b.cfg.Containers.Enabled && containerPool == nil
|
b.sandboxRequired = b.cfg.Containers.Enabled && containerPool == nil
|
||||||
|
return closeDocker
|
||||||
}
|
}
|
||||||
|
|
||||||
// setupLLMAndChat wires the runtime, tool registry, model management,
|
// setupLLMAndChat wires the runtime, tool registry, model management,
|
||||||
@@ -995,16 +1005,9 @@ func (b *boot) registerMinimaxTool(cfg config.Config, log *slog.Logger) {
|
|||||||
// it shares with it.
|
// it shares with it.
|
||||||
func (b *boot) buildDaemon() {
|
func (b *boot) buildDaemon() {
|
||||||
cfg, log := b.cfg, b.log
|
cfg, log := b.cfg, b.log
|
||||||
// ConnectedNATS must carry the endpoint the daemon's own client connected
|
|
||||||
// with at startup, not the raw config: for embedded mode the config URL is
|
|
||||||
// empty, but container env must never see an empty NATS_URL. (Containers
|
|
||||||
// are rejected in embedded mode by validation, but the field should mean
|
|
||||||
// the same thing regardless of mode.)
|
|
||||||
connectedNATS := cfg.NATS
|
|
||||||
connectedNATS.URL = b.natsURL
|
|
||||||
b.d = &daemon.Daemon{
|
b.d = &daemon.Daemon{
|
||||||
Cfg: config.NewHolder(cfg),
|
Cfg: config.NewHolder(cfg),
|
||||||
ConnectedNATS: connectedNATS,
|
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken},
|
||||||
Store: b.st,
|
Store: b.st,
|
||||||
Bus: b.bus,
|
Bus: b.bus,
|
||||||
Forge: b.forgeClient,
|
Forge: b.forgeClient,
|
||||||
@@ -1323,11 +1326,14 @@ func (b *boot) startServices(ctx context.Context) error {
|
|||||||
// setupBackends wires the optional runtime backends: the NATS client
|
// setupBackends wires the optional runtime backends: the NATS client
|
||||||
// and the container pool.
|
// and the container pool.
|
||||||
func (b *boot) setupBackends(ctx context.Context) error {
|
func (b *boot) setupBackends(ctx context.Context) error {
|
||||||
if err := b.connectNATS(ctx); err != nil {
|
closeContainers := b.setupContainers(ctx)
|
||||||
return err
|
err := b.connectNATS(ctx)
|
||||||
}
|
// Container discovery must precede embedded NATS so it can bind the
|
||||||
b.setupContainers(ctx)
|
// worker bridge gateway, but cleanup registration comes afterwards.
|
||||||
return nil
|
// cleanups run LIFO: workers stop first, then the daemon client closes,
|
||||||
|
// then the embedded server shuts down.
|
||||||
|
b.addCleanup(closeContainers)
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildAgentAndWorkflows builds the agent runner and the workflow
|
// buildAgentAndWorkflows builds the agent runner and the workflow
|
||||||
|
|||||||
@@ -0,0 +1,152 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"go/ast"
|
||||||
|
"go/parser"
|
||||||
|
"go/token"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/moby/moby/client"
|
||||||
|
natsio "github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/config"
|
||||||
|
"github.com/samcharles93/archie-core/internal/container"
|
||||||
|
)
|
||||||
|
|
||||||
|
// setupContainers must discover the bridge before embedded NATS starts, but
|
||||||
|
// its cleanup registration comes afterwards. boot.cleanup is LIFO, so this
|
||||||
|
// ordering stops worker containers before closing the daemon client and the
|
||||||
|
// embedded server they are connected to.
|
||||||
|
func TestSetupBackendsRegistersContainerCleanupAfterNATS(t *testing.T) {
|
||||||
|
fileset := token.NewFileSet()
|
||||||
|
file, err := parser.ParseFile(fileset, "bootstrap.go", nil, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
setupContainers := methodBody(t, file, "setupContainers")
|
||||||
|
if methodCallPosition(setupContainers, "addCleanup") != token.NoPos {
|
||||||
|
t.Fatal("setupContainers registers cleanup before NATS; LIFO would stop the broker before workers")
|
||||||
|
}
|
||||||
|
setupBackends := methodBody(t, file, "setupBackends")
|
||||||
|
connect := methodCallPosition(setupBackends, "connectNATS")
|
||||||
|
register := methodCallPosition(setupBackends, "addCleanup")
|
||||||
|
if connect == token.NoPos || register == token.NoPos || register < connect {
|
||||||
|
t.Fatalf("setupBackends call positions: connectNATS=%d addCleanup=%d, want container cleanup registered after connect", connect, register)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func methodBody(t *testing.T, file *ast.File, name string) *ast.BlockStmt {
|
||||||
|
t.Helper()
|
||||||
|
for _, decl := range file.Decls {
|
||||||
|
fn, ok := decl.(*ast.FuncDecl)
|
||||||
|
if ok && fn.Recv != nil && fn.Name.Name == name {
|
||||||
|
return fn.Body
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("method %s not found", name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func methodCallPosition(body *ast.BlockStmt, name string) token.Pos {
|
||||||
|
position := token.NoPos
|
||||||
|
ast.Inspect(body, func(node ast.Node) bool {
|
||||||
|
call, ok := node.(*ast.CallExpr)
|
||||||
|
if !ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
selector, ok := call.Fun.(*ast.SelectorExpr)
|
||||||
|
if ok && selector.Sel.Name == name && position == token.NoPos {
|
||||||
|
position = call.Pos()
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
return position
|
||||||
|
}
|
||||||
|
|
||||||
|
// This is the native-daemon deployment seam: Docker supplies the host-side
|
||||||
|
// gateway of the worker bridge, embedded NATS binds that address, and the
|
||||||
|
// runtime credential shared with workers is required by the same listener.
|
||||||
|
func TestConnectEmbeddedNATSUsesManagedWorkerBridge(t *testing.T) {
|
||||||
|
const gateway = "127.0.0.1"
|
||||||
|
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch {
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/networks/workers"):
|
||||||
|
writeDockerResponse(t, w, map[string]any{
|
||||||
|
"Name": "workers",
|
||||||
|
"Driver": "bridge",
|
||||||
|
"Scope": "local",
|
||||||
|
"IPAM": map[string]any{
|
||||||
|
"Config": []map[string]string{{
|
||||||
|
"Subnet": "127.0.0.0/8",
|
||||||
|
"Gateway": gateway,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/containers/json"):
|
||||||
|
writeDockerResponse(t, w, []any{})
|
||||||
|
default:
|
||||||
|
http.Error(w, fmt.Sprintf("unexpected Docker API path %s", r.URL.Path), http.StatusNotFound)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
t.Cleanup(dockerAPI.Close)
|
||||||
|
|
||||||
|
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = dockerClient.Close() })
|
||||||
|
|
||||||
|
pool, err := container.NewPool(t.Context(), container.Config{
|
||||||
|
DockerClient: dockerClient,
|
||||||
|
Network: "workers",
|
||||||
|
RequireHostGateway: true,
|
||||||
|
}, slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("container.NewPool = %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = pool.Close() })
|
||||||
|
|
||||||
|
b := &boot{
|
||||||
|
cfg: config.Config{
|
||||||
|
DBPath: filepath.Join(t.TempDir(), "archie.db"),
|
||||||
|
NATS: config.NATSConfig{Mode: config.NATSModeEmbedded},
|
||||||
|
},
|
||||||
|
log: slog.New(slog.DiscardHandler),
|
||||||
|
containerPool: pool,
|
||||||
|
}
|
||||||
|
if err := b.connectNATS(t.Context()); err != nil {
|
||||||
|
t.Fatalf("connectNATS = %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(b.cleanup)
|
||||||
|
|
||||||
|
if !strings.HasPrefix(b.natsURL, "nats://"+gateway+":") {
|
||||||
|
t.Errorf("connected NATS URL = %q, want bridge gateway %s", b.natsURL, gateway)
|
||||||
|
}
|
||||||
|
if b.natsToken == "" {
|
||||||
|
t.Fatal("connected NATS token is empty")
|
||||||
|
}
|
||||||
|
if unauthenticated, connectErr := natsio.Connect(b.natsURL); connectErr == nil {
|
||||||
|
unauthenticated.Close()
|
||||||
|
t.Fatal("unauthenticated worker connection succeeded")
|
||||||
|
}
|
||||||
|
authenticated, err := natsio.Connect(b.natsURL, natsio.Token(b.natsToken))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("authenticated worker connection = %v", err)
|
||||||
|
}
|
||||||
|
authenticated.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeDockerResponse(t *testing.T, w http.ResponseWriter, value any) {
|
||||||
|
t.Helper()
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
if err := json.NewEncoder(w).Encode(value); err != nil {
|
||||||
|
t.Errorf("encode Docker response: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
+8
-7
@@ -979,13 +979,14 @@ func startContainers(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pool, err := container.NewPool(ctx, container.Config{
|
pool, err := container.NewPool(ctx, container.Config{
|
||||||
Image: cfg.Containers.Image,
|
Image: cfg.Containers.Image,
|
||||||
MaxConcurrency: cfg.Containers.MaxConcurrency,
|
MaxConcurrency: cfg.Containers.MaxConcurrency,
|
||||||
MaxUptime: cfg.Containers.MaxUptime.Std(),
|
MaxUptime: cfg.Containers.MaxUptime.Std(),
|
||||||
PullPolicy: cfg.Containers.PullPolicy,
|
PullPolicy: cfg.Containers.PullPolicy,
|
||||||
Network: cfg.Containers.Network,
|
Network: cfg.Containers.Network,
|
||||||
DockerClient: dockerCli,
|
DockerClient: dockerCli,
|
||||||
}, cfg.NATS.URL, log)
|
RequireHostGateway: cfg.NATS.Mode == config.NATSModeEmbedded,
|
||||||
|
}, log)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// A missing image is recoverable by hand. The daemon sends no registry
|
// A missing image is recoverable by hand. The daemon sends no registry
|
||||||
// credentials on pull (internal/container/pool.go), so a private
|
// credentials on pull (internal/container/pool.go), so a private
|
||||||
|
|||||||
+4
-3
@@ -231,14 +231,14 @@ gate_max_failures = 5
|
|||||||
# NATS JetStream for task distribution and reaction delivery. mode selects
|
# NATS JetStream for task distribution and reaction delivery. mode selects
|
||||||
# how NATS is provided:
|
# how NATS is provided:
|
||||||
# "embedded" – in-process nats-server (the default; no external server)
|
# "embedded" – in-process nats-server (the default; no external server)
|
||||||
# "external" – connect to url (required for agent.mode="nats" / [containers])
|
# "external" – connect to url
|
||||||
# "off" – no NATS; the SQLite ClaimNext flow is used for tasks
|
# "off" – no NATS; the SQLite ClaimNext flow is used for tasks
|
||||||
# Leave the section absent (or set no url) to run embedded; set mode="off"
|
# Leave the section absent (or set no url) to run embedded; set mode="off"
|
||||||
# only to opt out of NATS entirely.
|
# only to opt out of NATS entirely.
|
||||||
# [nats]
|
# [nats]
|
||||||
# mode = "embedded" # "embedded" | "external" | "off"
|
# mode = "embedded" # "embedded" | "external" | "off"
|
||||||
# url = "nats://localhost:4222" # required when mode = "external"
|
# url = "nats://localhost:4222" # required when mode = "external"
|
||||||
# token_env = "NATS_TOKEN" # optional; named variable must be set
|
# token_env = "NATS_TOKEN" # optional external-server credential; embedded generates one per start
|
||||||
|
|
||||||
# Unbound webhook capture: POST /webhooks/capture/{source} accepts and
|
# Unbound webhook capture: POST /webhooks/capture/{source} accepts and
|
||||||
# stores any inbound payload archie doesn't yet have a binding for, so it can
|
# stores any inbound payload archie doesn't yet have a binding for, so it can
|
||||||
@@ -255,7 +255,8 @@ gate_max_failures = 5
|
|||||||
# rate_per_second = 1.0 # per-remote-address token bucket
|
# rate_per_second = 1.0 # per-remote-address token bucket
|
||||||
# rate_burst = 5
|
# rate_burst = 5
|
||||||
|
|
||||||
# Docker sandbox execution of archie-agent. Requires [nats] and agent.mode = "nats".
|
# Docker sandbox execution of archie-agent. Requires NATS and agent.mode = "nats";
|
||||||
|
# embedded and external broker modes are both supported.
|
||||||
# [containers]
|
# [containers]
|
||||||
# enabled = true
|
# enabled = true
|
||||||
# image = "archie-agent:latest" # Docker image to run
|
# image = "archie-agent:latest" # Docker image to run
|
||||||
|
|||||||
+61
-71
@@ -1,86 +1,76 @@
|
|||||||
# Embedded NATS -- decision
|
# Embedded NATS execution -- decision
|
||||||
|
|
||||||
**Status:** Decided, not yet implemented
|
**Status:** Amended; implementation in progress
|
||||||
**Date:** 2026-08-22
|
|
||||||
**Beads issue:** `archie-core-7d5u.2`
|
|
||||||
|
|
||||||
`event-sources-and-reactions.md` settled the reaction-delivery mechanism (reuse
|
**Date:** 2026-08-24
|
||||||
`internal/infrastructure/eventbus/nats`, a second stream under a fan-out
|
|
||||||
retention policy) and left the deployment shape to this issue. This document
|
|
||||||
decides that shape. `github.com/nats-io/nats-server/v2` is already a direct
|
|
||||||
dependency, used today only in `_test.go` files; the gap is that production
|
|
||||||
startup only knows "external URL configured" vs "no NATS, use SQLite
|
|
||||||
`ClaimNext`".
|
|
||||||
|
|
||||||
## 1. Embedded runs alongside SQLite, not instead of it
|
**Beads epic:** `archie-core-q15y`
|
||||||
|
|
||||||
The SQLite store remains authoritative for task state. NATS -- external or
|
This decision supersedes the 2026-08-22 position that embedded NATS was only a
|
||||||
embedded -- is the *distribution* path, and `drainNATS` already falls back to
|
distribution mechanism and could not be an agent-execution topology. That was
|
||||||
`Store.ClaimNext` for requeued tasks (waiting_human approval, retry-parked)
|
true only because the embedded listener was unauthenticated loopback. It left a
|
||||||
that never went through a NATS publish. So wiring embedded NATS does not
|
second production architecture in which `archied` ran autonomous workflows in
|
||||||
remove the SQLite flow; it makes `Daemon.Tasks != nil` true in single-process
|
its own process, weakening the isolation boundary that managed task containers
|
||||||
deployments, which routes poll→enqueue through `pollNATS` and drain through
|
exist to provide.
|
||||||
`drainNATS` exactly as an external-NATS deployment already does. `ClaimNext`
|
|
||||||
stays as the requeue fallback inside `drainNATS`. No new drain path, no second
|
|
||||||
eventbus type.
|
|
||||||
|
|
||||||
## 2. Config: `[nats] mode = "embedded" | "external" | "off"`
|
## One execution architecture, two broker deployments
|
||||||
|
|
||||||
`NATSConfig` gains a `mode` field. `url` no longer doubles as the on/off
|
Every autonomous repository workflow is handed over as one complete task via
|
||||||
switch -- that coupling is what made "no NATS" a permanent stance instead of a
|
NATS and runs in a task-scoped `archie-agent` container. `archied` remains a
|
||||||
deployment shape.
|
native, long-lived orchestration and interactive-chat process; its operator
|
||||||
|
configured host filesystem access is unchanged. It owns task admission,
|
||||||
|
state, forge and worktree operations, container lifecycle, and the broker when
|
||||||
|
the embedded deployment is selected. It does not run an autonomous model loop.
|
||||||
|
|
||||||
| mode | meaning |
|
NATS has two supported deployment shapes:
|
||||||
|------------|----------------------------------------------------------------|
|
|
||||||
| `external` | connect to `url` (today's behaviour). `url` is required. |
|
|
||||||
| `embedded` | start an in-process nats-server; the client connects to it. |
|
|
||||||
| `off` | no NATS at all; the old SQLite-only flow (`ClaimNext`). |
|
|
||||||
|
|
||||||
Default resolution: `url` set → `external`; `url` empty and `mode` unset or
|
| mode | meaning |
|
||||||
`embedded` → `embedded`; `mode = "off"` → `off`. `mode = "external"` with no
|
| --- | --- |
|
||||||
`url` is a validation error; `mode = "off"`/`"embedded"` with a `url` set is a
|
| `embedded` | `archied` owns an in-process JetStream server (default). |
|
||||||
validation error. This is a meaning change and is stated as one: `url` empty
|
| `external` | `archied` and workers connect to the configured NATS URL. |
|
||||||
*used to mean* "no NATS, SQLite only"; it *now means* "embedded NATS", and
|
|
||||||
getting the old behaviour requires the explicit opt-out `mode = "off"`.
|
|
||||||
|
|
||||||
`agent.mode = "nats"` and `[containers]` require `mode = "external"` (a
|
Both shapes have identical task semantics: one container and one full-task
|
||||||
container cannot reach an in-process loopback server, and embedded NATS is not
|
request per workflow. Broker placement must not select a different executor.
|
||||||
an agent-execution topology).
|
The legacy `nats.mode = "off"`, `agent.mode = "inprocess"`, per-stage NATS, and
|
||||||
|
subprocess execution paths are migration inputs only and will be removed rather
|
||||||
|
than retained as production fallbacks.
|
||||||
|
|
||||||
## 3. The daemon owns the embedded server's lifecycle
|
SQLite remains authoritative for task state. NATS is the sole autonomous work
|
||||||
|
handoff. SQLite recovery may identify work that needs republishing, but it must
|
||||||
|
not cause `archied` to call `workflow.Run` locally.
|
||||||
|
|
||||||
`internal/infrastructure/eventbus/nats` gains an `EmbeddedServer` type that
|
## Embedded reachability and authentication
|
||||||
wraps `nats-server/v2/server.Server`, exposing only `ClientURL()` and
|
|
||||||
`Shutdown()`. The composition root (`cmd/archied/bootstrap.go`) starts it in
|
|
||||||
`connectNATS` (before dialing the client), binds it to `127.0.0.1` on a random
|
|
||||||
port, enables JetStream with a store dir under the daemon's data directory
|
|
||||||
(`<dir of DBPath>/nats`), and registers a cleanup that shuts the server down
|
|
||||||
*after* the client closes. Cleanup ordering is LIFO: the server shutdown is
|
|
||||||
registered before the client close, so the client closes first.
|
|
||||||
|
|
||||||
## Retention is a Config field, not a constant
|
An embedded listener uses a cryptographically random bearer token generated for
|
||||||
|
each daemon start. The token exists only in process memory and the environment
|
||||||
|
of managed worker containers; it is never written to config or logs.
|
||||||
|
|
||||||
`nats.Config` gains `Retention *jetstream.RetentionPolicy`, defaulting to
|
For a native daemon with managed containers, startup resolves the Docker bridge
|
||||||
`jetstream.WorkQueuePolicy` in `withDefaults`, replacing the hardcoded value in
|
the workers will join. The listener binds only to that bridge's host gateway
|
||||||
`Connect` (`client.go:65`). `WorkQueuePolicy` is correct for `ARCHIE_TASKS`
|
address on a random port. Both the native daemon and workers use that same
|
||||||
(one consumer claims each message) and wrong for reactions (fan-out). This is
|
reachable endpoint, and every client supplies the generated token. Binding a
|
||||||
the enabler for the second reaction stream: `event-sources-and-reactions.md`
|
Docker bridge address keeps the listener off ordinary LAN interfaces without
|
||||||
requires it, and nothing here creates the reaction stream itself -- that is
|
publishing a host port. A configured user bridge is respected; otherwise the
|
||||||
provisioned by the beads that build reaction producers and consumers, which do
|
Docker default bridge is used. A network without a usable local IPv4 gateway is
|
||||||
not exist yet. A test proves the parameterization: two overlapping `Subscribe`
|
a startup error for autonomous execution.
|
||||||
consumers both receive a message under `LimitsPolicy` (fan-out) and starve
|
|
||||||
under `WorkQueuePolicy`, the exact trap `CLAUDE.md` documents.
|
|
||||||
|
|
||||||
## Packages this touches
|
If Docker is unavailable, interactive chat may still start and embedded NATS
|
||||||
|
may remain loopback-only, but autonomous tasks park with an explicit container
|
||||||
|
capability failure. They never fall back to host execution.
|
||||||
|
|
||||||
- `internal/infrastructure/eventbus/nats`: `Retention` on `Config`, an
|
## Ownership and lifecycle
|
||||||
`EmbeddedServer` type.
|
|
||||||
- `internal/config`: `NATSConfig.Mode`.
|
- `internal/infrastructure/eventbus/nats` owns the authenticated embedded
|
||||||
- `internal/infrastructure/configuration`: `mode` defaulting and validation.
|
server and exposes its runtime endpoint, not the server implementation.
|
||||||
- `cmd/archied/bootstrap.go`: start/stop the embedded server; set
|
- `internal/container` resolves the worker bridge and gateway and owns task
|
||||||
`Daemon.ConnectedNATS.URL` to the embedded client URL so container env (if
|
container lifecycle.
|
||||||
ever enabled) never receives an empty endpoint.
|
- `internal/app` composition selects embedded or external broker deployment,
|
||||||
- `config.example.toml`: document `mode`. The `deployments/*` profiles are
|
freezes the connected runtime endpoint for worker environments, and wires the
|
||||||
unchanged on purpose: their absent `[nats]` section now selects embedded by
|
full-task handoff.
|
||||||
default, which is the intended single-process behaviour this issue exists to
|
- `cmd/archied` remains process input only as the domain migration reaches this
|
||||||
provide, not a regression to paper over with `mode = "off"`.
|
composition path.
|
||||||
|
|
||||||
|
The daemon starts the embedded server before its client, closes clients before
|
||||||
|
the server, and keeps JetStream data beneath the daemon data directory. NATS URL
|
||||||
|
and token are startup-built values: a reload cannot send new workers to a broker
|
||||||
|
different from the one the daemon is using.
|
||||||
@@ -623,8 +623,7 @@ const (
|
|||||||
// delivery. The mode selects how NATS is provided:
|
// delivery. The mode selects how NATS is provided:
|
||||||
//
|
//
|
||||||
// "embedded" – an in-process nats-server (default; no external server)
|
// "embedded" – an in-process nats-server (default; no external server)
|
||||||
// "external" – connect to URL (required; the only mode agent.mode="nats"
|
// "external" – connect to URL (required)
|
||||||
// and [containers] can use)
|
|
||||||
// "off" – no NATS; the SQLite ClaimNext flow is used unchanged
|
// "off" – no NATS; the SQLite ClaimNext flow is used unchanged
|
||||||
//
|
//
|
||||||
// An empty Mode resolves from URL: URL set means "external", URL empty means
|
// An empty Mode resolves from URL: URL set means "external", URL empty means
|
||||||
@@ -637,8 +636,8 @@ type NATSConfig struct {
|
|||||||
// URL is the NATS server address, e.g. "nats://localhost:4222".
|
// URL is the NATS server address, e.g. "nats://localhost:4222".
|
||||||
// Required when Mode is "external"; must be empty otherwise.
|
// Required when Mode is "external"; must be empty otherwise.
|
||||||
URL string `toml:"url" yaml:"url"`
|
URL string `toml:"url" yaml:"url"`
|
||||||
// TokenEnv optionally names an env var holding a NATS auth token.
|
// TokenEnv optionally names an env var holding an external NATS auth token.
|
||||||
// Empty means no authentication. Ignored when Mode is "embedded".
|
// Empty means no authentication. Embedded mode generates a per-start token.
|
||||||
TokenEnv string `toml:"token_env" yaml:"token_env"`
|
TokenEnv string `toml:"token_env" yaml:"token_env"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -685,13 +684,11 @@ type ContainerConfig struct {
|
|||||||
VolumeTTL Duration `toml:"volume_ttl" yaml:"volume_ttl"`
|
VolumeTTL Duration `toml:"volume_ttl" yaml:"volume_ttl"`
|
||||||
// PullPolicy controls image pulling: "missing" (default) or "always".
|
// PullPolicy controls image pulling: "missing" (default) or "always".
|
||||||
PullPolicy string `toml:"pull_policy" yaml:"pull_policy"`
|
PullPolicy string `toml:"pull_policy" yaml:"pull_policy"`
|
||||||
// Network is the Docker network spawned agent containers join, so they
|
// Network is the Docker network spawned agent containers join. Empty first
|
||||||
// can resolve sibling compose services (nats, etc.) by name. Empty
|
// tries the daemon's own network (for containerized composition), then uses
|
||||||
// falls back to auto-detecting the daemon's own network, which is
|
// Docker's default bridge. Embedded NATS binds the selected bridge's host
|
||||||
// best-effort and silently yields the default bridge network (no
|
// gateway; external deployments can set a Compose network explicitly when
|
||||||
// hostname resolution) if detection fails — set this explicitly in
|
// workers must resolve broker service names (e.g. "archie-core_default").
|
||||||
// compose/production deployments (e.g. "archie-core_default") rather
|
|
||||||
// than relying on auto-detection.
|
|
||||||
Network string `toml:"network" yaml:"network"`
|
Network string `toml:"network" yaml:"network"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/moby/moby/client"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NetworkName is the Docker network managed worker containers join.
|
||||||
|
func (p *Pool) NetworkName() string { return p.network }
|
||||||
|
|
||||||
|
// HostGateway is the host-side IPv4 gateway of NetworkName. It is non-empty
|
||||||
|
// only when Config.RequireHostGateway requested embedded broker reachability.
|
||||||
|
func (p *Pool) HostGateway() string { return p.hostGateway }
|
||||||
|
|
||||||
|
func resolveHostGateway(
|
||||||
|
ctx context.Context,
|
||||||
|
cli *client.Client,
|
||||||
|
networkName string,
|
||||||
|
required bool,
|
||||||
|
) (string, error) {
|
||||||
|
if !required {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
result, err := cli.NetworkInspect(ctx, networkName, client.NetworkInspectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("inspect Docker network %q for embedded NATS: %w", networkName, err)
|
||||||
|
}
|
||||||
|
network := result.Network.Network
|
||||||
|
if network.Driver != "bridge" {
|
||||||
|
return "", fmt.Errorf("docker network %q uses driver %q, want bridge for embedded NATS", networkName, network.Driver)
|
||||||
|
}
|
||||||
|
if network.Scope != "" && network.Scope != "local" {
|
||||||
|
return "", fmt.Errorf("docker network %q has scope %q, want local for embedded NATS", networkName, network.Scope)
|
||||||
|
}
|
||||||
|
for _, ipam := range network.IPAM.Config {
|
||||||
|
gateway := ipam.Gateway
|
||||||
|
if gateway.IsValid() && gateway.Is4() && !gateway.IsUnspecified() {
|
||||||
|
return gateway.String(), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("docker network %q has no usable IPv4 host gateway for embedded NATS", networkName)
|
||||||
|
}
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/moby/moby/client"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A native archied process is not itself attached to a Docker network. When
|
||||||
|
// embedded NATS must be reachable by managed workers, the pool therefore uses
|
||||||
|
// Docker's default bridge and exposes its host-side gateway to composition.
|
||||||
|
func TestNewPoolResolvesDefaultBridgeGatewayForNativeDaemon(t *testing.T) {
|
||||||
|
hostname, err := os.Hostname()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
const gateway = "172.31.0.1"
|
||||||
|
|
||||||
|
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch {
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/containers/"+hostname+"/json"):
|
||||||
|
http.Error(w, `{"message":"no such container"}`, http.StatusNotFound)
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/networks/bridge"):
|
||||||
|
writeDockerJSON(t, w, map[string]any{
|
||||||
|
"Name": "bridge",
|
||||||
|
"Driver": "bridge",
|
||||||
|
"Scope": "local",
|
||||||
|
"IPAM": map[string]any{
|
||||||
|
"Config": []map[string]string{{
|
||||||
|
"Subnet": "172.31.0.0/16",
|
||||||
|
"Gateway": gateway,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/containers/json"):
|
||||||
|
writeDockerJSON(t, w, []any{})
|
||||||
|
default:
|
||||||
|
http.Error(w, fmt.Sprintf("unexpected Docker API path %s", r.URL.Path), http.StatusNotFound)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
t.Cleanup(dockerAPI.Close)
|
||||||
|
|
||||||
|
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = dockerClient.Close() })
|
||||||
|
|
||||||
|
pool, err := NewPool(t.Context(), Config{
|
||||||
|
DockerClient: dockerClient,
|
||||||
|
RequireHostGateway: true,
|
||||||
|
}, discardLogger())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPool = %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = pool.Close() })
|
||||||
|
|
||||||
|
access, ok := any(pool).(interface {
|
||||||
|
NetworkName() string
|
||||||
|
HostGateway() string
|
||||||
|
})
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("Pool does not expose its resolved worker network and host gateway")
|
||||||
|
}
|
||||||
|
if got := access.NetworkName(); got != "bridge" {
|
||||||
|
t.Errorf("NetworkName() = %q, want bridge", got)
|
||||||
|
}
|
||||||
|
if got := access.HostGateway(); got != gateway {
|
||||||
|
t.Errorf("HostGateway() = %q, want %q", got, gateway)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewPoolRejectsUnreachableEmbeddedNetwork(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
statusCode int
|
||||||
|
network map[string]any
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "inspect failure",
|
||||||
|
statusCode: http.StatusInternalServerError,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non bridge driver",
|
||||||
|
network: map[string]any{
|
||||||
|
"Name": "workers", "Driver": "overlay", "Scope": "local",
|
||||||
|
"IPAM": map[string]any{"Config": []map[string]string{{"Gateway": "172.31.0.1"}}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non local scope",
|
||||||
|
network: map[string]any{
|
||||||
|
"Name": "workers", "Driver": "bridge", "Scope": "swarm",
|
||||||
|
"IPAM": map[string]any{"Config": []map[string]string{{"Gateway": "172.31.0.1"}}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no IPv4 gateway",
|
||||||
|
network: map[string]any{
|
||||||
|
"Name": "workers", "Driver": "bridge", "Scope": "local",
|
||||||
|
"IPAM": map[string]any{"Config": []map[string]string{{"Gateway": "fd00::1"}}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch {
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/networks/workers"):
|
||||||
|
if tt.statusCode != 0 {
|
||||||
|
http.Error(w, `{"message":"network inspect failed"}`, tt.statusCode)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeDockerJSON(t, w, tt.network)
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/containers/json"):
|
||||||
|
writeDockerJSON(t, w, []any{})
|
||||||
|
default:
|
||||||
|
http.Error(w, fmt.Sprintf("unexpected Docker API path %s", r.URL.Path), http.StatusNotFound)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
t.Cleanup(dockerAPI.Close)
|
||||||
|
|
||||||
|
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = dockerClient.Close() })
|
||||||
|
|
||||||
|
pool, err := NewPool(t.Context(), Config{
|
||||||
|
DockerClient: dockerClient,
|
||||||
|
Network: "workers",
|
||||||
|
RequireHostGateway: true,
|
||||||
|
}, discardLogger())
|
||||||
|
if err == nil {
|
||||||
|
_ = pool.Close()
|
||||||
|
t.Fatal("NewPool = nil error, want embedded reachability validation failure")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeDockerJSON(t *testing.T, w http.ResponseWriter, value any) {
|
||||||
|
t.Helper()
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
if err := json.NewEncoder(w).Encode(value); err != nil {
|
||||||
|
t.Errorf("encode Docker response: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func discardLogger() *slog.Logger {
|
||||||
|
return slog.New(slog.DiscardHandler)
|
||||||
|
}
|
||||||
+35
-17
@@ -72,17 +72,19 @@ func WriteTaskJSON(workspace string, payload TaskPayload) error {
|
|||||||
|
|
||||||
// Pool manages a set of Docker containers running archie-agent.
|
// Pool manages a set of Docker containers running archie-agent.
|
||||||
type Pool struct {
|
type Pool struct {
|
||||||
cli *client.Client
|
cli *client.Client
|
||||||
ownCli bool // true when the pool created the client (owns Close)
|
ownCli bool // true when the pool created the client (owns Close)
|
||||||
cfg Config
|
cfg Config
|
||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
natsURL string
|
// network is the Docker network spawned containers join. External broker
|
||||||
// network is the user-defined Docker network spawned containers join,
|
// deployments may leave it empty to use Docker's implicit default bridge;
|
||||||
// detected from the daemon's own container so they can resolve
|
// embedded deployments resolve that default explicitly as "bridge" so its
|
||||||
// sibling services (nats, etc.) by compose service name. Empty when
|
// host gateway can be inspected.
|
||||||
// the daemon isn't running in a container on such a network, or when
|
|
||||||
// detection fails -- Docker then falls back to its default bridge.
|
|
||||||
network string
|
network string
|
||||||
|
// hostGateway is the local IPv4 gateway of network. It is populated when
|
||||||
|
// embedded broker reachability is required, so native composition can bind
|
||||||
|
// the listener to an address visible only to workers on this bridge.
|
||||||
|
hostGateway string
|
||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
active int
|
active int
|
||||||
@@ -107,6 +109,9 @@ type Config struct {
|
|||||||
// NewPool creates its own via client.New(client.FromEnv). Pass a
|
// NewPool creates its own via client.New(client.FromEnv). Pass a
|
||||||
// shared client to avoid multiple independent connections.
|
// shared client to avoid multiple independent connections.
|
||||||
DockerClient *client.Client
|
DockerClient *client.Client
|
||||||
|
// RequireHostGateway resolves and validates a local Docker bridge gateway.
|
||||||
|
// Embedded NATS needs this address; external brokers do not.
|
||||||
|
RequireHostGateway bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewPool connects to the Docker daemon, optionally pulls the image,
|
// NewPool connects to the Docker daemon, optionally pulls the image,
|
||||||
@@ -114,7 +119,7 @@ type Config struct {
|
|||||||
//
|
//
|
||||||
// If cfg.DockerClient is set, it is reused (caller owns Close).
|
// If cfg.DockerClient is set, it is reused (caller owns Close).
|
||||||
// Otherwise a new client is created via client.FromEnv (pool owns Close).
|
// Otherwise a new client is created via client.FromEnv (pool owns Close).
|
||||||
func NewPool(ctx context.Context, cfg Config, natsURL string, log *slog.Logger) (*Pool, error) {
|
func NewPool(ctx context.Context, cfg Config, log *slog.Logger) (*Pool, error) {
|
||||||
cli := cfg.DockerClient
|
cli := cfg.DockerClient
|
||||||
if cli == nil {
|
if cli == nil {
|
||||||
var err error
|
var err error
|
||||||
@@ -128,14 +133,27 @@ func NewPool(ctx context.Context, cfg Config, natsURL string, log *slog.Logger)
|
|||||||
if network == "" {
|
if network == "" {
|
||||||
network = selfNetwork(ctx, cli, log)
|
network = selfNetwork(ctx, cli, log)
|
||||||
}
|
}
|
||||||
|
if cfg.RequireHostGateway && network == "" {
|
||||||
|
// A native daemon is not itself attached to a Docker network. Its
|
||||||
|
// managed workers use Docker's default bridge unless configured
|
||||||
|
// otherwise, so that is the bridge whose host gateway must be bound.
|
||||||
|
network = "bridge"
|
||||||
|
}
|
||||||
|
hostGateway, err := resolveHostGateway(ctx, cli, network, cfg.RequireHostGateway)
|
||||||
|
if err != nil {
|
||||||
|
if cfg.DockerClient == nil {
|
||||||
|
_ = cli.Close()
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
p := &Pool{
|
p := &Pool{
|
||||||
cli: cli,
|
cli: cli,
|
||||||
cfg: cfg,
|
cfg: cfg,
|
||||||
log: log,
|
log: log,
|
||||||
natsURL: natsURL,
|
ownCli: cfg.DockerClient == nil,
|
||||||
ownCli: cfg.DockerClient == nil,
|
network: network,
|
||||||
network: network,
|
hostGateway: hostGateway,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Pull image if needed.
|
// Pull image if needed.
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
package daemon
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
dockercontainer "github.com/moby/moby/api/types/container"
|
||||||
|
"github.com/moby/moby/client"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/config"
|
||||||
|
archiecontainer "github.com/samcharles93/archie-core/internal/container"
|
||||||
|
)
|
||||||
|
|
||||||
|
// This pins the last handoff seam, not just its two halves: the runtime NATS
|
||||||
|
// endpoint frozen in Daemon becomes the environment on a container created on
|
||||||
|
// the same Docker network whose gateway embedded NATS binds.
|
||||||
|
func TestConnectedNATSEndpointReachesManagedContainerCreate(t *testing.T) {
|
||||||
|
const (
|
||||||
|
networkName = "workers"
|
||||||
|
gateway = "172.31.0.1"
|
||||||
|
containerID = "12345678901234567890123456789012"
|
||||||
|
token = "runtime-only-token"
|
||||||
|
)
|
||||||
|
var createRequest dockercontainer.CreateRequest
|
||||||
|
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch {
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/networks/"+networkName):
|
||||||
|
writeContainerDockerJSON(t, w, map[string]any{
|
||||||
|
"Name": networkName,
|
||||||
|
"Driver": "bridge",
|
||||||
|
"Scope": "local",
|
||||||
|
"IPAM": map[string]any{
|
||||||
|
"Config": []map[string]string{{
|
||||||
|
"Subnet": "172.31.0.0/16",
|
||||||
|
"Gateway": gateway,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/containers/create"):
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&createRequest); err != nil {
|
||||||
|
t.Errorf("decode container create request: %v", err)
|
||||||
|
}
|
||||||
|
writeContainerDockerJSON(t, w, dockercontainer.CreateResponse{ID: containerID})
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/containers/"+containerID+"/start"):
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/containers/json"):
|
||||||
|
writeContainerDockerJSON(t, w, []any{})
|
||||||
|
default:
|
||||||
|
http.Error(w, fmt.Sprintf("unexpected Docker API path %s", r.URL.Path), http.StatusNotFound)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
t.Cleanup(dockerAPI.Close)
|
||||||
|
|
||||||
|
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = dockerClient.Close() })
|
||||||
|
|
||||||
|
pool, err := archiecontainer.NewPool(t.Context(), archiecontainer.Config{
|
||||||
|
Image: "archie-agent:test",
|
||||||
|
Network: networkName,
|
||||||
|
DockerClient: dockerClient,
|
||||||
|
RequireHostGateway: true,
|
||||||
|
}, slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("container.NewPool = %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = pool.Close() })
|
||||||
|
|
||||||
|
d := &Daemon{
|
||||||
|
Cfg: config.NewHolder(config.Config{}),
|
||||||
|
ConnectedNATS: NATSEndpoint{
|
||||||
|
URL: "nats://" + gateway + ":4222",
|
||||||
|
Token: token,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if _, err := pool.Acquire(t.Context(), nil, d.containerEnv(nil)); err != nil {
|
||||||
|
t.Fatalf("Pool.Acquire = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if createRequest.HostConfig == nil || string(createRequest.HostConfig.NetworkMode) != networkName {
|
||||||
|
t.Fatalf("container network = %v, want %q", createRequest.HostConfig, networkName)
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"NATS_URL=nats://" + gateway + ":4222",
|
||||||
|
"NATS_TOKEN=" + token,
|
||||||
|
} {
|
||||||
|
if !slices.Contains(createRequest.Env, want) {
|
||||||
|
t.Errorf("container env = %q, want %q", createRequest.Env, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeContainerDockerJSON(t *testing.T, w http.ResponseWriter, value any) {
|
||||||
|
t.Helper()
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
if err := json.NewEncoder(w).Encode(value); err != nil {
|
||||||
|
t.Errorf("encode Docker response: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
+15
-11
@@ -54,17 +54,23 @@ type WorktreeGrantIssuer interface {
|
|||||||
Issue(task *store.Task) (token string, revoke func(), err error)
|
Issue(task *store.Task) (token string, revoke func(), err error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NATSEndpoint is the broker address and credential archied connected with at
|
||||||
|
// startup. It is runtime state rather than configuration: embedded mode
|
||||||
|
// generates both values, and external secret references are resolved once.
|
||||||
|
type NATSEndpoint struct {
|
||||||
|
URL string
|
||||||
|
Token string
|
||||||
|
}
|
||||||
|
|
||||||
type Daemon struct {
|
type Daemon struct {
|
||||||
// Cfg publishes the running configuration. Read through Cfg() so a
|
// Cfg publishes the running configuration. Read through Cfg() so a
|
||||||
// reload swaps the published snapshot atomically. See config.Holder.
|
// reload swaps the published snapshot atomically. See config.Holder.
|
||||||
Cfg *config.Holder
|
Cfg *config.Holder
|
||||||
// ConnectedNATS is the [nats] configuration the daemon's own client
|
// ConnectedNATS is the endpoint and credential the daemon's own client
|
||||||
// connected with at startup (the endpoint it publishes on). Container
|
// connected with at startup. Container env is built from this, not from
|
||||||
// env is built from this, not from the live config, so a reload of
|
// live config or environment, so reload cannot point new containers at a
|
||||||
// nats.url/token_env cannot point new containers at a server the
|
// broker the daemon is not publishing on. Zero value means no NATS.
|
||||||
// daemon is not publishing on. Zero value means NATS is not
|
ConnectedNATS NATSEndpoint
|
||||||
// configured.
|
|
||||||
ConnectedNATS config.NATSConfig
|
|
||||||
Store store.TaskStore
|
Store store.TaskStore
|
||||||
Forge forge.Forge
|
Forge forge.Forge
|
||||||
Trees *worktree.Manager
|
Trees *worktree.Manager
|
||||||
@@ -1168,10 +1174,8 @@ func (d *Daemon) containerEnv(task *store.Task) []string {
|
|||||||
// the live config: a reloaded [nats] section must not point new
|
// the live config: a reloaded [nats] section must not point new
|
||||||
// containers at a server the daemon is not publishing on.
|
// containers at a server the daemon is not publishing on.
|
||||||
env = append(env, "NATS_URL="+d.ConnectedNATS.URL)
|
env = append(env, "NATS_URL="+d.ConnectedNATS.URL)
|
||||||
if tokenEnv := d.ConnectedNATS.TokenEnv; tokenEnv != "" {
|
if token := d.ConnectedNATS.Token; token != "" {
|
||||||
if token := os.Getenv(tokenEnv); token != "" {
|
env = append(env, "NATS_TOKEN="+token)
|
||||||
env = append(env, "NATS_TOKEN="+token)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
// The agent process runs as root inside the container (no USER in the
|
// The agent process runs as root inside the container (no USER in the
|
||||||
// Dockerfile, no userns-remap), so a commit it writes to the bind-mounted
|
// Dockerfile, no userns-remap), so a commit it writes to the bind-mounted
|
||||||
|
|||||||
@@ -305,20 +305,17 @@ func TestAllowConcurrentForTaskPrefersOwningIdentityRepo(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestContainerEnvIncludesConfiguredNATSCredentials(t *testing.T) {
|
func TestContainerEnvIncludesConfiguredNATSCredentials(t *testing.T) {
|
||||||
t.Setenv("ARCHIE_NATS_SECRET", "test-nats-token")
|
|
||||||
|
|
||||||
// ConnectedNATS is the endpoint the daemon's own client connected with
|
// ConnectedNATS is the endpoint the daemon's own client connected with
|
||||||
// at startup; containerEnv deliberately reads this, not the live Cfg,
|
// at startup; containerEnv deliberately reads this frozen runtime value,
|
||||||
// so a reload of nats.url cannot point new containers at a server the
|
// not live config or environment.
|
||||||
// daemon is not publishing on.
|
|
||||||
natsCfg := config.NATSConfig{
|
|
||||||
URL: "nats://nats.example:4222",
|
|
||||||
TokenEnv: "ARCHIE_NATS_SECRET",
|
|
||||||
}
|
|
||||||
d := &Daemon{
|
d := &Daemon{
|
||||||
Cfg: config.NewHolder(config.Config{}),
|
Cfg: config.NewHolder(config.Config{}),
|
||||||
ConnectedNATS: natsCfg,
|
ConnectedNATS: NATSEndpoint{URL: "nats://nats.example:4222", Token: "test-nats-token"},
|
||||||
}
|
}
|
||||||
|
// Runtime credentials are frozen when the daemon connects. Mutating the
|
||||||
|
// source environment later must not send new workers to the same endpoint
|
||||||
|
// with a credential the broker never adopted.
|
||||||
|
t.Setenv("ARCHIE_NATS_SECRET", "changed-after-startup")
|
||||||
|
|
||||||
got := d.containerEnv(nil)
|
got := d.containerEnv(nil)
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
|
|||||||
@@ -278,10 +278,10 @@ func effectiveNATSMode(cfg *config.Config) string {
|
|||||||
return config.NATSModeEmbedded
|
return config.NATSModeEmbedded
|
||||||
}
|
}
|
||||||
|
|
||||||
// validateNATS checks the nats mode and its consistency with url and the
|
// validateNATS checks the nats mode and its consistency with url. It resolves
|
||||||
// agent execution mode. It resolves an unset mode from url without mutating
|
// an unset mode from url without mutating cfg, so Validate (which documents it
|
||||||
// cfg, so Validate (which documents it does not apply defaults) accepts a
|
// does not apply defaults) accepts a hand-built config the same way Loader.File
|
||||||
// hand-built config the same way Loader.File accepts its on-disk form.
|
// accepts its on-disk form.
|
||||||
func validateNATS(cfg *config.Config) error {
|
func validateNATS(cfg *config.Config) error {
|
||||||
mode := effectiveNATSMode(cfg)
|
mode := effectiveNATSMode(cfg)
|
||||||
if !oneOf(mode, natsModes) {
|
if !oneOf(mode, natsModes) {
|
||||||
@@ -297,17 +297,15 @@ func validateNATS(cfg *config.Config) error {
|
|||||||
return fmt.Errorf("%w: nats.url must be empty when nats.mode is %q", ErrInvalidInput, mode)
|
return fmt.Errorf("%w: nats.url must be empty when nats.mode is %q", ErrInvalidInput, mode)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Agent execution over NATS needs workers that can reach the server. An
|
if cfg.Agent.Mode == agentModeNATS && mode == config.NATSModeEmbedded && !cfg.Containers.Enabled {
|
||||||
// embedded server binds loopback on a random port, which workers cannot
|
return fmt.Errorf("%w: containers.enabled must be true when agent.mode is %q and nats.mode is %q", ErrInvalidInput, agentModeNATS, config.NATSModeEmbedded)
|
||||||
// discover, so agent.mode="nats" requires an external URL.
|
|
||||||
if cfg.Agent.Mode == agentModeNATS && mode != config.NATSModeExternal {
|
|
||||||
return fmt.Errorf("%w: nats.mode must be %q when agent.mode is %q", ErrInvalidInput, config.NATSModeExternal, agentModeNATS)
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// validateContainers checks the combination that container execution
|
// validateContainers checks the combination container execution requires.
|
||||||
// requires: a NATS transport and somewhere to reach it.
|
// Embedded and external NATS are both worker-reachable deployment shapes;
|
||||||
|
// composition resolves the Docker bridge gateway for embedded mode.
|
||||||
func validateContainers(cfg *config.Config) error {
|
func validateContainers(cfg *config.Config) error {
|
||||||
if !cfg.Containers.Enabled {
|
if !cfg.Containers.Enabled {
|
||||||
return nil
|
return nil
|
||||||
@@ -318,8 +316,8 @@ func validateContainers(cfg *config.Config) error {
|
|||||||
if cfg.Agent.Mode != agentModeNATS {
|
if cfg.Agent.Mode != agentModeNATS {
|
||||||
return fmt.Errorf("%w: agent.mode must be %q when containers.enabled is true", ErrInvalidInput, agentModeNATS)
|
return fmt.Errorf("%w: agent.mode must be %q when containers.enabled is true", ErrInvalidInput, agentModeNATS)
|
||||||
}
|
}
|
||||||
if effectiveNATSMode(cfg) != config.NATSModeExternal {
|
if effectiveNATSMode(cfg) == config.NATSModeOff {
|
||||||
return fmt.Errorf("%w: nats.mode must be %q when containers.enabled is true (containers cannot reach an embedded loopback server)", ErrInvalidInput, config.NATSModeExternal)
|
return fmt.Errorf("%w: nats.mode must not be %q when containers.enabled is true", ErrInvalidInput, config.NATSModeOff)
|
||||||
}
|
}
|
||||||
if cfg.Containers.VolumeTTL < 0 {
|
if cfg.Containers.VolumeTTL < 0 {
|
||||||
return fmt.Errorf("%w: containers.volume_ttl must not be negative", ErrInvalidInput)
|
return fmt.Errorf("%w: containers.volume_ttl must not be negative", ErrInvalidInput)
|
||||||
|
|||||||
@@ -192,14 +192,16 @@ func TestValidate_RejectsTheSameProblemsAsLoaderLoad(t *testing.T) {
|
|||||||
|
|
||||||
// TestValidateNATS pins the nats mode contract from docs/prds/embedded-nats.md:
|
// TestValidateNATS pins the nats mode contract from docs/prds/embedded-nats.md:
|
||||||
// an unset mode resolves from url without mutating cfg, external requires url,
|
// an unset mode resolves from url without mutating cfg, external requires url,
|
||||||
// embedded/off forbid a url, and agent.mode="nats" requires external.
|
// embedded/off forbid a url, and NATS agent execution works with either
|
||||||
|
// embedded or external broker deployment.
|
||||||
func TestValidateNATS(t *testing.T) {
|
func TestValidateNATS(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
mode string
|
mode string
|
||||||
url string
|
url string
|
||||||
agent string
|
agent string
|
||||||
wantErr bool
|
containers bool
|
||||||
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
{name: "unset mode with url resolves external", url: "nats://localhost:4222"},
|
{name: "unset mode with url resolves external", url: "nats://localhost:4222"},
|
||||||
{name: "unset mode without url resolves embedded"},
|
{name: "unset mode without url resolves embedded"},
|
||||||
@@ -210,7 +212,8 @@ func TestValidateNATS(t *testing.T) {
|
|||||||
{name: "embedded with url", mode: config.NATSModeEmbedded, url: "nats://localhost:4222", wantErr: true},
|
{name: "embedded with url", mode: config.NATSModeEmbedded, url: "nats://localhost:4222", wantErr: true},
|
||||||
{name: "off with url", mode: config.NATSModeOff, url: "nats://localhost:4222", wantErr: true},
|
{name: "off with url", mode: config.NATSModeOff, url: "nats://localhost:4222", wantErr: true},
|
||||||
{name: "unknown mode", mode: "not-a-mode", wantErr: true},
|
{name: "unknown mode", mode: "not-a-mode", wantErr: true},
|
||||||
{name: "agent nats with embedded", mode: config.NATSModeEmbedded, agent: "nats", wantErr: true},
|
{name: "agent nats with embedded requires containers", mode: config.NATSModeEmbedded, agent: "nats", wantErr: true},
|
||||||
|
{name: "agent nats with embedded containers", mode: config.NATSModeEmbedded, agent: "nats", containers: true},
|
||||||
{name: "agent nats with external", mode: config.NATSModeExternal, url: "nats://localhost:4222", agent: "nats"},
|
{name: "agent nats with external", mode: config.NATSModeExternal, url: "nats://localhost:4222", agent: "nats"},
|
||||||
}
|
}
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
@@ -220,6 +223,9 @@ func TestValidateNATS(t *testing.T) {
|
|||||||
if tc.agent != "" {
|
if tc.agent != "" {
|
||||||
cfg.Agent.Mode = tc.agent
|
cfg.Agent.Mode = tc.agent
|
||||||
}
|
}
|
||||||
|
if tc.containers {
|
||||||
|
cfg.Containers.Enabled = true
|
||||||
|
}
|
||||||
err := validateNATS(&cfg)
|
err := validateNATS(&cfg)
|
||||||
if tc.wantErr && err == nil {
|
if tc.wantErr && err == nil {
|
||||||
t.Fatal("validateNATS() = nil, want an error")
|
t.Fatal("validateNATS() = nil, want an error")
|
||||||
@@ -252,11 +258,11 @@ func TestValidateContainersResolvesNATSMode(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("embedded mode rejects containers", func(t *testing.T) {
|
t.Run("embedded mode supports containers", func(t *testing.T) {
|
||||||
cfg := base()
|
cfg := base()
|
||||||
cfg.NATS.Mode = config.NATSModeEmbedded
|
cfg.NATS.Mode = config.NATSModeEmbedded
|
||||||
if err := Validate(&cfg); err == nil {
|
if err := Validate(&cfg); err != nil {
|
||||||
t.Fatal("Validate() = nil, want an error for containers with embedded nats")
|
t.Fatalf("Validate() = %v, want nil for managed containers with embedded nats", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package nats
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"time"
|
"time"
|
||||||
@@ -19,14 +20,16 @@ import (
|
|||||||
// owns the lifecycle: StartEmbedded before dialing the client, Shutdown after
|
// owns the lifecycle: StartEmbedded before dialing the client, Shutdown after
|
||||||
// the client closes.
|
// the client closes.
|
||||||
type EmbeddedServer struct {
|
type EmbeddedServer struct {
|
||||||
srv *server.Server
|
srv *server.Server
|
||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
|
token string
|
||||||
}
|
}
|
||||||
|
|
||||||
// EmbeddedOptions configures StartEmbedded.
|
// EmbeddedOptions configures StartEmbedded.
|
||||||
type EmbeddedOptions struct {
|
type EmbeddedOptions struct {
|
||||||
// Host is the bind address. Empty means "127.0.0.1" (loopback only,
|
// Host is the bind address. Empty means "127.0.0.1" (loopback only).
|
||||||
// which is correct: an embedded server carries no authentication).
|
// Container-backed deployments pass the host gateway address of the
|
||||||
|
// Docker bridge their workers join.
|
||||||
Host string
|
Host string
|
||||||
// Port is the listen port. Zero means a random port.
|
// Port is the listen port. Zero means a random port.
|
||||||
Port int
|
Port int
|
||||||
@@ -50,13 +53,15 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger)
|
|||||||
if log == nil {
|
if log == nil {
|
||||||
log = slog.New(slog.DiscardHandler)
|
log = slog.New(slog.DiscardHandler)
|
||||||
}
|
}
|
||||||
|
token := rand.Text()
|
||||||
|
|
||||||
srv, err := server.NewServer(&server.Options{
|
srv, err := server.NewServer(&server.Options{
|
||||||
Host: opts.Host,
|
Host: opts.Host,
|
||||||
Port: opts.Port,
|
Port: opts.Port,
|
||||||
JetStream: true,
|
JetStream: true,
|
||||||
StoreDir: opts.StoreDir,
|
StoreDir: opts.StoreDir,
|
||||||
NoSigs: true, // the daemon owns signals, not the embedded server
|
Authorization: token,
|
||||||
|
NoSigs: true, // the daemon owns signals, not the embedded server
|
||||||
// NoLog leaves the server's logger nil (ConfigureLogger is only ever
|
// NoLog leaves the server's logger nil (ConfigureLogger is only ever
|
||||||
// called on reload). A nil logger makes the server's Fatalf a no-op
|
// called on reload). A nil logger makes the server's Fatalf a no-op
|
||||||
// rather than an os.Exit, so a JetStream startup failure cannot crash
|
// rather than an os.Exit, so a JetStream startup failure cannot crash
|
||||||
@@ -83,7 +88,7 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger)
|
|||||||
}
|
}
|
||||||
|
|
||||||
log.Info("embedded nats ready", "url", srv.ClientURL())
|
log.Info("embedded nats ready", "url", srv.ClientURL())
|
||||||
return &EmbeddedServer{srv: srv, log: log}, nil
|
return &EmbeddedServer{srv: srv, log: log, token: token}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ClientURL returns the address the server is listening on. It is the URL the
|
// ClientURL returns the address the server is listening on. It is the URL the
|
||||||
@@ -91,6 +96,11 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger)
|
|||||||
// connection used.
|
// connection used.
|
||||||
func (e *EmbeddedServer) ClientURL() string { return e.srv.ClientURL() }
|
func (e *EmbeddedServer) ClientURL() string { return e.srv.ClientURL() }
|
||||||
|
|
||||||
|
// Token returns the per-start credential required by every embedded NATS
|
||||||
|
// client. Composition keeps it in memory and passes it only to the daemon's
|
||||||
|
// client and managed worker environments; it must never be logged or persisted.
|
||||||
|
func (e *EmbeddedServer) Token() string { return e.token }
|
||||||
|
|
||||||
// Shutdown stops the server and waits for it to finish. It is safe to call on
|
// Shutdown stops the server and waits for it to finish. It is safe to call on
|
||||||
// a nil server so callers can defer it unconditionally.
|
// a nil server so callers can defer it unconditionally.
|
||||||
func (e *EmbeddedServer) Shutdown() {
|
func (e *EmbeddedServer) Shutdown() {
|
||||||
|
|||||||
@@ -21,11 +21,20 @@ func TestStartEmbeddedLifecycle(t *testing.T) {
|
|||||||
if url == "" {
|
if url == "" {
|
||||||
t.Fatal("ClientURL() = empty, want the bound address")
|
t.Fatal("ClientURL() = empty, want the bound address")
|
||||||
}
|
}
|
||||||
|
if unauthenticated, connectErr := nats.Connect(url); connectErr == nil {
|
||||||
|
unauthenticated.Close()
|
||||||
|
t.Fatal("unauthenticated connection succeeded, want embedded NATS to require its per-start token")
|
||||||
|
}
|
||||||
|
token := srv.Token()
|
||||||
|
if token == "" {
|
||||||
|
t.Fatal("Token() = empty, want a generated runtime token")
|
||||||
|
}
|
||||||
|
|
||||||
// Connecting and provisioning a stream proves JetStream is live, not just
|
// Connecting and provisioning a stream proves JetStream is live, not just
|
||||||
// the core NATS socket.
|
// the core NATS socket.
|
||||||
c, err := Connect(t.Context(), Config{
|
c, err := Connect(t.Context(), Config{
|
||||||
URL: url,
|
URL: url,
|
||||||
|
Token: token,
|
||||||
Subjects: []string{"embedded.test.>"},
|
Subjects: []string{"embedded.test.>"},
|
||||||
FilterSubject: "embedded.test.>",
|
FilterSubject: "embedded.test.>",
|
||||||
}, discardLogger())
|
}, discardLogger())
|
||||||
|
|||||||
Reference in new issue
Block a user