fix(nats): containers log in with their run credential and reach only their task's subjects

This commit is contained in:
sam committed 2026-10-04 19:33:14 +11:00
1 parent 6c208f2b0a
commit 4699280c9d
11 files changed
+321 -19

No files matched your search

+8 -6
View File
@@ -112,9 +112,17 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies
dependencies.markSafe(ctx, workDir, log)
// The task is known before connecting: without a broker token the
// worker logs in as its task, with its run credential.
taskID, err := dependencies.bootID(workDir)
if err != nil {
log.Error("task boot failed", "err", err)
return &StartupError{Operation: "task boot failed", Err: err}
}
transport, err := dependencies.connect(ctx, agentnats.Config{
URL: settings.NATSURL,
Token: settings.NATSToken,
TaskID: taskID,
StateStoreURL: settings.StateStoreTarget,
StateStoreToken: settings.StateStoreToken,
}, log)
@@ -124,12 +132,6 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies
return &StartupError{Operation: operation, Err: err}
}
defer transport.Close()
taskID, err := dependencies.bootID(workDir)
if err != nil {
log.Error("task boot failed", "err", err)
return &StartupError{Operation: "task boot failed", Err: err}
}
log = slog.New(agentexec.NewSystemLogHandler(log.Handler(), transport.LogPublisher(), taskID))
log.Info("system log publisher attached", "task", taskID)
+4 -1
View File
@@ -424,6 +424,9 @@ func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) {
// A restarted broker keeps the previous endpoint, so the Gateway and
// running agents reconnect to it instead of a broker that no longer exists.
opts := nats.EmbeddedOptions{Host: host, StoreDir: filepath.Join(cfg.StateDir, "nats")}
if b.stateStoreGrants != nil {
opts.Tasks = taskBroker{runs: b.stateStoreGrants.Client}
}
if previous, err := servicekit.ReadNATSEndpoint(cfg.StateDir); err == nil {
opts.Token = previous.Token
if u, err := url.Parse(previous.URL); err == nil {
@@ -595,7 +598,7 @@ func (b *boot) buildDaemon() {
log := b.log
b.d = &daemon.Daemon{
Cfg: b.cfgHolder,
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken},
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken, Scoped: b.cfg.NATS.Mode != config.NATSModeExternal},
ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), Token: b.stateStoreToken},
Store: b.stateStore,
Bus: b.bus,
+13
View File
@@ -47,6 +47,9 @@ func (b *boot) setupReadinessProbes() {
if b.accessLive != nil {
probes = append(probes, readiness.NewProblemProbe("access_policies", b.accessLive))
}
if cfg.NATS.Mode == config.NATSModeExternal {
probes = append(probes, unscopedBroker{})
}
b.healthRegistry = health.NewRegistry(probes...)
}
@@ -87,3 +90,13 @@ func diskProbePath(cfg config.Config) string {
return "."
}
}
// unscopedBroker reports that containers hold the external broker's own
// token: only the embedded broker can limit a container to its task.
type unscopedBroker struct{}
func (unscopedBroker) Name() string { return "nats_credential_scope" }
func (unscopedBroker) Check(context.Context) health.Result {
return health.Result{Status: health.StatusDegraded, Detail: "external NATS: agent containers receive the broker token, which reaches every subject; use the embedded broker to scope them to their task"}
}
+37
View File
@@ -0,0 +1,37 @@
package archied
import (
"context"
"fmt"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/forgerpc"
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
"github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
"github.com/samcharles93/archie-core/internal/worktreerpc"
)
// taskBroker opens a task's own broker subjects to its run credential: its
// handoff, its logs and events, and its identity's forge and push servers.
type taskBroker struct {
runs interface {
TaskForCredential(ctx context.Context, token string) (*workflow.Task, error)
}
}
func (b taskBroker) TaskSubjects(ctx context.Context, taskID int64, credential string) (nats.TaskSubjects, error) {
task, err := b.runs.TaskForCredential(ctx, credential)
if err != nil {
return nats.TaskSubjects{}, err
}
if task.ID != taskID {
return nats.TaskSubjects{}, fmt.Errorf("run credential is for task %d, not %d", task.ID, taskID)
}
publish := append([]string{
agentexec.SubjectForSystem(taskID),
agentexec.SubjectForEvents(taskID),
worktreerpc.SubjectFor(task.Identity, worktreerpc.SubjectPush),
}, forgerpc.Subjects(task.Identity)...)
return nats.TaskSubjects{Publish: publish, Subscribe: []string{agentnats.SubjectForTask(taskID)}}, nil
}
+14 -1
View File
@@ -63,6 +63,19 @@ type RunCredentialIssuer interface {
type NATSEndpoint struct {
URL string
Token string
// Scoped is true on the embedded broker, where a container logs in with
// its run credential and reaches only its own task's subjects.
Scoped bool
}
// ContainerToken is the broker token a container is handed: none on a
// scoped broker. An external broker cannot scope a container, so it still
// gets the broker's token (reported as a health issue by archied).
func (e NATSEndpoint) ContainerToken() string {
if e.Scoped {
return ""
}
return e.Token
}
// StateStoreEndpoint is the State Store target and token passed to agent
@@ -1937,7 +1950,7 @@ func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken string) []str
// the live config: a reloaded [nats] section must not point new
// containers at a server the daemon is not publishing on.
env = append(env, "NATS_URL="+d.ConnectedNATS.URL)
if token := d.ConnectedNATS.Token; token != "" {
if token := d.ConnectedNATS.ContainerToken(); token != "" {
env = append(env, "NATS_TOKEN="+token)
}
// Pass the State Store target with this task's scoped token, never the
+1 -1
View File
@@ -45,7 +45,7 @@ func (d *Daemon) runKitTask(ctx context.Context, task *workflow.Task, repo confi
}
defer revokeCredential()
env, err := kitrun.WorkerEnv(kitrun.Endpoints{
NATS: d.ConnectedNATS.URL, NATSToken: d.ConnectedNATS.Token,
NATS: d.ConnectedNATS.URL, NATSToken: d.ConnectedNATS.ContainerToken(),
StateStore: d.ConnectedStateStore.URL, StateStoreToken: credential,
}, os.Getuid(), os.Getgid())
if err != nil {
+14
View File
@@ -29,6 +29,20 @@ const (
SubjectReplyToReview = "archie.forge.reply_to_review"
)
// Subjects returns every forge RPC subject identity's server answers: the
// set a task of that identity may publish to.
func Subjects(identity string) []string {
bases := []string{
SubjectComment, SubjectCloseIssue, SubjectCreatePR, SubjectLinkBranch,
SubjectSetStateLabel, SubjectCreateReviewComment, SubjectReplyToReview,
}
out := make([]string, len(bases))
for i, base := range bases {
out[i] = SubjectFor(identity, base)
}
return out
}
// SubjectFor returns base scoped to identity, or base when identity is
// empty.
func SubjectFor(identity, base string) string {
@@ -18,6 +18,7 @@ import (
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/forgerpc"
eventbus "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
"github.com/samcharles93/archie-core/internal/taskrun"
"github.com/samcharles93/archie-core/internal/worktreerpc"
@@ -28,8 +29,12 @@ import (
// fully on gRPC for its store calls, and the legacy NATS storerpc path is
// deleted
type Config struct {
URL string
Token string
URL string
// Token is a broker-wide token, set only on an external broker. Without
// it the worker logs in as TaskID with StateStoreToken, its run
// credential, which reaches only its own task's subjects.
Token string
TaskID int64
StateStoreURL string
StateStoreToken string
@@ -65,8 +70,13 @@ func Connect(ctx context.Context, config Config, log *slog.Logger) (*Transport,
return nil, err
}
options := []natsio.Option{natsio.Name("archie-agent")}
if config.Token != "" {
switch {
case config.Token != "":
options = append(options, natsio.Token(config.Token))
case config.StateStoreToken != "":
options = append(options,
natsio.UserInfo(fmt.Sprintf("%s%d", eventbus.TaskUserPrefix, config.TaskID), config.StateStoreToken),
natsio.CustomInboxPrefix(eventbus.TaskInboxPrefix(config.TaskID)))
}
conn, err := natsio.Connect(config.URL, options...)
if err != nil {
@@ -0,0 +1,82 @@
package nats
import (
"context"
"crypto/subtle"
"strconv"
"strings"
"time"
"github.com/nats-io/nats-server/v2/server"
)
// TaskUserPrefix names a task's broker user: "task-<id>". Its password is the
// task's run credential.
const TaskUserPrefix = "task-"
// TaskInboxPrefix is the inbox prefix a task's connection must use: the only
// inbox it may subscribe to.
func TaskInboxPrefix(taskID int64) string {
return "_INBOX." + TaskUserPrefix + strconv.FormatInt(taskID, 10)
}
// TaskSubjects is what one task's connection may publish to and subscribe to,
// beside its own inbox and its replies.
type TaskSubjects struct {
Publish []string
Subscribe []string
}
// TaskAuthority answers which subjects a run credential opens for its task.
// An error refuses the connection.
type TaskAuthority interface {
TaskSubjects(ctx context.Context, taskID int64, credential string) (TaskSubjects, error)
}
// taskAuthTimeout bounds the credential lookup a connection waits on.
const taskAuthTimeout = 2 * time.Second
// authenticator admits the instance token with every permission, and a task
// user only to its own task's subjects, so no container holds a credential
// that reaches past its run.
type authenticator struct {
token string
tasks TaskAuthority
}
func (a authenticator) Check(c server.ClientAuthentication) bool {
opts := c.GetOpts()
if opts.Token != "" {
if subtle.ConstantTimeCompare([]byte(opts.Token), []byte(a.token)) != 1 {
return false
}
c.RegisterUser(&server.User{Username: "archie"})
return true
}
taskID, ok := taskUser(opts.Username)
if !ok || opts.Password == "" || a.tasks == nil {
return false
}
ctx, cancel := context.WithTimeout(context.Background(), taskAuthTimeout)
defer cancel()
subjects, err := a.tasks.TaskSubjects(ctx, taskID, opts.Password)
if err != nil {
return false
}
c.RegisterUser(&server.User{Username: opts.Username, Permissions: &server.Permissions{
Publish: &server.SubjectPermission{Allow: subjects.Publish},
Subscribe: &server.SubjectPermission{Allow: append(subjects.Subscribe, TaskInboxPrefix(taskID)+".>")},
// Lets the task answer the one request it is sent, on the requester's
// reply subject.
Response: &server.ResponsePermission{MaxMsgs: 1},
}})
return true
}
func taskUser(name string) (int64, bool) {
id, err := strconv.ParseInt(strings.TrimPrefix(name, TaskUserPrefix), 10, 64)
if !strings.HasPrefix(name, TaskUserPrefix) || err != nil || id <= 0 {
return 0, false
}
return id, true
}
@@ -0,0 +1,123 @@
package nats
import (
"context"
"errors"
"slices"
"strings"
"testing"
"time"
natsio "github.com/nats-io/nats.go"
)
type taskSeven struct{}
func (taskSeven) TaskSubjects(_ context.Context, taskID int64, credential string) (TaskSubjects, error) {
if taskID != 7 || credential != "cred-7" {
return TaskSubjects{}, errors.New("unknown")
}
return TaskSubjects{Publish: []string{"archie.agent.7.events"}, Subscribe: []string{"archie.taskrun.7"}}, nil
}
// TestBrokerScopesTaskCredentials pins that the instance token reaches
// everything and a task's run credential reaches only its own subjects, on a
// real embedded broker.
func TestBrokerScopesTaskCredentials(t *testing.T) {
srv, err := StartEmbedded(context.Background(), EmbeddedOptions{Token: "instance", StoreDir: t.TempDir(), Tasks: taskSeven{}}, nil)
if err != nil {
t.Fatal(err)
}
defer srv.Shutdown()
tests := []struct {
name string
opts []natsio.Option
connects bool
allowed []string // "pub:" or "sub:" subjects that must work
refused []string // that must be refused
}{
{
"instance token reaches everything",
[]natsio.Option{natsio.Token("instance")},
true,
[]string{"pub:archie.taskrun.8", "sub:_INBOX.>", "js:"},
nil,
},
{
"task credential reaches its own subjects only",
[]natsio.Option{natsio.UserInfo("task-7", "cred-7"), natsio.CustomInboxPrefix(TaskInboxPrefix(7))},
true,
[]string{"pub:archie.agent.7.events", "sub:archie.taskrun.7", "sub:" + TaskInboxPrefix(7) + ".x"},
[]string{"pub:archie.taskrun.8", "pub:archie.agent.8.events", "sub:archie.taskrun.8", "sub:_INBOX.>"},
},
{"credential for another task", []natsio.Option{natsio.UserInfo("task-8", "cred-7")}, false, nil, nil},
{"unknown credential", []natsio.Option{natsio.UserInfo("task-7", "guess")}, false, nil, nil},
{"wrong instance token", []natsio.Option{natsio.Token("guess")}, false, nil, nil},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
violations := make(chan string, 16)
opts := append(slices.Clone(tt.opts), natsio.ErrorHandler(func(_ *natsio.Conn, _ *natsio.Subscription, err error) {
violations <- err.Error()
}))
nc, err := natsio.Connect(srv.ClientURL(), opts...)
if !tt.connects {
if err == nil {
nc.Close()
t.Fatal("connected; want refused")
}
return
}
if err != nil {
t.Fatal(err)
}
defer nc.Close()
for _, op := range tt.allowed {
if v := attempt(t, nc, op, violations); v != "" {
t.Errorf("%s refused: %s", op, v)
}
}
for _, op := range tt.refused {
if v := attempt(t, nc, op, violations); !strings.Contains(v, "Permissions Violation") {
t.Errorf("%s allowed; want refused", op)
}
}
})
}
}
// attempt performs one "pub:", "sub:" or "js:" operation and returns the permission
// violation it drew, if any.
func attempt(t *testing.T, nc *natsio.Conn, op string, violations chan string) string {
t.Helper()
kind, subject, _ := strings.Cut(op, ":")
switch kind {
case "js":
// The daemon's task queue lives in JetStream.
js, err := nc.JetStream()
if err == nil {
_, err = js.AccountInfo()
}
if err != nil {
return err.Error()
}
case "pub":
_ = nc.Publish(subject, []byte("x"))
default:
sub, err := nc.SubscribeSync(subject)
if err != nil {
return err.Error()
}
defer func() { _ = sub.Unsubscribe() }()
}
if err := nc.FlushTimeout(time.Second); err != nil {
t.Fatal(err)
}
select {
case v := <-violations:
return v
case <-time.After(100 * time.Millisecond):
return ""
}
}
@@ -26,8 +26,11 @@ type EmbeddedOptions struct {
Host string
// Port is the listen port. Zero means a random port.
Port int
// Token is the client token. Empty generates one.
// Token is the instance token: every permission. Empty generates one.
Token string
// Tasks admits task users ("task-<id>" with the run credential) to their
// own task's subjects. Nil admits the instance token only.
Tasks TaskAuthority
// StoreDir is where JetStream persists streams. Empty means a temporary
// directory (streams do not survive a reboot). The daemon passes a dir
// under its data directory so the ARCHIE_TASKS and reaction streams are
@@ -54,12 +57,14 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger)
}
srv, err := server.NewServer(&server.Options{
Host: opts.Host,
Port: opts.Port,
JetStream: true,
StoreDir: opts.StoreDir,
Authorization: token,
NoSigs: true, // the daemon owns signals, not the embedded server
Host: opts.Host,
Port: opts.Port,
JetStream: true,
StoreDir: opts.StoreDir,
// Replaces token authorization: the instance token and task users
// are both checked here.
CustomClientAuthentication: authenticator{token: token, tasks: opts.Tasks},
NoSigs: true, // the daemon owns signals, not the embedded server
// NoLog leaves the server's logger nil (ConfigureLogger is only ever
// called on reload). A nil logger makes the server's Fatalf a no-op
// rather than an os.Exit, so a JetStream startup failure cannot crash