fix(nats): containers log in with their run credential and reach only their task's subjects
This commit is contained in:
1 parent
6c208f2b0a
commit
4699280c9d
11 files changed
+321
-19
No files matched your search
@@ -112,9 +112,17 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies
|
||||
|
||||
dependencies.markSafe(ctx, workDir, log)
|
||||
|
||||
// The task is known before connecting: without a broker token the
|
||||
// worker logs in as its task, with its run credential.
|
||||
taskID, err := dependencies.bootID(workDir)
|
||||
if err != nil {
|
||||
log.Error("task boot failed", "err", err)
|
||||
return &StartupError{Operation: "task boot failed", Err: err}
|
||||
}
|
||||
transport, err := dependencies.connect(ctx, agentnats.Config{
|
||||
URL: settings.NATSURL,
|
||||
Token: settings.NATSToken,
|
||||
TaskID: taskID,
|
||||
StateStoreURL: settings.StateStoreTarget,
|
||||
StateStoreToken: settings.StateStoreToken,
|
||||
}, log)
|
||||
@@ -124,12 +132,6 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies
|
||||
return &StartupError{Operation: operation, Err: err}
|
||||
}
|
||||
defer transport.Close()
|
||||
|
||||
taskID, err := dependencies.bootID(workDir)
|
||||
if err != nil {
|
||||
log.Error("task boot failed", "err", err)
|
||||
return &StartupError{Operation: "task boot failed", Err: err}
|
||||
}
|
||||
log = slog.New(agentexec.NewSystemLogHandler(log.Handler(), transport.LogPublisher(), taskID))
|
||||
log.Info("system log publisher attached", "task", taskID)
|
||||
|
||||
|
||||
@@ -424,6 +424,9 @@ func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) {
|
||||
// A restarted broker keeps the previous endpoint, so the Gateway and
|
||||
// running agents reconnect to it instead of a broker that no longer exists.
|
||||
opts := nats.EmbeddedOptions{Host: host, StoreDir: filepath.Join(cfg.StateDir, "nats")}
|
||||
if b.stateStoreGrants != nil {
|
||||
opts.Tasks = taskBroker{runs: b.stateStoreGrants.Client}
|
||||
}
|
||||
if previous, err := servicekit.ReadNATSEndpoint(cfg.StateDir); err == nil {
|
||||
opts.Token = previous.Token
|
||||
if u, err := url.Parse(previous.URL); err == nil {
|
||||
@@ -595,7 +598,7 @@ func (b *boot) buildDaemon() {
|
||||
log := b.log
|
||||
b.d = &daemon.Daemon{
|
||||
Cfg: b.cfgHolder,
|
||||
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken},
|
||||
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken, Scoped: b.cfg.NATS.Mode != config.NATSModeExternal},
|
||||
ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), Token: b.stateStoreToken},
|
||||
Store: b.stateStore,
|
||||
Bus: b.bus,
|
||||
|
||||
@@ -47,6 +47,9 @@ func (b *boot) setupReadinessProbes() {
|
||||
if b.accessLive != nil {
|
||||
probes = append(probes, readiness.NewProblemProbe("access_policies", b.accessLive))
|
||||
}
|
||||
if cfg.NATS.Mode == config.NATSModeExternal {
|
||||
probes = append(probes, unscopedBroker{})
|
||||
}
|
||||
b.healthRegistry = health.NewRegistry(probes...)
|
||||
}
|
||||
|
||||
@@ -87,3 +90,13 @@ func diskProbePath(cfg config.Config) string {
|
||||
return "."
|
||||
}
|
||||
}
|
||||
|
||||
// unscopedBroker reports that containers hold the external broker's own
|
||||
// token: only the embedded broker can limit a container to its task.
|
||||
type unscopedBroker struct{}
|
||||
|
||||
func (unscopedBroker) Name() string { return "nats_credential_scope" }
|
||||
|
||||
func (unscopedBroker) Check(context.Context) health.Result {
|
||||
return health.Result{Status: health.StatusDegraded, Detail: "external NATS: agent containers receive the broker token, which reaches every subject; use the embedded broker to scope them to their task"}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/forgerpc"
|
||||
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
|
||||
"github.com/samcharles93/archie-core/internal/worktreerpc"
|
||||
)
|
||||
|
||||
// taskBroker opens a task's own broker subjects to its run credential: its
|
||||
// handoff, its logs and events, and its identity's forge and push servers.
|
||||
type taskBroker struct {
|
||||
runs interface {
|
||||
TaskForCredential(ctx context.Context, token string) (*workflow.Task, error)
|
||||
}
|
||||
}
|
||||
|
||||
func (b taskBroker) TaskSubjects(ctx context.Context, taskID int64, credential string) (nats.TaskSubjects, error) {
|
||||
task, err := b.runs.TaskForCredential(ctx, credential)
|
||||
if err != nil {
|
||||
return nats.TaskSubjects{}, err
|
||||
}
|
||||
if task.ID != taskID {
|
||||
return nats.TaskSubjects{}, fmt.Errorf("run credential is for task %d, not %d", task.ID, taskID)
|
||||
}
|
||||
publish := append([]string{
|
||||
agentexec.SubjectForSystem(taskID),
|
||||
agentexec.SubjectForEvents(taskID),
|
||||
worktreerpc.SubjectFor(task.Identity, worktreerpc.SubjectPush),
|
||||
}, forgerpc.Subjects(task.Identity)...)
|
||||
return nats.TaskSubjects{Publish: publish, Subscribe: []string{agentnats.SubjectForTask(taskID)}}, nil
|
||||
}
|
||||
@@ -63,6 +63,19 @@ type RunCredentialIssuer interface {
|
||||
type NATSEndpoint struct {
|
||||
URL string
|
||||
Token string
|
||||
// Scoped is true on the embedded broker, where a container logs in with
|
||||
// its run credential and reaches only its own task's subjects.
|
||||
Scoped bool
|
||||
}
|
||||
|
||||
// ContainerToken is the broker token a container is handed: none on a
|
||||
// scoped broker. An external broker cannot scope a container, so it still
|
||||
// gets the broker's token (reported as a health issue by archied).
|
||||
func (e NATSEndpoint) ContainerToken() string {
|
||||
if e.Scoped {
|
||||
return ""
|
||||
}
|
||||
return e.Token
|
||||
}
|
||||
|
||||
// StateStoreEndpoint is the State Store target and token passed to agent
|
||||
@@ -1937,7 +1950,7 @@ func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken string) []str
|
||||
// the live config: a reloaded [nats] section must not point new
|
||||
// containers at a server the daemon is not publishing on.
|
||||
env = append(env, "NATS_URL="+d.ConnectedNATS.URL)
|
||||
if token := d.ConnectedNATS.Token; token != "" {
|
||||
if token := d.ConnectedNATS.ContainerToken(); token != "" {
|
||||
env = append(env, "NATS_TOKEN="+token)
|
||||
}
|
||||
// Pass the State Store target with this task's scoped token, never the
|
||||
|
||||
@@ -45,7 +45,7 @@ func (d *Daemon) runKitTask(ctx context.Context, task *workflow.Task, repo confi
|
||||
}
|
||||
defer revokeCredential()
|
||||
env, err := kitrun.WorkerEnv(kitrun.Endpoints{
|
||||
NATS: d.ConnectedNATS.URL, NATSToken: d.ConnectedNATS.Token,
|
||||
NATS: d.ConnectedNATS.URL, NATSToken: d.ConnectedNATS.ContainerToken(),
|
||||
StateStore: d.ConnectedStateStore.URL, StateStoreToken: credential,
|
||||
}, os.Getuid(), os.Getgid())
|
||||
if err != nil {
|
||||
|
||||
@@ -29,6 +29,20 @@ const (
|
||||
SubjectReplyToReview = "archie.forge.reply_to_review"
|
||||
)
|
||||
|
||||
// Subjects returns every forge RPC subject identity's server answers: the
|
||||
// set a task of that identity may publish to.
|
||||
func Subjects(identity string) []string {
|
||||
bases := []string{
|
||||
SubjectComment, SubjectCloseIssue, SubjectCreatePR, SubjectLinkBranch,
|
||||
SubjectSetStateLabel, SubjectCreateReviewComment, SubjectReplyToReview,
|
||||
}
|
||||
out := make([]string, len(bases))
|
||||
for i, base := range bases {
|
||||
out[i] = SubjectFor(identity, base)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// SubjectFor returns base scoped to identity, or base when identity is
|
||||
// empty.
|
||||
func SubjectFor(identity, base string) string {
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/forgerpc"
|
||||
eventbus "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||
"github.com/samcharles93/archie-core/internal/taskrun"
|
||||
"github.com/samcharles93/archie-core/internal/worktreerpc"
|
||||
@@ -28,8 +29,12 @@ import (
|
||||
// fully on gRPC for its store calls, and the legacy NATS storerpc path is
|
||||
// deleted
|
||||
type Config struct {
|
||||
URL string
|
||||
Token string
|
||||
URL string
|
||||
// Token is a broker-wide token, set only on an external broker. Without
|
||||
// it the worker logs in as TaskID with StateStoreToken, its run
|
||||
// credential, which reaches only its own task's subjects.
|
||||
Token string
|
||||
TaskID int64
|
||||
|
||||
StateStoreURL string
|
||||
StateStoreToken string
|
||||
@@ -65,8 +70,13 @@ func Connect(ctx context.Context, config Config, log *slog.Logger) (*Transport,
|
||||
return nil, err
|
||||
}
|
||||
options := []natsio.Option{natsio.Name("archie-agent")}
|
||||
if config.Token != "" {
|
||||
switch {
|
||||
case config.Token != "":
|
||||
options = append(options, natsio.Token(config.Token))
|
||||
case config.StateStoreToken != "":
|
||||
options = append(options,
|
||||
natsio.UserInfo(fmt.Sprintf("%s%d", eventbus.TaskUserPrefix, config.TaskID), config.StateStoreToken),
|
||||
natsio.CustomInboxPrefix(eventbus.TaskInboxPrefix(config.TaskID)))
|
||||
}
|
||||
conn, err := natsio.Connect(config.URL, options...)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package nats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
)
|
||||
|
||||
// TaskUserPrefix names a task's broker user: "task-<id>". Its password is the
|
||||
// task's run credential.
|
||||
const TaskUserPrefix = "task-"
|
||||
|
||||
// TaskInboxPrefix is the inbox prefix a task's connection must use: the only
|
||||
// inbox it may subscribe to.
|
||||
func TaskInboxPrefix(taskID int64) string {
|
||||
return "_INBOX." + TaskUserPrefix + strconv.FormatInt(taskID, 10)
|
||||
}
|
||||
|
||||
// TaskSubjects is what one task's connection may publish to and subscribe to,
|
||||
// beside its own inbox and its replies.
|
||||
type TaskSubjects struct {
|
||||
Publish []string
|
||||
Subscribe []string
|
||||
}
|
||||
|
||||
// TaskAuthority answers which subjects a run credential opens for its task.
|
||||
// An error refuses the connection.
|
||||
type TaskAuthority interface {
|
||||
TaskSubjects(ctx context.Context, taskID int64, credential string) (TaskSubjects, error)
|
||||
}
|
||||
|
||||
// taskAuthTimeout bounds the credential lookup a connection waits on.
|
||||
const taskAuthTimeout = 2 * time.Second
|
||||
|
||||
// authenticator admits the instance token with every permission, and a task
|
||||
// user only to its own task's subjects, so no container holds a credential
|
||||
// that reaches past its run.
|
||||
type authenticator struct {
|
||||
token string
|
||||
tasks TaskAuthority
|
||||
}
|
||||
|
||||
func (a authenticator) Check(c server.ClientAuthentication) bool {
|
||||
opts := c.GetOpts()
|
||||
if opts.Token != "" {
|
||||
if subtle.ConstantTimeCompare([]byte(opts.Token), []byte(a.token)) != 1 {
|
||||
return false
|
||||
}
|
||||
c.RegisterUser(&server.User{Username: "archie"})
|
||||
return true
|
||||
}
|
||||
taskID, ok := taskUser(opts.Username)
|
||||
if !ok || opts.Password == "" || a.tasks == nil {
|
||||
return false
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), taskAuthTimeout)
|
||||
defer cancel()
|
||||
subjects, err := a.tasks.TaskSubjects(ctx, taskID, opts.Password)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
c.RegisterUser(&server.User{Username: opts.Username, Permissions: &server.Permissions{
|
||||
Publish: &server.SubjectPermission{Allow: subjects.Publish},
|
||||
Subscribe: &server.SubjectPermission{Allow: append(subjects.Subscribe, TaskInboxPrefix(taskID)+".>")},
|
||||
// Lets the task answer the one request it is sent, on the requester's
|
||||
// reply subject.
|
||||
Response: &server.ResponsePermission{MaxMsgs: 1},
|
||||
}})
|
||||
return true
|
||||
}
|
||||
|
||||
func taskUser(name string) (int64, bool) {
|
||||
id, err := strconv.ParseInt(strings.TrimPrefix(name, TaskUserPrefix), 10, 64)
|
||||
if !strings.HasPrefix(name, TaskUserPrefix) || err != nil || id <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package nats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
natsio "github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
type taskSeven struct{}
|
||||
|
||||
func (taskSeven) TaskSubjects(_ context.Context, taskID int64, credential string) (TaskSubjects, error) {
|
||||
if taskID != 7 || credential != "cred-7" {
|
||||
return TaskSubjects{}, errors.New("unknown")
|
||||
}
|
||||
return TaskSubjects{Publish: []string{"archie.agent.7.events"}, Subscribe: []string{"archie.taskrun.7"}}, nil
|
||||
}
|
||||
|
||||
// TestBrokerScopesTaskCredentials pins that the instance token reaches
|
||||
// everything and a task's run credential reaches only its own subjects, on a
|
||||
// real embedded broker.
|
||||
func TestBrokerScopesTaskCredentials(t *testing.T) {
|
||||
srv, err := StartEmbedded(context.Background(), EmbeddedOptions{Token: "instance", StoreDir: t.TempDir(), Tasks: taskSeven{}}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer srv.Shutdown()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
opts []natsio.Option
|
||||
connects bool
|
||||
allowed []string // "pub:" or "sub:" subjects that must work
|
||||
refused []string // that must be refused
|
||||
}{
|
||||
{
|
||||
"instance token reaches everything",
|
||||
[]natsio.Option{natsio.Token("instance")},
|
||||
true,
|
||||
[]string{"pub:archie.taskrun.8", "sub:_INBOX.>", "js:"},
|
||||
nil,
|
||||
},
|
||||
{
|
||||
"task credential reaches its own subjects only",
|
||||
[]natsio.Option{natsio.UserInfo("task-7", "cred-7"), natsio.CustomInboxPrefix(TaskInboxPrefix(7))},
|
||||
true,
|
||||
[]string{"pub:archie.agent.7.events", "sub:archie.taskrun.7", "sub:" + TaskInboxPrefix(7) + ".x"},
|
||||
[]string{"pub:archie.taskrun.8", "pub:archie.agent.8.events", "sub:archie.taskrun.8", "sub:_INBOX.>"},
|
||||
},
|
||||
{"credential for another task", []natsio.Option{natsio.UserInfo("task-8", "cred-7")}, false, nil, nil},
|
||||
{"unknown credential", []natsio.Option{natsio.UserInfo("task-7", "guess")}, false, nil, nil},
|
||||
{"wrong instance token", []natsio.Option{natsio.Token("guess")}, false, nil, nil},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
violations := make(chan string, 16)
|
||||
opts := append(slices.Clone(tt.opts), natsio.ErrorHandler(func(_ *natsio.Conn, _ *natsio.Subscription, err error) {
|
||||
violations <- err.Error()
|
||||
}))
|
||||
nc, err := natsio.Connect(srv.ClientURL(), opts...)
|
||||
if !tt.connects {
|
||||
if err == nil {
|
||||
nc.Close()
|
||||
t.Fatal("connected; want refused")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
for _, op := range tt.allowed {
|
||||
if v := attempt(t, nc, op, violations); v != "" {
|
||||
t.Errorf("%s refused: %s", op, v)
|
||||
}
|
||||
}
|
||||
for _, op := range tt.refused {
|
||||
if v := attempt(t, nc, op, violations); !strings.Contains(v, "Permissions Violation") {
|
||||
t.Errorf("%s allowed; want refused", op)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// attempt performs one "pub:", "sub:" or "js:" operation and returns the permission
|
||||
// violation it drew, if any.
|
||||
func attempt(t *testing.T, nc *natsio.Conn, op string, violations chan string) string {
|
||||
t.Helper()
|
||||
kind, subject, _ := strings.Cut(op, ":")
|
||||
switch kind {
|
||||
case "js":
|
||||
// The daemon's task queue lives in JetStream.
|
||||
js, err := nc.JetStream()
|
||||
if err == nil {
|
||||
_, err = js.AccountInfo()
|
||||
}
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
case "pub":
|
||||
_ = nc.Publish(subject, []byte("x"))
|
||||
default:
|
||||
sub, err := nc.SubscribeSync(subject)
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
}
|
||||
if err := nc.FlushTimeout(time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case v := <-violations:
|
||||
return v
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
return ""
|
||||
}
|
||||
}
|
||||
@@ -26,8 +26,11 @@ type EmbeddedOptions struct {
|
||||
Host string
|
||||
// Port is the listen port. Zero means a random port.
|
||||
Port int
|
||||
// Token is the client token. Empty generates one.
|
||||
// Token is the instance token: every permission. Empty generates one.
|
||||
Token string
|
||||
// Tasks admits task users ("task-<id>" with the run credential) to their
|
||||
// own task's subjects. Nil admits the instance token only.
|
||||
Tasks TaskAuthority
|
||||
// StoreDir is where JetStream persists streams. Empty means a temporary
|
||||
// directory (streams do not survive a reboot). The daemon passes a dir
|
||||
// under its data directory so the ARCHIE_TASKS and reaction streams are
|
||||
@@ -54,12 +57,14 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger)
|
||||
}
|
||||
|
||||
srv, err := server.NewServer(&server.Options{
|
||||
Host: opts.Host,
|
||||
Port: opts.Port,
|
||||
JetStream: true,
|
||||
StoreDir: opts.StoreDir,
|
||||
Authorization: token,
|
||||
NoSigs: true, // the daemon owns signals, not the embedded server
|
||||
Host: opts.Host,
|
||||
Port: opts.Port,
|
||||
JetStream: true,
|
||||
StoreDir: opts.StoreDir,
|
||||
// Replaces token authorization: the instance token and task users
|
||||
// are both checked here.
|
||||
CustomClientAuthentication: authenticator{token: token, tasks: opts.Tasks},
|
||||
NoSigs: true, // the daemon owns signals, not the embedded server
|
||||
// NoLog leaves the server's logger nil (ConfigureLogger is only ever
|
||||
// called on reload). A nil logger makes the server's Fatalf a no-op
|
||||
// rather than an os.Exit, so a JetStream startup failure cannot crash
|
||||
|
||||
Reference in new issue
Block a user