feat(archied): a rotated connection reaches the next operation

The connections kind joins the live resource watches. On a new revision
archied re-resolves the instance forge from the settings it just published and
replaces every holder of the client: the boot forge, the worktree token, the
daemon's root forge and the root task RPC servers, which are now returned from
registration so a task that is already connected asks the new credential
rather than the one the process booted with.

This commit also carries the execution-identity agent's concurrent edits to
these three files, which the shared working tree had not committed yet.
This commit is contained in:
sam committed 2026-10-17 03:09:43 +11:00
1 parent c56caa45c2
commit 0315057d4c
3 files changed
+82 -30

No files matched your search

+41 -13
View File
@@ -36,6 +36,7 @@ import (
"github.com/samcharles93/archie-core/internal/domain/applystatus" "github.com/samcharles93/archie-core/internal/domain/applystatus"
"github.com/samcharles93/archie-core/internal/domain/health" "github.com/samcharles93/archie-core/internal/domain/health"
"github.com/samcharles93/archie-core/internal/domain/identity" "github.com/samcharles93/archie-core/internal/domain/identity"
"github.com/samcharles93/archie-core/internal/domain/org"
"github.com/samcharles93/archie-core/internal/domain/presence" "github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/domain/scheduling" "github.com/samcharles93/archie-core/internal/domain/scheduling"
"github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/storecontract"
@@ -91,6 +92,9 @@ type boot struct {
forgeClient forge.Forge forgeClient forge.Forge
forgeExt *forgeExtensions forgeExt *forgeExtensions
token string token string
// rootRPC is the root (identity-less) task RPC server pair, held so a
// live connection change reaches the servers a task already asks.
rootRPC *rootTaskRPC
// stateStore is the State Store contract adapter every daemon and gateway // stateStore is the State Store contract adapter every daemon and gateway
// store consumer depends on. It is ALWAYS the remote *staterpc.Client // store consumer depends on. It is ALWAYS the remote *staterpc.Client
@@ -267,6 +271,28 @@ func (b *boot) openForge(ctx context.Context) {
b.forgeClient, b.token = resolveForge(ctx, b.cfg.Forge, "default", b.secrets, b.forgeExt, b.log) b.forgeClient, b.token = resolveForge(ctx, b.cfg.Forge, "default", b.secrets, b.forgeExt, b.log)
} }
// reopenForge re-resolves the instance's forge connection from the settings
// the layering just published. Every holder of the client is replaced, so a
// rotated credential, a changed host or a removed connection reaches the next
// operation rather than dying with the process that booted. It runs from the
// connections watch alone, after the config holder carries the change; the
// worktree manager is shared by pointer, so its token is the only field to
// replace.
func (b *boot) reopenForge(ctx context.Context) {
client, token := resolveForge(ctx, b.cfgHolder.Get().Forge, "default", b.secrets, b.forgeExt, b.log)
b.forgeClient, b.token = client, token
if b.trees != nil {
b.trees.Token = token
}
if b.d != nil {
b.d.Forge = client
}
if b.rootRPC != nil {
b.rootRPC.forge.Forge = client
b.rootRPC.trees.Trees = b.trees
}
}
func (b *boot) openStateStoreAdapter(ctx context.Context) error { func (b *boot) openStateStoreAdapter(ctx context.Context) error {
target := strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target) target := strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target)
if target == "" { if target == "" {
@@ -302,13 +328,13 @@ func (b *boot) openStateStage(ctx context.Context) error {
} }
func (b *boot) openAccessChain(ctx context.Context) error { func (b *boot) openAccessChain(ctx context.Context) error {
source, ok := b.stateStore.(access.PolicySource) source, ok := b.stateStore.(access.PolicyStore)
if !ok { if !ok {
return nil return fmt.Errorf("state store policy reader unavailable")
} }
live, err := infraaccess.NewLive(ctx, source.Policies, b.log) live := infraaccess.NewPending(source.ListPolicies, b.log)
if err != nil { if err := live.Reload(ctx); err != nil {
return fmt.Errorf("load stored policies: %w", err) b.log.Warn("workflow policies unavailable; execution remains denied", "err", err)
} }
b.accessChain, b.accessLive = live, live b.accessChain, b.accessLive = live, live
go live.Run(ctx, applystatus.RestampInterval) go live.Run(ctx, applystatus.RestampInterval)
@@ -538,11 +564,12 @@ func (b *boot) registerNATSRPC() error {
log.Error("nats connection unavailable for task RPC", "err", err) log.Error("nats connection unavailable for task RPC", "err", err)
return err return err
} }
unsubscribe, err := registerTaskRPCServers(coreConn, b.forgeClient, b.trees, b.identityRunners, b.stateStoreGrants.Client, installedAuthorities{b.stateStoreGrants.Client}, log) unsubscribe, rootRPC, err := registerTaskRPCServers(coreConn, b.forgeClient, b.trees, b.identityRunners, b.stateStoreGrants.Client, installedAuthorities{b.stateStoreGrants.Client}, log)
if err != nil { if err != nil {
log.Error("task RPC server registration failed", "err", err) log.Error("task RPC server registration failed", "err", err)
return err return err
} }
b.rootRPC = rootRPC
b.addCleanup(unsubscribe) b.addCleanup(unsubscribe)
stopLogs, err := logrpc.RegisterDaemon(coreConn, b.logFeed, log) stopLogs, err := logrpc.RegisterDaemon(coreConn, b.logFeed, log)
@@ -605,6 +632,9 @@ func (b *boot) buildDaemon() {
if identities, ok := b.stateStore.(identity.Repository); ok { if identities, ok := b.stateStore.(identity.Repository); ok {
b.d.IdentityRepository = identities b.d.IdentityRepository = identities
} }
if assignments, ok := b.stateStore.(org.AgentAssignments); ok {
b.d.AgentAssignments = assignments
}
if ms, ok := b.stateStore.(storecontract.MappingStore); ok { if ms, ok := b.stateStore.(storecontract.MappingStore); ok {
b.d.Mappings = ms b.d.Mappings = ms
@@ -628,13 +658,11 @@ func (b *boot) buildDaemon() {
b.d.WithdrawTask = b.taskActions().Withdraw b.d.WithdrawTask = b.taskActions().Withdraw
b.d.Access = b.accessChain b.d.Access = b.accessChain
if b.accessChain != nil { if principals, ok := b.stateStore.(access.PrincipalSource); ok {
if principals, ok := b.stateStore.(access.PrincipalSource); ok { b.d.Principals = principals
b.d.Principals = principals }
} if denials, ok := b.stateStore.(access.DenialRecorder); ok {
if denials, ok := b.stateStore.(access.DenialRecorder); ok { b.d.Denials = denials
b.d.Denials = denials
}
} }
b.d.PlaybookLedger = playbookLedger(b.stateStore, b.log) b.d.PlaybookLedger = playbookLedger(b.stateStore, b.log)
b.setupForgeWebhook() b.setupForgeWebhook()
+13 -1
View File
@@ -260,7 +260,11 @@ func (b *boot) startLiveSettings(ctx context.Context) error {
// directory consumers hold the boot value: the watch refuses such a change // directory consumers hold the boot value: the watch refuses such a change
// rather than layering it (refuseSkillsDirChange), so the kind is labelled // rather than layering it (refuseSkillsDirChange), so the kind is labelled
// restart-required. Each kind is watched in its own goroutine, exactly the // restart-required. Each kind is watched in its own goroutine, exactly the
// shape the workflow-execution-settings watch established. // shape the workflow-execution-settings watch established. ForgeConnectionsKind
// is on it because the forge client and the worktree token are re-resolved
// from the republished config by reopenForge, which replaces every holder of
// the client (the daemon's root forge and the root task RPC servers), so a
// rotated credential reaches the next operation without a restart.
var runtimeResourceKinds = []string{ var runtimeResourceKinds = []string{
controlplane.TaskPolicyKind, controlplane.TaskPolicyKind,
controlplane.ProviderSettingsKind, controlplane.ProviderSettingsKind,
@@ -275,6 +279,7 @@ var runtimeResourceKinds = []string{
controlplane.CredentialBindingsKind, controlplane.CredentialBindingsKind,
controlplane.IdentityGrantsKind, controlplane.IdentityGrantsKind,
controlplane.AgentProfileKind, controlplane.AgentProfileKind,
controlplane.ForgeConnectionsKind,
} }
// startRuntimeResourceWatches keeps a watch per live kind established for the // startRuntimeResourceWatches keeps a watch per live kind established for the
@@ -387,6 +392,13 @@ func (b *boot) applyRuntimeResourceUpdate(ctx context.Context, kind string, upda
// lower must be honoured as running tasks finish. // lower must be honoured as running tasks finish.
b.resizeTaskDispatcher(cfg.Containers.MaxConcurrency) b.resizeTaskDispatcher(cfg.Containers.MaxConcurrency)
} }
if kind == controlplane.ForgeConnectionsKind {
// The forge client is built from the credential rather than taken from
// the published config, so the change that published it must reach the
// client too: a rotated token otherwise keeps authenticating with the
// boot value until the process restarts.
b.reopenForge(ctx)
}
return nil return nil
} }
+28 -16
View File
@@ -359,7 +359,7 @@ func subscribeAgentEvents(nc *natsio.Conn, bus *events.Bus, log *slog.Logger) (u
// tasks, and one server pair per identity answers that identity's scoped // tasks, and one server pair per identity answers that identity's scoped
// subjects so a container-mode task owned by a non-root identity has its RPC // subjects so a container-mode task owned by a non-root identity has its RPC
// calls served by its own forge client and worktree manager. // calls served by its own forge client and worktree manager.
func registerTaskRPCServers(nc *natsio.Conn, forgeClient forge.Forge, trees *worktree.Manager, identities []*daemon.IdentityRunner, runs worktreerpc.Runs, authorities forgerpc.Authorities, log *slog.Logger) (unsubscribe func(), err error) { func registerTaskRPCServers(nc *natsio.Conn, forgeClient forge.Forge, trees *worktree.Manager, identities []*daemon.IdentityRunner, runs worktreerpc.Runs, authorities forgerpc.Authorities, log *slog.Logger) (unsubscribe func(), root *rootTaskRPC, err error) {
unsubs := make([]func(), 0, 2+2*len(identities)) unsubs := make([]func(), 0, 2+2*len(identities))
unsubAll := func() { unsubAll := func() {
for _, u := range unsubs { for _, u := range unsubs {
@@ -367,49 +367,61 @@ func registerTaskRPCServers(nc *natsio.Conn, forgeClient forge.Forge, trees *wor
} }
} }
registerForge := func(fg forge.Forge, identity string) error { registerForge := func(fg forge.Forge, identity string) (*forgerpc.Server, error) {
srv := &forgerpc.Server{Forge: fg, Runs: runs, Authorities: authorities, Log: log.With("rpc_identity", identity)} srv := &forgerpc.Server{Forge: fg, Runs: runs, Authorities: authorities, Log: log.With("rpc_identity", identity)}
u, err := srv.RegisterFor(nc, identity) u, err := srv.RegisterFor(nc, identity)
if err != nil { if err != nil {
return fmt.Errorf("register forgerpc%s: %w", identitySuffix(identity), err) return nil, fmt.Errorf("register forgerpc%s: %w", identitySuffix(identity), err)
} }
unsubs = append(unsubs, u) unsubs = append(unsubs, u)
return nil return srv, nil
} }
registerTrees := func(mgr *worktree.Manager, identity string) error { registerTrees := func(mgr *worktree.Manager, identity string) (*worktreerpc.Server, error) {
srv := &worktreerpc.Server{Trees: mgr, Runs: runs, Log: log.With("rpc_identity", identity)} srv := &worktreerpc.Server{Trees: mgr, Runs: runs, Log: log.With("rpc_identity", identity)}
u, err := srv.RegisterFor(nc, identity) u, err := srv.RegisterFor(nc, identity)
if err != nil { if err != nil {
return fmt.Errorf("register worktreerpc%s: %w", identitySuffix(identity), err) return nil, fmt.Errorf("register worktreerpc%s: %w", identitySuffix(identity), err)
} }
unsubs = append(unsubs, u) unsubs = append(unsubs, u)
return nil return srv, nil
} }
// Root (identity-less) forge and worktree servers. // Root (identity-less) forge and worktree servers.
if err := registerForge(forgeClient, ""); err != nil { rootForge, err := registerForge(forgeClient, "")
if err != nil {
unsubAll() unsubAll()
return nil, err return nil, nil, err
} }
if err := registerTrees(trees, ""); err != nil { rootTrees, err := registerTrees(trees, "")
if err != nil {
unsubAll() unsubAll()
return nil, err return nil, nil, err
} }
root = &rootTaskRPC{forge: rootForge, trees: rootTrees}
// One forge/worktree server pair per identity, on identity-scoped // One forge/worktree server pair per identity, on identity-scoped
// subjects. // subjects.
for _, id := range identities { for _, id := range identities {
if err := registerForge(id.Forge, id.Name); err != nil { if _, err := registerForge(id.Forge, string(id.ID)); err != nil {
unsubAll() unsubAll()
return nil, err return nil, nil, err
} }
if err := registerTrees(id.Trees, id.Name); err != nil { if _, err := registerTrees(id.Trees, string(id.ID)); err != nil {
unsubAll() unsubAll()
return nil, err return nil, nil, err
} }
} }
return unsubAll, nil return unsubAll, root, nil
}
// rootTaskRPC is the pair of root (identity-less) task RPC servers. It is
// held so a live connection change replaces the forge the servers answer
// with: a task already connected asks the new credential, rather than the
// one the process booted with.
type rootTaskRPC struct {
forge *forgerpc.Server
trees *worktreerpc.Server
} }
// identitySuffix renders a log suffix for an identity-scoped registration, // identitySuffix renders a log suffix for an identity-scoped registration,