feat(archied): a rotated connection reaches the next operation
The connections kind joins the live resource watches. On a new revision archied re-resolves the instance forge from the settings it just published and replaces every holder of the client: the boot forge, the worktree token, the daemon's root forge and the root task RPC servers, which are now returned from registration so a task that is already connected asks the new credential rather than the one the process booted with. This commit also carries the execution-identity agent's concurrent edits to these three files, which the shared working tree had not committed yet.
This commit is contained in:
3 files changed
+82
-30
No files matched your search
@@ -36,6 +36,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||
"github.com/samcharles93/archie-core/internal/domain/health"
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/org"
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
"github.com/samcharles93/archie-core/internal/domain/scheduling"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
@@ -91,6 +92,9 @@ type boot struct {
|
||||
forgeClient forge.Forge
|
||||
forgeExt *forgeExtensions
|
||||
token string
|
||||
// rootRPC is the root (identity-less) task RPC server pair, held so a
|
||||
// live connection change reaches the servers a task already asks.
|
||||
rootRPC *rootTaskRPC
|
||||
|
||||
// stateStore is the State Store contract adapter every daemon and gateway
|
||||
// store consumer depends on. It is ALWAYS the remote *staterpc.Client
|
||||
@@ -267,6 +271,28 @@ func (b *boot) openForge(ctx context.Context) {
|
||||
b.forgeClient, b.token = resolveForge(ctx, b.cfg.Forge, "default", b.secrets, b.forgeExt, b.log)
|
||||
}
|
||||
|
||||
// reopenForge re-resolves the instance's forge connection from the settings
|
||||
// the layering just published. Every holder of the client is replaced, so a
|
||||
// rotated credential, a changed host or a removed connection reaches the next
|
||||
// operation rather than dying with the process that booted. It runs from the
|
||||
// connections watch alone, after the config holder carries the change; the
|
||||
// worktree manager is shared by pointer, so its token is the only field to
|
||||
// replace.
|
||||
func (b *boot) reopenForge(ctx context.Context) {
|
||||
client, token := resolveForge(ctx, b.cfgHolder.Get().Forge, "default", b.secrets, b.forgeExt, b.log)
|
||||
b.forgeClient, b.token = client, token
|
||||
if b.trees != nil {
|
||||
b.trees.Token = token
|
||||
}
|
||||
if b.d != nil {
|
||||
b.d.Forge = client
|
||||
}
|
||||
if b.rootRPC != nil {
|
||||
b.rootRPC.forge.Forge = client
|
||||
b.rootRPC.trees.Trees = b.trees
|
||||
}
|
||||
}
|
||||
|
||||
func (b *boot) openStateStoreAdapter(ctx context.Context) error {
|
||||
target := strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target)
|
||||
if target == "" {
|
||||
@@ -302,13 +328,13 @@ func (b *boot) openStateStage(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (b *boot) openAccessChain(ctx context.Context) error {
|
||||
source, ok := b.stateStore.(access.PolicySource)
|
||||
source, ok := b.stateStore.(access.PolicyStore)
|
||||
if !ok {
|
||||
return nil
|
||||
return fmt.Errorf("state store policy reader unavailable")
|
||||
}
|
||||
live, err := infraaccess.NewLive(ctx, source.Policies, b.log)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load stored policies: %w", err)
|
||||
live := infraaccess.NewPending(source.ListPolicies, b.log)
|
||||
if err := live.Reload(ctx); err != nil {
|
||||
b.log.Warn("workflow policies unavailable; execution remains denied", "err", err)
|
||||
}
|
||||
b.accessChain, b.accessLive = live, live
|
||||
go live.Run(ctx, applystatus.RestampInterval)
|
||||
@@ -538,11 +564,12 @@ func (b *boot) registerNATSRPC() error {
|
||||
log.Error("nats connection unavailable for task RPC", "err", err)
|
||||
return err
|
||||
}
|
||||
unsubscribe, err := registerTaskRPCServers(coreConn, b.forgeClient, b.trees, b.identityRunners, b.stateStoreGrants.Client, installedAuthorities{b.stateStoreGrants.Client}, log)
|
||||
unsubscribe, rootRPC, err := registerTaskRPCServers(coreConn, b.forgeClient, b.trees, b.identityRunners, b.stateStoreGrants.Client, installedAuthorities{b.stateStoreGrants.Client}, log)
|
||||
if err != nil {
|
||||
log.Error("task RPC server registration failed", "err", err)
|
||||
return err
|
||||
}
|
||||
b.rootRPC = rootRPC
|
||||
b.addCleanup(unsubscribe)
|
||||
|
||||
stopLogs, err := logrpc.RegisterDaemon(coreConn, b.logFeed, log)
|
||||
@@ -605,6 +632,9 @@ func (b *boot) buildDaemon() {
|
||||
if identities, ok := b.stateStore.(identity.Repository); ok {
|
||||
b.d.IdentityRepository = identities
|
||||
}
|
||||
if assignments, ok := b.stateStore.(org.AgentAssignments); ok {
|
||||
b.d.AgentAssignments = assignments
|
||||
}
|
||||
|
||||
if ms, ok := b.stateStore.(storecontract.MappingStore); ok {
|
||||
b.d.Mappings = ms
|
||||
@@ -628,13 +658,11 @@ func (b *boot) buildDaemon() {
|
||||
b.d.WithdrawTask = b.taskActions().Withdraw
|
||||
|
||||
b.d.Access = b.accessChain
|
||||
if b.accessChain != nil {
|
||||
if principals, ok := b.stateStore.(access.PrincipalSource); ok {
|
||||
b.d.Principals = principals
|
||||
}
|
||||
if denials, ok := b.stateStore.(access.DenialRecorder); ok {
|
||||
b.d.Denials = denials
|
||||
}
|
||||
if principals, ok := b.stateStore.(access.PrincipalSource); ok {
|
||||
b.d.Principals = principals
|
||||
}
|
||||
if denials, ok := b.stateStore.(access.DenialRecorder); ok {
|
||||
b.d.Denials = denials
|
||||
}
|
||||
b.d.PlaybookLedger = playbookLedger(b.stateStore, b.log)
|
||||
b.setupForgeWebhook()
|
||||
|
||||
@@ -260,7 +260,11 @@ func (b *boot) startLiveSettings(ctx context.Context) error {
|
||||
// directory consumers hold the boot value: the watch refuses such a change
|
||||
// rather than layering it (refuseSkillsDirChange), so the kind is labelled
|
||||
// restart-required. Each kind is watched in its own goroutine, exactly the
|
||||
// shape the workflow-execution-settings watch established.
|
||||
// shape the workflow-execution-settings watch established. ForgeConnectionsKind
|
||||
// is on it because the forge client and the worktree token are re-resolved
|
||||
// from the republished config by reopenForge, which replaces every holder of
|
||||
// the client (the daemon's root forge and the root task RPC servers), so a
|
||||
// rotated credential reaches the next operation without a restart.
|
||||
var runtimeResourceKinds = []string{
|
||||
controlplane.TaskPolicyKind,
|
||||
controlplane.ProviderSettingsKind,
|
||||
@@ -275,6 +279,7 @@ var runtimeResourceKinds = []string{
|
||||
controlplane.CredentialBindingsKind,
|
||||
controlplane.IdentityGrantsKind,
|
||||
controlplane.AgentProfileKind,
|
||||
controlplane.ForgeConnectionsKind,
|
||||
}
|
||||
|
||||
// startRuntimeResourceWatches keeps a watch per live kind established for the
|
||||
@@ -387,6 +392,13 @@ func (b *boot) applyRuntimeResourceUpdate(ctx context.Context, kind string, upda
|
||||
// lower must be honoured as running tasks finish.
|
||||
b.resizeTaskDispatcher(cfg.Containers.MaxConcurrency)
|
||||
}
|
||||
if kind == controlplane.ForgeConnectionsKind {
|
||||
// The forge client is built from the credential rather than taken from
|
||||
// the published config, so the change that published it must reach the
|
||||
// client too: a rotated token otherwise keeps authenticating with the
|
||||
// boot value until the process restarts.
|
||||
b.reopenForge(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -359,7 +359,7 @@ func subscribeAgentEvents(nc *natsio.Conn, bus *events.Bus, log *slog.Logger) (u
|
||||
// tasks, and one server pair per identity answers that identity's scoped
|
||||
// subjects so a container-mode task owned by a non-root identity has its RPC
|
||||
// calls served by its own forge client and worktree manager.
|
||||
func registerTaskRPCServers(nc *natsio.Conn, forgeClient forge.Forge, trees *worktree.Manager, identities []*daemon.IdentityRunner, runs worktreerpc.Runs, authorities forgerpc.Authorities, log *slog.Logger) (unsubscribe func(), err error) {
|
||||
func registerTaskRPCServers(nc *natsio.Conn, forgeClient forge.Forge, trees *worktree.Manager, identities []*daemon.IdentityRunner, runs worktreerpc.Runs, authorities forgerpc.Authorities, log *slog.Logger) (unsubscribe func(), root *rootTaskRPC, err error) {
|
||||
unsubs := make([]func(), 0, 2+2*len(identities))
|
||||
unsubAll := func() {
|
||||
for _, u := range unsubs {
|
||||
@@ -367,49 +367,61 @@ func registerTaskRPCServers(nc *natsio.Conn, forgeClient forge.Forge, trees *wor
|
||||
}
|
||||
}
|
||||
|
||||
registerForge := func(fg forge.Forge, identity string) error {
|
||||
registerForge := func(fg forge.Forge, identity string) (*forgerpc.Server, error) {
|
||||
srv := &forgerpc.Server{Forge: fg, Runs: runs, Authorities: authorities, Log: log.With("rpc_identity", identity)}
|
||||
u, err := srv.RegisterFor(nc, identity)
|
||||
if err != nil {
|
||||
return fmt.Errorf("register forgerpc%s: %w", identitySuffix(identity), err)
|
||||
return nil, fmt.Errorf("register forgerpc%s: %w", identitySuffix(identity), err)
|
||||
}
|
||||
unsubs = append(unsubs, u)
|
||||
return nil
|
||||
return srv, nil
|
||||
}
|
||||
registerTrees := func(mgr *worktree.Manager, identity string) error {
|
||||
registerTrees := func(mgr *worktree.Manager, identity string) (*worktreerpc.Server, error) {
|
||||
srv := &worktreerpc.Server{Trees: mgr, Runs: runs, Log: log.With("rpc_identity", identity)}
|
||||
u, err := srv.RegisterFor(nc, identity)
|
||||
if err != nil {
|
||||
return fmt.Errorf("register worktreerpc%s: %w", identitySuffix(identity), err)
|
||||
return nil, fmt.Errorf("register worktreerpc%s: %w", identitySuffix(identity), err)
|
||||
}
|
||||
unsubs = append(unsubs, u)
|
||||
return nil
|
||||
return srv, nil
|
||||
}
|
||||
|
||||
// Root (identity-less) forge and worktree servers.
|
||||
if err := registerForge(forgeClient, ""); err != nil {
|
||||
rootForge, err := registerForge(forgeClient, "")
|
||||
if err != nil {
|
||||
unsubAll()
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := registerTrees(trees, ""); err != nil {
|
||||
rootTrees, err := registerTrees(trees, "")
|
||||
if err != nil {
|
||||
unsubAll()
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
root = &rootTaskRPC{forge: rootForge, trees: rootTrees}
|
||||
|
||||
// One forge/worktree server pair per identity, on identity-scoped
|
||||
// subjects.
|
||||
for _, id := range identities {
|
||||
if err := registerForge(id.Forge, id.Name); err != nil {
|
||||
if _, err := registerForge(id.Forge, string(id.ID)); err != nil {
|
||||
unsubAll()
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := registerTrees(id.Trees, id.Name); err != nil {
|
||||
if _, err := registerTrees(id.Trees, string(id.ID)); err != nil {
|
||||
unsubAll()
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return unsubAll, nil
|
||||
return unsubAll, root, nil
|
||||
}
|
||||
|
||||
// rootTaskRPC is the pair of root (identity-less) task RPC servers. It is
|
||||
// held so a live connection change replaces the forge the servers answer
|
||||
// with: a task already connected asks the new credential, rather than the
|
||||
// one the process booted with.
|
||||
type rootTaskRPC struct {
|
||||
forge *forgerpc.Server
|
||||
trees *worktreerpc.Server
|
||||
}
|
||||
|
||||
// identitySuffix renders a log suffix for an identity-scoped registration,
|
||||
|
||||
Reference in new issue
Block a user