feat(archied): a rotated connection reaches the next operation
The connections kind joins the live resource watches. On a new revision archied re-resolves the instance forge from the settings it just published and replaces every holder of the client: the boot forge, the worktree token, the daemon's root forge and the root task RPC servers, which are now returned from registration so a task that is already connected asks the new credential rather than the one the process booted with. This commit also carries the execution-identity agent's concurrent edits to these three files, which the shared working tree had not committed yet.
This commit is contained in:
3 files changed
+82
-30
No files matched your search
@@ -36,6 +36,7 @@ import (
|
|||||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||||
"github.com/samcharles93/archie-core/internal/domain/health"
|
"github.com/samcharles93/archie-core/internal/domain/health"
|
||||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||||
|
"github.com/samcharles93/archie-core/internal/domain/org"
|
||||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||||
"github.com/samcharles93/archie-core/internal/domain/scheduling"
|
"github.com/samcharles93/archie-core/internal/domain/scheduling"
|
||||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||||
@@ -91,6 +92,9 @@ type boot struct {
|
|||||||
forgeClient forge.Forge
|
forgeClient forge.Forge
|
||||||
forgeExt *forgeExtensions
|
forgeExt *forgeExtensions
|
||||||
token string
|
token string
|
||||||
|
// rootRPC is the root (identity-less) task RPC server pair, held so a
|
||||||
|
// live connection change reaches the servers a task already asks.
|
||||||
|
rootRPC *rootTaskRPC
|
||||||
|
|
||||||
// stateStore is the State Store contract adapter every daemon and gateway
|
// stateStore is the State Store contract adapter every daemon and gateway
|
||||||
// store consumer depends on. It is ALWAYS the remote *staterpc.Client
|
// store consumer depends on. It is ALWAYS the remote *staterpc.Client
|
||||||
@@ -267,6 +271,28 @@ func (b *boot) openForge(ctx context.Context) {
|
|||||||
b.forgeClient, b.token = resolveForge(ctx, b.cfg.Forge, "default", b.secrets, b.forgeExt, b.log)
|
b.forgeClient, b.token = resolveForge(ctx, b.cfg.Forge, "default", b.secrets, b.forgeExt, b.log)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reopenForge re-resolves the instance's forge connection from the settings
|
||||||
|
// the layering just published. Every holder of the client is replaced, so a
|
||||||
|
// rotated credential, a changed host or a removed connection reaches the next
|
||||||
|
// operation rather than dying with the process that booted. It runs from the
|
||||||
|
// connections watch alone, after the config holder carries the change; the
|
||||||
|
// worktree manager is shared by pointer, so its token is the only field to
|
||||||
|
// replace.
|
||||||
|
func (b *boot) reopenForge(ctx context.Context) {
|
||||||
|
client, token := resolveForge(ctx, b.cfgHolder.Get().Forge, "default", b.secrets, b.forgeExt, b.log)
|
||||||
|
b.forgeClient, b.token = client, token
|
||||||
|
if b.trees != nil {
|
||||||
|
b.trees.Token = token
|
||||||
|
}
|
||||||
|
if b.d != nil {
|
||||||
|
b.d.Forge = client
|
||||||
|
}
|
||||||
|
if b.rootRPC != nil {
|
||||||
|
b.rootRPC.forge.Forge = client
|
||||||
|
b.rootRPC.trees.Trees = b.trees
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (b *boot) openStateStoreAdapter(ctx context.Context) error {
|
func (b *boot) openStateStoreAdapter(ctx context.Context) error {
|
||||||
target := strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target)
|
target := strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target)
|
||||||
if target == "" {
|
if target == "" {
|
||||||
@@ -302,13 +328,13 @@ func (b *boot) openStateStage(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (b *boot) openAccessChain(ctx context.Context) error {
|
func (b *boot) openAccessChain(ctx context.Context) error {
|
||||||
source, ok := b.stateStore.(access.PolicySource)
|
source, ok := b.stateStore.(access.PolicyStore)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return fmt.Errorf("state store policy reader unavailable")
|
||||||
}
|
}
|
||||||
live, err := infraaccess.NewLive(ctx, source.Policies, b.log)
|
live := infraaccess.NewPending(source.ListPolicies, b.log)
|
||||||
if err != nil {
|
if err := live.Reload(ctx); err != nil {
|
||||||
return fmt.Errorf("load stored policies: %w", err)
|
b.log.Warn("workflow policies unavailable; execution remains denied", "err", err)
|
||||||
}
|
}
|
||||||
b.accessChain, b.accessLive = live, live
|
b.accessChain, b.accessLive = live, live
|
||||||
go live.Run(ctx, applystatus.RestampInterval)
|
go live.Run(ctx, applystatus.RestampInterval)
|
||||||
@@ -538,11 +564,12 @@ func (b *boot) registerNATSRPC() error {
|
|||||||
log.Error("nats connection unavailable for task RPC", "err", err)
|
log.Error("nats connection unavailable for task RPC", "err", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
unsubscribe, err := registerTaskRPCServers(coreConn, b.forgeClient, b.trees, b.identityRunners, b.stateStoreGrants.Client, installedAuthorities{b.stateStoreGrants.Client}, log)
|
unsubscribe, rootRPC, err := registerTaskRPCServers(coreConn, b.forgeClient, b.trees, b.identityRunners, b.stateStoreGrants.Client, installedAuthorities{b.stateStoreGrants.Client}, log)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error("task RPC server registration failed", "err", err)
|
log.Error("task RPC server registration failed", "err", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
b.rootRPC = rootRPC
|
||||||
b.addCleanup(unsubscribe)
|
b.addCleanup(unsubscribe)
|
||||||
|
|
||||||
stopLogs, err := logrpc.RegisterDaemon(coreConn, b.logFeed, log)
|
stopLogs, err := logrpc.RegisterDaemon(coreConn, b.logFeed, log)
|
||||||
@@ -605,6 +632,9 @@ func (b *boot) buildDaemon() {
|
|||||||
if identities, ok := b.stateStore.(identity.Repository); ok {
|
if identities, ok := b.stateStore.(identity.Repository); ok {
|
||||||
b.d.IdentityRepository = identities
|
b.d.IdentityRepository = identities
|
||||||
}
|
}
|
||||||
|
if assignments, ok := b.stateStore.(org.AgentAssignments); ok {
|
||||||
|
b.d.AgentAssignments = assignments
|
||||||
|
}
|
||||||
|
|
||||||
if ms, ok := b.stateStore.(storecontract.MappingStore); ok {
|
if ms, ok := b.stateStore.(storecontract.MappingStore); ok {
|
||||||
b.d.Mappings = ms
|
b.d.Mappings = ms
|
||||||
@@ -628,13 +658,11 @@ func (b *boot) buildDaemon() {
|
|||||||
b.d.WithdrawTask = b.taskActions().Withdraw
|
b.d.WithdrawTask = b.taskActions().Withdraw
|
||||||
|
|
||||||
b.d.Access = b.accessChain
|
b.d.Access = b.accessChain
|
||||||
if b.accessChain != nil {
|
if principals, ok := b.stateStore.(access.PrincipalSource); ok {
|
||||||
if principals, ok := b.stateStore.(access.PrincipalSource); ok {
|
b.d.Principals = principals
|
||||||
b.d.Principals = principals
|
}
|
||||||
}
|
if denials, ok := b.stateStore.(access.DenialRecorder); ok {
|
||||||
if denials, ok := b.stateStore.(access.DenialRecorder); ok {
|
b.d.Denials = denials
|
||||||
b.d.Denials = denials
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
b.d.PlaybookLedger = playbookLedger(b.stateStore, b.log)
|
b.d.PlaybookLedger = playbookLedger(b.stateStore, b.log)
|
||||||
b.setupForgeWebhook()
|
b.setupForgeWebhook()
|
||||||
|
|||||||
@@ -260,7 +260,11 @@ func (b *boot) startLiveSettings(ctx context.Context) error {
|
|||||||
// directory consumers hold the boot value: the watch refuses such a change
|
// directory consumers hold the boot value: the watch refuses such a change
|
||||||
// rather than layering it (refuseSkillsDirChange), so the kind is labelled
|
// rather than layering it (refuseSkillsDirChange), so the kind is labelled
|
||||||
// restart-required. Each kind is watched in its own goroutine, exactly the
|
// restart-required. Each kind is watched in its own goroutine, exactly the
|
||||||
// shape the workflow-execution-settings watch established.
|
// shape the workflow-execution-settings watch established. ForgeConnectionsKind
|
||||||
|
// is on it because the forge client and the worktree token are re-resolved
|
||||||
|
// from the republished config by reopenForge, which replaces every holder of
|
||||||
|
// the client (the daemon's root forge and the root task RPC servers), so a
|
||||||
|
// rotated credential reaches the next operation without a restart.
|
||||||
var runtimeResourceKinds = []string{
|
var runtimeResourceKinds = []string{
|
||||||
controlplane.TaskPolicyKind,
|
controlplane.TaskPolicyKind,
|
||||||
controlplane.ProviderSettingsKind,
|
controlplane.ProviderSettingsKind,
|
||||||
@@ -275,6 +279,7 @@ var runtimeResourceKinds = []string{
|
|||||||
controlplane.CredentialBindingsKind,
|
controlplane.CredentialBindingsKind,
|
||||||
controlplane.IdentityGrantsKind,
|
controlplane.IdentityGrantsKind,
|
||||||
controlplane.AgentProfileKind,
|
controlplane.AgentProfileKind,
|
||||||
|
controlplane.ForgeConnectionsKind,
|
||||||
}
|
}
|
||||||
|
|
||||||
// startRuntimeResourceWatches keeps a watch per live kind established for the
|
// startRuntimeResourceWatches keeps a watch per live kind established for the
|
||||||
@@ -387,6 +392,13 @@ func (b *boot) applyRuntimeResourceUpdate(ctx context.Context, kind string, upda
|
|||||||
// lower must be honoured as running tasks finish.
|
// lower must be honoured as running tasks finish.
|
||||||
b.resizeTaskDispatcher(cfg.Containers.MaxConcurrency)
|
b.resizeTaskDispatcher(cfg.Containers.MaxConcurrency)
|
||||||
}
|
}
|
||||||
|
if kind == controlplane.ForgeConnectionsKind {
|
||||||
|
// The forge client is built from the credential rather than taken from
|
||||||
|
// the published config, so the change that published it must reach the
|
||||||
|
// client too: a rotated token otherwise keeps authenticating with the
|
||||||
|
// boot value until the process restarts.
|
||||||
|
b.reopenForge(ctx)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -359,7 +359,7 @@ func subscribeAgentEvents(nc *natsio.Conn, bus *events.Bus, log *slog.Logger) (u
|
|||||||
// tasks, and one server pair per identity answers that identity's scoped
|
// tasks, and one server pair per identity answers that identity's scoped
|
||||||
// subjects so a container-mode task owned by a non-root identity has its RPC
|
// subjects so a container-mode task owned by a non-root identity has its RPC
|
||||||
// calls served by its own forge client and worktree manager.
|
// calls served by its own forge client and worktree manager.
|
||||||
func registerTaskRPCServers(nc *natsio.Conn, forgeClient forge.Forge, trees *worktree.Manager, identities []*daemon.IdentityRunner, runs worktreerpc.Runs, authorities forgerpc.Authorities, log *slog.Logger) (unsubscribe func(), err error) {
|
func registerTaskRPCServers(nc *natsio.Conn, forgeClient forge.Forge, trees *worktree.Manager, identities []*daemon.IdentityRunner, runs worktreerpc.Runs, authorities forgerpc.Authorities, log *slog.Logger) (unsubscribe func(), root *rootTaskRPC, err error) {
|
||||||
unsubs := make([]func(), 0, 2+2*len(identities))
|
unsubs := make([]func(), 0, 2+2*len(identities))
|
||||||
unsubAll := func() {
|
unsubAll := func() {
|
||||||
for _, u := range unsubs {
|
for _, u := range unsubs {
|
||||||
@@ -367,49 +367,61 @@ func registerTaskRPCServers(nc *natsio.Conn, forgeClient forge.Forge, trees *wor
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
registerForge := func(fg forge.Forge, identity string) error {
|
registerForge := func(fg forge.Forge, identity string) (*forgerpc.Server, error) {
|
||||||
srv := &forgerpc.Server{Forge: fg, Runs: runs, Authorities: authorities, Log: log.With("rpc_identity", identity)}
|
srv := &forgerpc.Server{Forge: fg, Runs: runs, Authorities: authorities, Log: log.With("rpc_identity", identity)}
|
||||||
u, err := srv.RegisterFor(nc, identity)
|
u, err := srv.RegisterFor(nc, identity)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("register forgerpc%s: %w", identitySuffix(identity), err)
|
return nil, fmt.Errorf("register forgerpc%s: %w", identitySuffix(identity), err)
|
||||||
}
|
}
|
||||||
unsubs = append(unsubs, u)
|
unsubs = append(unsubs, u)
|
||||||
return nil
|
return srv, nil
|
||||||
}
|
}
|
||||||
registerTrees := func(mgr *worktree.Manager, identity string) error {
|
registerTrees := func(mgr *worktree.Manager, identity string) (*worktreerpc.Server, error) {
|
||||||
srv := &worktreerpc.Server{Trees: mgr, Runs: runs, Log: log.With("rpc_identity", identity)}
|
srv := &worktreerpc.Server{Trees: mgr, Runs: runs, Log: log.With("rpc_identity", identity)}
|
||||||
u, err := srv.RegisterFor(nc, identity)
|
u, err := srv.RegisterFor(nc, identity)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("register worktreerpc%s: %w", identitySuffix(identity), err)
|
return nil, fmt.Errorf("register worktreerpc%s: %w", identitySuffix(identity), err)
|
||||||
}
|
}
|
||||||
unsubs = append(unsubs, u)
|
unsubs = append(unsubs, u)
|
||||||
return nil
|
return srv, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Root (identity-less) forge and worktree servers.
|
// Root (identity-less) forge and worktree servers.
|
||||||
if err := registerForge(forgeClient, ""); err != nil {
|
rootForge, err := registerForge(forgeClient, "")
|
||||||
|
if err != nil {
|
||||||
unsubAll()
|
unsubAll()
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
if err := registerTrees(trees, ""); err != nil {
|
rootTrees, err := registerTrees(trees, "")
|
||||||
|
if err != nil {
|
||||||
unsubAll()
|
unsubAll()
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
root = &rootTaskRPC{forge: rootForge, trees: rootTrees}
|
||||||
|
|
||||||
// One forge/worktree server pair per identity, on identity-scoped
|
// One forge/worktree server pair per identity, on identity-scoped
|
||||||
// subjects.
|
// subjects.
|
||||||
for _, id := range identities {
|
for _, id := range identities {
|
||||||
if err := registerForge(id.Forge, id.Name); err != nil {
|
if _, err := registerForge(id.Forge, string(id.ID)); err != nil {
|
||||||
unsubAll()
|
unsubAll()
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
if err := registerTrees(id.Trees, id.Name); err != nil {
|
if _, err := registerTrees(id.Trees, string(id.ID)); err != nil {
|
||||||
unsubAll()
|
unsubAll()
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return unsubAll, nil
|
return unsubAll, root, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// rootTaskRPC is the pair of root (identity-less) task RPC servers. It is
|
||||||
|
// held so a live connection change replaces the forge the servers answer
|
||||||
|
// with: a task already connected asks the new credential, rather than the
|
||||||
|
// one the process booted with.
|
||||||
|
type rootTaskRPC struct {
|
||||||
|
forge *forgerpc.Server
|
||||||
|
trees *worktreerpc.Server
|
||||||
}
|
}
|
||||||
|
|
||||||
// identitySuffix renders a log suffix for an identity-scoped registration,
|
// identitySuffix renders a log suffix for an identity-scoped registration,
|
||||||
|
|||||||
Reference in new issue
Block a user