diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 13728ad98..19fb3cf13 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -36,6 +36,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/applystatus" "github.com/samcharles93/archie-core/internal/domain/health" "github.com/samcharles93/archie-core/internal/domain/identity" + "github.com/samcharles93/archie-core/internal/domain/org" "github.com/samcharles93/archie-core/internal/domain/presence" "github.com/samcharles93/archie-core/internal/domain/scheduling" "github.com/samcharles93/archie-core/internal/domain/storecontract" @@ -91,6 +92,9 @@ type boot struct { forgeClient forge.Forge forgeExt *forgeExtensions token string + // rootRPC is the root (identity-less) task RPC server pair, held so a + // live connection change reaches the servers a task already asks. + rootRPC *rootTaskRPC // stateStore is the State Store contract adapter every daemon and gateway // store consumer depends on. It is ALWAYS the remote *staterpc.Client @@ -267,6 +271,28 @@ func (b *boot) openForge(ctx context.Context) { b.forgeClient, b.token = resolveForge(ctx, b.cfg.Forge, "default", b.secrets, b.forgeExt, b.log) } +// reopenForge re-resolves the instance's forge connection from the settings +// the layering just published. Every holder of the client is replaced, so a +// rotated credential, a changed host or a removed connection reaches the next +// operation rather than dying with the process that booted. It runs from the +// connections watch alone, after the config holder carries the change; the +// worktree manager is shared by pointer, so its token is the only field to +// replace. +func (b *boot) reopenForge(ctx context.Context) { + client, token := resolveForge(ctx, b.cfgHolder.Get().Forge, "default", b.secrets, b.forgeExt, b.log) + b.forgeClient, b.token = client, token + if b.trees != nil { + b.trees.Token = token + } + if b.d != nil { + b.d.Forge = client + } + if b.rootRPC != nil { + b.rootRPC.forge.Forge = client + b.rootRPC.trees.Trees = b.trees + } +} + func (b *boot) openStateStoreAdapter(ctx context.Context) error { target := strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target) if target == "" { @@ -302,13 +328,13 @@ func (b *boot) openStateStage(ctx context.Context) error { } func (b *boot) openAccessChain(ctx context.Context) error { - source, ok := b.stateStore.(access.PolicySource) + source, ok := b.stateStore.(access.PolicyStore) if !ok { - return nil + return fmt.Errorf("state store policy reader unavailable") } - live, err := infraaccess.NewLive(ctx, source.Policies, b.log) - if err != nil { - return fmt.Errorf("load stored policies: %w", err) + live := infraaccess.NewPending(source.ListPolicies, b.log) + if err := live.Reload(ctx); err != nil { + b.log.Warn("workflow policies unavailable; execution remains denied", "err", err) } b.accessChain, b.accessLive = live, live go live.Run(ctx, applystatus.RestampInterval) @@ -538,11 +564,12 @@ func (b *boot) registerNATSRPC() error { log.Error("nats connection unavailable for task RPC", "err", err) return err } - unsubscribe, err := registerTaskRPCServers(coreConn, b.forgeClient, b.trees, b.identityRunners, b.stateStoreGrants.Client, installedAuthorities{b.stateStoreGrants.Client}, log) + unsubscribe, rootRPC, err := registerTaskRPCServers(coreConn, b.forgeClient, b.trees, b.identityRunners, b.stateStoreGrants.Client, installedAuthorities{b.stateStoreGrants.Client}, log) if err != nil { log.Error("task RPC server registration failed", "err", err) return err } + b.rootRPC = rootRPC b.addCleanup(unsubscribe) stopLogs, err := logrpc.RegisterDaemon(coreConn, b.logFeed, log) @@ -605,6 +632,9 @@ func (b *boot) buildDaemon() { if identities, ok := b.stateStore.(identity.Repository); ok { b.d.IdentityRepository = identities } + if assignments, ok := b.stateStore.(org.AgentAssignments); ok { + b.d.AgentAssignments = assignments + } if ms, ok := b.stateStore.(storecontract.MappingStore); ok { b.d.Mappings = ms @@ -628,13 +658,11 @@ func (b *boot) buildDaemon() { b.d.WithdrawTask = b.taskActions().Withdraw b.d.Access = b.accessChain - if b.accessChain != nil { - if principals, ok := b.stateStore.(access.PrincipalSource); ok { - b.d.Principals = principals - } - if denials, ok := b.stateStore.(access.DenialRecorder); ok { - b.d.Denials = denials - } + if principals, ok := b.stateStore.(access.PrincipalSource); ok { + b.d.Principals = principals + } + if denials, ok := b.stateStore.(access.DenialRecorder); ok { + b.d.Denials = denials } b.d.PlaybookLedger = playbookLedger(b.stateStore, b.log) b.setupForgeWebhook() diff --git a/internal/app/archied/control_plane.go b/internal/app/archied/control_plane.go index 8cf46d858..52badd096 100644 --- a/internal/app/archied/control_plane.go +++ b/internal/app/archied/control_plane.go @@ -260,7 +260,11 @@ func (b *boot) startLiveSettings(ctx context.Context) error { // directory consumers hold the boot value: the watch refuses such a change // rather than layering it (refuseSkillsDirChange), so the kind is labelled // restart-required. Each kind is watched in its own goroutine, exactly the -// shape the workflow-execution-settings watch established. +// shape the workflow-execution-settings watch established. ForgeConnectionsKind +// is on it because the forge client and the worktree token are re-resolved +// from the republished config by reopenForge, which replaces every holder of +// the client (the daemon's root forge and the root task RPC servers), so a +// rotated credential reaches the next operation without a restart. var runtimeResourceKinds = []string{ controlplane.TaskPolicyKind, controlplane.ProviderSettingsKind, @@ -275,6 +279,7 @@ var runtimeResourceKinds = []string{ controlplane.CredentialBindingsKind, controlplane.IdentityGrantsKind, controlplane.AgentProfileKind, + controlplane.ForgeConnectionsKind, } // startRuntimeResourceWatches keeps a watch per live kind established for the @@ -387,6 +392,13 @@ func (b *boot) applyRuntimeResourceUpdate(ctx context.Context, kind string, upda // lower must be honoured as running tasks finish. b.resizeTaskDispatcher(cfg.Containers.MaxConcurrency) } + if kind == controlplane.ForgeConnectionsKind { + // The forge client is built from the credential rather than taken from + // the published config, so the change that published it must reach the + // client too: a rotated token otherwise keeps authenticating with the + // boot value until the process restarts. + b.reopenForge(ctx) + } return nil } diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index 27d4b2c8c..9fdc24b64 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -359,7 +359,7 @@ func subscribeAgentEvents(nc *natsio.Conn, bus *events.Bus, log *slog.Logger) (u // tasks, and one server pair per identity answers that identity's scoped // subjects so a container-mode task owned by a non-root identity has its RPC // calls served by its own forge client and worktree manager. -func registerTaskRPCServers(nc *natsio.Conn, forgeClient forge.Forge, trees *worktree.Manager, identities []*daemon.IdentityRunner, runs worktreerpc.Runs, authorities forgerpc.Authorities, log *slog.Logger) (unsubscribe func(), err error) { +func registerTaskRPCServers(nc *natsio.Conn, forgeClient forge.Forge, trees *worktree.Manager, identities []*daemon.IdentityRunner, runs worktreerpc.Runs, authorities forgerpc.Authorities, log *slog.Logger) (unsubscribe func(), root *rootTaskRPC, err error) { unsubs := make([]func(), 0, 2+2*len(identities)) unsubAll := func() { for _, u := range unsubs { @@ -367,49 +367,61 @@ func registerTaskRPCServers(nc *natsio.Conn, forgeClient forge.Forge, trees *wor } } - registerForge := func(fg forge.Forge, identity string) error { + registerForge := func(fg forge.Forge, identity string) (*forgerpc.Server, error) { srv := &forgerpc.Server{Forge: fg, Runs: runs, Authorities: authorities, Log: log.With("rpc_identity", identity)} u, err := srv.RegisterFor(nc, identity) if err != nil { - return fmt.Errorf("register forgerpc%s: %w", identitySuffix(identity), err) + return nil, fmt.Errorf("register forgerpc%s: %w", identitySuffix(identity), err) } unsubs = append(unsubs, u) - return nil + return srv, nil } - registerTrees := func(mgr *worktree.Manager, identity string) error { + registerTrees := func(mgr *worktree.Manager, identity string) (*worktreerpc.Server, error) { srv := &worktreerpc.Server{Trees: mgr, Runs: runs, Log: log.With("rpc_identity", identity)} u, err := srv.RegisterFor(nc, identity) if err != nil { - return fmt.Errorf("register worktreerpc%s: %w", identitySuffix(identity), err) + return nil, fmt.Errorf("register worktreerpc%s: %w", identitySuffix(identity), err) } unsubs = append(unsubs, u) - return nil + return srv, nil } // Root (identity-less) forge and worktree servers. - if err := registerForge(forgeClient, ""); err != nil { + rootForge, err := registerForge(forgeClient, "") + if err != nil { unsubAll() - return nil, err + return nil, nil, err } - if err := registerTrees(trees, ""); err != nil { + rootTrees, err := registerTrees(trees, "") + if err != nil { unsubAll() - return nil, err + return nil, nil, err } + root = &rootTaskRPC{forge: rootForge, trees: rootTrees} // One forge/worktree server pair per identity, on identity-scoped // subjects. for _, id := range identities { - if err := registerForge(id.Forge, id.Name); err != nil { + if _, err := registerForge(id.Forge, string(id.ID)); err != nil { unsubAll() - return nil, err + return nil, nil, err } - if err := registerTrees(id.Trees, id.Name); err != nil { + if _, err := registerTrees(id.Trees, string(id.ID)); err != nil { unsubAll() - return nil, err + return nil, nil, err } } - return unsubAll, nil + return unsubAll, root, nil +} + +// rootTaskRPC is the pair of root (identity-less) task RPC servers. It is +// held so a live connection change replaces the forge the servers answer +// with: a task already connected asks the new credential, rather than the +// one the process booted with. +type rootTaskRPC struct { + forge *forgerpc.Server + trees *worktreerpc.Server } // identitySuffix renders a log suffix for an identity-scoped registration,