Files

904 lines
33 KiB
Go

// bootstrap.go owns the composition of the daemon: run() is the entry
// point and driver, and each phase of wiring lives in a method on boot.
// The struct exists because the composition root assembles ~30 pieces of
// state that every phase touches; passing them as parameters would make
// each phase function longer than the code it contains.
package archied
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net"
"net/url"
"os"
"os/signal"
"path/filepath"
"slices"
"strconv"
"strings"
"sync/atomic"
"syscall"
"time"
"github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/container"
controlpb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
"github.com/samcharles93/archie-core/internal/daemon"
"github.com/samcharles93/archie-core/internal/domain/access"
"github.com/samcharles93/archie-core/internal/domain/agent"
"github.com/samcharles93/archie-core/internal/domain/alert"
"github.com/samcharles93/archie-core/internal/domain/applystatus"
"github.com/samcharles93/archie-core/internal/domain/health"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/domain/scheduling"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/domain/workintake"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/forge"
infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/controlplanestore"
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
"github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
"github.com/samcharles93/archie-core/internal/infrastructure/extension"
"github.com/samcharles93/archie-core/internal/infrastructure/issuerpc"
"github.com/samcharles93/archie-core/internal/infrastructure/logrpc"
"github.com/samcharles93/archie-core/internal/infrastructure/messagingrpc"
"github.com/samcharles93/archie-core/internal/infrastructure/modelcatalog"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
"github.com/samcharles93/archie-core/internal/infrastructure/taskactions"
"github.com/samcharles93/archie-core/internal/logging"
"github.com/samcharles93/archie-core/internal/secret"
"github.com/samcharles93/archie-core/internal/storage"
"github.com/samcharles93/archie-core/internal/webui"
"github.com/samcharles93/archie-core/internal/worktree"
)
// boot is the mutable wiring state assembled by the run() composition
// phases. Cleanup funcs are registered in creation order and run in
// reverse at shutdown, matching the LIFO ordering of the deferred calls
// they replace.
type boot struct {
catalog *modelcatalog.Catalog
cfg config.Config
log *slog.Logger
playbooks *controlplanestore.LivePlaybooks
// stderrLog keeps setupLogging on stderr, never touching cfg.Log.File. An
// offline command reads the bootstrap config but is not the daemon: creating
// the deployment's log, appending a line stamped component="daemon", or
// rotating a log the daemon is writing are side effects a diagnosis must not
// have, and it needs no durable copy of its own.
stderrLog bool
loader *configuration.Loader
doc *configuration.Document
currentProvenance atomic.Pointer[configuration.Provenance]
health *healthSurface
logFeed *logging.Feed
taskLogs *logging.TaskRegistry
secrets *secret.Registry
forgeClient forge.Forge
forgeExt *forgeExtensions
token string
// stateStore is the State Store contract adapter every daemon and gateway
// store consumer depends on. It is ALWAYS the remote *staterpc.Client
// dialed to the standalone archie-state-store gRPC service at
// [services.state].target. It is set by openStateStoreAdapter, which
// requires [services.state].target to be set.
stateStore storecontract.TaskStore
// stateClient is stateStore's concrete type, which the daemon's typed store
// fields are assigned from directly.
stateClient *staterpc.Client
taskModels daemon.TaskModels
taskInstructions daemon.TaskInstructions
// accessChain is the daemon's policy engine, built once from the stored
// policies (openAccessChain); accessProblems is what the readiness
// surface reports. Both are nil when no policy store is wired.
accessChain access.Authorizer
accessLive *infraaccess.Live
// stateStoreGrants issues each task's run credential
// (daemon.RunCredentialIssuer), wrapping the same *staterpc.Client as
// stateStore, which also verifies a credential for a worktree push.
stateStoreGrants *staterpc.GrantIssuer
controlPlane *controlplanestore.Client
// controlPlaneRPC is the State Store's control-plane transport, kept so the
// daemon root can build its workflow-definitions client from it
// (openDaemonWorkflowDefinitions). The gateway root never builds that client:
// it resolves no workflow step type.
controlPlaneRPC controlpb.ControlPlaneServiceClient
// workflowDefinitions is the one control-plane surface that resolves
// workflow step types. It is separate from controlPlane because it is the
// only surface that needs the process's step vocabulary.
workflowDefinitions *controlplanestore.WorkflowDefinitionsClient
// executionSettings is the last workflow execution settings the control
// plane published. They arrive on a watch rather than in the file
// document, so reloadConfig re-applies them from here.
executionSettings atomic.Pointer[workflow.ExecutionSettings]
// applyStatus reports which control-plane resource version this process
// is running. Nil until the State Store adapter is open, and nil-safe,
// so apply points call it without a guard.
applyStatus *applystatus.Reporter
// processName is which binary this composition is running as, and is the
// name apply-status records carry. Set by each entry point, because one
// boot builds both archied and archie-gateway.
processName string
bus *events.Bus
// cfgHolder is the daemon's one configuration Holder. Boot owns it and
// the daemon reads through it, so a reload swaps one snapshot and every
// reader sees it: there is no second holder to keep in step.
cfgHolder *config.Holder
// chat is the dashboard-shaped chat surface. It survives the UI
// extraction because the readiness gateway probe and the Telegram task
// actor both read the same contract.
chat *webui.ChatService
// healthRegistry backs /health/detailed on the daemon's own listener.
healthRegistry *health.Registry
// lastReload reports the most recent reload outcome for the published
// configuration projection. Nil until reload wiring installs it.
lastReload func() config.ReloadStatus
natsClient *nats.Client
// reactionClient pulls the ARCHIE_REACTIONS fan-out stream; the daemon
// drains it each cycle to turn review reactions into remediate runs.
reactionClient *nats.Client
// natsURL is the endpoint the daemon's own client connected with at
// startup. For external mode it is cfg.NATS.URL; for embedded mode it is
// the embedded server's ClientURL(). Recorded so Daemon.ConnectedNATS
// carries the real endpoint rather than an empty config URL.
natsURL string
containerPool *container.Pool
// taskRuntime is what task agents run in: the container pool, or an
// OpenShell gateway's sandboxes. Nil parks every task.
taskRuntime daemon.TaskRuntime
// sandboxes provisions each task container's isolated network and runs
// the egress relay that is its only way out.
sandboxes *egress.Networks
// modelEgress gives native tasks model access through the egress proxy;
// nil when the proxy cannot start.
modelEgress daemon.ModelEgress
// egressProxyAddr is where the egress proxy listens, the address the relay
// forwards its proxy port to. Empty when the proxy is not running.
egressProxyAddr string
storeBackend storage.Backend
// runtimeVersions records the version of every control-plane kind boot's
// layering applied. Set once at boot before the runtime-resource watches
// start, which read it as their resume points.
runtimeVersions map[string]int64
// messaging delivers outbound chat messages through the Messaging Service.
messaging *messagingrpc.Client
startGateways []func()
trees *worktree.Manager
d *daemon.Daemon
// schedulingEngine is the cron/scheduling ticker engine (setupScheduling).
schedulingEngine *scheduling.Engine
// agentStatus records the most recent version/install-type an
// archie-agent worker reported about itself. Allocated up front, before
// any gateway goroutine (Telegram, webui) that might read it via
// RunningVersions starts, so those closures never see a nil pointer or
// race its assignment -- only its own internal mutex guards concurrent
// Observe/Snapshot calls once tasks start completing.
agentStatus *daemon.AgentStatus
// shutdown cancels the daemon's root context, the exact path a SIGTERM
// takes to graceful shutdown. The drain monitor invokes it when it honours
// an external drain request, so a marker-driven drain reuses the operator
// signal path rather than inventing a new one.
shutdown context.CancelFunc
// alertChecks re-evaluates the configuration alerts when the configuration
// changes. Nil until the daemon's run loop starts them.
alertChecks *alertChecks
// credentialWatch reports bound credentials that stop resolving. Nil
// when the State Store has no alert surface.
credentialWatch *credentialWatch
cleanups []func()
}
func newBootstrap() *boot {
return &boot{log: slog.New(slog.NewJSONHandler(os.Stderr, nil)), agentStatus: &daemon.AgentStatus{}}
}
func (b *boot) cleanup() {
for _, v := range slices.Backward(b.cleanups) {
v()
}
}
func (b *boot) loadConfig(ctx context.Context, cfgPath, overlayPath string) error {
loader, doc, err := servicekit.Resolve(b.log, cfgPath, overlayPath)
if err != nil {
return err
}
b.loader, b.doc = loader, doc
b.catalog = modelcatalog.NewCatalog(cfgPath)
b.cfg = b.doc.Config
b.cfgHolder = config.NewHolder(b.cfg)
b.currentProvenance.Store(&b.doc.Provenance)
return b.setupLogging(ctx)
}
// setupLogging stays on stderr for an offline command (stderrLog): creating,
// appending to or rotating the deployment's log is a side effect a diagnosis
// must not have.
func (b *boot) setupLogging(ctx context.Context) error {
if b.stderrLog {
return nil
}
logs, err := servicekit.Logging(ctx, b.cfg, "daemon")
if err != nil {
return err
}
b.log, b.logFeed, b.taskLogs = logs.Log, logs.Feed, logs.TaskLogs
b.addCleanup(func() { _ = logs.Closer.Close() })
return nil
}
// openStores wires the secret registry and the conversation store. The
// registry starts with the env engine only; extension engines join it once the
// State Store client exists (openStateStoreAdapter), so nothing here resolves a
// reference that may name one.
func (b *boot) openStores(ctx context.Context) error {
b.secrets = secret.NewRegistry()
return nil
}
// openForge builds the forge client. It runs after the runtime settings are
// layered, so a token held by an extension engine can resolve.
func (b *boot) openForge(ctx context.Context) {
if packages, ok := b.stateStore.(extension.Packages); ok {
b.forgeExt = newForgeExtensions(extension.Source{Query: b.controlPlane, Packages: packages}, b.log)
b.addCleanup(b.forgeExt.close)
}
b.forgeClient, b.token = resolveForge(ctx, b.cfg.Forge, "default", b.secrets, b.forgeExt, b.log)
}
// reopenForge re-resolves the instance's forge connection from the settings
// the layering just published. Every holder of the client is replaced, so a
// rotated credential, a changed host or a removed connection reaches the next
// operation rather than dying with the process that booted. It runs from the
// connections watch alone, after the config holder carries the change; the
// worktree manager is shared by pointer, so its token is the only field to
// replace.
func (b *boot) reopenForge(ctx context.Context) {
client, token := resolveForge(ctx, b.cfgHolder.Get().Forge, "default", b.secrets, b.forgeExt, b.log)
b.forgeClient, b.token = client, token
if b.trees != nil {
b.trees.Token = token
}
if b.d != nil {
b.d.Forge = client
}
}
func (b *boot) openStateStoreAdapter(ctx context.Context) error {
target := strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target)
if target == "" {
return fmt.Errorf("services.state.target is required: archied/archie-gateway no longer own archie.db; the standalone archie-state-store process owns it (docs/prds/state-store-contract.md §12 step 7)")
}
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, b.cfg, staterpc.WithCallDeadline(stateCallDeadline))
if err != nil {
b.log.ErrorContext(ctx, "state store adapter", "err", err)
return err
}
b.stateStore = client
b.stateClient = client
if alerts, ok := b.stateStore.(alert.Store); ok {
b.credentialWatch = &credentialWatch{reporter: alert.Reporter{Raiser: alerts, Log: b.log, By: "archied"}}
}
b.taskModels = client
b.taskInstructions = client
b.controlPlaneRPC = client.ControlPlane()
b.controlPlane = controlplanestore.NewRPCClient(b.controlPlaneRPC)
b.applyStatus = applystatus.New(b.processName, client, b.log)
b.stateStoreGrants = &staterpc.GrantIssuer{Client: client}
b.addCleanup(cleanup)
b.startSecretEngines(ctx, client)
return nil
}
func (b *boot) openStateStage(ctx context.Context) error {
if err := b.openStores(ctx); err != nil {
return err
}
if err := b.openDaemonStateSurfaces(ctx); err != nil {
return err
}
return b.openAccessChain(ctx)
}
func (b *boot) openAccessChain(ctx context.Context) error {
source, ok := b.stateStore.(access.PolicyStore)
if !ok {
return fmt.Errorf("state store policy reader unavailable")
}
live := infraaccess.NewPending(source.PoliciesVersion, source.ListPolicies, b.log)
if err := live.Reload(ctx); err != nil {
b.log.WarnContext(ctx, "workflow policies unavailable; execution remains denied", "err", err)
}
b.accessChain, b.accessLive = live, live
return nil
}
// handleRequeue replays a parked/waiting task by id, then signals an
// early success exit unless -once keeps the daemon running.
func (b *boot) handleRequeue(ctx context.Context, requeue int64, once bool) (bool, error) {
if requeue <= 0 {
return false, nil
}
if err := manualRequeueTask(ctx, b.stateStore, requeue); err != nil {
b.log.ErrorContext(ctx, "requeue failed", "task", requeue, "err", err)
return false, err
}
b.log.InfoContext(ctx, "task requeued", "task", requeue)
return !once, nil
}
func (b *boot) seedSoul(ctx context.Context, cfgPath string) {
result, err := configuration.SeedSoul(cfgPath, agent.ShippedSoul())
if err != nil {
b.log.WarnContext(ctx, "soul: starter file unavailable", "err", err)
return
}
if result.Action == configuration.SoulCreated {
b.log.InfoContext(ctx, "soul: starter file written", "path", result.Path, "action", result.Action)
}
}
func (b *boot) setupObservability(ctx context.Context) {
cfg, log := b.cfg, b.log
bus := events.NewBus()
b.bus = bus
b.addCleanup(func() { bus.Close() })
b.startUpdateRelay(ctx, updateReportPath(cfg.WorkDir, "webui"))
sink := bus.Subscribe(256)
go persistEvents(ctx, sink, b.stateStore, log)
}
const reactionStreamMaxAge = 24 * time.Hour
func (b *boot) connectNATS(ctx context.Context) error { // embedded broker discovery and startup require explicit fallback branches
cfg, log := b.cfg, b.log
url := cfg.NATS.URL
var natsToken string
if cfg.NATS.Mode == config.NATSModeExternal { //nolint:nestif // embedded broker discovery and startup require explicit fallback branches
if url == "" {
return fmt.Errorf("nats.url is required when nats.mode is external")
}
var err error
natsToken, err = servicekit.NATSToken(cfg.NATS, os.Getenv)
if err != nil {
log.ErrorContext(ctx, "nats credentials", "err", err)
return err
}
if err := b.startCallout(ctx, url, natsToken); err != nil {
return err
}
} else {
endpoint, readErr := servicekit.ReadNATSEndpoint(cfg.StateDir)
if readErr == nil {
url, natsToken = endpoint.URL, endpoint.Token
if probe, err := nats.Connect(ctx, nats.Config{URL: url, Token: natsToken, Subjects: []string{workintake.SubjectTaskWildcard}, FilterSubject: workintake.SubjectTaskWildcard}, log); err == nil {
probe.Close()
url, natsToken = endpoint.URL, endpoint.Token
} else {
url, natsToken, err = b.startEmbeddedNATS(ctx)
if err != nil {
return err
}
}
} else {
var err error
url, natsToken, err = b.startEmbeddedNATS(ctx)
if err != nil {
return err
}
}
}
natsClient, err := nats.Connect(ctx, nats.Config{
URL: url,
Token: natsToken,
Subjects: []string{workintake.SubjectTaskWildcard},
FilterSubject: workintake.SubjectTaskWildcard,
}, log)
if err != nil {
log.ErrorContext(ctx, "nats connect failed", "err", err)
return err
}
reactionMaxAge := reactionStreamMaxAge
reactionClient, err := nats.Connect(ctx, nats.Config{
URL: url,
Token: natsToken,
StreamName: nats.DefaultReactionStreamName,
Subjects: []string{workintake.SubjectReactionWildcard},
FilterSubject: workintake.SubjectReactionWildcard,
Retention: nats.FanOutRetention(),
MaxAge: &reactionMaxAge,
}, log)
if err != nil {
log.ErrorContext(ctx, "nats reaction stream connect failed", "err", err)
natsClient.Close()
return err
}
if err := b.startSpanRelay(ctx, url, natsToken); err != nil {
log.ErrorContext(ctx, "span relay failed", "err", err)
natsClient.Close()
reactionClient.Close()
return err
}
b.natsClient = natsClient
b.natsURL = url
b.addCleanup(func() { natsClient.Close() })
b.addCleanup(func() { reactionClient.Close() })
b.reactionClient = reactionClient
log.InfoContext(ctx, "nats connected", "url", url, "task_stream", nats.DefaultStreamName, "reaction_stream", nats.DefaultReactionStreamName)
return nil
}
// startCallout makes the external broker defer every login to archie, so a task
// container logs in with its run credential and reaches only its own subjects.
// It runs before any other login, because those are decided here.
func (b *boot) startCallout(ctx context.Context, url, token string) error {
seed := os.Getenv(b.cfg.NATS.CalloutIssuerSeedEnv)
if seed == "" {
return fmt.Errorf("%s is required when nats.callout_issuer_seed_env is configured", b.cfg.NATS.CalloutIssuerSeedEnv)
}
opts := nats.CalloutOptions{URL: url, Token: token, IssuerSeed: seed}
if b.stateStoreGrants != nil {
opts.Tasks = taskBroker{runs: b.stateStoreGrants.Client}
}
svc, err := nats.StartCallout(ctx, opts, b.log)
if err != nil {
b.log.ErrorContext(ctx, "nats callout start failed", "err", err)
return err
}
b.addCleanup(svc.Close)
return nil
}
// natsHost is the address the embedded broker binds. OpenShell sandboxes run
// on the host network and reach the broker by the services host name, so it
// binds the address that name resolves to; Docker containers reach it on the
// bridge gateway.
func natsHost(ctx context.Context, cfg config.Config, pool *container.Pool) (string, error) {
shell := cfg.Containers.OpenShell
if cfg.Containers.Runtime == config.RuntimeOpenShell && shell.ServicesHost != "" {
addrs, err := net.DefaultResolver.LookupHost(ctx, shell.ServicesHost)
if err != nil || len(addrs) == 0 {
return "", fmt.Errorf("embedded nats: resolve services host %s: %w", shell.ServicesHost, err)
}
for _, a := range addrs {
if ip := net.ParseIP(a); ip != nil && ip.To4() != nil {
return a, nil
}
}
return addrs[0], nil
}
if pool != nil {
return pool.HostGateway(), nil
}
return "", nil
}
func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) {
cfg, log := b.cfg, b.log
if cfg.StateDir == "" {
return "", "", errors.New("embedded nats: state_dir is required")
}
host, err := natsHost(ctx, cfg, b.containerPool)
if err != nil {
return "", "", err
}
// A restarted broker keeps the previous endpoint, so the Gateway and
// running agents reconnect to it instead of a broker that no longer exists.
opts := nats.EmbeddedOptions{Host: host, StoreDir: filepath.Join(cfg.StateDir, "nats")}
if b.stateStoreGrants != nil {
opts.Tasks = taskBroker{runs: b.stateStoreGrants.Client}
}
if previous, err := servicekit.ReadNATSEndpoint(cfg.StateDir); err == nil {
opts.Token = previous.Token
if u, err := url.Parse(previous.URL); err == nil {
opts.Port, _ = strconv.Atoi(u.Port())
}
}
srv, err := nats.StartEmbedded(ctx, opts, log)
if err != nil && opts.Port != 0 {
log.WarnContext(ctx, "embedded nats: previous port unavailable; taking a new one", "port", opts.Port, "err", err)
opts.Port = 0
srv, err = nats.StartEmbedded(ctx, opts, log)
}
if err != nil {
log.ErrorContext(ctx, "embedded nats start failed", "err", err)
return "", "", err
}
b.addCleanup(func() { srv.Shutdown() })
log.InfoContext(ctx, "embedded nats started", "url", srv.ClientURL())
if err := servicekit.WriteNATSEndpoint(cfg.StateDir, srv.ClientURL(), srv.Token()); err != nil {
srv.Shutdown()
return "", "", err
}
return srv.ClientURL(), srv.Token(), nil
}
func (b *boot) setupContainers(ctx context.Context) func() {
containerPool, sandboxes, storeBackend, closeDocker := startContainers(ctx, b.cfgHolder, b.secrets, b.log)
b.containerPool = containerPool
b.sandboxes = sandboxes
b.storeBackend = storeBackend
b.taskRuntime = daemon.DockerRuntime(containerPool)
if b.cfg.Containers.Runtime != config.RuntimeOpenShell {
return closeDocker
}
rt, client, closeOpenShell, err := startOpenShell(ctx, b.cfgHolder, b.secrets, b.log)
if err != nil {
b.log.ErrorContext(ctx, "autonomous workflows unavailable: openshell runtime", "err", err)
b.taskRuntime = nil
return closeDocker
}
b.taskRuntime = daemon.OpenShellRuntime(rt)
settings := b.cfg.Containers.OpenShell
bindings := runCredentialResolver{runs: b.stateStoreGrants.Client, models: b.taskModels, config: b.cfgHolder, secrets: b.secrets, credentials: b.credentialWatch}
egress := newOpenShellModelEgress(client, settings.Workspace, b.cfg.Instance, b.secrets, bindings, b.log)
b.modelEgress = egress
sweepCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
if err := egress.Sweep(sweepCtx); err != nil {
b.log.WarnContext(ctx, "openshell: orphaned gateway providers remain", "err", err)
}
return func() { closeOpenShell(); closeDocker() }
}
// setupGatewayClient dials the Gateway, which owns every model call.
func (b *boot) setupGatewayClient(ctx context.Context) error {
contract, cleanup, err := composeChatContract(b.cfg)
if err != nil {
return err
}
b.addCleanup(cleanup)
b.chat = &webui.ChatService{Contract: contract}
messaging, closeMessaging, err := composeMessaging(b.cfg)
if err != nil {
return err
}
b.addCleanup(closeMessaging)
b.messaging = messaging
posted := func(taskID int64) {
b.bus.Publish(ctx, events.Event{At: time.Now().UTC(), Kind: events.KindChatOutcome, TaskID: taskID})
}
go postOutcomes(ctx, b.stateStore, deliverOutcome(contract, messaging), posted, b.log)
b.setupReadinessProbes()
return nil
}
func (b *boot) buildWorktreeManager() {
cfg := b.cfg
b.trees = &worktree.Manager{
WorkDir: cfg.WorkDir,
Token: b.token,
BotUser: cfg.BotUser,
BotEmail: cfg.BotEmail,
BaseURL: cfg.Forge.Host,
}
}
func (b *boot) registerNATSRPC(ctx context.Context) error {
log := b.log
if b.natsClient == nil {
return nil
}
coreConn, err := b.natsClient.CoreConn()
if err != nil {
log.ErrorContext(ctx, "nats connection unavailable for task RPC", "err", err)
return err
}
unsubscribe, err := registerTaskRPCServers(ctx, coreConn, func(ctx context.Context, t *workflow.Task, op string) (daemon.RepositoryAccess, error) {
if b.d == nil {
return daemon.RepositoryAccess{}, errors.New("repository connections starting")
}
return b.d.Repository(ctx, t, op)
}, func(ctx context.Context, org, channel, chatID, text string) error {
if b.messaging == nil {
return errors.New("messaging service not configured")
}
return b.messaging.Deliver(ctx, org, channel, chatID, text)
}, b.stateStoreGrants.Client, installedAuthorities{b.stateStoreGrants.Client}, log)
if err != nil {
log.ErrorContext(ctx, "task RPC server registration failed", "err", err)
return err
}
b.addCleanup(unsubscribe)
stopLogs, err := logrpc.RegisterDaemon(ctx, coreConn, b.logFeed, log)
if err != nil {
return fmt.Errorf("register daemon logs: %w", err)
}
b.addCleanup(stopLogs)
unsubscribeSystemLogs, err := subscribeSystemLogs(ctx, coreConn, b.taskLogs, log)
if err != nil {
log.ErrorContext(ctx, "system log subscribe failed", "err", err)
return err
}
b.addCleanup(unsubscribeSystemLogs)
unsubscribeAgentEvents, err := subscribeAgentEvents(ctx, coreConn, b.bus, log)
if err != nil {
log.ErrorContext(ctx, "agent event subscribe failed", "err", err)
return err
}
b.addCleanup(unsubscribeAgentEvents)
unsubscribeTaskActions, err := taskactions.Register(ctx, coreConn, b.taskActions(), log)
if err != nil {
log.ErrorContext(ctx, "gateway task action register failed", "err", err)
return err
}
b.addCleanup(unsubscribeTaskActions)
unsubscribeIssues, err := (&issuerpc.Server{Forge: b.issueForge, Log: log}).Register(ctx, coreConn)
if err != nil {
log.ErrorContext(ctx, "issue RPC register failed", "err", err)
return err
}
b.addCleanup(unsubscribeIssues)
return nil
}
// issueForge is the forge of owner/repo's own connection.
func (b *boot) issueForge(ctx context.Context, owner, repo string) (forge.IssueForge, error) {
return b.repositoryForge(ctx, owner, repo)
}
func (b *boot) buildDaemon(ctx context.Context) {
log := b.log
workspaces, _ := b.stateStore.(access.WorkspaceLister)
d := &daemon.Daemon{
ResolveRepository: b.repositoryAccess,
Cfg: b.cfgHolder,
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL},
ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), CA: b.stateStoreCA(ctx)},
Store: b.stateStore,
Identity: daemon.NewIdentityGate(b.stateClient, b.stateClient, b.accessChain, b.stateClient, b.stateClient, b.orgPrincipals, workspaces, log),
PlaybookLedger: b.stateClient,
Bus: b.bus,
Forge: b.forgeClient,
Trees: b.trees,
PackageAuthorities: installedAuthorities{b.stateStoreGrants.Client},
Storage: b.storeBackend,
Log: log,
TaskTransport: b.natsClient,
RunCredentials: b.stateStoreGrants,
Runtime: b.taskRuntime,
ModelEgress: b.modelEgress,
TaskModels: b.taskModels,
TaskInstructions: b.taskInstructions,
TaskLogs: b.taskLogs,
AgentStatus: b.agentStatus,
WorkflowDefinitions: b.workflowDefinitions,
Playbooks: b.livePlaybooks(),
}
d.ReviewScanner = daemon.NewReviewScanner(b.stateStore, b.reactionClient, b.natsClient.PublishUnique, b.bus, d, d.Identity, d, b.repositoryForge, log)
d.BindingDispatch = daemon.NewBindingDispatch(daemon.BindingDispatchDeps{
Store: b.stateStore,
Mappings: b.stateClient,
Bindings: b.stateClient,
Dispatcher: b.stateClient,
Matches: b.stateClient,
Creator: b.stateClient,
Enablement: b.controlPlane,
Definitions: b.workflowDefinitions,
Access: d.Identity,
Denials: d.Identity,
Cfg: b.cfgHolder,
Orgs: d.Identity,
OrgPrincipals: b.orgPrincipals,
Log: log,
})
d.WireTaskBus()
d.IssuePoller = daemon.NewIssuePoller(b.cfgHolder, b.stateStore, log, b.issueSources, b.taskActions().Withdraw, d.TaskBus.PublishTask, d.Identity, d.BindingDispatch)
b.d = d
}
func (b *boot) publishConfig(ctx context.Context, cfg config.Config) {
b.cfgHolder.Set(cfg)
// The daemon owns the published configuration view.
if b.processName != applystatus.Gateway {
b.publishConfigSnapshot(ctx)
b.recheckAlerts(ctx)
}
}
func (b *boot) configViewInput() webui.ConfigViewInput {
catalog := b.catalogState()
in := webui.ConfigViewInput{
Config: b.cfgHolder.Get(),
Catalog: catalogView(catalog),
SecretEngines: b.secrets.List(),
}
if b.lastReload != nil {
status := b.lastReload()
in.Reload = &status
}
return in
}
func (b *boot) publishConfigSnapshot(ctx context.Context) {
snapshots, ok := b.stateStore.(storecontract.ConfigSnapshotStore)
if !ok || b.cfgHolder == nil {
return
}
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
document, err := json.Marshal(webui.BuildConfigView(b.configViewInput()))
if err != nil {
b.log.WarnContext(ctx, "config snapshot not published", "err", err)
return
}
err = snapshots.PutConfigSnapshot(ctx, storecontract.ConfigSnapshot{
Schema: webui.ConfigViewSchema,
Document: document,
PublishedAt: time.Now().UTC(),
})
if err != nil {
b.log.WarnContext(ctx, "config snapshot not published", "err", err)
}
}
func (b *boot) wireConfigPublishing(ctx context.Context, cfgPath, overlayPath string) {
log := b.log
reloadController := newReloadController(b.loader, cfgPath, overlayPath, b.reloadConfig)
b.lastReload = func() config.ReloadStatus {
st := reloadController.Status()
return st
}
reloadCh := make(chan os.Signal, 1)
signal.Notify(reloadCh, syscall.SIGHUP)
b.addCleanup(func() { signal.Stop(reloadCh) })
go reloadLoop(ctx, reloadCh, reloadController, log)
}
func (b *boot) startServices(ctx context.Context) error {
log := b.log
if err := b.d.Startup(ctx); err != nil {
log.ErrorContext(ctx, "startup", "err", err)
return err
}
if b.schedulingEngine != nil {
if err := b.schedulingEngine.Start(ctx); err != nil {
log.ErrorContext(ctx, "scheduling engine startup", "err", err)
}
}
for _, startGateway := range b.startGateways {
startGateway()
}
return nil
}
func (b *boot) setupBackends(ctx context.Context) error {
closeContainers := b.setupContainers(ctx)
b.addCleanup(closeContainers)
err := b.connectNATS(ctx)
if err == nil && b.cfg.Containers.Runtime != config.RuntimeOpenShell {
b.setupModelEgress(ctx)
b.startEgressRelay(ctx)
}
return err
}
func exitCode(err error) int {
if err != nil {
return 1
}
return 0
}
func (b *boot) runLoop(ctx context.Context, once bool) error {
b.health.markServing()
b.announceReady(ctx)
if once {
b.d.RunOnce(ctx)
return nil
}
b.log.InfoContext(ctx, "archied running", "repos", len(b.cfg.Repos), "poll", b.cfg.PollInterval.Std().String(), "label", b.cfg.Label)
b.startDrainMonitor(ctx)
b.startWatchdog(ctx)
b.startAlerts(ctx)
if err := b.d.Run(ctx); err != nil && ctx.Err() == nil {
b.log.ErrorContext(ctx, "daemon exited", "err", err)
return err
}
return nil
}
func (b *boot) addCleanup(fn func()) {
b.cleanups = append(b.cleanups, fn)
}
func (b *boot) catalogState() modelcatalog.Snapshot { return b.catalog.State() }
// loadCatalog layers the model catalog under the file config. A catalog that
// cannot be read leaves the configured providers and models in effect.
func (b *boot) loadCatalog(ctx context.Context) {
snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfgHolder.Get().Providers)
if err != nil {
b.log.WarnContext(ctx, "model catalog unavailable; using configured providers and models", "err", err)
return
}
models := modelcatalog.Apply(&b.cfg, snapshot)
// Layer the catalog over the live config, not b.cfg: b.cfg predates the
// execution-settings watch, so publishing it would put the file's budgets
// back over the control plane's.
live := b.cfgHolder.Get().Clone()
modelcatalog.Apply(&live, snapshot)
b.cfgHolder.Set(live)
b.catalog.Set(snapshot)
b.log.InfoContext(ctx, "model catalog loaded", "providers", len(snapshot.Providers), "models", len(models))
}
func (b *boot) refreshModelCatalog(ctx context.Context) error {
snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfgHolder.Get().Providers)
if err != nil {
return err
}
base := b.cfgHolder.Get().Clone()
models := modelcatalog.Apply(&base, snapshot)
cfg, _, err := b.runtimeConfig(ctx, base)
if err != nil {
return err
}
b.catalog.Set(snapshot)
b.publishConfig(ctx, cfg)
b.log.InfoContext(ctx, "model catalog refreshed", "providers", len(snapshot.Providers), "models", len(models))
return nil
}
func (b *boot) startModelCatalogRefresh(ctx context.Context) {
modelcatalog.Every(ctx, modelcatalog.RefreshInterval, func() {
if err := b.refreshModelCatalog(ctx); err != nil {
b.log.WarnContext(ctx, "model catalog refresh failed; keeping the loaded catalog", "err", err)
}
})
}
func (b *boot) livePlaybooks() *controlplanestore.LivePlaybooks {
b.playbooks = controlplanestore.NewLivePlaybooks(b.controlPlaneRPC, b.log)
b.playbooks.ApplyStatus = b.applyStatus
return b.playbooks
}
// playbookRefreshInterval is how often archied re-reads the resources it
// would otherwise load only on first use, so their apply status stays reported.
const playbookRefreshInterval = 30 * time.Second
// stateStoreCA is the CA a task container verifies the State Store with;
// nil without [tls].
func (b *boot) stateStoreCA(ctx context.Context) []byte {
id, err := servicekit.Identity(b.cfg.TLS, presence.Daemon)
if err != nil {
b.log.WarnContext(ctx, "state store CA for containers", "err", err)
return nil
}
return id.CAPEM()
}