904 lines
33 KiB
Go
904 lines
33 KiB
Go
// bootstrap.go owns the composition of the daemon: run() is the entry
|
|
// point and driver, and each phase of wiring lives in a method on boot.
|
|
// The struct exists because the composition root assembles ~30 pieces of
|
|
// state that every phase touches; passing them as parameters would make
|
|
// each phase function longer than the code it contains.
|
|
package archied
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net"
|
|
"net/url"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
|
"github.com/samcharles93/archie-core/internal/config"
|
|
"github.com/samcharles93/archie-core/internal/container"
|
|
controlpb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
|
|
"github.com/samcharles93/archie-core/internal/daemon"
|
|
"github.com/samcharles93/archie-core/internal/domain/access"
|
|
"github.com/samcharles93/archie-core/internal/domain/agent"
|
|
"github.com/samcharles93/archie-core/internal/domain/alert"
|
|
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
|
"github.com/samcharles93/archie-core/internal/domain/health"
|
|
"github.com/samcharles93/archie-core/internal/domain/presence"
|
|
"github.com/samcharles93/archie-core/internal/domain/scheduling"
|
|
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
|
"github.com/samcharles93/archie-core/internal/domain/workintake"
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
"github.com/samcharles93/archie-core/internal/forge"
|
|
infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/controlplanestore"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/extension"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/issuerpc"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/logrpc"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/messagingrpc"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/modelcatalog"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/taskactions"
|
|
"github.com/samcharles93/archie-core/internal/logging"
|
|
"github.com/samcharles93/archie-core/internal/secret"
|
|
"github.com/samcharles93/archie-core/internal/storage"
|
|
"github.com/samcharles93/archie-core/internal/webui"
|
|
"github.com/samcharles93/archie-core/internal/worktree"
|
|
)
|
|
|
|
// boot is the mutable wiring state assembled by the run() composition
|
|
// phases. Cleanup funcs are registered in creation order and run in
|
|
// reverse at shutdown, matching the LIFO ordering of the deferred calls
|
|
// they replace.
|
|
type boot struct {
|
|
catalog *modelcatalog.Catalog
|
|
cfg config.Config
|
|
log *slog.Logger
|
|
playbooks *controlplanestore.LivePlaybooks
|
|
|
|
// stderrLog keeps setupLogging on stderr, never touching cfg.Log.File. An
|
|
// offline command reads the bootstrap config but is not the daemon: creating
|
|
// the deployment's log, appending a line stamped component="daemon", or
|
|
// rotating a log the daemon is writing are side effects a diagnosis must not
|
|
// have, and it needs no durable copy of its own.
|
|
stderrLog bool
|
|
|
|
loader *configuration.Loader
|
|
doc *configuration.Document
|
|
currentProvenance atomic.Pointer[configuration.Provenance]
|
|
|
|
health *healthSurface
|
|
logFeed *logging.Feed
|
|
taskLogs *logging.TaskRegistry
|
|
|
|
secrets *secret.Registry
|
|
forgeClient forge.Forge
|
|
forgeExt *forgeExtensions
|
|
token string
|
|
|
|
// stateStore is the State Store contract adapter every daemon and gateway
|
|
// store consumer depends on. It is ALWAYS the remote *staterpc.Client
|
|
// dialed to the standalone archie-state-store gRPC service at
|
|
// [services.state].target. It is set by openStateStoreAdapter, which
|
|
// requires [services.state].target to be set.
|
|
stateStore storecontract.TaskStore
|
|
// stateClient is stateStore's concrete type, which the daemon's typed store
|
|
// fields are assigned from directly.
|
|
stateClient *staterpc.Client
|
|
taskModels daemon.TaskModels
|
|
taskInstructions daemon.TaskInstructions
|
|
// accessChain is the daemon's policy engine, built once from the stored
|
|
// policies (openAccessChain); accessProblems is what the readiness
|
|
// surface reports. Both are nil when no policy store is wired.
|
|
accessChain access.Authorizer
|
|
accessLive *infraaccess.Live
|
|
// stateStoreGrants issues each task's run credential
|
|
// (daemon.RunCredentialIssuer), wrapping the same *staterpc.Client as
|
|
// stateStore, which also verifies a credential for a worktree push.
|
|
stateStoreGrants *staterpc.GrantIssuer
|
|
controlPlane *controlplanestore.Client
|
|
// controlPlaneRPC is the State Store's control-plane transport, kept so the
|
|
// daemon root can build its workflow-definitions client from it
|
|
// (openDaemonWorkflowDefinitions). The gateway root never builds that client:
|
|
// it resolves no workflow step type.
|
|
controlPlaneRPC controlpb.ControlPlaneServiceClient
|
|
// workflowDefinitions is the one control-plane surface that resolves
|
|
// workflow step types. It is separate from controlPlane because it is the
|
|
// only surface that needs the process's step vocabulary.
|
|
workflowDefinitions *controlplanestore.WorkflowDefinitionsClient
|
|
// executionSettings is the last workflow execution settings the control
|
|
// plane published. They arrive on a watch rather than in the file
|
|
// document, so reloadConfig re-applies them from here.
|
|
executionSettings atomic.Pointer[workflow.ExecutionSettings]
|
|
// applyStatus reports which control-plane resource version this process
|
|
// is running. Nil until the State Store adapter is open, and nil-safe,
|
|
// so apply points call it without a guard.
|
|
applyStatus *applystatus.Reporter
|
|
// processName is which binary this composition is running as, and is the
|
|
// name apply-status records carry. Set by each entry point, because one
|
|
// boot builds both archied and archie-gateway.
|
|
processName string
|
|
|
|
bus *events.Bus
|
|
// cfgHolder is the daemon's one configuration Holder. Boot owns it and
|
|
// the daemon reads through it, so a reload swaps one snapshot and every
|
|
// reader sees it: there is no second holder to keep in step.
|
|
cfgHolder *config.Holder
|
|
// chat is the dashboard-shaped chat surface. It survives the UI
|
|
// extraction because the readiness gateway probe and the Telegram task
|
|
// actor both read the same contract.
|
|
chat *webui.ChatService
|
|
// healthRegistry backs /health/detailed on the daemon's own listener.
|
|
healthRegistry *health.Registry
|
|
// lastReload reports the most recent reload outcome for the published
|
|
// configuration projection. Nil until reload wiring installs it.
|
|
lastReload func() config.ReloadStatus
|
|
|
|
natsClient *nats.Client
|
|
// reactionClient pulls the ARCHIE_REACTIONS fan-out stream; the daemon
|
|
// drains it each cycle to turn review reactions into remediate runs.
|
|
reactionClient *nats.Client
|
|
// natsURL is the endpoint the daemon's own client connected with at
|
|
// startup. For external mode it is cfg.NATS.URL; for embedded mode it is
|
|
// the embedded server's ClientURL(). Recorded so Daemon.ConnectedNATS
|
|
// carries the real endpoint rather than an empty config URL.
|
|
natsURL string
|
|
|
|
containerPool *container.Pool
|
|
// taskRuntime is what task agents run in: the container pool, or an
|
|
// OpenShell gateway's sandboxes. Nil parks every task.
|
|
taskRuntime daemon.TaskRuntime
|
|
// sandboxes provisions each task container's isolated network and runs
|
|
// the egress relay that is its only way out.
|
|
sandboxes *egress.Networks
|
|
// modelEgress gives native tasks model access through the egress proxy;
|
|
// nil when the proxy cannot start.
|
|
modelEgress daemon.ModelEgress
|
|
// egressProxyAddr is where the egress proxy listens, the address the relay
|
|
// forwards its proxy port to. Empty when the proxy is not running.
|
|
egressProxyAddr string
|
|
storeBackend storage.Backend
|
|
|
|
// runtimeVersions records the version of every control-plane kind boot's
|
|
// layering applied. Set once at boot before the runtime-resource watches
|
|
// start, which read it as their resume points.
|
|
runtimeVersions map[string]int64
|
|
|
|
// messaging delivers outbound chat messages through the Messaging Service.
|
|
messaging *messagingrpc.Client
|
|
|
|
startGateways []func()
|
|
trees *worktree.Manager
|
|
d *daemon.Daemon
|
|
// schedulingEngine is the cron/scheduling ticker engine (setupScheduling).
|
|
schedulingEngine *scheduling.Engine
|
|
|
|
// agentStatus records the most recent version/install-type an
|
|
// archie-agent worker reported about itself. Allocated up front, before
|
|
// any gateway goroutine (Telegram, webui) that might read it via
|
|
// RunningVersions starts, so those closures never see a nil pointer or
|
|
// race its assignment -- only its own internal mutex guards concurrent
|
|
// Observe/Snapshot calls once tasks start completing.
|
|
agentStatus *daemon.AgentStatus
|
|
|
|
// shutdown cancels the daemon's root context, the exact path a SIGTERM
|
|
// takes to graceful shutdown. The drain monitor invokes it when it honours
|
|
// an external drain request, so a marker-driven drain reuses the operator
|
|
// signal path rather than inventing a new one.
|
|
shutdown context.CancelFunc
|
|
|
|
// alertChecks re-evaluates the configuration alerts when the configuration
|
|
// changes. Nil until the daemon's run loop starts them.
|
|
alertChecks *alertChecks
|
|
// credentialWatch reports bound credentials that stop resolving. Nil
|
|
// when the State Store has no alert surface.
|
|
credentialWatch *credentialWatch
|
|
|
|
cleanups []func()
|
|
}
|
|
|
|
func newBootstrap() *boot {
|
|
return &boot{log: slog.New(slog.NewJSONHandler(os.Stderr, nil)), agentStatus: &daemon.AgentStatus{}}
|
|
}
|
|
|
|
func (b *boot) cleanup() {
|
|
for _, v := range slices.Backward(b.cleanups) {
|
|
v()
|
|
}
|
|
}
|
|
|
|
func (b *boot) loadConfig(ctx context.Context, cfgPath, overlayPath string) error {
|
|
loader, doc, err := servicekit.Resolve(b.log, cfgPath, overlayPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b.loader, b.doc = loader, doc
|
|
b.catalog = modelcatalog.NewCatalog(cfgPath)
|
|
b.cfg = b.doc.Config
|
|
b.cfgHolder = config.NewHolder(b.cfg)
|
|
b.currentProvenance.Store(&b.doc.Provenance)
|
|
return b.setupLogging(ctx)
|
|
}
|
|
|
|
// setupLogging stays on stderr for an offline command (stderrLog): creating,
|
|
// appending to or rotating the deployment's log is a side effect a diagnosis
|
|
// must not have.
|
|
func (b *boot) setupLogging(ctx context.Context) error {
|
|
if b.stderrLog {
|
|
return nil
|
|
}
|
|
logs, err := servicekit.Logging(ctx, b.cfg, "daemon")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b.log, b.logFeed, b.taskLogs = logs.Log, logs.Feed, logs.TaskLogs
|
|
b.addCleanup(func() { _ = logs.Closer.Close() })
|
|
return nil
|
|
}
|
|
|
|
// openStores wires the secret registry and the conversation store. The
|
|
// registry starts with the env engine only; extension engines join it once the
|
|
// State Store client exists (openStateStoreAdapter), so nothing here resolves a
|
|
// reference that may name one.
|
|
func (b *boot) openStores(ctx context.Context) error {
|
|
b.secrets = secret.NewRegistry()
|
|
return nil
|
|
}
|
|
|
|
// openForge builds the forge client. It runs after the runtime settings are
|
|
// layered, so a token held by an extension engine can resolve.
|
|
func (b *boot) openForge(ctx context.Context) {
|
|
if packages, ok := b.stateStore.(extension.Packages); ok {
|
|
b.forgeExt = newForgeExtensions(extension.Source{Query: b.controlPlane, Packages: packages}, b.log)
|
|
b.addCleanup(b.forgeExt.close)
|
|
}
|
|
b.forgeClient, b.token = resolveForge(ctx, b.cfg.Forge, "default", b.secrets, b.forgeExt, b.log)
|
|
}
|
|
|
|
// reopenForge re-resolves the instance's forge connection from the settings
|
|
// the layering just published. Every holder of the client is replaced, so a
|
|
// rotated credential, a changed host or a removed connection reaches the next
|
|
// operation rather than dying with the process that booted. It runs from the
|
|
// connections watch alone, after the config holder carries the change; the
|
|
// worktree manager is shared by pointer, so its token is the only field to
|
|
// replace.
|
|
func (b *boot) reopenForge(ctx context.Context) {
|
|
client, token := resolveForge(ctx, b.cfgHolder.Get().Forge, "default", b.secrets, b.forgeExt, b.log)
|
|
b.forgeClient, b.token = client, token
|
|
if b.trees != nil {
|
|
b.trees.Token = token
|
|
}
|
|
if b.d != nil {
|
|
b.d.Forge = client
|
|
}
|
|
}
|
|
|
|
func (b *boot) openStateStoreAdapter(ctx context.Context) error {
|
|
target := strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target)
|
|
if target == "" {
|
|
return fmt.Errorf("services.state.target is required: archied/archie-gateway no longer own archie.db; the standalone archie-state-store process owns it (docs/prds/state-store-contract.md §12 step 7)")
|
|
}
|
|
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, b.cfg, staterpc.WithCallDeadline(stateCallDeadline))
|
|
if err != nil {
|
|
b.log.ErrorContext(ctx, "state store adapter", "err", err)
|
|
return err
|
|
}
|
|
b.stateStore = client
|
|
b.stateClient = client
|
|
if alerts, ok := b.stateStore.(alert.Store); ok {
|
|
b.credentialWatch = &credentialWatch{reporter: alert.Reporter{Raiser: alerts, Log: b.log, By: "archied"}}
|
|
}
|
|
b.taskModels = client
|
|
b.taskInstructions = client
|
|
b.controlPlaneRPC = client.ControlPlane()
|
|
b.controlPlane = controlplanestore.NewRPCClient(b.controlPlaneRPC)
|
|
b.applyStatus = applystatus.New(b.processName, client, b.log)
|
|
b.stateStoreGrants = &staterpc.GrantIssuer{Client: client}
|
|
b.addCleanup(cleanup)
|
|
b.startSecretEngines(ctx, client)
|
|
return nil
|
|
}
|
|
|
|
func (b *boot) openStateStage(ctx context.Context) error {
|
|
if err := b.openStores(ctx); err != nil {
|
|
return err
|
|
}
|
|
if err := b.openDaemonStateSurfaces(ctx); err != nil {
|
|
return err
|
|
}
|
|
return b.openAccessChain(ctx)
|
|
}
|
|
|
|
func (b *boot) openAccessChain(ctx context.Context) error {
|
|
source, ok := b.stateStore.(access.PolicyStore)
|
|
if !ok {
|
|
return fmt.Errorf("state store policy reader unavailable")
|
|
}
|
|
live := infraaccess.NewPending(source.PoliciesVersion, source.ListPolicies, b.log)
|
|
if err := live.Reload(ctx); err != nil {
|
|
b.log.WarnContext(ctx, "workflow policies unavailable; execution remains denied", "err", err)
|
|
}
|
|
b.accessChain, b.accessLive = live, live
|
|
return nil
|
|
}
|
|
|
|
// handleRequeue replays a parked/waiting task by id, then signals an
|
|
// early success exit unless -once keeps the daemon running.
|
|
func (b *boot) handleRequeue(ctx context.Context, requeue int64, once bool) (bool, error) {
|
|
if requeue <= 0 {
|
|
return false, nil
|
|
}
|
|
if err := manualRequeueTask(ctx, b.stateStore, requeue); err != nil {
|
|
b.log.ErrorContext(ctx, "requeue failed", "task", requeue, "err", err)
|
|
return false, err
|
|
}
|
|
b.log.InfoContext(ctx, "task requeued", "task", requeue)
|
|
return !once, nil
|
|
}
|
|
|
|
func (b *boot) seedSoul(ctx context.Context, cfgPath string) {
|
|
result, err := configuration.SeedSoul(cfgPath, agent.ShippedSoul())
|
|
if err != nil {
|
|
b.log.WarnContext(ctx, "soul: starter file unavailable", "err", err)
|
|
return
|
|
}
|
|
if result.Action == configuration.SoulCreated {
|
|
b.log.InfoContext(ctx, "soul: starter file written", "path", result.Path, "action", result.Action)
|
|
}
|
|
}
|
|
|
|
func (b *boot) setupObservability(ctx context.Context) {
|
|
cfg, log := b.cfg, b.log
|
|
bus := events.NewBus()
|
|
b.bus = bus
|
|
b.addCleanup(func() { bus.Close() })
|
|
b.startUpdateRelay(ctx, updateReportPath(cfg.WorkDir, "webui"))
|
|
sink := bus.Subscribe(256)
|
|
go persistEvents(ctx, sink, b.stateStore, log)
|
|
}
|
|
|
|
const reactionStreamMaxAge = 24 * time.Hour
|
|
|
|
func (b *boot) connectNATS(ctx context.Context) error { // embedded broker discovery and startup require explicit fallback branches
|
|
cfg, log := b.cfg, b.log
|
|
url := cfg.NATS.URL
|
|
var natsToken string
|
|
if cfg.NATS.Mode == config.NATSModeExternal { //nolint:nestif // embedded broker discovery and startup require explicit fallback branches
|
|
if url == "" {
|
|
return fmt.Errorf("nats.url is required when nats.mode is external")
|
|
}
|
|
var err error
|
|
natsToken, err = servicekit.NATSToken(cfg.NATS, os.Getenv)
|
|
if err != nil {
|
|
log.ErrorContext(ctx, "nats credentials", "err", err)
|
|
return err
|
|
}
|
|
if err := b.startCallout(ctx, url, natsToken); err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
endpoint, readErr := servicekit.ReadNATSEndpoint(cfg.StateDir)
|
|
if readErr == nil {
|
|
url, natsToken = endpoint.URL, endpoint.Token
|
|
if probe, err := nats.Connect(ctx, nats.Config{URL: url, Token: natsToken, Subjects: []string{workintake.SubjectTaskWildcard}, FilterSubject: workintake.SubjectTaskWildcard}, log); err == nil {
|
|
probe.Close()
|
|
url, natsToken = endpoint.URL, endpoint.Token
|
|
} else {
|
|
url, natsToken, err = b.startEmbeddedNATS(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
} else {
|
|
var err error
|
|
url, natsToken, err = b.startEmbeddedNATS(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
natsClient, err := nats.Connect(ctx, nats.Config{
|
|
URL: url,
|
|
Token: natsToken,
|
|
Subjects: []string{workintake.SubjectTaskWildcard},
|
|
FilterSubject: workintake.SubjectTaskWildcard,
|
|
}, log)
|
|
if err != nil {
|
|
log.ErrorContext(ctx, "nats connect failed", "err", err)
|
|
return err
|
|
}
|
|
|
|
reactionMaxAge := reactionStreamMaxAge
|
|
reactionClient, err := nats.Connect(ctx, nats.Config{
|
|
URL: url,
|
|
Token: natsToken,
|
|
StreamName: nats.DefaultReactionStreamName,
|
|
Subjects: []string{workintake.SubjectReactionWildcard},
|
|
FilterSubject: workintake.SubjectReactionWildcard,
|
|
Retention: nats.FanOutRetention(),
|
|
MaxAge: &reactionMaxAge,
|
|
}, log)
|
|
if err != nil {
|
|
log.ErrorContext(ctx, "nats reaction stream connect failed", "err", err)
|
|
natsClient.Close()
|
|
return err
|
|
}
|
|
|
|
if err := b.startSpanRelay(ctx, url, natsToken); err != nil {
|
|
log.ErrorContext(ctx, "span relay failed", "err", err)
|
|
natsClient.Close()
|
|
reactionClient.Close()
|
|
return err
|
|
}
|
|
b.natsClient = natsClient
|
|
b.natsURL = url
|
|
b.addCleanup(func() { natsClient.Close() })
|
|
b.addCleanup(func() { reactionClient.Close() })
|
|
b.reactionClient = reactionClient
|
|
log.InfoContext(ctx, "nats connected", "url", url, "task_stream", nats.DefaultStreamName, "reaction_stream", nats.DefaultReactionStreamName)
|
|
return nil
|
|
}
|
|
|
|
// startCallout makes the external broker defer every login to archie, so a task
|
|
// container logs in with its run credential and reaches only its own subjects.
|
|
// It runs before any other login, because those are decided here.
|
|
func (b *boot) startCallout(ctx context.Context, url, token string) error {
|
|
seed := os.Getenv(b.cfg.NATS.CalloutIssuerSeedEnv)
|
|
if seed == "" {
|
|
return fmt.Errorf("%s is required when nats.callout_issuer_seed_env is configured", b.cfg.NATS.CalloutIssuerSeedEnv)
|
|
}
|
|
opts := nats.CalloutOptions{URL: url, Token: token, IssuerSeed: seed}
|
|
if b.stateStoreGrants != nil {
|
|
opts.Tasks = taskBroker{runs: b.stateStoreGrants.Client}
|
|
}
|
|
svc, err := nats.StartCallout(ctx, opts, b.log)
|
|
if err != nil {
|
|
b.log.ErrorContext(ctx, "nats callout start failed", "err", err)
|
|
return err
|
|
}
|
|
b.addCleanup(svc.Close)
|
|
return nil
|
|
}
|
|
|
|
// natsHost is the address the embedded broker binds. OpenShell sandboxes run
|
|
// on the host network and reach the broker by the services host name, so it
|
|
// binds the address that name resolves to; Docker containers reach it on the
|
|
// bridge gateway.
|
|
func natsHost(ctx context.Context, cfg config.Config, pool *container.Pool) (string, error) {
|
|
shell := cfg.Containers.OpenShell
|
|
if cfg.Containers.Runtime == config.RuntimeOpenShell && shell.ServicesHost != "" {
|
|
addrs, err := net.DefaultResolver.LookupHost(ctx, shell.ServicesHost)
|
|
if err != nil || len(addrs) == 0 {
|
|
return "", fmt.Errorf("embedded nats: resolve services host %s: %w", shell.ServicesHost, err)
|
|
}
|
|
for _, a := range addrs {
|
|
if ip := net.ParseIP(a); ip != nil && ip.To4() != nil {
|
|
return a, nil
|
|
}
|
|
}
|
|
return addrs[0], nil
|
|
}
|
|
if pool != nil {
|
|
return pool.HostGateway(), nil
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) {
|
|
cfg, log := b.cfg, b.log
|
|
if cfg.StateDir == "" {
|
|
return "", "", errors.New("embedded nats: state_dir is required")
|
|
}
|
|
host, err := natsHost(ctx, cfg, b.containerPool)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
// A restarted broker keeps the previous endpoint, so the Gateway and
|
|
// running agents reconnect to it instead of a broker that no longer exists.
|
|
opts := nats.EmbeddedOptions{Host: host, StoreDir: filepath.Join(cfg.StateDir, "nats")}
|
|
if b.stateStoreGrants != nil {
|
|
opts.Tasks = taskBroker{runs: b.stateStoreGrants.Client}
|
|
}
|
|
if previous, err := servicekit.ReadNATSEndpoint(cfg.StateDir); err == nil {
|
|
opts.Token = previous.Token
|
|
if u, err := url.Parse(previous.URL); err == nil {
|
|
opts.Port, _ = strconv.Atoi(u.Port())
|
|
}
|
|
}
|
|
srv, err := nats.StartEmbedded(ctx, opts, log)
|
|
if err != nil && opts.Port != 0 {
|
|
log.WarnContext(ctx, "embedded nats: previous port unavailable; taking a new one", "port", opts.Port, "err", err)
|
|
opts.Port = 0
|
|
srv, err = nats.StartEmbedded(ctx, opts, log)
|
|
}
|
|
if err != nil {
|
|
log.ErrorContext(ctx, "embedded nats start failed", "err", err)
|
|
return "", "", err
|
|
}
|
|
b.addCleanup(func() { srv.Shutdown() })
|
|
log.InfoContext(ctx, "embedded nats started", "url", srv.ClientURL())
|
|
if err := servicekit.WriteNATSEndpoint(cfg.StateDir, srv.ClientURL(), srv.Token()); err != nil {
|
|
srv.Shutdown()
|
|
return "", "", err
|
|
}
|
|
return srv.ClientURL(), srv.Token(), nil
|
|
}
|
|
|
|
func (b *boot) setupContainers(ctx context.Context) func() {
|
|
containerPool, sandboxes, storeBackend, closeDocker := startContainers(ctx, b.cfgHolder, b.secrets, b.log)
|
|
b.containerPool = containerPool
|
|
b.sandboxes = sandboxes
|
|
b.storeBackend = storeBackend
|
|
b.taskRuntime = daemon.DockerRuntime(containerPool)
|
|
if b.cfg.Containers.Runtime != config.RuntimeOpenShell {
|
|
return closeDocker
|
|
}
|
|
rt, client, closeOpenShell, err := startOpenShell(ctx, b.cfgHolder, b.secrets, b.log)
|
|
if err != nil {
|
|
b.log.ErrorContext(ctx, "autonomous workflows unavailable: openshell runtime", "err", err)
|
|
b.taskRuntime = nil
|
|
return closeDocker
|
|
}
|
|
b.taskRuntime = daemon.OpenShellRuntime(rt)
|
|
settings := b.cfg.Containers.OpenShell
|
|
bindings := runCredentialResolver{runs: b.stateStoreGrants.Client, models: b.taskModels, config: b.cfgHolder, secrets: b.secrets, credentials: b.credentialWatch}
|
|
egress := newOpenShellModelEgress(client, settings.Workspace, b.cfg.Instance, b.secrets, bindings, b.log)
|
|
b.modelEgress = egress
|
|
sweepCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
|
defer cancel()
|
|
if err := egress.Sweep(sweepCtx); err != nil {
|
|
b.log.WarnContext(ctx, "openshell: orphaned gateway providers remain", "err", err)
|
|
}
|
|
return func() { closeOpenShell(); closeDocker() }
|
|
}
|
|
|
|
// setupGatewayClient dials the Gateway, which owns every model call.
|
|
func (b *boot) setupGatewayClient(ctx context.Context) error {
|
|
contract, cleanup, err := composeChatContract(b.cfg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b.addCleanup(cleanup)
|
|
b.chat = &webui.ChatService{Contract: contract}
|
|
messaging, closeMessaging, err := composeMessaging(b.cfg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b.addCleanup(closeMessaging)
|
|
b.messaging = messaging
|
|
posted := func(taskID int64) {
|
|
b.bus.Publish(ctx, events.Event{At: time.Now().UTC(), Kind: events.KindChatOutcome, TaskID: taskID})
|
|
}
|
|
go postOutcomes(ctx, b.stateStore, deliverOutcome(contract, messaging), posted, b.log)
|
|
b.setupReadinessProbes()
|
|
return nil
|
|
}
|
|
|
|
func (b *boot) buildWorktreeManager() {
|
|
cfg := b.cfg
|
|
b.trees = &worktree.Manager{
|
|
WorkDir: cfg.WorkDir,
|
|
Token: b.token,
|
|
BotUser: cfg.BotUser,
|
|
BotEmail: cfg.BotEmail,
|
|
BaseURL: cfg.Forge.Host,
|
|
}
|
|
}
|
|
|
|
func (b *boot) registerNATSRPC(ctx context.Context) error {
|
|
log := b.log
|
|
if b.natsClient == nil {
|
|
return nil
|
|
}
|
|
coreConn, err := b.natsClient.CoreConn()
|
|
if err != nil {
|
|
log.ErrorContext(ctx, "nats connection unavailable for task RPC", "err", err)
|
|
return err
|
|
}
|
|
unsubscribe, err := registerTaskRPCServers(ctx, coreConn, func(ctx context.Context, t *workflow.Task, op string) (daemon.RepositoryAccess, error) {
|
|
if b.d == nil {
|
|
return daemon.RepositoryAccess{}, errors.New("repository connections starting")
|
|
}
|
|
return b.d.Repository(ctx, t, op)
|
|
}, func(ctx context.Context, org, channel, chatID, text string) error {
|
|
if b.messaging == nil {
|
|
return errors.New("messaging service not configured")
|
|
}
|
|
return b.messaging.Deliver(ctx, org, channel, chatID, text)
|
|
}, b.stateStoreGrants.Client, installedAuthorities{b.stateStoreGrants.Client}, log)
|
|
if err != nil {
|
|
log.ErrorContext(ctx, "task RPC server registration failed", "err", err)
|
|
return err
|
|
}
|
|
b.addCleanup(unsubscribe)
|
|
|
|
stopLogs, err := logrpc.RegisterDaemon(ctx, coreConn, b.logFeed, log)
|
|
if err != nil {
|
|
return fmt.Errorf("register daemon logs: %w", err)
|
|
}
|
|
b.addCleanup(stopLogs)
|
|
unsubscribeSystemLogs, err := subscribeSystemLogs(ctx, coreConn, b.taskLogs, log)
|
|
if err != nil {
|
|
log.ErrorContext(ctx, "system log subscribe failed", "err", err)
|
|
return err
|
|
}
|
|
b.addCleanup(unsubscribeSystemLogs)
|
|
|
|
unsubscribeAgentEvents, err := subscribeAgentEvents(ctx, coreConn, b.bus, log)
|
|
if err != nil {
|
|
log.ErrorContext(ctx, "agent event subscribe failed", "err", err)
|
|
return err
|
|
}
|
|
b.addCleanup(unsubscribeAgentEvents)
|
|
|
|
unsubscribeTaskActions, err := taskactions.Register(ctx, coreConn, b.taskActions(), log)
|
|
if err != nil {
|
|
log.ErrorContext(ctx, "gateway task action register failed", "err", err)
|
|
return err
|
|
}
|
|
b.addCleanup(unsubscribeTaskActions)
|
|
|
|
unsubscribeIssues, err := (&issuerpc.Server{Forge: b.issueForge, Log: log}).Register(ctx, coreConn)
|
|
if err != nil {
|
|
log.ErrorContext(ctx, "issue RPC register failed", "err", err)
|
|
return err
|
|
}
|
|
b.addCleanup(unsubscribeIssues)
|
|
return nil
|
|
}
|
|
|
|
// issueForge is the forge of owner/repo's own connection.
|
|
func (b *boot) issueForge(ctx context.Context, owner, repo string) (forge.IssueForge, error) {
|
|
return b.repositoryForge(ctx, owner, repo)
|
|
}
|
|
|
|
func (b *boot) buildDaemon(ctx context.Context) {
|
|
log := b.log
|
|
workspaces, _ := b.stateStore.(access.WorkspaceLister)
|
|
d := &daemon.Daemon{
|
|
ResolveRepository: b.repositoryAccess,
|
|
Cfg: b.cfgHolder,
|
|
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL},
|
|
ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), CA: b.stateStoreCA(ctx)},
|
|
Store: b.stateStore,
|
|
Identity: daemon.NewIdentityGate(b.stateClient, b.stateClient, b.accessChain, b.stateClient, b.stateClient, b.orgPrincipals, workspaces, log),
|
|
PlaybookLedger: b.stateClient,
|
|
Bus: b.bus,
|
|
Forge: b.forgeClient,
|
|
Trees: b.trees,
|
|
PackageAuthorities: installedAuthorities{b.stateStoreGrants.Client},
|
|
Storage: b.storeBackend,
|
|
Log: log,
|
|
TaskTransport: b.natsClient,
|
|
RunCredentials: b.stateStoreGrants,
|
|
Runtime: b.taskRuntime,
|
|
ModelEgress: b.modelEgress,
|
|
TaskModels: b.taskModels,
|
|
TaskInstructions: b.taskInstructions,
|
|
TaskLogs: b.taskLogs,
|
|
AgentStatus: b.agentStatus,
|
|
WorkflowDefinitions: b.workflowDefinitions,
|
|
Playbooks: b.livePlaybooks(),
|
|
}
|
|
|
|
d.ReviewScanner = daemon.NewReviewScanner(b.stateStore, b.reactionClient, b.natsClient.PublishUnique, b.bus, d, d.Identity, d, b.repositoryForge, log)
|
|
d.BindingDispatch = daemon.NewBindingDispatch(daemon.BindingDispatchDeps{
|
|
Store: b.stateStore,
|
|
Mappings: b.stateClient,
|
|
Bindings: b.stateClient,
|
|
Dispatcher: b.stateClient,
|
|
Matches: b.stateClient,
|
|
Creator: b.stateClient,
|
|
Enablement: b.controlPlane,
|
|
Definitions: b.workflowDefinitions,
|
|
Access: d.Identity,
|
|
Denials: d.Identity,
|
|
Cfg: b.cfgHolder,
|
|
Orgs: d.Identity,
|
|
OrgPrincipals: b.orgPrincipals,
|
|
Log: log,
|
|
})
|
|
d.WireTaskBus()
|
|
d.IssuePoller = daemon.NewIssuePoller(b.cfgHolder, b.stateStore, log, b.issueSources, b.taskActions().Withdraw, d.TaskBus.PublishTask, d.Identity, d.BindingDispatch)
|
|
b.d = d
|
|
}
|
|
|
|
func (b *boot) publishConfig(ctx context.Context, cfg config.Config) {
|
|
b.cfgHolder.Set(cfg)
|
|
// The daemon owns the published configuration view.
|
|
if b.processName != applystatus.Gateway {
|
|
b.publishConfigSnapshot(ctx)
|
|
b.recheckAlerts(ctx)
|
|
}
|
|
}
|
|
|
|
func (b *boot) configViewInput() webui.ConfigViewInput {
|
|
catalog := b.catalogState()
|
|
in := webui.ConfigViewInput{
|
|
Config: b.cfgHolder.Get(),
|
|
Catalog: catalogView(catalog),
|
|
SecretEngines: b.secrets.List(),
|
|
}
|
|
if b.lastReload != nil {
|
|
status := b.lastReload()
|
|
in.Reload = &status
|
|
}
|
|
return in
|
|
}
|
|
|
|
func (b *boot) publishConfigSnapshot(ctx context.Context) {
|
|
snapshots, ok := b.stateStore.(storecontract.ConfigSnapshotStore)
|
|
if !ok || b.cfgHolder == nil {
|
|
return
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
|
defer cancel()
|
|
|
|
document, err := json.Marshal(webui.BuildConfigView(b.configViewInput()))
|
|
if err != nil {
|
|
b.log.WarnContext(ctx, "config snapshot not published", "err", err)
|
|
return
|
|
}
|
|
err = snapshots.PutConfigSnapshot(ctx, storecontract.ConfigSnapshot{
|
|
Schema: webui.ConfigViewSchema,
|
|
Document: document,
|
|
PublishedAt: time.Now().UTC(),
|
|
})
|
|
if err != nil {
|
|
b.log.WarnContext(ctx, "config snapshot not published", "err", err)
|
|
}
|
|
}
|
|
|
|
func (b *boot) wireConfigPublishing(ctx context.Context, cfgPath, overlayPath string) {
|
|
log := b.log
|
|
reloadController := newReloadController(b.loader, cfgPath, overlayPath, b.reloadConfig)
|
|
b.lastReload = func() config.ReloadStatus {
|
|
st := reloadController.Status()
|
|
return st
|
|
}
|
|
|
|
reloadCh := make(chan os.Signal, 1)
|
|
signal.Notify(reloadCh, syscall.SIGHUP)
|
|
b.addCleanup(func() { signal.Stop(reloadCh) })
|
|
go reloadLoop(ctx, reloadCh, reloadController, log)
|
|
}
|
|
|
|
func (b *boot) startServices(ctx context.Context) error {
|
|
log := b.log
|
|
if err := b.d.Startup(ctx); err != nil {
|
|
log.ErrorContext(ctx, "startup", "err", err)
|
|
return err
|
|
}
|
|
if b.schedulingEngine != nil {
|
|
if err := b.schedulingEngine.Start(ctx); err != nil {
|
|
log.ErrorContext(ctx, "scheduling engine startup", "err", err)
|
|
}
|
|
}
|
|
for _, startGateway := range b.startGateways {
|
|
startGateway()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *boot) setupBackends(ctx context.Context) error {
|
|
closeContainers := b.setupContainers(ctx)
|
|
b.addCleanup(closeContainers)
|
|
err := b.connectNATS(ctx)
|
|
if err == nil && b.cfg.Containers.Runtime != config.RuntimeOpenShell {
|
|
b.setupModelEgress(ctx)
|
|
b.startEgressRelay(ctx)
|
|
}
|
|
return err
|
|
}
|
|
|
|
func exitCode(err error) int {
|
|
if err != nil {
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func (b *boot) runLoop(ctx context.Context, once bool) error {
|
|
b.health.markServing()
|
|
b.announceReady(ctx)
|
|
if once {
|
|
b.d.RunOnce(ctx)
|
|
return nil
|
|
}
|
|
b.log.InfoContext(ctx, "archied running", "repos", len(b.cfg.Repos), "poll", b.cfg.PollInterval.Std().String(), "label", b.cfg.Label)
|
|
b.startDrainMonitor(ctx)
|
|
b.startWatchdog(ctx)
|
|
b.startAlerts(ctx)
|
|
if err := b.d.Run(ctx); err != nil && ctx.Err() == nil {
|
|
b.log.ErrorContext(ctx, "daemon exited", "err", err)
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *boot) addCleanup(fn func()) {
|
|
b.cleanups = append(b.cleanups, fn)
|
|
}
|
|
|
|
func (b *boot) catalogState() modelcatalog.Snapshot { return b.catalog.State() }
|
|
|
|
// loadCatalog layers the model catalog under the file config. A catalog that
|
|
// cannot be read leaves the configured providers and models in effect.
|
|
func (b *boot) loadCatalog(ctx context.Context) {
|
|
snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfgHolder.Get().Providers)
|
|
if err != nil {
|
|
b.log.WarnContext(ctx, "model catalog unavailable; using configured providers and models", "err", err)
|
|
return
|
|
}
|
|
models := modelcatalog.Apply(&b.cfg, snapshot)
|
|
// Layer the catalog over the live config, not b.cfg: b.cfg predates the
|
|
// execution-settings watch, so publishing it would put the file's budgets
|
|
// back over the control plane's.
|
|
live := b.cfgHolder.Get().Clone()
|
|
modelcatalog.Apply(&live, snapshot)
|
|
b.cfgHolder.Set(live)
|
|
b.catalog.Set(snapshot)
|
|
b.log.InfoContext(ctx, "model catalog loaded", "providers", len(snapshot.Providers), "models", len(models))
|
|
}
|
|
|
|
func (b *boot) refreshModelCatalog(ctx context.Context) error {
|
|
snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfgHolder.Get().Providers)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
base := b.cfgHolder.Get().Clone()
|
|
models := modelcatalog.Apply(&base, snapshot)
|
|
cfg, _, err := b.runtimeConfig(ctx, base)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b.catalog.Set(snapshot)
|
|
b.publishConfig(ctx, cfg)
|
|
b.log.InfoContext(ctx, "model catalog refreshed", "providers", len(snapshot.Providers), "models", len(models))
|
|
return nil
|
|
}
|
|
|
|
func (b *boot) startModelCatalogRefresh(ctx context.Context) {
|
|
modelcatalog.Every(ctx, modelcatalog.RefreshInterval, func() {
|
|
if err := b.refreshModelCatalog(ctx); err != nil {
|
|
b.log.WarnContext(ctx, "model catalog refresh failed; keeping the loaded catalog", "err", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func (b *boot) livePlaybooks() *controlplanestore.LivePlaybooks {
|
|
b.playbooks = controlplanestore.NewLivePlaybooks(b.controlPlaneRPC, b.log)
|
|
b.playbooks.ApplyStatus = b.applyStatus
|
|
return b.playbooks
|
|
}
|
|
|
|
// playbookRefreshInterval is how often archied re-reads the resources it
|
|
// would otherwise load only on first use, so their apply status stays reported.
|
|
const playbookRefreshInterval = 30 * time.Second
|
|
|
|
// stateStoreCA is the CA a task container verifies the State Store with;
|
|
// nil without [tls].
|
|
func (b *boot) stateStoreCA(ctx context.Context) []byte {
|
|
id, err := servicekit.Identity(b.cfg.TLS, presence.Daemon)
|
|
if err != nil {
|
|
b.log.WarnContext(ctx, "state store CA for containers", "err", err)
|
|
return nil
|
|
}
|
|
return id.CAPEM()
|
|
}
|