Files
archie/cmd/archied/main.go
T
sam d80401d595 feat: add config overlay and shared RPC/Yaegi packages
- Introduce `-config-overlay` flag and `config.LoadOverlay()` to merge a base config with an overlay file, only overriding fields the overlay sets. This enables docker-compose to mount the base config from the host and apply only deployment-specific overrides.
- Create `internal/natsrpc` package with reusable NATS request/reply primitives (Envelope, Client, Call, RegisterAll, Respond). Migrate storerpc, worktreerpc, and forgerpc to use it, eliminating duplicated marshal/unmarshal and error handling.
- Create `internal/yaegiutil` package with shared interpreter construction (stdlib + extra symbols), generic export resolution (Resolve), and panic recovery (Safe). Migrate all Yaegi-based surfaces (gate scripts, daemon plugins, skill plugins, workflow stages, skill scripts) to use it, reducing boilerplate and centralizing panic safety.
- Simplify `config.docker.toml` to contain only overlay fields (paths, container config) and remove the duplicated repos list – the base config is now the single source of truth.
- Update docker-compose.yml to pass both -config and -config-overlay flags and mount the base config from the host.
- Add tests for LoadOverlay, DockerConfigOverlaysExampleConfig, Envelope, and yaegiutil utilities.
2026-07-24 12:57:10 +10:00

345 lines
9.9 KiB
Go

// Command archied is the archie orchestrator daemon: it watches GitHub
// for issues labelled for archie, works each one in an isolated
// worktree through its routed workflow, and opens pull requests for
// human review.
package main
import (
"context"
"flag"
"fmt"
"log/slog"
"os"
"os/signal"
"path/filepath"
"syscall"
natsio "github.com/nats-io/nats.go"
"github.com/moby/moby/client"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/container"
"github.com/samcharles93/archie-core/internal/daemon"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/forge"
"github.com/samcharles93/archie-core/internal/forgerpc"
"github.com/samcharles93/archie-core/internal/nats"
"github.com/samcharles93/archie-core/internal/plugin"
"github.com/samcharles93/archie-core/internal/plugin/pluginextract"
"github.com/samcharles93/archie-core/internal/storage"
"github.com/samcharles93/archie-core/internal/store"
"github.com/samcharles93/archie-core/internal/storerpc"
"github.com/samcharles93/archie-core/internal/webui"
"github.com/samcharles93/archie-core/internal/workflow/skillbuild"
"github.com/samcharles93/archie-core/internal/workflow/wfeval"
"github.com/samcharles93/archie-core/internal/worktree"
"github.com/samcharles93/archie-core/internal/worktreerpc"
)
func main() {
os.Exit(run())
}
func run() int {
defaultCfg := filepath.Join(configHome(), "archie", "config.toml")
cfgPath := flag.String("config", defaultCfg, "path to config.toml")
overlayPath := flag.String("config-overlay", "", "path to a config.toml overlay applied on top of -config (only the fields it sets are overridden)")
once := flag.Bool("once", false, "run a single poll+process cycle and exit (systemd timer / testing)")
requeue := flag.Int64("requeue", 0, "requeue a parked/waiting task by id (keeps its workflow), then exit unless -once is also set")
flag.Parse()
log := slog.New(slog.NewJSONHandler(os.Stderr, nil))
cfg, err := config.LoadOverlay(*cfgPath, *overlayPath)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
token := os.Getenv(cfg.Forge.TokenEnv)
if token == "" {
fmt.Fprintln(os.Stderr, cfg.Forge.TokenEnv+" is required")
return 1
}
forgeClient, err := forge.New(cfg.Forge.Type, token, cfg.Forge.Host, log)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
st, err := store.Open(cfg.DBPath)
if err != nil {
log.Error("open store", "err", err)
return 1
}
defer func() {
if err := st.Close(); err != nil {
log.Error("close store", "err", err)
}
}()
if *requeue > 0 {
if err := st.Requeue(context.Background(), *requeue, "manual", ""); err != nil {
log.Error("requeue failed", "task", *requeue, "err", err)
return 1
}
log.Info("task requeued", "task", *requeue)
if !*once {
return 0
}
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
// Observability: every event is logged to SQLite (stamped with its
// row id) and then fanned out to live dashboard connections.
bus := events.NewBus()
defer bus.Close()
web := &webui.Server{Store: st, Log: log}
sink := bus.Subscribe(256)
go func() {
for e := range sink.C {
id, err := st.InsertEvent(context.Background(), e)
if err != nil {
log.Error("event sink insert failed", "err", err)
continue
}
e.ID = id
web.Broadcast(e)
}
}()
if l := cfg.Web.Listen; l != "" && l != "off" {
go func() {
if err := web.Run(ctx, l); err != nil {
log.Error("web ui failed", "err", err)
}
}()
}
// NATS client (optional — SQLite flow unchanged when [nats] is absent).
var natsClient *nats.Client
if cfg.NATS.URL != "" {
natsToken, err := configuredNATSToken(cfg.NATS, os.Getenv)
if err != nil {
log.Error("nats credentials", "err", err)
return 1
}
natsClient, err = nats.Connect(ctx, cfg.NATS.URL, natsToken, log)
if err != nil {
log.Error("nats connect failed", "err", err)
return 1
}
defer natsClient.Close()
log.Info("nats connected", "url", cfg.NATS.URL)
}
// Container pool (optional — no containers when [containers] is absent).
var containerPool *container.Pool
var storeBackend storage.Backend
if cfg.Containers.Enabled {
// Single Docker client shared between pool and storage backend.
dockerCli, err := client.New(client.FromEnv)
if err != nil {
log.Error("docker client failed", "err", err)
return 1
}
defer dockerCli.Close()
containerPool, err = container.NewPool(ctx, container.Config{
Image: cfg.Containers.Image,
MaxConcurrency: cfg.Containers.MaxConcurrency,
MaxUptime: cfg.Containers.MaxUptime.Std(),
PullPolicy: cfg.Containers.PullPolicy,
DockerClient: dockerCli,
}, cfg.NATS.URL, log)
if err != nil {
log.Error("container pool failed", "err", err)
return 1
}
defer containerPool.Close()
storeBackend = storage.NewDockerBackend(dockerCli)
}
providers := executionProviders(cfg)
llm := agentexec.NewRuntime(providers)
var agentRunner agentexec.Runner
if llm != nil {
switch cfg.Agent.Mode {
case "subprocess":
agentRunner = &agentexec.SubprocessRunner{
Command: cfg.Agent.Command,
Environ: os.Environ(),
AdditionalEnv: cfg.Agent.Env,
Diagnostics: os.Stderr,
Providers: providers,
}
case "inprocess":
agentRunner = agentexec.NewInProcessRunner(llm, log)
case "nats":
if natsClient == nil {
log.Error("agent.mode is nats but [nats] is not configured")
return 1
}
agentRunner = &agentexec.NATSRunner{
Nats: natsClient,
Providers: providers,
Log: log,
}
}
}
// Build the workflow registry from the skill catalog. Plugin-defined
// workflows override built-ins of the same name; built-ins fill gaps.
skillsBase := cfg.SkillsDir
if skillsBase == "" {
skillsBase = cfg.WorkDir
}
registry, err := skillbuild.BuildRegistry(skillsBase)
if err != nil {
log.Error("skill registry build failed", "err", err)
return 1
}
log.Info("workflow registry built", "workflows", len(registry))
// Load daemon plugins from the configured plugin directory (Layer 2).
// Failed plugins are skipped — the daemon starts with the remaining set.
var pluginReg *plugin.Registry
if cfg.PluginDir != "" {
plugins, err := plugin.LoadDir(cfg.PluginDir, pluginextract.Symbols)
if err != nil {
log.Error("plugin load failed", "dir", cfg.PluginDir, "err", err)
return 1
}
pluginReg = &plugin.Registry{}
for _, p := range plugins {
pluginReg.Register(p)
name, version := safePluginInfo(p)
log.Info("daemon plugin loaded", "name", name, "version", version)
}
}
trees := &worktree.Manager{
WorkDir: cfg.WorkDir,
Token: token,
BotUser: cfg.BotUser,
BotEmail: cfg.BotEmail,
BaseURL: cfg.Forge.Host,
}
// Let archie-agent containers (which hold no DB connection, forge
// token, or push credential) proxy Store/Forge/worktree operations
// back to archied over NATS.
if natsClient != nil {
unsubscribe, err := registerTaskRPCServers(natsClient.Conn(), st, forgeClient, trees, log)
if err != nil {
log.Error("task RPC server registration failed", "err", err)
return 1
}
defer unsubscribe()
}
d := &daemon.Daemon{
Cfg: cfg,
Store: st,
Bus: bus,
Forge: forgeClient,
Trees: trees,
Runtime: llm,
Agent: agentRunner,
Workflows: registry,
PluginRegistry: pluginReg,
Storage: storeBackend,
Log: log,
CustomStages: wfeval.Discover,
Nats: natsClient,
ContainerPool: containerPool,
}
if err := d.Startup(ctx); err != nil {
log.Error("startup", "err", err)
return 1
}
if *once {
d.Cycle(ctx)
return 0
}
log.Info("archied running", "repos", len(cfg.Repos), "poll", cfg.PollInterval.Std().String(), "label", cfg.Label)
if err := d.Run(ctx); err != nil && ctx.Err() == nil {
log.Error("daemon exited", "err", err)
return 1
}
return 0
}
func executionProviders(cfg config.Config) map[string]agentexec.Provider {
return agentexec.ProvidersFromConfig(cfg.Providers)
}
// registerTaskRPCServers subscribes the storerpc/forgerpc/worktreerpc
// handlers on nc so an archie-agent container (which holds no DB
// connection, forge token, or push credential) can proxy those operations
// back to archied. The returned func unsubscribes all three.
func registerTaskRPCServers(nc *natsio.Conn, st *store.Store, forgeClient forge.Forge, trees *worktree.Manager, log *slog.Logger) (unsubscribe func(), err error) {
storeServer := &storerpc.Server{Store: st, Log: log}
unsubStore, err := storeServer.Register(nc)
if err != nil {
return nil, fmt.Errorf("register storerpc: %w", err)
}
forgeServer := &forgerpc.Server{Forge: forgeClient, Log: log}
unsubForge, err := forgeServer.Register(nc)
if err != nil {
unsubStore()
return nil, fmt.Errorf("register forgerpc: %w", err)
}
treesServer := &worktreerpc.Server{Trees: trees, Log: log}
unsubTrees, err := treesServer.Register(nc)
if err != nil {
unsubStore()
unsubForge()
return nil, fmt.Errorf("register worktreerpc: %w", err)
}
return func() {
unsubStore()
unsubForge()
unsubTrees()
}, nil
}
func configuredNATSToken(cfg config.NATSConfig, getenv func(string) string) (string, error) {
if cfg.TokenEnv == "" {
return "", nil
}
token := getenv(cfg.TokenEnv)
if token == "" {
return "", fmt.Errorf("%s is required when nats.token_env is configured", cfg.TokenEnv)
}
return token, nil
}
func configHome() string {
if x := os.Getenv("XDG_CONFIG_HOME"); x != "" {
return x
}
home, _ := os.UserHomeDir()
return filepath.Join(home, ".config")
}
// safePluginInfo calls Name() and Version() on a plugin, recovering from
// panics. A panicking plugin must not crash the daemon at startup.
func safePluginInfo(p plugin.Plugin) (name, version string) {
defer func() {
if r := recover(); r != nil {
name = "(panic)"
version = "(panic)"
}
}()
return p.Name(), p.Version()
}