- Introduce `-config-overlay` flag and `config.LoadOverlay()` to merge a base config with an overlay file, only overriding fields the overlay sets. This enables docker-compose to mount the base config from the host and apply only deployment-specific overrides. - Create `internal/natsrpc` package with reusable NATS request/reply primitives (Envelope, Client, Call, RegisterAll, Respond). Migrate storerpc, worktreerpc, and forgerpc to use it, eliminating duplicated marshal/unmarshal and error handling. - Create `internal/yaegiutil` package with shared interpreter construction (stdlib + extra symbols), generic export resolution (Resolve), and panic recovery (Safe). Migrate all Yaegi-based surfaces (gate scripts, daemon plugins, skill plugins, workflow stages, skill scripts) to use it, reducing boilerplate and centralizing panic safety. - Simplify `config.docker.toml` to contain only overlay fields (paths, container config) and remove the duplicated repos list – the base config is now the single source of truth. - Update docker-compose.yml to pass both -config and -config-overlay flags and mount the base config from the host. - Add tests for LoadOverlay, DockerConfigOverlaysExampleConfig, Envelope, and yaegiutil utilities.
345 lines
9.9 KiB
Go
345 lines
9.9 KiB
Go
// Command archied is the archie orchestrator daemon: it watches GitHub
|
|
// for issues labelled for archie, works each one in an isolated
|
|
// worktree through its routed workflow, and opens pull requests for
|
|
// human review.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"syscall"
|
|
|
|
natsio "github.com/nats-io/nats.go"
|
|
|
|
"github.com/moby/moby/client"
|
|
"github.com/samcharles93/archie-core/internal/agentexec"
|
|
"github.com/samcharles93/archie-core/internal/config"
|
|
"github.com/samcharles93/archie-core/internal/container"
|
|
"github.com/samcharles93/archie-core/internal/daemon"
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
"github.com/samcharles93/archie-core/internal/forge"
|
|
"github.com/samcharles93/archie-core/internal/forgerpc"
|
|
"github.com/samcharles93/archie-core/internal/nats"
|
|
|
|
"github.com/samcharles93/archie-core/internal/plugin"
|
|
"github.com/samcharles93/archie-core/internal/plugin/pluginextract"
|
|
"github.com/samcharles93/archie-core/internal/storage"
|
|
"github.com/samcharles93/archie-core/internal/store"
|
|
"github.com/samcharles93/archie-core/internal/storerpc"
|
|
"github.com/samcharles93/archie-core/internal/webui"
|
|
"github.com/samcharles93/archie-core/internal/workflow/skillbuild"
|
|
"github.com/samcharles93/archie-core/internal/workflow/wfeval"
|
|
"github.com/samcharles93/archie-core/internal/worktree"
|
|
"github.com/samcharles93/archie-core/internal/worktreerpc"
|
|
)
|
|
|
|
func main() {
|
|
os.Exit(run())
|
|
}
|
|
|
|
func run() int {
|
|
defaultCfg := filepath.Join(configHome(), "archie", "config.toml")
|
|
cfgPath := flag.String("config", defaultCfg, "path to config.toml")
|
|
overlayPath := flag.String("config-overlay", "", "path to a config.toml overlay applied on top of -config (only the fields it sets are overridden)")
|
|
once := flag.Bool("once", false, "run a single poll+process cycle and exit (systemd timer / testing)")
|
|
requeue := flag.Int64("requeue", 0, "requeue a parked/waiting task by id (keeps its workflow), then exit unless -once is also set")
|
|
flag.Parse()
|
|
|
|
log := slog.New(slog.NewJSONHandler(os.Stderr, nil))
|
|
|
|
cfg, err := config.LoadOverlay(*cfgPath, *overlayPath)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, err)
|
|
return 1
|
|
}
|
|
|
|
token := os.Getenv(cfg.Forge.TokenEnv)
|
|
if token == "" {
|
|
fmt.Fprintln(os.Stderr, cfg.Forge.TokenEnv+" is required")
|
|
return 1
|
|
}
|
|
forgeClient, err := forge.New(cfg.Forge.Type, token, cfg.Forge.Host, log)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, err)
|
|
return 1
|
|
}
|
|
|
|
st, err := store.Open(cfg.DBPath)
|
|
if err != nil {
|
|
log.Error("open store", "err", err)
|
|
return 1
|
|
}
|
|
defer func() {
|
|
if err := st.Close(); err != nil {
|
|
log.Error("close store", "err", err)
|
|
}
|
|
}()
|
|
|
|
if *requeue > 0 {
|
|
if err := st.Requeue(context.Background(), *requeue, "manual", ""); err != nil {
|
|
log.Error("requeue failed", "task", *requeue, "err", err)
|
|
return 1
|
|
}
|
|
log.Info("task requeued", "task", *requeue)
|
|
if !*once {
|
|
return 0
|
|
}
|
|
}
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
// Observability: every event is logged to SQLite (stamped with its
|
|
// row id) and then fanned out to live dashboard connections.
|
|
bus := events.NewBus()
|
|
defer bus.Close()
|
|
web := &webui.Server{Store: st, Log: log}
|
|
sink := bus.Subscribe(256)
|
|
go func() {
|
|
for e := range sink.C {
|
|
id, err := st.InsertEvent(context.Background(), e)
|
|
if err != nil {
|
|
log.Error("event sink insert failed", "err", err)
|
|
continue
|
|
}
|
|
e.ID = id
|
|
web.Broadcast(e)
|
|
}
|
|
}()
|
|
if l := cfg.Web.Listen; l != "" && l != "off" {
|
|
go func() {
|
|
if err := web.Run(ctx, l); err != nil {
|
|
log.Error("web ui failed", "err", err)
|
|
}
|
|
}()
|
|
}
|
|
|
|
// NATS client (optional — SQLite flow unchanged when [nats] is absent).
|
|
var natsClient *nats.Client
|
|
if cfg.NATS.URL != "" {
|
|
natsToken, err := configuredNATSToken(cfg.NATS, os.Getenv)
|
|
if err != nil {
|
|
log.Error("nats credentials", "err", err)
|
|
return 1
|
|
}
|
|
natsClient, err = nats.Connect(ctx, cfg.NATS.URL, natsToken, log)
|
|
if err != nil {
|
|
log.Error("nats connect failed", "err", err)
|
|
return 1
|
|
}
|
|
defer natsClient.Close()
|
|
log.Info("nats connected", "url", cfg.NATS.URL)
|
|
}
|
|
|
|
// Container pool (optional — no containers when [containers] is absent).
|
|
var containerPool *container.Pool
|
|
var storeBackend storage.Backend
|
|
if cfg.Containers.Enabled {
|
|
// Single Docker client shared between pool and storage backend.
|
|
dockerCli, err := client.New(client.FromEnv)
|
|
if err != nil {
|
|
log.Error("docker client failed", "err", err)
|
|
return 1
|
|
}
|
|
defer dockerCli.Close()
|
|
|
|
containerPool, err = container.NewPool(ctx, container.Config{
|
|
Image: cfg.Containers.Image,
|
|
MaxConcurrency: cfg.Containers.MaxConcurrency,
|
|
MaxUptime: cfg.Containers.MaxUptime.Std(),
|
|
PullPolicy: cfg.Containers.PullPolicy,
|
|
DockerClient: dockerCli,
|
|
}, cfg.NATS.URL, log)
|
|
if err != nil {
|
|
log.Error("container pool failed", "err", err)
|
|
return 1
|
|
}
|
|
defer containerPool.Close()
|
|
|
|
storeBackend = storage.NewDockerBackend(dockerCli)
|
|
}
|
|
|
|
providers := executionProviders(cfg)
|
|
llm := agentexec.NewRuntime(providers)
|
|
var agentRunner agentexec.Runner
|
|
if llm != nil {
|
|
switch cfg.Agent.Mode {
|
|
case "subprocess":
|
|
agentRunner = &agentexec.SubprocessRunner{
|
|
Command: cfg.Agent.Command,
|
|
Environ: os.Environ(),
|
|
AdditionalEnv: cfg.Agent.Env,
|
|
Diagnostics: os.Stderr,
|
|
Providers: providers,
|
|
}
|
|
case "inprocess":
|
|
agentRunner = agentexec.NewInProcessRunner(llm, log)
|
|
case "nats":
|
|
if natsClient == nil {
|
|
log.Error("agent.mode is nats but [nats] is not configured")
|
|
return 1
|
|
}
|
|
agentRunner = &agentexec.NATSRunner{
|
|
Nats: natsClient,
|
|
Providers: providers,
|
|
Log: log,
|
|
}
|
|
}
|
|
}
|
|
// Build the workflow registry from the skill catalog. Plugin-defined
|
|
// workflows override built-ins of the same name; built-ins fill gaps.
|
|
skillsBase := cfg.SkillsDir
|
|
if skillsBase == "" {
|
|
skillsBase = cfg.WorkDir
|
|
}
|
|
registry, err := skillbuild.BuildRegistry(skillsBase)
|
|
if err != nil {
|
|
log.Error("skill registry build failed", "err", err)
|
|
return 1
|
|
}
|
|
log.Info("workflow registry built", "workflows", len(registry))
|
|
|
|
// Load daemon plugins from the configured plugin directory (Layer 2).
|
|
// Failed plugins are skipped — the daemon starts with the remaining set.
|
|
var pluginReg *plugin.Registry
|
|
if cfg.PluginDir != "" {
|
|
plugins, err := plugin.LoadDir(cfg.PluginDir, pluginextract.Symbols)
|
|
if err != nil {
|
|
log.Error("plugin load failed", "dir", cfg.PluginDir, "err", err)
|
|
return 1
|
|
}
|
|
pluginReg = &plugin.Registry{}
|
|
for _, p := range plugins {
|
|
pluginReg.Register(p)
|
|
name, version := safePluginInfo(p)
|
|
log.Info("daemon plugin loaded", "name", name, "version", version)
|
|
}
|
|
}
|
|
|
|
trees := &worktree.Manager{
|
|
WorkDir: cfg.WorkDir,
|
|
Token: token,
|
|
BotUser: cfg.BotUser,
|
|
BotEmail: cfg.BotEmail,
|
|
BaseURL: cfg.Forge.Host,
|
|
}
|
|
|
|
// Let archie-agent containers (which hold no DB connection, forge
|
|
// token, or push credential) proxy Store/Forge/worktree operations
|
|
// back to archied over NATS.
|
|
if natsClient != nil {
|
|
unsubscribe, err := registerTaskRPCServers(natsClient.Conn(), st, forgeClient, trees, log)
|
|
if err != nil {
|
|
log.Error("task RPC server registration failed", "err", err)
|
|
return 1
|
|
}
|
|
defer unsubscribe()
|
|
}
|
|
|
|
d := &daemon.Daemon{
|
|
Cfg: cfg,
|
|
Store: st,
|
|
Bus: bus,
|
|
Forge: forgeClient,
|
|
Trees: trees,
|
|
Runtime: llm,
|
|
Agent: agentRunner,
|
|
Workflows: registry,
|
|
PluginRegistry: pluginReg,
|
|
Storage: storeBackend,
|
|
Log: log,
|
|
CustomStages: wfeval.Discover,
|
|
Nats: natsClient,
|
|
ContainerPool: containerPool,
|
|
}
|
|
|
|
if err := d.Startup(ctx); err != nil {
|
|
log.Error("startup", "err", err)
|
|
return 1
|
|
}
|
|
|
|
if *once {
|
|
d.Cycle(ctx)
|
|
return 0
|
|
}
|
|
log.Info("archied running", "repos", len(cfg.Repos), "poll", cfg.PollInterval.Std().String(), "label", cfg.Label)
|
|
if err := d.Run(ctx); err != nil && ctx.Err() == nil {
|
|
log.Error("daemon exited", "err", err)
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func executionProviders(cfg config.Config) map[string]agentexec.Provider {
|
|
return agentexec.ProvidersFromConfig(cfg.Providers)
|
|
}
|
|
|
|
// registerTaskRPCServers subscribes the storerpc/forgerpc/worktreerpc
|
|
// handlers on nc so an archie-agent container (which holds no DB
|
|
// connection, forge token, or push credential) can proxy those operations
|
|
// back to archied. The returned func unsubscribes all three.
|
|
func registerTaskRPCServers(nc *natsio.Conn, st *store.Store, forgeClient forge.Forge, trees *worktree.Manager, log *slog.Logger) (unsubscribe func(), err error) {
|
|
storeServer := &storerpc.Server{Store: st, Log: log}
|
|
unsubStore, err := storeServer.Register(nc)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("register storerpc: %w", err)
|
|
}
|
|
|
|
forgeServer := &forgerpc.Server{Forge: forgeClient, Log: log}
|
|
unsubForge, err := forgeServer.Register(nc)
|
|
if err != nil {
|
|
unsubStore()
|
|
return nil, fmt.Errorf("register forgerpc: %w", err)
|
|
}
|
|
|
|
treesServer := &worktreerpc.Server{Trees: trees, Log: log}
|
|
unsubTrees, err := treesServer.Register(nc)
|
|
if err != nil {
|
|
unsubStore()
|
|
unsubForge()
|
|
return nil, fmt.Errorf("register worktreerpc: %w", err)
|
|
}
|
|
|
|
return func() {
|
|
unsubStore()
|
|
unsubForge()
|
|
unsubTrees()
|
|
}, nil
|
|
}
|
|
|
|
func configuredNATSToken(cfg config.NATSConfig, getenv func(string) string) (string, error) {
|
|
if cfg.TokenEnv == "" {
|
|
return "", nil
|
|
}
|
|
token := getenv(cfg.TokenEnv)
|
|
if token == "" {
|
|
return "", fmt.Errorf("%s is required when nats.token_env is configured", cfg.TokenEnv)
|
|
}
|
|
return token, nil
|
|
}
|
|
|
|
func configHome() string {
|
|
if x := os.Getenv("XDG_CONFIG_HOME"); x != "" {
|
|
return x
|
|
}
|
|
home, _ := os.UserHomeDir()
|
|
return filepath.Join(home, ".config")
|
|
}
|
|
|
|
// safePluginInfo calls Name() and Version() on a plugin, recovering from
|
|
// panics. A panicking plugin must not crash the daemon at startup.
|
|
func safePluginInfo(p plugin.Plugin) (name, version string) {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
name = "(panic)"
|
|
version = "(panic)"
|
|
}
|
|
}()
|
|
return p.Name(), p.Version()
|
|
}
|