Files

476 lines
14 KiB
Go

package curator
import (
"context"
"errors"
"fmt"
"maps"
"reflect"
"slices"
"sync"
"time"
"github.com/samcharles93/archie-core/internal/events"
)
const (
defaultPassTimeout = 10 * time.Minute
defaultWakeBuffer = 64
// minPassGap is the minimum time between the end of one pass and the
// start of the next. It exists so a curator that suggests a past or
// immediate NextCheckIn cannot hot-spin the loop; nudges still wake a
// loop early, so pending input is never delayed by it.
minPassGap = time.Second
)
// RuntimeConfig tunes the curator runtime. All fields are optional; zero
// values use the documented defaults.
type RuntimeConfig struct {
// PassTimeout bounds a single pass via context deadline. Zero uses
// defaultPassTimeout (10m).
PassTimeout time.Duration
// MaxConcurrentPasses caps simultaneous passes across all curators.
// Zero uses 1: passes serialize behind a shared slot so a pass burst
// can never saturate the model runtime against chat turns.
MaxConcurrentPasses int
}
// Runtime runs one loop per registered curator, with per-pass timeouts and
// panic recovery. Wake events are best-effort. Curators may be added and
// removed while it runs.
type Runtime struct {
registry *Registry
cfg RuntimeConfig
sem chan struct{}
// ops serialises Start, Stop, Add and Remove; mu guards the fields below
// and is never held across a pass or an engine lifecycle call.
ops sync.Mutex
mu sync.Mutex
started bool
loops map[string]*loop
nudges map[string]chan struct{}
}
// loop is one curator's running goroutine. It stays in Runtime.loops until
// the goroutine has returned, so a loop that outlives a bounded stop still
// blocks a second loop of the same name.
type loop struct {
cancel context.CancelFunc
done chan struct{}
}
// NewRuntime builds a runtime over the registry. The runtime shares the
// registry's clock (so loop timing is fake-clock testable) and event sink
// (so every run and action stays attributable).
func NewRuntime(registry *Registry, cfg RuntimeConfig) *Runtime {
if cfg.PassTimeout <= 0 {
cfg.PassTimeout = defaultPassTimeout
}
if cfg.MaxConcurrentPasses <= 0 {
cfg.MaxConcurrentPasses = 1
}
return &Runtime{
registry: registry,
cfg: cfg,
loops: make(map[string]*loop),
nudges: make(map[string]chan struct{}),
sem: make(chan struct{}, cfg.MaxConcurrentPasses),
}
}
// errDraining reports a curator whose previous loop has not yet returned.
var errDraining = errors.New("curator loop still stopping")
// Start launches one loop per registered curator and starts the curators'
// lifecycle. A curator whose Start fails is skipped by the registry and
// reported in the joined error; every other curator still runs. Starting an
// already-started runtime, or one whose previous loops have not returned, is
// an error.
func (rt *Runtime) Start(ctx context.Context) error {
rt.ops.Lock()
defer rt.ops.Unlock()
rt.mu.Lock()
if rt.started {
rt.mu.Unlock()
return errors.New("curator runtime already started")
}
if n := len(rt.loops); n > 0 {
rt.mu.Unlock()
return fmt.Errorf("%w: %d from the previous run", errDraining, n)
}
rt.started = true
rt.mu.Unlock()
startErr := rt.registry.Start(ctx)
if log := rt.registry.Host().Log; log != nil {
if startErr != nil {
log.ErrorContext(ctx, "curator runtime start completed with failures", "err", startErr)
} else {
log.InfoContext(ctx, "curator runtime started", "curators", len(rt.registry.Names()))
}
}
for _, name := range rt.registry.Names() {
c, _ := rt.registry.Get(name)
rt.launch(ctx, name, c)
}
return startErr
}
// launch starts name's loop under ctx. Remove and Stop cancel it.
func (rt *Runtime) launch(ctx context.Context, name string, c CuratorEngine) {
rt.mu.Lock()
defer rt.mu.Unlock()
var wake chan struct{} // nil for non-input curators: the case never fires
if c.Manifest().OnInput {
wake = make(chan struct{}, defaultWakeBuffer)
rt.nudges[name] = wake
}
ctx, cancel := context.WithCancel(ctx)
l := &loop{cancel: cancel, done: make(chan struct{})}
rt.loops[name] = l
go func() {
defer func() {
close(l.done)
rt.mu.Lock()
if rt.loops[name] == l {
delete(rt.loops, name)
}
rt.mu.Unlock()
}()
rt.runLoop(ctx, name, c, wake)
}()
}
// Add registers c and, when the runtime is running, starts it and its loop.
func (rt *Runtime) Add(ctx context.Context, c CuratorEngine) error {
rt.ops.Lock()
defer rt.ops.Unlock()
rt.mu.Lock()
_, draining := rt.loops[c.Name()]
started := rt.started
rt.mu.Unlock()
if draining {
return fmt.Errorf("%w: %s", errDraining, c.Name())
}
if err := rt.registry.Register(c); err != nil {
return err
}
if !started {
return nil
}
err := rt.registry.StartOne(ctx, c.Name())
// The loop belongs to the runtime, not to the call that added it.
rt.launch(context.WithoutCancel(ctx), c.Name(), c)
return err
}
// Remove cancels name's loop, including an in-flight pass, waits for it
// within ctx, and unregisters the curator. The curator is unregistered even
// when the wait times out; its loop then exits once the pass returns, and
// until it does the name cannot be added again.
func (rt *Runtime) Remove(ctx context.Context, name string) error {
rt.ops.Lock()
defer rt.ops.Unlock()
rt.mu.Lock()
l := rt.loops[name]
delete(rt.nudges, name)
rt.mu.Unlock()
var errs []error
if l != nil {
l.cancel()
if err := waitLoops(ctx, l); err != nil {
errs = append(errs, fmt.Errorf("curator %s: %w", name, err))
}
}
errs = append(errs, rt.registry.Unregister(ctx, name))
return errors.Join(errs...)
}
// waitLoops blocks until every loop has returned or ctx ends.
func waitLoops(ctx context.Context, loops ...*loop) error {
for _, l := range loops {
select {
case <-l.done:
case <-ctx.Done():
return fmt.Errorf("%w: %w", errDraining, ctx.Err())
}
}
return nil
}
// Nudge is a best-effort wake: input-driven curators check their trigger
// early instead of waiting for the next interval. A full channel or an
// unknown name drops the nudge — waking is an acceleration, never the
// source of truth. An empty curator name nudges every input-driven curator.
func (rt *Runtime) Nudge(curator string) {
rt.mu.Lock()
defer rt.mu.Unlock()
for name, ch := range rt.nudges {
if curator != "" && curator != name {
continue
}
select {
case ch <- struct{}{}:
default:
}
}
}
// Stop cancels every loop, waits for passes to finish within ctx, then stops
// the curators in reverse order. Loops that outlive ctx stay tracked, so a
// later Start refuses rather than running a second copy.
func (rt *Runtime) Stop(ctx context.Context) error {
rt.ops.Lock()
defer rt.ops.Unlock()
rt.mu.Lock()
if !rt.started {
rt.mu.Unlock()
return nil
}
rt.started = false
loops := slices.Collect(maps.Values(rt.loops))
for _, l := range loops {
l.cancel()
}
clear(rt.nudges)
rt.mu.Unlock()
if err := waitLoops(ctx, loops...); err != nil {
if log := rt.registry.Host().Log; log != nil {
log.ErrorContext(ctx, "curator runtime shutdown timed out", "err", err)
}
return fmt.Errorf("curator runtime shutdown: %w", errors.Join(err, rt.registry.Stop(ctx)))
}
err := rt.registry.Stop(ctx)
if log := rt.registry.Host().Log; log != nil {
if err != nil {
log.ErrorContext(ctx, "curator runtime stopped with failures", "err", err)
} else {
log.InfoContext(ctx, "curator runtime stopped")
}
}
return err
}
// runLoop is one curator's lifetime: sleep until eligible or nudged, ask
// Check whether input is pending, run a bounded pass when it is, and back
// off to the cooldown after an idle or failed pass. Panics in Check and
// Pass are recovered here; the loop survives them.
func (rt *Runtime) runLoop(ctx context.Context, name string, c CuratorEngine, wakeCh <-chan struct{}) {
m := c.Manifest()
cooldown := m.Cooldown
if cooldown <= 0 {
cooldown = 2 * m.Interval
}
clock := rt.registry.Host().Clock
var next time.Time // zero means immediately eligible
var lastPass time.Time
for {
now := clock.Now()
reason := "check-in"
if !next.IsZero() && now.Before(next) {
woke, done := waitNext(ctx, wakeCh, clock.After(next.Sub(now)))
if done {
return
}
if woke {
reason = "wake"
}
}
due, err := rt.safeCheck(ctx, c)
if err != nil {
rt.fail(ctx, name, "check", err)
next = clock.Now().Add(cooldown)
continue
}
if !due {
next = clock.Now().Add(m.Interval)
if log := rt.registry.Host().Log; log != nil {
log.DebugContext(ctx, "curator not due", "curator", name, "next", next)
}
continue
}
if !rt.acquire(ctx) {
return // shutting down while waiting for a pass slot
}
passCtx, cancel := context.WithTimeout(ctx, rt.cfg.PassTimeout)
result, perr := rt.safePass(passCtx, c, PassInput{Reason: reason, Since: lastPass, LastPass: lastPass})
cancel()
rt.release()
now = clock.Now()
lastPass = now
if perr != nil {
rt.fail(ctx, name, "pass", perr)
next = now.Add(cooldown)
} else {
next = scheduleNext(now, m, cooldown, result)
rt.emitRun(ctx, name, now, result)
}
}
}
// waitNext blocks until a nudge, the sleep timer, or shutdown. woke means
// a nudge fired — re-check the trigger now; done means the context was
// cancelled — exit the loop.
func waitNext(ctx context.Context, wakeCh <-chan struct{}, timer <-chan time.Time) (woke, done bool) {
select {
case <-ctx.Done():
return false, true
case <-wakeCh:
return true, false
case <-timer:
return false, false
}
}
// scheduleNext computes the next eligible time after a successful pass. A
// curator-suggested NextCheckIn in the past (or immediate) would hot-spin
// the loop, so it is clamped to the minimum pass gap; an idle pass (no
// actions) backs off to the cooldown.
func scheduleNext(now time.Time, m Manifest, cooldown time.Duration, result PassResult) time.Time {
switch {
case !result.NextCheckIn.IsZero():
next := result.NextCheckIn
if earliest := now.Add(minPassGap); next.Before(earliest) {
next = earliest
}
return next
case len(result.Actions) == 0:
return now.Add(cooldown)
default:
return now.Add(m.Interval)
}
}
func (rt *Runtime) acquire(ctx context.Context) bool {
select {
case rt.sem <- struct{}{}:
return true
case <-ctx.Done():
return false
}
}
func (rt *Runtime) release() { <-rt.sem }
func (rt *Runtime) safeCheck(ctx context.Context, c CuratorEngine) (due bool, err error) {
defer func() {
if p := recover(); p != nil {
err = fmt.Errorf("check panic: %v", p)
}
}()
return c.Check(ctx)
}
func (rt *Runtime) safePass(ctx context.Context, c CuratorEngine, in PassInput) (result PassResult, err error) {
defer func() {
if p := recover(); p != nil {
err = fmt.Errorf("pass panic: %v", p)
}
}()
return c.Pass(ctx, in)
}
// fail records a Check/Pass failure on the event stream. The event bus is
// non-blocking by contract (bounded dropping buffers), so even a failing
// curator cannot backpressure anything.
func (rt *Runtime) fail(ctx context.Context, name, phase string, err error) {
if log := rt.registry.Host().Log; log != nil {
log.ErrorContext(ctx, "curator failed", "curator", name, "phase", phase, "err", err)
}
if sink := rt.registry.Host().Events; sink != nil {
sink.Emit(events.KindCuratorError, name, map[string]any{
"curator": name,
"phase": phase,
"err": err.Error(),
})
}
}
// emitRun publishes one run summary plus one event per action, so what ran,
// when, what it changed, and why all stay attributable.
func (rt *Runtime) emitRun(ctx context.Context, name string, at time.Time, result PassResult) {
rt.registry.RecordActivity(name, at, result.Actions)
host := rt.registry.Host()
if log := host.Log; log != nil {
log.InfoContext(ctx, "curator pass completed", "curator", name, "actions", len(result.Actions), "at", at)
for _, a := range result.Actions {
log.InfoContext(ctx, "curator action", "curator", name, "type", a.Type, "detail", a.Detail, "reason", a.Reason, "at", a.At)
}
}
sink := host.Events
if sink == nil {
return
}
sink.Emit(events.KindCuratorRun, name, map[string]any{
"curator": name,
"actions": len(result.Actions),
"at": at,
})
for _, a := range result.Actions {
sink.Emit(events.KindCuratorAction, name, map[string]any{
"curator": name,
"type": a.Type,
"detail": a.Detail,
"reason": a.Reason,
"at": a.At,
})
}
}
// ReplaceDefinitions applies a full set of data-defined curators: it
// validates the set first, then removes, adds or replaces only the curators
// that changed. Built-in engines, unchanged curators and the registry's
// observation surface keep running.
func (rt *Runtime) ReplaceDefinitions(ctx context.Context, definitions []Definition) error {
want := make(map[string]Definition, len(definitions))
candidate := NewRegistry(rt.registry.Host())
for _, def := range definitions {
if !def.Enabled {
continue
}
if held, ok := rt.registry.Get(def.Name); ok {
if _, custom := held.(*DefinitionEngine); !custom {
return fmt.Errorf("%w: %s", ErrDuplicate, def.Name)
}
}
if err := candidate.Register(NewDefinitionEngine(def)); err != nil {
return err
}
want[def.Name] = def
}
stopCtx, cancel := context.WithTimeout(ctx, defaultPassTimeout)
defer cancel()
var errs []error
for _, name := range rt.registry.Names() {
held, _ := rt.registry.Get(name)
engine, custom := held.(*DefinitionEngine)
if !custom {
continue
}
if def, keep := want[name]; keep && reflect.DeepEqual(def, engine.def) {
delete(want, name)
continue
}
if err := rt.Remove(stopCtx, name); err != nil {
errs = append(errs, err)
}
}
for _, name := range slices.Sorted(maps.Keys(want)) {
if err := rt.Add(ctx, NewDefinitionEngine(want[name])); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}