476 lines
14 KiB
Go
476 lines
14 KiB
Go
package curator
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"maps"
|
|
"reflect"
|
|
"slices"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
)
|
|
|
|
const (
|
|
defaultPassTimeout = 10 * time.Minute
|
|
defaultWakeBuffer = 64
|
|
// minPassGap is the minimum time between the end of one pass and the
|
|
// start of the next. It exists so a curator that suggests a past or
|
|
// immediate NextCheckIn cannot hot-spin the loop; nudges still wake a
|
|
// loop early, so pending input is never delayed by it.
|
|
minPassGap = time.Second
|
|
)
|
|
|
|
// RuntimeConfig tunes the curator runtime. All fields are optional; zero
|
|
// values use the documented defaults.
|
|
type RuntimeConfig struct {
|
|
// PassTimeout bounds a single pass via context deadline. Zero uses
|
|
// defaultPassTimeout (10m).
|
|
PassTimeout time.Duration
|
|
// MaxConcurrentPasses caps simultaneous passes across all curators.
|
|
// Zero uses 1: passes serialize behind a shared slot so a pass burst
|
|
// can never saturate the model runtime against chat turns.
|
|
MaxConcurrentPasses int
|
|
}
|
|
|
|
// Runtime runs one loop per registered curator, with per-pass timeouts and
|
|
// panic recovery. Wake events are best-effort. Curators may be added and
|
|
// removed while it runs.
|
|
type Runtime struct {
|
|
registry *Registry
|
|
cfg RuntimeConfig
|
|
sem chan struct{}
|
|
|
|
// ops serialises Start, Stop, Add and Remove; mu guards the fields below
|
|
// and is never held across a pass or an engine lifecycle call.
|
|
ops sync.Mutex
|
|
mu sync.Mutex
|
|
started bool
|
|
loops map[string]*loop
|
|
nudges map[string]chan struct{}
|
|
}
|
|
|
|
// loop is one curator's running goroutine. It stays in Runtime.loops until
|
|
// the goroutine has returned, so a loop that outlives a bounded stop still
|
|
// blocks a second loop of the same name.
|
|
type loop struct {
|
|
cancel context.CancelFunc
|
|
done chan struct{}
|
|
}
|
|
|
|
// NewRuntime builds a runtime over the registry. The runtime shares the
|
|
// registry's clock (so loop timing is fake-clock testable) and event sink
|
|
// (so every run and action stays attributable).
|
|
func NewRuntime(registry *Registry, cfg RuntimeConfig) *Runtime {
|
|
if cfg.PassTimeout <= 0 {
|
|
cfg.PassTimeout = defaultPassTimeout
|
|
}
|
|
if cfg.MaxConcurrentPasses <= 0 {
|
|
cfg.MaxConcurrentPasses = 1
|
|
}
|
|
return &Runtime{
|
|
registry: registry,
|
|
cfg: cfg,
|
|
loops: make(map[string]*loop),
|
|
nudges: make(map[string]chan struct{}),
|
|
sem: make(chan struct{}, cfg.MaxConcurrentPasses),
|
|
}
|
|
}
|
|
|
|
// errDraining reports a curator whose previous loop has not yet returned.
|
|
var errDraining = errors.New("curator loop still stopping")
|
|
|
|
// Start launches one loop per registered curator and starts the curators'
|
|
// lifecycle. A curator whose Start fails is skipped by the registry and
|
|
// reported in the joined error; every other curator still runs. Starting an
|
|
// already-started runtime, or one whose previous loops have not returned, is
|
|
// an error.
|
|
func (rt *Runtime) Start(ctx context.Context) error {
|
|
rt.ops.Lock()
|
|
defer rt.ops.Unlock()
|
|
rt.mu.Lock()
|
|
if rt.started {
|
|
rt.mu.Unlock()
|
|
return errors.New("curator runtime already started")
|
|
}
|
|
if n := len(rt.loops); n > 0 {
|
|
rt.mu.Unlock()
|
|
return fmt.Errorf("%w: %d from the previous run", errDraining, n)
|
|
}
|
|
rt.started = true
|
|
rt.mu.Unlock()
|
|
|
|
startErr := rt.registry.Start(ctx)
|
|
if log := rt.registry.Host().Log; log != nil {
|
|
if startErr != nil {
|
|
log.ErrorContext(ctx, "curator runtime start completed with failures", "err", startErr)
|
|
} else {
|
|
log.InfoContext(ctx, "curator runtime started", "curators", len(rt.registry.Names()))
|
|
}
|
|
}
|
|
for _, name := range rt.registry.Names() {
|
|
c, _ := rt.registry.Get(name)
|
|
rt.launch(ctx, name, c)
|
|
}
|
|
return startErr
|
|
}
|
|
|
|
// launch starts name's loop under ctx. Remove and Stop cancel it.
|
|
func (rt *Runtime) launch(ctx context.Context, name string, c CuratorEngine) {
|
|
rt.mu.Lock()
|
|
defer rt.mu.Unlock()
|
|
var wake chan struct{} // nil for non-input curators: the case never fires
|
|
if c.Manifest().OnInput {
|
|
wake = make(chan struct{}, defaultWakeBuffer)
|
|
rt.nudges[name] = wake
|
|
}
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
l := &loop{cancel: cancel, done: make(chan struct{})}
|
|
rt.loops[name] = l
|
|
go func() {
|
|
defer func() {
|
|
close(l.done)
|
|
rt.mu.Lock()
|
|
if rt.loops[name] == l {
|
|
delete(rt.loops, name)
|
|
}
|
|
rt.mu.Unlock()
|
|
}()
|
|
rt.runLoop(ctx, name, c, wake)
|
|
}()
|
|
}
|
|
|
|
// Add registers c and, when the runtime is running, starts it and its loop.
|
|
func (rt *Runtime) Add(ctx context.Context, c CuratorEngine) error {
|
|
rt.ops.Lock()
|
|
defer rt.ops.Unlock()
|
|
rt.mu.Lock()
|
|
_, draining := rt.loops[c.Name()]
|
|
started := rt.started
|
|
rt.mu.Unlock()
|
|
if draining {
|
|
return fmt.Errorf("%w: %s", errDraining, c.Name())
|
|
}
|
|
if err := rt.registry.Register(c); err != nil {
|
|
return err
|
|
}
|
|
if !started {
|
|
return nil
|
|
}
|
|
err := rt.registry.StartOne(ctx, c.Name())
|
|
// The loop belongs to the runtime, not to the call that added it.
|
|
rt.launch(context.WithoutCancel(ctx), c.Name(), c)
|
|
return err
|
|
}
|
|
|
|
// Remove cancels name's loop, including an in-flight pass, waits for it
|
|
// within ctx, and unregisters the curator. The curator is unregistered even
|
|
// when the wait times out; its loop then exits once the pass returns, and
|
|
// until it does the name cannot be added again.
|
|
func (rt *Runtime) Remove(ctx context.Context, name string) error {
|
|
rt.ops.Lock()
|
|
defer rt.ops.Unlock()
|
|
rt.mu.Lock()
|
|
l := rt.loops[name]
|
|
delete(rt.nudges, name)
|
|
rt.mu.Unlock()
|
|
|
|
var errs []error
|
|
if l != nil {
|
|
l.cancel()
|
|
if err := waitLoops(ctx, l); err != nil {
|
|
errs = append(errs, fmt.Errorf("curator %s: %w", name, err))
|
|
}
|
|
}
|
|
errs = append(errs, rt.registry.Unregister(ctx, name))
|
|
return errors.Join(errs...)
|
|
}
|
|
|
|
// waitLoops blocks until every loop has returned or ctx ends.
|
|
func waitLoops(ctx context.Context, loops ...*loop) error {
|
|
for _, l := range loops {
|
|
select {
|
|
case <-l.done:
|
|
case <-ctx.Done():
|
|
return fmt.Errorf("%w: %w", errDraining, ctx.Err())
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Nudge is a best-effort wake: input-driven curators check their trigger
|
|
// early instead of waiting for the next interval. A full channel or an
|
|
// unknown name drops the nudge — waking is an acceleration, never the
|
|
// source of truth. An empty curator name nudges every input-driven curator.
|
|
func (rt *Runtime) Nudge(curator string) {
|
|
rt.mu.Lock()
|
|
defer rt.mu.Unlock()
|
|
for name, ch := range rt.nudges {
|
|
if curator != "" && curator != name {
|
|
continue
|
|
}
|
|
select {
|
|
case ch <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
}
|
|
|
|
// Stop cancels every loop, waits for passes to finish within ctx, then stops
|
|
// the curators in reverse order. Loops that outlive ctx stay tracked, so a
|
|
// later Start refuses rather than running a second copy.
|
|
func (rt *Runtime) Stop(ctx context.Context) error {
|
|
rt.ops.Lock()
|
|
defer rt.ops.Unlock()
|
|
rt.mu.Lock()
|
|
if !rt.started {
|
|
rt.mu.Unlock()
|
|
return nil
|
|
}
|
|
rt.started = false
|
|
loops := slices.Collect(maps.Values(rt.loops))
|
|
for _, l := range loops {
|
|
l.cancel()
|
|
}
|
|
clear(rt.nudges)
|
|
rt.mu.Unlock()
|
|
|
|
if err := waitLoops(ctx, loops...); err != nil {
|
|
if log := rt.registry.Host().Log; log != nil {
|
|
log.ErrorContext(ctx, "curator runtime shutdown timed out", "err", err)
|
|
}
|
|
return fmt.Errorf("curator runtime shutdown: %w", errors.Join(err, rt.registry.Stop(ctx)))
|
|
}
|
|
err := rt.registry.Stop(ctx)
|
|
if log := rt.registry.Host().Log; log != nil {
|
|
if err != nil {
|
|
log.ErrorContext(ctx, "curator runtime stopped with failures", "err", err)
|
|
} else {
|
|
log.InfoContext(ctx, "curator runtime stopped")
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
|
|
// runLoop is one curator's lifetime: sleep until eligible or nudged, ask
|
|
// Check whether input is pending, run a bounded pass when it is, and back
|
|
// off to the cooldown after an idle or failed pass. Panics in Check and
|
|
// Pass are recovered here; the loop survives them.
|
|
func (rt *Runtime) runLoop(ctx context.Context, name string, c CuratorEngine, wakeCh <-chan struct{}) {
|
|
m := c.Manifest()
|
|
cooldown := m.Cooldown
|
|
if cooldown <= 0 {
|
|
cooldown = 2 * m.Interval
|
|
}
|
|
clock := rt.registry.Host().Clock
|
|
|
|
var next time.Time // zero means immediately eligible
|
|
var lastPass time.Time
|
|
|
|
for {
|
|
now := clock.Now()
|
|
reason := "check-in"
|
|
if !next.IsZero() && now.Before(next) {
|
|
woke, done := waitNext(ctx, wakeCh, clock.After(next.Sub(now)))
|
|
if done {
|
|
return
|
|
}
|
|
if woke {
|
|
reason = "wake"
|
|
}
|
|
}
|
|
|
|
due, err := rt.safeCheck(ctx, c)
|
|
if err != nil {
|
|
rt.fail(name, "check", err)
|
|
next = clock.Now().Add(cooldown)
|
|
continue
|
|
}
|
|
if !due {
|
|
next = clock.Now().Add(m.Interval)
|
|
if log := rt.registry.Host().Log; log != nil {
|
|
log.DebugContext(ctx, "curator not due", "curator", name, "next", next)
|
|
}
|
|
continue
|
|
}
|
|
|
|
if !rt.acquire(ctx) {
|
|
return // shutting down while waiting for a pass slot
|
|
}
|
|
passCtx, cancel := context.WithTimeout(ctx, rt.cfg.PassTimeout)
|
|
result, perr := rt.safePass(passCtx, c, PassInput{Reason: reason, Since: lastPass, LastPass: lastPass})
|
|
cancel()
|
|
rt.release()
|
|
|
|
now = clock.Now()
|
|
lastPass = now
|
|
if perr != nil {
|
|
rt.fail(name, "pass", perr)
|
|
next = now.Add(cooldown)
|
|
} else {
|
|
next = scheduleNext(now, m, cooldown, result)
|
|
rt.emitRun(name, now, result)
|
|
}
|
|
}
|
|
}
|
|
|
|
// waitNext blocks until a nudge, the sleep timer, or shutdown. woke means
|
|
// a nudge fired — re-check the trigger now; done means the context was
|
|
// cancelled — exit the loop.
|
|
func waitNext(ctx context.Context, wakeCh <-chan struct{}, timer <-chan time.Time) (woke, done bool) {
|
|
select {
|
|
case <-ctx.Done():
|
|
return false, true
|
|
case <-wakeCh:
|
|
return true, false
|
|
case <-timer:
|
|
return false, false
|
|
}
|
|
}
|
|
|
|
// scheduleNext computes the next eligible time after a successful pass. A
|
|
// curator-suggested NextCheckIn in the past (or immediate) would hot-spin
|
|
// the loop, so it is clamped to the minimum pass gap; an idle pass (no
|
|
// actions) backs off to the cooldown.
|
|
func scheduleNext(now time.Time, m Manifest, cooldown time.Duration, result PassResult) time.Time {
|
|
switch {
|
|
case !result.NextCheckIn.IsZero():
|
|
next := result.NextCheckIn
|
|
if earliest := now.Add(minPassGap); next.Before(earliest) {
|
|
next = earliest
|
|
}
|
|
return next
|
|
case len(result.Actions) == 0:
|
|
return now.Add(cooldown)
|
|
default:
|
|
return now.Add(m.Interval)
|
|
}
|
|
}
|
|
|
|
func (rt *Runtime) acquire(ctx context.Context) bool {
|
|
select {
|
|
case rt.sem <- struct{}{}:
|
|
return true
|
|
case <-ctx.Done():
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (rt *Runtime) release() { <-rt.sem }
|
|
|
|
func (rt *Runtime) safeCheck(ctx context.Context, c CuratorEngine) (due bool, err error) {
|
|
defer func() {
|
|
if p := recover(); p != nil {
|
|
err = fmt.Errorf("check panic: %v", p)
|
|
}
|
|
}()
|
|
return c.Check(ctx)
|
|
}
|
|
|
|
func (rt *Runtime) safePass(ctx context.Context, c CuratorEngine, in PassInput) (result PassResult, err error) {
|
|
defer func() {
|
|
if p := recover(); p != nil {
|
|
err = fmt.Errorf("pass panic: %v", p)
|
|
}
|
|
}()
|
|
return c.Pass(ctx, in)
|
|
}
|
|
|
|
// fail records a Check/Pass failure on the event stream. The event bus is
|
|
// non-blocking by contract (bounded dropping buffers), so even a failing
|
|
// curator cannot backpressure anything.
|
|
func (rt *Runtime) fail(name, phase string, err error) {
|
|
if log := rt.registry.Host().Log; log != nil {
|
|
log.Error("curator failed", "curator", name, "phase", phase, "err", err)
|
|
}
|
|
if sink := rt.registry.Host().Events; sink != nil {
|
|
sink.Emit(events.KindCuratorError, name, map[string]any{
|
|
"curator": name,
|
|
"phase": phase,
|
|
"err": err.Error(),
|
|
})
|
|
}
|
|
}
|
|
|
|
// emitRun publishes one run summary plus one event per action, so what ran,
|
|
// when, what it changed, and why all stay attributable.
|
|
func (rt *Runtime) emitRun(name string, at time.Time, result PassResult) {
|
|
rt.registry.RecordActivity(name, at, result.Actions)
|
|
|
|
host := rt.registry.Host()
|
|
if log := host.Log; log != nil {
|
|
log.Info("curator pass completed", "curator", name, "actions", len(result.Actions), "at", at)
|
|
for _, a := range result.Actions {
|
|
log.Info("curator action", "curator", name, "type", a.Type, "detail", a.Detail, "reason", a.Reason, "at", a.At)
|
|
}
|
|
}
|
|
|
|
sink := host.Events
|
|
if sink == nil {
|
|
return
|
|
}
|
|
sink.Emit(events.KindCuratorRun, name, map[string]any{
|
|
"curator": name,
|
|
"actions": len(result.Actions),
|
|
"at": at,
|
|
})
|
|
for _, a := range result.Actions {
|
|
sink.Emit(events.KindCuratorAction, name, map[string]any{
|
|
"curator": name,
|
|
"type": a.Type,
|
|
"detail": a.Detail,
|
|
"reason": a.Reason,
|
|
"at": a.At,
|
|
})
|
|
}
|
|
}
|
|
|
|
// ReplaceDefinitions applies a full set of data-defined curators: it
|
|
// validates the set first, then removes, adds or replaces only the curators
|
|
// that changed. Built-in engines, unchanged curators and the registry's
|
|
// observation surface keep running.
|
|
func (rt *Runtime) ReplaceDefinitions(ctx context.Context, definitions []Definition) error {
|
|
want := make(map[string]Definition, len(definitions))
|
|
candidate := NewRegistry(rt.registry.Host())
|
|
for _, def := range definitions {
|
|
if !def.Enabled {
|
|
continue
|
|
}
|
|
if held, ok := rt.registry.Get(def.Name); ok {
|
|
if _, custom := held.(*DefinitionEngine); !custom {
|
|
return fmt.Errorf("%w: %s", ErrDuplicate, def.Name)
|
|
}
|
|
}
|
|
if err := candidate.Register(NewDefinitionEngine(def)); err != nil {
|
|
return err
|
|
}
|
|
want[def.Name] = def
|
|
}
|
|
|
|
stopCtx, cancel := context.WithTimeout(ctx, defaultPassTimeout)
|
|
defer cancel()
|
|
var errs []error
|
|
for _, name := range rt.registry.Names() {
|
|
held, _ := rt.registry.Get(name)
|
|
engine, custom := held.(*DefinitionEngine)
|
|
if !custom {
|
|
continue
|
|
}
|
|
if def, keep := want[name]; keep && reflect.DeepEqual(def, engine.def) {
|
|
delete(want, name)
|
|
continue
|
|
}
|
|
if err := rt.Remove(stopCtx, name); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
for _, name := range slices.Sorted(maps.Keys(want)) {
|
|
if err := rt.Add(ctx, NewDefinitionEngine(want[name])); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
return errors.Join(errs...)
|
|
}
|