feat(curator): add and remove curators live; definition updates touch only changed curators

This commit is contained in:
sam committed 2026-10-25 16:45:34 +11:00
1 parent cdf87c2ade
commit 541dca807d
4 files changed
+261 -72

No files matched your search

-7
View File
@@ -118,10 +118,3 @@ func (r *Registry) filter(m Manifest) Registrar {
func agentic(m Manifest) bool {
return len(m.Tools) > 0 || m.Skills || m.MemoryEngine != "" || m.Conversations
}
func (r *Registry) replaceDefinitions(candidate *Registry) {
r.Replace(func(c CuratorEngine) bool {
_, custom := c.(*DefinitionEngine)
return custom
}, candidate.Registry)
}
+154 -40
View File
@@ -4,6 +4,9 @@ import (
"context"
"errors"
"fmt"
"maps"
"reflect"
"slices"
"sync"
"time"
@@ -33,17 +36,28 @@ type RuntimeConfig struct {
}
// Runtime runs one loop per registered curator, with per-pass timeouts and
// panic recovery. Wake events are best-effort.
// panic recovery. Wake events are best-effort. Curators may be added and
// removed while it runs.
type Runtime struct {
registry *Registry
cfg RuntimeConfig
sem chan struct{}
// ops serialises Start, Stop, Add and Remove; mu guards the fields below
// and is never held across a pass or an engine lifecycle call.
ops sync.Mutex
mu sync.Mutex
started bool
cancel context.CancelFunc
wg sync.WaitGroup
loops map[string]*loop
nudges map[string]chan struct{}
sem chan struct{}
}
// loop is one curator's running goroutine. It stays in Runtime.loops until
// the goroutine has returned, so a loop that outlives a bounded stop still
// blocks a second loop of the same name.
type loop struct {
cancel context.CancelFunc
done chan struct{}
}
// NewRuntime builds a runtime over the registry. The runtime shares the
@@ -59,31 +73,33 @@ func NewRuntime(registry *Registry, cfg RuntimeConfig) *Runtime {
return &Runtime{
registry: registry,
cfg: cfg,
loops: make(map[string]*loop),
nudges: make(map[string]chan struct{}),
sem: make(chan struct{}, cfg.MaxConcurrentPasses),
}
}
// errDraining reports a curator whose previous loop has not yet returned.
var errDraining = errors.New("curator loop still stopping")
// Start launches one loop per registered curator and starts the curators'
// lifecycle. A curator whose Start fails is skipped by the registry and
// reported in the joined error; every other curator still runs. Starting an
// already-started runtime is an error.
// already-started runtime, or one whose previous loops have not returned, is
// an error.
func (rt *Runtime) Start(ctx context.Context) error {
rt.ops.Lock()
defer rt.ops.Unlock()
rt.mu.Lock()
if rt.started {
rt.mu.Unlock()
return errors.New("curator runtime already started")
}
rt.started = true
loopCtx, cancel := context.WithCancel(ctx)
rt.cancel = cancel
rt.nudges = make(map[string]chan struct{})
for _, name := range rt.registry.Names() {
c, _ := rt.registry.Get(name)
if c.Manifest().OnInput {
rt.nudges[name] = make(chan struct{}, defaultWakeBuffer)
}
if n := len(rt.loops); n > 0 {
rt.mu.Unlock()
return fmt.Errorf("%w: %d from the previous run", errDraining, n)
}
rt.started = true
rt.mu.Unlock()
startErr := rt.registry.Start(ctx)
@@ -96,12 +112,94 @@ func (rt *Runtime) Start(ctx context.Context) error {
}
for _, name := range rt.registry.Names() {
c, _ := rt.registry.Get(name)
rt.wg.Add(1)
go rt.runLoop(loopCtx, name, c)
rt.launch(ctx, name, c)
}
return startErr
}
// launch starts name's loop under ctx. Remove and Stop cancel it.
func (rt *Runtime) launch(ctx context.Context, name string, c CuratorEngine) {
rt.mu.Lock()
defer rt.mu.Unlock()
var wake chan struct{} // nil for non-input curators: the case never fires
if c.Manifest().OnInput {
wake = make(chan struct{}, defaultWakeBuffer)
rt.nudges[name] = wake
}
ctx, cancel := context.WithCancel(ctx)
l := &loop{cancel: cancel, done: make(chan struct{})}
rt.loops[name] = l
go func() {
defer func() {
close(l.done)
rt.mu.Lock()
if rt.loops[name] == l {
delete(rt.loops, name)
}
rt.mu.Unlock()
}()
rt.runLoop(ctx, name, c, wake)
}()
}
// Add registers c and, when the runtime is running, starts it and its loop.
func (rt *Runtime) Add(ctx context.Context, c CuratorEngine) error {
rt.ops.Lock()
defer rt.ops.Unlock()
rt.mu.Lock()
_, draining := rt.loops[c.Name()]
started := rt.started
rt.mu.Unlock()
if draining {
return fmt.Errorf("%w: %s", errDraining, c.Name())
}
if err := rt.registry.Register(c); err != nil {
return err
}
if !started {
return nil
}
err := rt.registry.StartOne(ctx, c.Name())
// The loop belongs to the runtime, not to the call that added it.
rt.launch(context.WithoutCancel(ctx), c.Name(), c)
return err
}
// Remove cancels name's loop, including an in-flight pass, waits for it
// within ctx, and unregisters the curator. The curator is unregistered even
// when the wait times out; its loop then exits once the pass returns, and
// until it does the name cannot be added again.
func (rt *Runtime) Remove(ctx context.Context, name string) error {
rt.ops.Lock()
defer rt.ops.Unlock()
rt.mu.Lock()
l := rt.loops[name]
delete(rt.nudges, name)
rt.mu.Unlock()
var errs []error
if l != nil {
l.cancel()
if err := waitLoops(ctx, l); err != nil {
errs = append(errs, fmt.Errorf("curator %s: %w", name, err))
}
}
errs = append(errs, rt.registry.Unregister(ctx, name))
return errors.Join(errs...)
}
// waitLoops blocks until every loop has returned or ctx ends.
func waitLoops(ctx context.Context, loops ...*loop) error {
for _, l := range loops {
select {
case <-l.done:
case <-ctx.Done():
return fmt.Errorf("%w: %w", errDraining, ctx.Err())
}
}
return nil
}
// Nudge is a best-effort wake: input-driven curators check their trigger
// early instead of waiting for the next interval. A full channel or an
// unknown name drops the nudge — waking is an acceleration, never the
@@ -121,31 +219,29 @@ func (rt *Runtime) Nudge(curator string) {
}
// Stop cancels every loop, waits for passes to finish within ctx, then stops
// the curators in reverse order.
// the curators in reverse order. Loops that outlive ctx stay tracked, so a
// later Start refuses rather than running a second copy.
func (rt *Runtime) Stop(ctx context.Context) error {
rt.ops.Lock()
defer rt.ops.Unlock()
rt.mu.Lock()
if !rt.started {
rt.mu.Unlock()
return nil
}
rt.started = false
cancel := rt.cancel
rt.cancel = nil
loops := slices.Collect(maps.Values(rt.loops))
for _, l := range loops {
l.cancel()
}
clear(rt.nudges)
rt.mu.Unlock()
cancel()
done := make(chan struct{})
go func() {
rt.wg.Wait()
close(done)
}()
select {
case <-done:
case <-ctx.Done():
if err := waitLoops(ctx, loops...); err != nil {
if log := rt.registry.Host().Log; log != nil {
log.Error("curator runtime shutdown timed out", "err", ctx.Err())
log.Error("curator runtime shutdown timed out", "err", err)
}
return fmt.Errorf("curator runtime shutdown: %w", ctx.Err())
return fmt.Errorf("curator runtime shutdown: %w", errors.Join(err, rt.registry.Stop(ctx)))
}
err := rt.registry.Stop(ctx)
if log := rt.registry.Host().Log; log != nil {
@@ -162,16 +258,13 @@ func (rt *Runtime) Stop(ctx context.Context) error {
// Check whether input is pending, run a bounded pass when it is, and back
// off to the cooldown after an idle or failed pass. Panics in Check and
// Pass are recovered here; the loop survives them.
func (rt *Runtime) runLoop(ctx context.Context, name string, c CuratorEngine) {
defer rt.wg.Done()
func (rt *Runtime) runLoop(ctx context.Context, name string, c CuratorEngine, wakeCh <-chan struct{}) {
m := c.Manifest()
cooldown := m.Cooldown
if cooldown <= 0 {
cooldown = 2 * m.Interval
}
clock := rt.registry.Host().Clock
wakeCh := rt.nudges[name] // nil for non-input curators: the case never fires
var next time.Time // zero means immediately eligible
var lastPass time.Time
@@ -334,9 +427,12 @@ func (rt *Runtime) emitRun(name string, at time.Time, result PassResult) {
}
}
// ReplaceDefinitions drains current passes before replacing data-defined
// curators. Built-in engines and the registry's observation surface survive.
// ReplaceDefinitions applies a full set of data-defined curators: it
// validates the set first, then removes, adds or replaces only the curators
// that changed. Built-in engines, unchanged curators and the registry's
// observation surface keep running.
func (rt *Runtime) ReplaceDefinitions(ctx context.Context, definitions []Definition) error {
want := make(map[string]Definition, len(definitions))
candidate := NewRegistry(rt.registry.Host())
for _, def := range definitions {
if !def.Enabled {
@@ -350,12 +446,30 @@ func (rt *Runtime) ReplaceDefinitions(ctx context.Context, definitions []Definit
if err := candidate.Register(NewDefinitionEngine(def)); err != nil {
return err
}
want[def.Name] = def
}
stopCtx, cancel := context.WithTimeout(ctx, defaultPassTimeout)
defer cancel()
if err := rt.Stop(stopCtx); err != nil {
return err
var errs []error
for _, name := range rt.registry.Names() {
held, _ := rt.registry.Get(name)
engine, custom := held.(*DefinitionEngine)
if !custom {
continue
}
if def, keep := want[name]; keep && reflect.DeepEqual(def, engine.def) {
delete(want, name)
continue
}
if err := rt.Remove(stopCtx, name); err != nil {
errs = append(errs, err)
}
}
rt.registry.replaceDefinitions(candidate)
return rt.Start(ctx)
for _, name := range slices.Sorted(maps.Keys(want)) {
if err := rt.Add(ctx, NewDefinitionEngine(want[name])); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
+62
View File
@@ -41,3 +41,65 @@ func TestReplaceLiveDefinitions(t *testing.T) {
})
}
}
// blockingEngine's Pass blocks until release closes, honouring its context
// only when obey is set.
type blockingEngine struct {
DefinitionEngine
obey bool
entered chan struct{}
release chan struct{}
}
func (e *blockingEngine) Pass(ctx context.Context, _ PassInput) (PassResult, error) {
close(e.entered)
if e.obey {
<-ctx.Done()
return PassResult{}, ctx.Err()
}
<-e.release
return PassResult{}, nil
}
func TestRemoveInFlightPass(t *testing.T) {
for _, tt := range []struct {
name string
obey bool
wantErr bool
readdErr bool
}{
{"pass honours cancellation", true, false, false},
{"pass ignores cancellation", false, true, true},
} {
t.Run(tt.name, func(t *testing.T) {
runtime := NewRuntime(NewRegistry(Registrar{}), RuntimeConfig{})
if err := runtime.Start(t.Context()); err != nil {
t.Fatal(err)
}
e := &blockingEngine{
def: Definition{Name: "c", Manifest: Manifest{Interval: time.Hour}},
obey: tt.obey, entered: make(chan struct{}), release: make(chan struct{}),
}
if err := runtime.Add(t.Context(), e); err != nil {
t.Fatal(err)
}
<-e.entered
ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
defer cancel()
if err := runtime.Remove(ctx, "c"); (err != nil) != tt.wantErr {
t.Fatalf("Remove error = %v", err)
}
if _, ok := runtime.registry.Get("c"); ok {
t.Fatal("still registered after Remove")
}
if err := runtime.Add(t.Context(), NewDefinitionEngine(e.def)); (err != nil) != tt.readdErr {
t.Fatalf("re-add error = %v", err)
}
close(e.release)
if err := runtime.Stop(context.Background()); err != nil {
t.Fatal(err)
}
})
}
}
+45 -25
View File
@@ -32,8 +32,7 @@ type Health struct {
var (
// ErrDuplicate is returned when an engine name is already registered.
ErrDuplicate = errors.New("duplicate engine")
// ErrStarted is returned when registration or lifecycle is attempted
// after the registry has started.
// ErrStarted is returned when a running registry is started again.
ErrStarted = errors.New("registry already started")
)
@@ -94,13 +93,11 @@ func IsNil(v any) bool {
return false
}
// Add binds e to host and registers it. A panicking Bind refuses the engine.
// Add binds e to host and registers it, unstarted. A panicking Bind refuses
// the engine. Adding to a running registry is allowed; StartOne starts it.
func (r *Registry[E, H]) Add(e E, host H) error {
r.mu.Lock()
defer r.mu.Unlock()
if r.state != stateIdle {
return fmt.Errorf("%s: %w", r.kind, ErrStarted)
}
name := e.Name()
if _, dup := r.engines[name]; dup {
return fmt.Errorf("%w: %s %s", ErrDuplicate, r.kind, name)
@@ -113,6 +110,48 @@ func (r *Registry[E, H]) Add(e E, host H) error {
return nil
}
// StartOne starts the named engine when the registry is running and the
// engine has not started; otherwise it does nothing. A failed Start marks the
// engine unhealthy, as Start does.
func (r *Registry[E, H]) StartOne(ctx context.Context, name string) error {
r.mu.Lock()
e, ok := r.engines[name]
if !ok || r.state != stateRunning || r.status[name].started {
r.mu.Unlock()
return nil
}
r.mu.Unlock()
err := guard(func() error { return e.Start(ctx) })
r.mu.Lock()
defer r.mu.Unlock()
if err != nil {
r.status[name] = status{startErr: err}
return r.fail(name, "start", err)
}
r.status[name] = status{started: true}
return nil
}
// Unregister removes the named engine, stopping it first when it started.
// The engine is removed even when its Stop fails; the failure is returned.
func (r *Registry[E, H]) Unregister(ctx context.Context, name string) error {
r.mu.Lock()
e, ok := r.engines[name]
st := r.status[name]
delete(r.engines, name)
delete(r.status, name)
r.order = slices.DeleteFunc(r.order, func(n string) bool { return n == name })
r.mu.Unlock()
if !ok || !st.started {
return nil
}
if err := guard(func() error { return e.Stop(ctx) }); err != nil {
return r.fail(name, "stop", err)
}
return nil
}
// Get returns the registered engine by name.
func (r *Registry[E, H]) Get(name string) (E, bool) {
r.mu.Lock()
@@ -201,25 +240,6 @@ func (r *Registry[E, H]) Stop(ctx context.Context) error {
return errors.Join(errs...)
}
// Replace removes the engines drop selects and appends every engine from
// src, keeping src's order. Call it only while stopped.
func (r *Registry[E, H]) Replace(drop func(E) bool, src *Registry[E, H]) {
r.mu.Lock()
defer r.mu.Unlock()
r.order = slices.DeleteFunc(r.order, func(name string) bool {
if !drop(r.engines[name]) {
return false
}
delete(r.engines, name)
delete(r.status, name)
return true
})
for _, name := range src.order {
r.engines[name] = src.engines[name]
r.order = append(r.order, name)
}
}
func (r *Registry[E, H]) fail(name, phase string, err error) error {
if r.log != nil {
r.log.Error(r.kind+" lifecycle failed", "name", name, "phase", phase, "err", err)