feat(curator): add and remove curators live; definition updates touch only changed curators
This commit is contained in:
1 parent
cdf87c2ade
commit
541dca807d
4 files changed
+261
-72
No files matched your search
@@ -118,10 +118,3 @@ func (r *Registry) filter(m Manifest) Registrar {
|
||||
func agentic(m Manifest) bool {
|
||||
return len(m.Tools) > 0 || m.Skills || m.MemoryEngine != "" || m.Conversations
|
||||
}
|
||||
|
||||
func (r *Registry) replaceDefinitions(candidate *Registry) {
|
||||
r.Replace(func(c CuratorEngine) bool {
|
||||
_, custom := c.(*DefinitionEngine)
|
||||
return custom
|
||||
}, candidate.Registry)
|
||||
}
|
||||
@@ -4,6 +4,9 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -33,17 +36,28 @@ type RuntimeConfig struct {
|
||||
}
|
||||
|
||||
// Runtime runs one loop per registered curator, with per-pass timeouts and
|
||||
// panic recovery. Wake events are best-effort.
|
||||
// panic recovery. Wake events are best-effort. Curators may be added and
|
||||
// removed while it runs.
|
||||
type Runtime struct {
|
||||
registry *Registry
|
||||
cfg RuntimeConfig
|
||||
sem chan struct{}
|
||||
|
||||
// ops serialises Start, Stop, Add and Remove; mu guards the fields below
|
||||
// and is never held across a pass or an engine lifecycle call.
|
||||
ops sync.Mutex
|
||||
mu sync.Mutex
|
||||
started bool
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
loops map[string]*loop
|
||||
nudges map[string]chan struct{}
|
||||
sem chan struct{}
|
||||
}
|
||||
|
||||
// loop is one curator's running goroutine. It stays in Runtime.loops until
|
||||
// the goroutine has returned, so a loop that outlives a bounded stop still
|
||||
// blocks a second loop of the same name.
|
||||
type loop struct {
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// NewRuntime builds a runtime over the registry. The runtime shares the
|
||||
@@ -59,31 +73,33 @@ func NewRuntime(registry *Registry, cfg RuntimeConfig) *Runtime {
|
||||
return &Runtime{
|
||||
registry: registry,
|
||||
cfg: cfg,
|
||||
loops: make(map[string]*loop),
|
||||
nudges: make(map[string]chan struct{}),
|
||||
sem: make(chan struct{}, cfg.MaxConcurrentPasses),
|
||||
}
|
||||
}
|
||||
|
||||
// errDraining reports a curator whose previous loop has not yet returned.
|
||||
var errDraining = errors.New("curator loop still stopping")
|
||||
|
||||
// Start launches one loop per registered curator and starts the curators'
|
||||
// lifecycle. A curator whose Start fails is skipped by the registry and
|
||||
// reported in the joined error; every other curator still runs. Starting an
|
||||
// already-started runtime is an error.
|
||||
// already-started runtime, or one whose previous loops have not returned, is
|
||||
// an error.
|
||||
func (rt *Runtime) Start(ctx context.Context) error {
|
||||
rt.ops.Lock()
|
||||
defer rt.ops.Unlock()
|
||||
rt.mu.Lock()
|
||||
if rt.started {
|
||||
rt.mu.Unlock()
|
||||
return errors.New("curator runtime already started")
|
||||
}
|
||||
rt.started = true
|
||||
loopCtx, cancel := context.WithCancel(ctx)
|
||||
rt.cancel = cancel
|
||||
rt.nudges = make(map[string]chan struct{})
|
||||
for _, name := range rt.registry.Names() {
|
||||
c, _ := rt.registry.Get(name)
|
||||
if c.Manifest().OnInput {
|
||||
rt.nudges[name] = make(chan struct{}, defaultWakeBuffer)
|
||||
}
|
||||
if n := len(rt.loops); n > 0 {
|
||||
rt.mu.Unlock()
|
||||
return fmt.Errorf("%w: %d from the previous run", errDraining, n)
|
||||
}
|
||||
rt.started = true
|
||||
rt.mu.Unlock()
|
||||
|
||||
startErr := rt.registry.Start(ctx)
|
||||
@@ -96,12 +112,94 @@ func (rt *Runtime) Start(ctx context.Context) error {
|
||||
}
|
||||
for _, name := range rt.registry.Names() {
|
||||
c, _ := rt.registry.Get(name)
|
||||
rt.wg.Add(1)
|
||||
go rt.runLoop(loopCtx, name, c)
|
||||
rt.launch(ctx, name, c)
|
||||
}
|
||||
return startErr
|
||||
}
|
||||
|
||||
// launch starts name's loop under ctx. Remove and Stop cancel it.
|
||||
func (rt *Runtime) launch(ctx context.Context, name string, c CuratorEngine) {
|
||||
rt.mu.Lock()
|
||||
defer rt.mu.Unlock()
|
||||
var wake chan struct{} // nil for non-input curators: the case never fires
|
||||
if c.Manifest().OnInput {
|
||||
wake = make(chan struct{}, defaultWakeBuffer)
|
||||
rt.nudges[name] = wake
|
||||
}
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
l := &loop{cancel: cancel, done: make(chan struct{})}
|
||||
rt.loops[name] = l
|
||||
go func() {
|
||||
defer func() {
|
||||
close(l.done)
|
||||
rt.mu.Lock()
|
||||
if rt.loops[name] == l {
|
||||
delete(rt.loops, name)
|
||||
}
|
||||
rt.mu.Unlock()
|
||||
}()
|
||||
rt.runLoop(ctx, name, c, wake)
|
||||
}()
|
||||
}
|
||||
|
||||
// Add registers c and, when the runtime is running, starts it and its loop.
|
||||
func (rt *Runtime) Add(ctx context.Context, c CuratorEngine) error {
|
||||
rt.ops.Lock()
|
||||
defer rt.ops.Unlock()
|
||||
rt.mu.Lock()
|
||||
_, draining := rt.loops[c.Name()]
|
||||
started := rt.started
|
||||
rt.mu.Unlock()
|
||||
if draining {
|
||||
return fmt.Errorf("%w: %s", errDraining, c.Name())
|
||||
}
|
||||
if err := rt.registry.Register(c); err != nil {
|
||||
return err
|
||||
}
|
||||
if !started {
|
||||
return nil
|
||||
}
|
||||
err := rt.registry.StartOne(ctx, c.Name())
|
||||
// The loop belongs to the runtime, not to the call that added it.
|
||||
rt.launch(context.WithoutCancel(ctx), c.Name(), c)
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove cancels name's loop, including an in-flight pass, waits for it
|
||||
// within ctx, and unregisters the curator. The curator is unregistered even
|
||||
// when the wait times out; its loop then exits once the pass returns, and
|
||||
// until it does the name cannot be added again.
|
||||
func (rt *Runtime) Remove(ctx context.Context, name string) error {
|
||||
rt.ops.Lock()
|
||||
defer rt.ops.Unlock()
|
||||
rt.mu.Lock()
|
||||
l := rt.loops[name]
|
||||
delete(rt.nudges, name)
|
||||
rt.mu.Unlock()
|
||||
|
||||
var errs []error
|
||||
if l != nil {
|
||||
l.cancel()
|
||||
if err := waitLoops(ctx, l); err != nil {
|
||||
errs = append(errs, fmt.Errorf("curator %s: %w", name, err))
|
||||
}
|
||||
}
|
||||
errs = append(errs, rt.registry.Unregister(ctx, name))
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// waitLoops blocks until every loop has returned or ctx ends.
|
||||
func waitLoops(ctx context.Context, loops ...*loop) error {
|
||||
for _, l := range loops {
|
||||
select {
|
||||
case <-l.done:
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("%w: %w", errDraining, ctx.Err())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Nudge is a best-effort wake: input-driven curators check their trigger
|
||||
// early instead of waiting for the next interval. A full channel or an
|
||||
// unknown name drops the nudge — waking is an acceleration, never the
|
||||
@@ -121,31 +219,29 @@ func (rt *Runtime) Nudge(curator string) {
|
||||
}
|
||||
|
||||
// Stop cancels every loop, waits for passes to finish within ctx, then stops
|
||||
// the curators in reverse order.
|
||||
// the curators in reverse order. Loops that outlive ctx stay tracked, so a
|
||||
// later Start refuses rather than running a second copy.
|
||||
func (rt *Runtime) Stop(ctx context.Context) error {
|
||||
rt.ops.Lock()
|
||||
defer rt.ops.Unlock()
|
||||
rt.mu.Lock()
|
||||
if !rt.started {
|
||||
rt.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
rt.started = false
|
||||
cancel := rt.cancel
|
||||
rt.cancel = nil
|
||||
loops := slices.Collect(maps.Values(rt.loops))
|
||||
for _, l := range loops {
|
||||
l.cancel()
|
||||
}
|
||||
clear(rt.nudges)
|
||||
rt.mu.Unlock()
|
||||
|
||||
cancel()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
rt.wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
if err := waitLoops(ctx, loops...); err != nil {
|
||||
if log := rt.registry.Host().Log; log != nil {
|
||||
log.Error("curator runtime shutdown timed out", "err", ctx.Err())
|
||||
log.Error("curator runtime shutdown timed out", "err", err)
|
||||
}
|
||||
return fmt.Errorf("curator runtime shutdown: %w", ctx.Err())
|
||||
return fmt.Errorf("curator runtime shutdown: %w", errors.Join(err, rt.registry.Stop(ctx)))
|
||||
}
|
||||
err := rt.registry.Stop(ctx)
|
||||
if log := rt.registry.Host().Log; log != nil {
|
||||
@@ -162,16 +258,13 @@ func (rt *Runtime) Stop(ctx context.Context) error {
|
||||
// Check whether input is pending, run a bounded pass when it is, and back
|
||||
// off to the cooldown after an idle or failed pass. Panics in Check and
|
||||
// Pass are recovered here; the loop survives them.
|
||||
func (rt *Runtime) runLoop(ctx context.Context, name string, c CuratorEngine) {
|
||||
defer rt.wg.Done()
|
||||
|
||||
func (rt *Runtime) runLoop(ctx context.Context, name string, c CuratorEngine, wakeCh <-chan struct{}) {
|
||||
m := c.Manifest()
|
||||
cooldown := m.Cooldown
|
||||
if cooldown <= 0 {
|
||||
cooldown = 2 * m.Interval
|
||||
}
|
||||
clock := rt.registry.Host().Clock
|
||||
wakeCh := rt.nudges[name] // nil for non-input curators: the case never fires
|
||||
|
||||
var next time.Time // zero means immediately eligible
|
||||
var lastPass time.Time
|
||||
@@ -334,9 +427,12 @@ func (rt *Runtime) emitRun(name string, at time.Time, result PassResult) {
|
||||
}
|
||||
}
|
||||
|
||||
// ReplaceDefinitions drains current passes before replacing data-defined
|
||||
// curators. Built-in engines and the registry's observation surface survive.
|
||||
// ReplaceDefinitions applies a full set of data-defined curators: it
|
||||
// validates the set first, then removes, adds or replaces only the curators
|
||||
// that changed. Built-in engines, unchanged curators and the registry's
|
||||
// observation surface keep running.
|
||||
func (rt *Runtime) ReplaceDefinitions(ctx context.Context, definitions []Definition) error {
|
||||
want := make(map[string]Definition, len(definitions))
|
||||
candidate := NewRegistry(rt.registry.Host())
|
||||
for _, def := range definitions {
|
||||
if !def.Enabled {
|
||||
@@ -350,12 +446,30 @@ func (rt *Runtime) ReplaceDefinitions(ctx context.Context, definitions []Definit
|
||||
if err := candidate.Register(NewDefinitionEngine(def)); err != nil {
|
||||
return err
|
||||
}
|
||||
want[def.Name] = def
|
||||
}
|
||||
|
||||
stopCtx, cancel := context.WithTimeout(ctx, defaultPassTimeout)
|
||||
defer cancel()
|
||||
if err := rt.Stop(stopCtx); err != nil {
|
||||
return err
|
||||
var errs []error
|
||||
for _, name := range rt.registry.Names() {
|
||||
held, _ := rt.registry.Get(name)
|
||||
engine, custom := held.(*DefinitionEngine)
|
||||
if !custom {
|
||||
continue
|
||||
}
|
||||
if def, keep := want[name]; keep && reflect.DeepEqual(def, engine.def) {
|
||||
delete(want, name)
|
||||
continue
|
||||
}
|
||||
if err := rt.Remove(stopCtx, name); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
rt.registry.replaceDefinitions(candidate)
|
||||
return rt.Start(ctx)
|
||||
for _, name := range slices.Sorted(maps.Keys(want)) {
|
||||
if err := rt.Add(ctx, NewDefinitionEngine(want[name])); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
@@ -41,3 +41,65 @@ func TestReplaceLiveDefinitions(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// blockingEngine's Pass blocks until release closes, honouring its context
|
||||
// only when obey is set.
|
||||
type blockingEngine struct {
|
||||
DefinitionEngine
|
||||
obey bool
|
||||
entered chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (e *blockingEngine) Pass(ctx context.Context, _ PassInput) (PassResult, error) {
|
||||
close(e.entered)
|
||||
if e.obey {
|
||||
<-ctx.Done()
|
||||
return PassResult{}, ctx.Err()
|
||||
}
|
||||
<-e.release
|
||||
return PassResult{}, nil
|
||||
}
|
||||
|
||||
func TestRemoveInFlightPass(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
obey bool
|
||||
wantErr bool
|
||||
readdErr bool
|
||||
}{
|
||||
{"pass honours cancellation", true, false, false},
|
||||
{"pass ignores cancellation", false, true, true},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
runtime := NewRuntime(NewRegistry(Registrar{}), RuntimeConfig{})
|
||||
if err := runtime.Start(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := &blockingEngine{
|
||||
def: Definition{Name: "c", Manifest: Manifest{Interval: time.Hour}},
|
||||
obey: tt.obey, entered: make(chan struct{}), release: make(chan struct{}),
|
||||
}
|
||||
if err := runtime.Add(t.Context(), e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
<-e.entered
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := runtime.Remove(ctx, "c"); (err != nil) != tt.wantErr {
|
||||
t.Fatalf("Remove error = %v", err)
|
||||
}
|
||||
if _, ok := runtime.registry.Get("c"); ok {
|
||||
t.Fatal("still registered after Remove")
|
||||
}
|
||||
if err := runtime.Add(t.Context(), NewDefinitionEngine(e.def)); (err != nil) != tt.readdErr {
|
||||
t.Fatalf("re-add error = %v", err)
|
||||
}
|
||||
close(e.release)
|
||||
if err := runtime.Stop(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -32,8 +32,7 @@ type Health struct {
|
||||
var (
|
||||
// ErrDuplicate is returned when an engine name is already registered.
|
||||
ErrDuplicate = errors.New("duplicate engine")
|
||||
// ErrStarted is returned when registration or lifecycle is attempted
|
||||
// after the registry has started.
|
||||
// ErrStarted is returned when a running registry is started again.
|
||||
ErrStarted = errors.New("registry already started")
|
||||
)
|
||||
|
||||
@@ -94,13 +93,11 @@ func IsNil(v any) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Add binds e to host and registers it. A panicking Bind refuses the engine.
|
||||
// Add binds e to host and registers it, unstarted. A panicking Bind refuses
|
||||
// the engine. Adding to a running registry is allowed; StartOne starts it.
|
||||
func (r *Registry[E, H]) Add(e E, host H) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.state != stateIdle {
|
||||
return fmt.Errorf("%s: %w", r.kind, ErrStarted)
|
||||
}
|
||||
name := e.Name()
|
||||
if _, dup := r.engines[name]; dup {
|
||||
return fmt.Errorf("%w: %s %s", ErrDuplicate, r.kind, name)
|
||||
@@ -113,6 +110,48 @@ func (r *Registry[E, H]) Add(e E, host H) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// StartOne starts the named engine when the registry is running and the
|
||||
// engine has not started; otherwise it does nothing. A failed Start marks the
|
||||
// engine unhealthy, as Start does.
|
||||
func (r *Registry[E, H]) StartOne(ctx context.Context, name string) error {
|
||||
r.mu.Lock()
|
||||
e, ok := r.engines[name]
|
||||
if !ok || r.state != stateRunning || r.status[name].started {
|
||||
r.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
err := guard(func() error { return e.Start(ctx) })
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if err != nil {
|
||||
r.status[name] = status{startErr: err}
|
||||
return r.fail(name, "start", err)
|
||||
}
|
||||
r.status[name] = status{started: true}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unregister removes the named engine, stopping it first when it started.
|
||||
// The engine is removed even when its Stop fails; the failure is returned.
|
||||
func (r *Registry[E, H]) Unregister(ctx context.Context, name string) error {
|
||||
r.mu.Lock()
|
||||
e, ok := r.engines[name]
|
||||
st := r.status[name]
|
||||
delete(r.engines, name)
|
||||
delete(r.status, name)
|
||||
r.order = slices.DeleteFunc(r.order, func(n string) bool { return n == name })
|
||||
r.mu.Unlock()
|
||||
if !ok || !st.started {
|
||||
return nil
|
||||
}
|
||||
if err := guard(func() error { return e.Stop(ctx) }); err != nil {
|
||||
return r.fail(name, "stop", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get returns the registered engine by name.
|
||||
func (r *Registry[E, H]) Get(name string) (E, bool) {
|
||||
r.mu.Lock()
|
||||
@@ -201,25 +240,6 @@ func (r *Registry[E, H]) Stop(ctx context.Context) error {
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// Replace removes the engines drop selects and appends every engine from
|
||||
// src, keeping src's order. Call it only while stopped.
|
||||
func (r *Registry[E, H]) Replace(drop func(E) bool, src *Registry[E, H]) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.order = slices.DeleteFunc(r.order, func(name string) bool {
|
||||
if !drop(r.engines[name]) {
|
||||
return false
|
||||
}
|
||||
delete(r.engines, name)
|
||||
delete(r.status, name)
|
||||
return true
|
||||
})
|
||||
for _, name := range src.order {
|
||||
r.engines[name] = src.engines[name]
|
||||
r.order = append(r.order, name)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry[E, H]) fail(name, phase string, err error) error {
|
||||
if r.log != nil {
|
||||
r.log.Error(r.kind+" lifecycle failed", "name", name, "phase", phase, "err", err)
|
||||
|
||||
Reference in new issue
Block a user