diff --git a/internal/domain/curator/registry.go b/internal/domain/curator/registry.go index cbddfcae5..a6e160187 100644 --- a/internal/domain/curator/registry.go +++ b/internal/domain/curator/registry.go @@ -118,10 +118,3 @@ func (r *Registry) filter(m Manifest) Registrar { func agentic(m Manifest) bool { return len(m.Tools) > 0 || m.Skills || m.MemoryEngine != "" || m.Conversations } - -func (r *Registry) replaceDefinitions(candidate *Registry) { - r.Replace(func(c CuratorEngine) bool { - _, custom := c.(*DefinitionEngine) - return custom - }, candidate.Registry) -} diff --git a/internal/domain/curator/runtime.go b/internal/domain/curator/runtime.go index f4b13d25d..371d8fb53 100644 --- a/internal/domain/curator/runtime.go +++ b/internal/domain/curator/runtime.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + "maps" + "reflect" + "slices" "sync" "time" @@ -33,17 +36,28 @@ type RuntimeConfig struct { } // Runtime runs one loop per registered curator, with per-pass timeouts and -// panic recovery. Wake events are best-effort. +// panic recovery. Wake events are best-effort. Curators may be added and +// removed while it runs. type Runtime struct { registry *Registry cfg RuntimeConfig + sem chan struct{} + // ops serialises Start, Stop, Add and Remove; mu guards the fields below + // and is never held across a pass or an engine lifecycle call. + ops sync.Mutex mu sync.Mutex started bool - cancel context.CancelFunc - wg sync.WaitGroup + loops map[string]*loop nudges map[string]chan struct{} - sem chan struct{} +} + +// loop is one curator's running goroutine. It stays in Runtime.loops until +// the goroutine has returned, so a loop that outlives a bounded stop still +// blocks a second loop of the same name. +type loop struct { + cancel context.CancelFunc + done chan struct{} } // NewRuntime builds a runtime over the registry. The runtime shares the @@ -59,31 +73,33 @@ func NewRuntime(registry *Registry, cfg RuntimeConfig) *Runtime { return &Runtime{ registry: registry, cfg: cfg, + loops: make(map[string]*loop), nudges: make(map[string]chan struct{}), sem: make(chan struct{}, cfg.MaxConcurrentPasses), } } +// errDraining reports a curator whose previous loop has not yet returned. +var errDraining = errors.New("curator loop still stopping") + // Start launches one loop per registered curator and starts the curators' // lifecycle. A curator whose Start fails is skipped by the registry and // reported in the joined error; every other curator still runs. Starting an -// already-started runtime is an error. +// already-started runtime, or one whose previous loops have not returned, is +// an error. func (rt *Runtime) Start(ctx context.Context) error { + rt.ops.Lock() + defer rt.ops.Unlock() rt.mu.Lock() if rt.started { rt.mu.Unlock() return errors.New("curator runtime already started") } - rt.started = true - loopCtx, cancel := context.WithCancel(ctx) - rt.cancel = cancel - rt.nudges = make(map[string]chan struct{}) - for _, name := range rt.registry.Names() { - c, _ := rt.registry.Get(name) - if c.Manifest().OnInput { - rt.nudges[name] = make(chan struct{}, defaultWakeBuffer) - } + if n := len(rt.loops); n > 0 { + rt.mu.Unlock() + return fmt.Errorf("%w: %d from the previous run", errDraining, n) } + rt.started = true rt.mu.Unlock() startErr := rt.registry.Start(ctx) @@ -96,12 +112,94 @@ func (rt *Runtime) Start(ctx context.Context) error { } for _, name := range rt.registry.Names() { c, _ := rt.registry.Get(name) - rt.wg.Add(1) - go rt.runLoop(loopCtx, name, c) + rt.launch(ctx, name, c) } return startErr } +// launch starts name's loop under ctx. Remove and Stop cancel it. +func (rt *Runtime) launch(ctx context.Context, name string, c CuratorEngine) { + rt.mu.Lock() + defer rt.mu.Unlock() + var wake chan struct{} // nil for non-input curators: the case never fires + if c.Manifest().OnInput { + wake = make(chan struct{}, defaultWakeBuffer) + rt.nudges[name] = wake + } + ctx, cancel := context.WithCancel(ctx) + l := &loop{cancel: cancel, done: make(chan struct{})} + rt.loops[name] = l + go func() { + defer func() { + close(l.done) + rt.mu.Lock() + if rt.loops[name] == l { + delete(rt.loops, name) + } + rt.mu.Unlock() + }() + rt.runLoop(ctx, name, c, wake) + }() +} + +// Add registers c and, when the runtime is running, starts it and its loop. +func (rt *Runtime) Add(ctx context.Context, c CuratorEngine) error { + rt.ops.Lock() + defer rt.ops.Unlock() + rt.mu.Lock() + _, draining := rt.loops[c.Name()] + started := rt.started + rt.mu.Unlock() + if draining { + return fmt.Errorf("%w: %s", errDraining, c.Name()) + } + if err := rt.registry.Register(c); err != nil { + return err + } + if !started { + return nil + } + err := rt.registry.StartOne(ctx, c.Name()) + // The loop belongs to the runtime, not to the call that added it. + rt.launch(context.WithoutCancel(ctx), c.Name(), c) + return err +} + +// Remove cancels name's loop, including an in-flight pass, waits for it +// within ctx, and unregisters the curator. The curator is unregistered even +// when the wait times out; its loop then exits once the pass returns, and +// until it does the name cannot be added again. +func (rt *Runtime) Remove(ctx context.Context, name string) error { + rt.ops.Lock() + defer rt.ops.Unlock() + rt.mu.Lock() + l := rt.loops[name] + delete(rt.nudges, name) + rt.mu.Unlock() + + var errs []error + if l != nil { + l.cancel() + if err := waitLoops(ctx, l); err != nil { + errs = append(errs, fmt.Errorf("curator %s: %w", name, err)) + } + } + errs = append(errs, rt.registry.Unregister(ctx, name)) + return errors.Join(errs...) +} + +// waitLoops blocks until every loop has returned or ctx ends. +func waitLoops(ctx context.Context, loops ...*loop) error { + for _, l := range loops { + select { + case <-l.done: + case <-ctx.Done(): + return fmt.Errorf("%w: %w", errDraining, ctx.Err()) + } + } + return nil +} + // Nudge is a best-effort wake: input-driven curators check their trigger // early instead of waiting for the next interval. A full channel or an // unknown name drops the nudge — waking is an acceleration, never the @@ -121,31 +219,29 @@ func (rt *Runtime) Nudge(curator string) { } // Stop cancels every loop, waits for passes to finish within ctx, then stops -// the curators in reverse order. +// the curators in reverse order. Loops that outlive ctx stay tracked, so a +// later Start refuses rather than running a second copy. func (rt *Runtime) Stop(ctx context.Context) error { + rt.ops.Lock() + defer rt.ops.Unlock() rt.mu.Lock() if !rt.started { rt.mu.Unlock() return nil } rt.started = false - cancel := rt.cancel - rt.cancel = nil + loops := slices.Collect(maps.Values(rt.loops)) + for _, l := range loops { + l.cancel() + } + clear(rt.nudges) rt.mu.Unlock() - cancel() - done := make(chan struct{}) - go func() { - rt.wg.Wait() - close(done) - }() - select { - case <-done: - case <-ctx.Done(): + if err := waitLoops(ctx, loops...); err != nil { if log := rt.registry.Host().Log; log != nil { - log.Error("curator runtime shutdown timed out", "err", ctx.Err()) + log.Error("curator runtime shutdown timed out", "err", err) } - return fmt.Errorf("curator runtime shutdown: %w", ctx.Err()) + return fmt.Errorf("curator runtime shutdown: %w", errors.Join(err, rt.registry.Stop(ctx))) } err := rt.registry.Stop(ctx) if log := rt.registry.Host().Log; log != nil { @@ -162,16 +258,13 @@ func (rt *Runtime) Stop(ctx context.Context) error { // Check whether input is pending, run a bounded pass when it is, and back // off to the cooldown after an idle or failed pass. Panics in Check and // Pass are recovered here; the loop survives them. -func (rt *Runtime) runLoop(ctx context.Context, name string, c CuratorEngine) { - defer rt.wg.Done() - +func (rt *Runtime) runLoop(ctx context.Context, name string, c CuratorEngine, wakeCh <-chan struct{}) { m := c.Manifest() cooldown := m.Cooldown if cooldown <= 0 { cooldown = 2 * m.Interval } clock := rt.registry.Host().Clock - wakeCh := rt.nudges[name] // nil for non-input curators: the case never fires var next time.Time // zero means immediately eligible var lastPass time.Time @@ -334,9 +427,12 @@ func (rt *Runtime) emitRun(name string, at time.Time, result PassResult) { } } -// ReplaceDefinitions drains current passes before replacing data-defined -// curators. Built-in engines and the registry's observation surface survive. +// ReplaceDefinitions applies a full set of data-defined curators: it +// validates the set first, then removes, adds or replaces only the curators +// that changed. Built-in engines, unchanged curators and the registry's +// observation surface keep running. func (rt *Runtime) ReplaceDefinitions(ctx context.Context, definitions []Definition) error { + want := make(map[string]Definition, len(definitions)) candidate := NewRegistry(rt.registry.Host()) for _, def := range definitions { if !def.Enabled { @@ -350,12 +446,30 @@ func (rt *Runtime) ReplaceDefinitions(ctx context.Context, definitions []Definit if err := candidate.Register(NewDefinitionEngine(def)); err != nil { return err } + want[def.Name] = def } + stopCtx, cancel := context.WithTimeout(ctx, defaultPassTimeout) defer cancel() - if err := rt.Stop(stopCtx); err != nil { - return err + var errs []error + for _, name := range rt.registry.Names() { + held, _ := rt.registry.Get(name) + engine, custom := held.(*DefinitionEngine) + if !custom { + continue + } + if def, keep := want[name]; keep && reflect.DeepEqual(def, engine.def) { + delete(want, name) + continue + } + if err := rt.Remove(stopCtx, name); err != nil { + errs = append(errs, err) + } } - rt.registry.replaceDefinitions(candidate) - return rt.Start(ctx) + for _, name := range slices.Sorted(maps.Keys(want)) { + if err := rt.Add(ctx, NewDefinitionEngine(want[name])); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) } diff --git a/internal/domain/curator/runtime_test.go b/internal/domain/curator/runtime_test.go index fcabe9b97..3603e6b32 100644 --- a/internal/domain/curator/runtime_test.go +++ b/internal/domain/curator/runtime_test.go @@ -41,3 +41,65 @@ func TestReplaceLiveDefinitions(t *testing.T) { }) } } + +// blockingEngine's Pass blocks until release closes, honouring its context +// only when obey is set. +type blockingEngine struct { + DefinitionEngine + obey bool + entered chan struct{} + release chan struct{} +} + +func (e *blockingEngine) Pass(ctx context.Context, _ PassInput) (PassResult, error) { + close(e.entered) + if e.obey { + <-ctx.Done() + return PassResult{}, ctx.Err() + } + <-e.release + return PassResult{}, nil +} + +func TestRemoveInFlightPass(t *testing.T) { + for _, tt := range []struct { + name string + obey bool + wantErr bool + readdErr bool + }{ + {"pass honours cancellation", true, false, false}, + {"pass ignores cancellation", false, true, true}, + } { + t.Run(tt.name, func(t *testing.T) { + runtime := NewRuntime(NewRegistry(Registrar{}), RuntimeConfig{}) + if err := runtime.Start(t.Context()); err != nil { + t.Fatal(err) + } + e := &blockingEngine{ + def: Definition{Name: "c", Manifest: Manifest{Interval: time.Hour}}, + obey: tt.obey, entered: make(chan struct{}), release: make(chan struct{}), + } + if err := runtime.Add(t.Context(), e); err != nil { + t.Fatal(err) + } + <-e.entered + + ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) + defer cancel() + if err := runtime.Remove(ctx, "c"); (err != nil) != tt.wantErr { + t.Fatalf("Remove error = %v", err) + } + if _, ok := runtime.registry.Get("c"); ok { + t.Fatal("still registered after Remove") + } + if err := runtime.Add(t.Context(), NewDefinitionEngine(e.def)); (err != nil) != tt.readdErr { + t.Fatalf("re-add error = %v", err) + } + close(e.release) + if err := runtime.Stop(context.Background()); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/internal/domain/lifecycle/registry.go b/internal/domain/lifecycle/registry.go index bb09769ec..5c4ab3f24 100644 --- a/internal/domain/lifecycle/registry.go +++ b/internal/domain/lifecycle/registry.go @@ -32,8 +32,7 @@ type Health struct { var ( // ErrDuplicate is returned when an engine name is already registered. ErrDuplicate = errors.New("duplicate engine") - // ErrStarted is returned when registration or lifecycle is attempted - // after the registry has started. + // ErrStarted is returned when a running registry is started again. ErrStarted = errors.New("registry already started") ) @@ -94,13 +93,11 @@ func IsNil(v any) bool { return false } -// Add binds e to host and registers it. A panicking Bind refuses the engine. +// Add binds e to host and registers it, unstarted. A panicking Bind refuses +// the engine. Adding to a running registry is allowed; StartOne starts it. func (r *Registry[E, H]) Add(e E, host H) error { r.mu.Lock() defer r.mu.Unlock() - if r.state != stateIdle { - return fmt.Errorf("%s: %w", r.kind, ErrStarted) - } name := e.Name() if _, dup := r.engines[name]; dup { return fmt.Errorf("%w: %s %s", ErrDuplicate, r.kind, name) @@ -113,6 +110,48 @@ func (r *Registry[E, H]) Add(e E, host H) error { return nil } +// StartOne starts the named engine when the registry is running and the +// engine has not started; otherwise it does nothing. A failed Start marks the +// engine unhealthy, as Start does. +func (r *Registry[E, H]) StartOne(ctx context.Context, name string) error { + r.mu.Lock() + e, ok := r.engines[name] + if !ok || r.state != stateRunning || r.status[name].started { + r.mu.Unlock() + return nil + } + r.mu.Unlock() + + err := guard(func() error { return e.Start(ctx) }) + r.mu.Lock() + defer r.mu.Unlock() + if err != nil { + r.status[name] = status{startErr: err} + return r.fail(name, "start", err) + } + r.status[name] = status{started: true} + return nil +} + +// Unregister removes the named engine, stopping it first when it started. +// The engine is removed even when its Stop fails; the failure is returned. +func (r *Registry[E, H]) Unregister(ctx context.Context, name string) error { + r.mu.Lock() + e, ok := r.engines[name] + st := r.status[name] + delete(r.engines, name) + delete(r.status, name) + r.order = slices.DeleteFunc(r.order, func(n string) bool { return n == name }) + r.mu.Unlock() + if !ok || !st.started { + return nil + } + if err := guard(func() error { return e.Stop(ctx) }); err != nil { + return r.fail(name, "stop", err) + } + return nil +} + // Get returns the registered engine by name. func (r *Registry[E, H]) Get(name string) (E, bool) { r.mu.Lock() @@ -201,25 +240,6 @@ func (r *Registry[E, H]) Stop(ctx context.Context) error { return errors.Join(errs...) } -// Replace removes the engines drop selects and appends every engine from -// src, keeping src's order. Call it only while stopped. -func (r *Registry[E, H]) Replace(drop func(E) bool, src *Registry[E, H]) { - r.mu.Lock() - defer r.mu.Unlock() - r.order = slices.DeleteFunc(r.order, func(name string) bool { - if !drop(r.engines[name]) { - return false - } - delete(r.engines, name) - delete(r.status, name) - return true - }) - for _, name := range src.order { - r.engines[name] = src.engines[name] - r.order = append(r.order, name) - } -} - func (r *Registry[E, H]) fail(name, phase string, err error) error { if r.log != nil { r.log.Error(r.kind+" lifecycle failed", "name", name, "phase", phase, "err", err)