diff --git a/internal/domain/scheduling/contract.go b/internal/domain/scheduling/contract.go new file mode 100644 index 000000000..fb604c262 --- /dev/null +++ b/internal/domain/scheduling/contract.go @@ -0,0 +1,114 @@ +// Package scheduling is the ticker engine: the timing half of archied's +// cron/scheduling capability (epic archie-core-1786637496320-249-be6d9a20). +// It owns one job and one job only — deciding *when* a scheduled job runs, +// and with what concurrency — and nothing about where jobs are stored, how +// they are authored, or how their output is delivered. +// +// The split is deliberate. A job store (file-backed, cross-process locked) +// and a delivery router are separate issues in that epic; this package +// declares the two contracts it needs from them (JobSource, Runner) and +// names no implementation. That is what lets the timing behaviour — pool +// semantics, overlap suppression, per-run timeouts, bounded shutdown — be +// tested against a fake clock with no store, no NATS and no model. +package scheduling + +import ( + "context" + "errors" + "fmt" + "strings" + "time" +) + +// Pool is how a job's runs are dispatched relative to other jobs. +type Pool string + +const ( + // PoolParallel runs the job concurrently with every other job, bounded + // only by the engine's MaxParallel cap. + PoolParallel Pool = "parallel" + // PoolSequential runs the job on a single shared worker, one at a time, + // in the order the source returned them. Use it for jobs that contend + // for a resource that tolerates no concurrency. + PoolSequential Pool = "sequential" +) + +// Validate rejects an unknown pool. The empty pool is valid and means +// PoolSequential: an unlabelled job is assumed to want the safer of the two, +// because wrongly serialising a job costs latency while wrongly parallelising +// one costs correctness. +func (p Pool) Validate() error { + switch p { + case "", PoolParallel, PoolSequential: + return nil + default: + return fmt.Errorf("scheduling: unknown pool %q", string(p)) + } +} + +// resolve maps the empty pool onto its documented default. +func (p Pool) resolve() Pool { + if p == "" { + return PoolSequential + } + return p +} + +// Job is one due unit of scheduled work as handed to the engine. It carries +// identity and dispatch policy only: the schedule itself (cron expression, +// next-run time, catch-up policy) belongs to the job store behind JobSource, +// which is why the engine never sees one. +type Job struct { + // ID identifies the job across ticks. It is the overlap key: while a + // run with this ID is queued or in flight, later ticks that report the + // same ID are skipped rather than stacked. + ID string + // Pool selects the dispatch pool. Empty means PoolSequential. + Pool Pool + // Detail is a human-readable label recorded on emitted events so a run + // stays attributable without the engine knowing what the job does. + Detail string +} + +// Validate checks the fields the engine relies on. +func (j Job) Validate() error { + if strings.TrimSpace(j.ID) == "" { + return errors.New("scheduling: job id must be non-empty") + } + return j.Pool.Validate() +} + +// JobSource yields the jobs eligible to run at a given instant. The engine +// calls Due once per tick and treats it as authoritative: it applies no +// schedule arithmetic of its own, so a source is free to implement cron +// expressions, fixed intervals or catch-up windows without the engine +// changing. +type JobSource interface { + Due(ctx context.Context, now time.Time) ([]Job, error) +} + +// Runner executes one due job. Implementations must respect ctx: the engine +// bounds every run with JobTimeout and cancels on shutdown, but a Runner that +// ignores cancellation can still delay Stop up to Stop's own deadline. +type Runner interface { + Run(ctx context.Context, job Job) error +} + +// Clock is injectable time: Now for timestamps and After for the tick sleep. +// Loop timing is tested against a fake clock, never wall-clock sleeps. +type Clock interface { + Now() time.Time + After(d time.Duration) <-chan time.Time +} + +type systemClock struct{} + +func (systemClock) Now() time.Time { return time.Now() } +func (systemClock) After(d time.Duration) <-chan time.Time { return time.After(d) } + +// Sink is the narrow event contract the engine needs. It matches +// events.Bus's emitter shape without importing it — the domain declares what +// it needs and app wires the implementation in. +type Sink interface { + Emit(kind, detail string, data map[string]any) +} diff --git a/internal/domain/scheduling/ticker.go b/internal/domain/scheduling/ticker.go new file mode 100644 index 000000000..587f028bf --- /dev/null +++ b/internal/domain/scheduling/ticker.go @@ -0,0 +1,359 @@ +package scheduling + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/samcharles93/archie-core/internal/events" +) + +// Defaults for the tunables an operator may leave unset. +const ( + // DefaultInterval is the tick cadence when none is configured. A minute + // is the finest granularity a cron-style schedule needs, and polling a + // local job store that often costs nothing. + DefaultInterval = time.Minute + // DefaultMaxParallel caps concurrent parallel-pool runs. It exists so a + // store that reports many jobs due at once (the first tick after a long + // downtime, typically) cannot launch an unbounded fan-out. + DefaultMaxParallel = 4 + // DefaultJobTimeout bounds one run. A scheduled job that has not + // finished in an hour is stuck, and leaving it in flight would suppress + // every later run of the same job forever. + DefaultJobTimeout = time.Hour +) + +// Event kinds emitted by the engine, re-exported so callers of this package +// need not import the events package to match on them. +const ( + KindJobRun = events.KindJobRun + KindJobError = events.KindJobError +) + +// Phases reported on KindJobError, naming which step failed. +const ( + phaseSource = "source" // JobSource.Due returned an error + phaseDispatch = "dispatch" // the job was rejected before it ever ran + phaseRun = "run" // Runner.Run failed, panicked or timed out +) + +// sequentialQueueDepth bounds the sequential pool's backlog. Overlap +// suppression already caps the backlog at one entry per distinct job, so this +// is a backstop against a pathological source, not the primary bound. +const sequentialQueueDepth = 256 + +// EngineConfig tunes the ticker engine. Every field is optional; zero values +// take the documented defaults. +type EngineConfig struct { + // Interval is the tick cadence: how often the engine asks its source + // what is due. Zero uses DefaultInterval; negative is an error. + Interval time.Duration + // MaxParallel caps simultaneous parallel-pool runs. Zero uses + // DefaultMaxParallel. It does not bound the sequential pool, which is + // one by definition. + MaxParallel int + // JobTimeout bounds a single run via context deadline. Zero uses + // DefaultJobTimeout. + JobTimeout time.Duration + // Clock is injectable time. Nil uses the system clock. + Clock Clock + // Events receives run and error events. Nil disables emission; the + // engine still runs, it is just unobservable. + Events Sink +} + +func (c EngineConfig) validate() error { + switch { + case c.Interval < 0: + return fmt.Errorf("scheduling: interval must not be negative, got %v", c.Interval) + case c.MaxParallel < 0: + return fmt.Errorf("scheduling: max parallel must not be negative, got %d", c.MaxParallel) + case c.JobTimeout < 0: + return fmt.Errorf("scheduling: job timeout must not be negative, got %v", c.JobTimeout) + } + return nil +} + +// withDefaults returns the config with zero values replaced. +func (c EngineConfig) withDefaults() EngineConfig { + if c.Interval == 0 { + c.Interval = DefaultInterval + } + if c.MaxParallel == 0 { + c.MaxParallel = DefaultMaxParallel + } + if c.JobTimeout == 0 { + c.JobTimeout = DefaultJobTimeout + } + if c.Clock == nil { + c.Clock = systemClock{} + } + return c +} + +// Engine is the ticker engine: one loop that asks the source what is due at +// each tick and dispatches each due job to the pool it declares. +// +// Two invariants make the timing safe to reason about: +// +// - A job never overlaps itself. While a run is queued or in flight, later +// ticks reporting the same ID are skipped, not stacked. A job slower than +// the tick interval therefore runs back-to-back, never fanned out. +// - The pools do not interfere. A blocked sequential job cannot delay a +// parallel one, and a saturated parallel cap cannot delay the sequential +// worker — they are separate dispatch paths sharing only the tick. +// +// Every failure is contained: a source error, a job error, a job panic and a +// job timeout are each reported as an event and leave the loop running. +type Engine struct { + source JobSource + runner Runner + cfg EngineConfig + + queue chan Job // sequential pool backlog + sem chan struct{} + + mu sync.Mutex + started bool + stopped bool + stopLoop context.CancelFunc + cancelRuns context.CancelFunc + active map[string]struct{} // queued or in flight, by job ID + wg sync.WaitGroup +} + +// NewEngine builds the engine over its source and runner. It fails on a nil +// dependency or a negative tunable; zero tunables take their defaults. +func NewEngine(source JobSource, runner Runner, cfg EngineConfig) (*Engine, error) { + if source == nil { + return nil, errors.New("scheduling: job source must not be nil") + } + if runner == nil { + return nil, errors.New("scheduling: runner must not be nil") + } + if err := cfg.validate(); err != nil { + return nil, err + } + cfg = cfg.withDefaults() + return &Engine{ + source: source, + runner: runner, + cfg: cfg, + queue: make(chan Job, sequentialQueueDepth), + sem: make(chan struct{}, cfg.MaxParallel), + active: make(map[string]struct{}), + }, nil +} + +// Interval reports the resolved tick cadence. +func (e *Engine) Interval() time.Duration { return e.cfg.Interval } + +// MaxParallel reports the resolved parallel-pool cap. +func (e *Engine) MaxParallel() int { return e.cfg.MaxParallel } + +// JobTimeout reports the resolved per-run timeout. +func (e *Engine) JobTimeout() time.Duration { return e.cfg.JobTimeout } + +// Start launches the tick loop and the sequential worker. +// +// An engine is single-use: starting one that is already started, or that has +// been stopped, is an error. Restarting is refused rather than supported +// because Stop leaves the dropped sequential backlog and the claims of runs +// it did not wait for behind — a restarted engine would re-run stale queued +// jobs and permanently suppress the jobs still marked claimed. Build a new +// engine instead; it is a struct, not a resource. +func (e *Engine) Start(ctx context.Context) error { + e.mu.Lock() + defer e.mu.Unlock() + if e.stopped { + return errors.New("scheduling: engine already stopped; build a new one") + } + if e.started { + return errors.New("scheduling: engine already started") + } + e.started = true + + loopCtx, stopLoop := context.WithCancel(ctx) + e.stopLoop = stopLoop + // Run contexts descend from Background, not from loopCtx, so that Stop + // drains gracefully: cancelling the loop must stop new work without + // yanking the context out from under a run that is already going. Stop + // cancels these only once its own deadline passes. + runCtx, cancelRuns := context.WithCancel(context.WithoutCancel(ctx)) + e.cancelRuns = cancelRuns + + e.wg.Add(2) + go e.tickLoop(loopCtx, runCtx) + go e.sequentialWorker(loopCtx, runCtx) + return nil +} + +// Stop halts ticking, drops any queued-but-unstarted runs, and waits for +// in-flight runs to finish, bounded by ctx. If ctx expires first, in-flight +// runs are cancelled and ctx's error is returned — a Runner that ignores +// cancellation can still outlive Stop, which is why the deadline is the +// caller's to set. Stopping an engine that never started is a no-op. +func (e *Engine) Stop(ctx context.Context) error { + e.mu.Lock() + if !e.started { + e.mu.Unlock() + return nil + } + e.started = false + e.stopped = true + stopLoop, cancelRuns := e.stopLoop, e.cancelRuns + e.stopLoop, e.cancelRuns = nil, nil + e.mu.Unlock() + + stopLoop() + done := make(chan struct{}) + go func() { + e.wg.Wait() + close(done) + }() + select { + case <-done: + cancelRuns() + return nil + case <-ctx.Done(): + cancelRuns() + return fmt.Errorf("scheduling: engine shutdown: %w", ctx.Err()) + } +} + +// tickLoop sleeps one interval, asks the source what is due, and dispatches +// it. The source is authoritative: the engine applies no schedule arithmetic +// of its own, so what "due" means stays the store's decision. +func (e *Engine) tickLoop(loopCtx, runCtx context.Context) { + defer e.wg.Done() + for { + select { + case <-loopCtx.Done(): + return + case <-e.cfg.Clock.After(e.cfg.Interval): + } + + jobs, err := e.source.Due(loopCtx, e.cfg.Clock.Now()) + if err != nil { + e.emitError(Job{}, phaseSource, err) + continue + } + for _, job := range jobs { + e.dispatch(loopCtx, runCtx, job) + } + } +} + +// dispatch routes one due job to its pool, skipping it if it is invalid or +// already queued or in flight. +func (e *Engine) dispatch(loopCtx, runCtx context.Context, job Job) { + if err := job.Validate(); err != nil { + e.emitError(job, phaseDispatch, err) + return + } + if !e.claim(job.ID) { + return // an earlier run of this job has not finished; skip this tick + } + + if job.Pool.resolve() == PoolParallel { + e.wg.Go(func() { + defer e.release(job.ID) + select { + case e.sem <- struct{}{}: + case <-loopCtx.Done(): + return // shutting down while waiting for a parallel slot + } + defer func() { <-e.sem }() + e.runJob(runCtx, job) + }) + return + } + + select { + case e.queue <- job: + default: + e.release(job.ID) + e.emitError(job, phaseDispatch, errors.New("scheduling: sequential queue full")) + } +} + +// sequentialWorker is the sequential pool: one goroutine draining the queue +// in the order the source reported the jobs, so serialised jobs never +// overlap each other. +func (e *Engine) sequentialWorker(loopCtx, runCtx context.Context) { + defer e.wg.Done() + for { + select { + case <-loopCtx.Done(): + return + case job := <-e.queue: + e.runJob(runCtx, job) + e.release(job.ID) + } + } +} + +// runJob executes one run under the configured timeout, recovering panics so +// a job that blows up takes nothing else with it. +func (e *Engine) runJob(ctx context.Context, job Job) { + runCtx, cancel := context.WithTimeout(ctx, e.cfg.JobTimeout) + defer cancel() + + start := e.cfg.Clock.Now() + err := e.safeRun(runCtx, job) + elapsed := e.cfg.Clock.Now().Sub(start) + if err != nil { + e.emitError(job, phaseRun, err) + return + } + e.emit(KindJobRun, job, map[string]any{ + "job": job.ID, + "pool": string(job.Pool.resolve()), + "duration_ms": elapsed.Milliseconds(), + }) +} + +func (e *Engine) safeRun(ctx context.Context, job Job) (err error) { + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("scheduling: job %q panicked: %v", job.ID, p) + } + }() + return e.runner.Run(ctx, job) +} + +// claim marks a job as queued-or-running, reporting false if it already was. +func (e *Engine) claim(id string) bool { + e.mu.Lock() + defer e.mu.Unlock() + if _, busy := e.active[id]; busy { + return false + } + e.active[id] = struct{}{} + return true +} + +func (e *Engine) release(id string) { + e.mu.Lock() + defer e.mu.Unlock() + delete(e.active, id) +} + +func (e *Engine) emitError(job Job, phase string, err error) { + e.emit(KindJobError, job, map[string]any{ + "job": job.ID, + "pool": string(job.Pool), + "phase": phase, + "err": err.Error(), + }) +} + +func (e *Engine) emit(kind string, job Job, data map[string]any) { + if e.cfg.Events == nil { + return + } + e.cfg.Events.Emit(kind, job.Detail, data) +} diff --git a/internal/domain/scheduling/ticker_test.go b/internal/domain/scheduling/ticker_test.go new file mode 100644 index 000000000..f3c06f398 --- /dev/null +++ b/internal/domain/scheduling/ticker_test.go @@ -0,0 +1,850 @@ +package scheduling + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +// --- fakes -------------------------------------------------------------------- + +// fakeClock is the engine's injectable clock: Now plus clock-driven timers +// that fire when Advance moves past their deadline. Tick timing is tested +// with this, never wall-clock sleeps. +type fakeClock struct { + mu sync.Mutex + now time.Time + timers []*fakeTimer +} + +type fakeTimer struct { + c chan time.Time + at time.Time +} + +func newFakeClock(start time.Time) *fakeClock { return &fakeClock{now: start} } + +func (fc *fakeClock) Now() time.Time { + fc.mu.Lock() + defer fc.mu.Unlock() + return fc.now +} + +func (fc *fakeClock) After(d time.Duration) <-chan time.Time { + fc.mu.Lock() + defer fc.mu.Unlock() + t := &fakeTimer{c: make(chan time.Time, 1), at: fc.now.Add(d)} + fc.timers = append(fc.timers, t) + return t.c +} + +// pending reports how many timers are waiting; the tick loop registers +// exactly one at a time, so it is the signal that the loop is back asleep. +func (fc *fakeClock) pending() int { + fc.mu.Lock() + defer fc.mu.Unlock() + return len(fc.timers) +} + +// Advance moves the clock forward and fires every timer whose deadline has +// passed, returning how many fired. +func (fc *fakeClock) Advance(d time.Duration) int { + fc.mu.Lock() + fc.now = fc.now.Add(d) + var fired []*fakeTimer + remain := fc.timers[:0] + for _, t := range fc.timers { + if !t.at.After(fc.now) { + fired = append(fired, t) + } else { + remain = append(remain, t) + } + } + fc.timers = remain + fc.mu.Unlock() + for _, t := range fired { + select { + case t.c <- t.at: + default: + } + } + return len(fired) +} + +// sourceFunc adapts a function to JobSource. +type sourceFunc func(ctx context.Context, now time.Time) ([]Job, error) + +func (f sourceFunc) Due(ctx context.Context, now time.Time) ([]Job, error) { return f(ctx, now) } + +// staticSource returns the same jobs every tick and counts the calls. +type staticSource struct { + jobs []Job + mu sync.Mutex + calls int +} + +func (s *staticSource) Due(context.Context, time.Time) ([]Job, error) { + s.mu.Lock() + s.calls++ + s.mu.Unlock() + return s.jobs, nil +} + +func (s *staticSource) callCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.calls +} + +// runnerFunc adapts a function to Runner. +type runnerFunc func(ctx context.Context, job Job) error + +func (f runnerFunc) Run(ctx context.Context, job Job) error { return f(ctx, job) } + +// recordingRunner records every run in order and optionally blocks on a gate +// keyed by job ID, so a test can hold a run in flight deterministically. +type recordingRunner struct { + mu sync.Mutex + started []string + done []string + inFlt int + maxFlt int + gates map[string]chan struct{} + err error +} + +func newRecordingRunner() *recordingRunner { + return &recordingRunner{gates: map[string]chan struct{}{}} +} + +// gate installs a release channel for a job; runs of that job block until +// release is called. +func (r *recordingRunner) gate(id string) { + r.mu.Lock() + defer r.mu.Unlock() + r.gates[id] = make(chan struct{}) +} + +func (r *recordingRunner) release(id string) { + r.mu.Lock() + ch := r.gates[id] + delete(r.gates, id) + r.mu.Unlock() + if ch != nil { + close(ch) + } +} + +func (r *recordingRunner) Run(ctx context.Context, job Job) error { + r.mu.Lock() + r.started = append(r.started, job.ID) + r.inFlt++ + if r.inFlt > r.maxFlt { + r.maxFlt = r.inFlt + } + gate := r.gates[job.ID] + err := r.err + r.mu.Unlock() + + if gate != nil { + select { + case <-gate: + case <-ctx.Done(): + err = ctx.Err() + } + } + + r.mu.Lock() + r.inFlt-- + r.done = append(r.done, job.ID) + r.mu.Unlock() + return err +} + +func (r *recordingRunner) startedIDs() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.started...) +} + +func (r *recordingRunner) doneIDs() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.done...) +} + +func (r *recordingRunner) peakInFlight() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.maxFlt +} + +// recordingSink collects emitted events. +type recordingSink struct { + mu sync.Mutex + events []sinkEvent +} + +type sinkEvent struct { + kind string + detail string + data map[string]any +} + +func (s *recordingSink) Emit(kind, detail string, data map[string]any) { + s.mu.Lock() + defer s.mu.Unlock() + s.events = append(s.events, sinkEvent{kind: kind, detail: detail, data: data}) +} + +func (s *recordingSink) countOf(kind string) int { + s.mu.Lock() + defer s.mu.Unlock() + n := 0 + for _, e := range s.events { + if e.kind == kind { + n++ + } + } + return n +} + +func (s *recordingSink) firstOf(kind string) (sinkEvent, bool) { + s.mu.Lock() + defer s.mu.Unlock() + for _, e := range s.events { + if e.kind == kind { + return e, true + } + } + return sinkEvent{}, false +} + +// --- helpers ------------------------------------------------------------------ + +const ( + testInterval = time.Minute + testTimeout = 2 * time.Second + pollGap = time.Millisecond +) + +func testStart() time.Time { return time.Date(2026, 8, 24, 12, 0, 0, 0, time.UTC) } + +// waitFor polls cond until it holds or the timeout expires. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(testTimeout) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(pollGap) + } + t.Fatalf("timed out waiting for %s", what) +} + +// tick advances the fake clock one interval and waits for the loop to arm its +// next timer, which happens only after the tick's dispatch has been decided. +func tick(t *testing.T, fc *fakeClock, e *Engine) { + t.Helper() + waitFor(t, "loop to arm its tick timer", func() bool { return fc.pending() > 0 }) + fc.Advance(testInterval) + waitFor(t, "loop to re-arm after tick", func() bool { return fc.pending() > 0 }) + _ = e +} + +// startEngine builds and starts an engine, registering Stop as cleanup. +func startEngine(t *testing.T, src JobSource, runner Runner, cfg EngineConfig) *Engine { + t.Helper() + e, err := NewEngine(src, runner, cfg) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + if err := e.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + _ = e.Stop(ctx) + }) + return e +} + +func countOccurrences(ids []string, want string) int { + n := 0 + for _, id := range ids { + if id == want { + n++ + } + } + return n +} + +// --- construction ------------------------------------------------------------- + +func TestNewEngineValidatesConfig(t *testing.T) { + src := &staticSource{} + runner := newRecordingRunner() + + tests := []struct { + name string + source JobSource + runner Runner + cfg EngineConfig + wantErr bool + }{ + {name: "nil source", source: nil, runner: runner, cfg: EngineConfig{}, wantErr: true}, + {name: "nil runner", source: src, runner: nil, cfg: EngineConfig{}, wantErr: true}, + {name: "negative interval", source: src, runner: runner, cfg: EngineConfig{Interval: -testInterval}, wantErr: true}, + {name: "negative max parallel", source: src, runner: runner, cfg: EngineConfig{MaxParallel: -1}, wantErr: true}, + {name: "negative job timeout", source: src, runner: runner, cfg: EngineConfig{JobTimeout: -testInterval}, wantErr: true}, + {name: "zero values are defaulted", source: src, runner: runner, cfg: EngineConfig{}, wantErr: false}, + { + name: "fully specified", source: src, runner: runner, + cfg: EngineConfig{Interval: testInterval, MaxParallel: 2, JobTimeout: testInterval}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e, err := NewEngine(tc.source, tc.runner, tc.cfg) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error, got engine %v", e) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if e == nil { + t.Fatal("expected engine, got nil") + } + }) + } +} + +func TestNewEngineAppliesDefaults(t *testing.T) { + e, err := NewEngine(&staticSource{}, newRecordingRunner(), EngineConfig{}) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + if got := e.Interval(); got != DefaultInterval { + t.Errorf("interval = %v, want %v", got, DefaultInterval) + } + if got := e.MaxParallel(); got != DefaultMaxParallel { + t.Errorf("max parallel = %d, want %d", got, DefaultMaxParallel) + } + if got := e.JobTimeout(); got != DefaultJobTimeout { + t.Errorf("job timeout = %v, want %v", got, DefaultJobTimeout) + } +} + +func TestEngineHonoursConfiguredInterval(t *testing.T) { + for _, interval := range []time.Duration{time.Second, testInterval, time.Hour} { + t.Run(interval.String(), func(t *testing.T) { + fc := newFakeClock(testStart()) + src := &staticSource{} + e := startEngine(t, src, newRecordingRunner(), EngineConfig{Interval: interval, Clock: fc}) + + waitFor(t, "first tick timer", func() bool { return fc.pending() > 0 }) + // One nanosecond short of the interval must not tick. + fc.Advance(interval - time.Nanosecond) + time.Sleep(10 * pollGap) + if got := src.callCount(); got != 0 { + t.Fatalf("source called %d times before the interval elapsed", got) + } + fc.Advance(time.Nanosecond) + waitFor(t, "source poll", func() bool { return src.callCount() >= 1 }) + if got := e.Interval(); got != interval { + t.Errorf("Interval() = %v, want %v", got, interval) + } + }) + } +} + +// --- dispatch ----------------------------------------------------------------- + +func TestEngineRunsDueJobs(t *testing.T) { + fc := newFakeClock(testStart()) + jobs := []Job{{ID: "a", Pool: PoolParallel}, {ID: "b", Pool: PoolSequential}} + src := &staticSource{jobs: jobs} + runner := newRecordingRunner() + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc}) + + tick(t, fc, e) + waitFor(t, "both jobs to finish", func() bool { return len(runner.doneIDs()) == len(jobs) }) + + for _, j := range jobs { + if countOccurrences(runner.doneIDs(), j.ID) != 1 { + t.Errorf("job %q ran %d times, want 1", j.ID, countOccurrences(runner.doneIDs(), j.ID)) + } + } +} + +func TestEngineParallelPoolRunsConcurrentlyUpToCap(t *testing.T) { + const maxParallel = 3 + fc := newFakeClock(testStart()) + jobs := make([]Job, 0, maxParallel+2) + runner := newRecordingRunner() + for i := range cap(jobs) { + id := string(rune('a' + i)) + jobs = append(jobs, Job{ID: id, Pool: PoolParallel}) + runner.gate(id) + } + src := &staticSource{jobs: jobs} + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, MaxParallel: maxParallel, Clock: fc}) + + tick(t, fc, e) + waitFor(t, "cap to be saturated", func() bool { return runner.peakInFlight() >= maxParallel }) + // Give any over-cap goroutine a chance to break the cap before asserting. + time.Sleep(10 * pollGap) + if got := runner.peakInFlight(); got != maxParallel { + t.Fatalf("peak in-flight = %d, want %d", got, maxParallel) + } + for _, j := range jobs { + runner.release(j.ID) + } + waitFor(t, "all jobs to finish", func() bool { return len(runner.doneIDs()) == len(jobs) }) +} + +func TestEngineSequentialPoolRunsOneAtATimeInOrder(t *testing.T) { + fc := newFakeClock(testStart()) + jobs := []Job{{ID: "first"}, {ID: "second", Pool: PoolSequential}, {ID: "third", Pool: PoolSequential}} + src := &staticSource{jobs: jobs} + runner := newRecordingRunner() + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, MaxParallel: len(jobs), Clock: fc}) + + tick(t, fc, e) + waitFor(t, "sequential jobs to finish", func() bool { return len(runner.doneIDs()) == len(jobs) }) + + if got := runner.peakInFlight(); got != 1 { + t.Errorf("sequential peak in-flight = %d, want 1", got) + } + for i, id := range runner.startedIDs() { + if id != jobs[i].ID { + t.Errorf("run %d = %q, want %q (source order)", i, id, jobs[i].ID) + } + } +} + +func TestEngineSequentialJobDoesNotBlockParallelJob(t *testing.T) { + fc := newFakeClock(testStart()) + src := &staticSource{jobs: []Job{{ID: "slow-seq", Pool: PoolSequential}, {ID: "quick-par", Pool: PoolParallel}}} + runner := newRecordingRunner() + runner.gate("slow-seq") + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc}) + + tick(t, fc, e) + waitFor(t, "parallel job to finish while sequential is blocked", func() bool { + return countOccurrences(runner.doneIDs(), "quick-par") == 1 + }) + runner.release("slow-seq") + waitFor(t, "sequential job to finish", func() bool { + return countOccurrences(runner.doneIDs(), "slow-seq") == 1 + }) +} + +func TestEngineSkipsJobAlreadyInFlight(t *testing.T) { + for _, pool := range []Pool{PoolParallel, PoolSequential} { + t.Run(string(pool), func(t *testing.T) { + const id = "overlapping" + fc := newFakeClock(testStart()) + src := &staticSource{jobs: []Job{{ID: id, Pool: pool}}} + runner := newRecordingRunner() + runner.gate(id) + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc}) + + tick(t, fc, e) + waitFor(t, "first run to start", func() bool { return len(runner.startedIDs()) == 1 }) + tick(t, fc, e) + tick(t, fc, e) + time.Sleep(10 * pollGap) + if got := len(runner.startedIDs()); got != 1 { + t.Fatalf("job started %d times while in flight, want 1", got) + } + runner.release(id) + waitFor(t, "first run to finish", func() bool { return len(runner.doneIDs()) == 1 }) + + // Once the run is done the job is eligible again. + tick(t, fc, e) + waitFor(t, "second run", func() bool { return len(runner.startedIDs()) == 2 }) + }) + } +} + +func TestEngineRejectsInvalidJob(t *testing.T) { + tests := []struct { + name string + job Job + }{ + {name: "empty id", job: Job{ID: " ", Pool: PoolParallel}}, + {name: "unknown pool", job: Job{ID: "bad-pool", Pool: Pool("elsewhere")}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fc := newFakeClock(testStart()) + sink := &recordingSink{} + src := &staticSource{jobs: []Job{tc.job, {ID: "healthy", Pool: PoolParallel}}} + runner := newRecordingRunner() + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc, Events: sink}) + + tick(t, fc, e) + waitFor(t, "healthy job to run", func() bool { + return countOccurrences(runner.doneIDs(), "healthy") == 1 + }) + if got := len(runner.startedIDs()); got != 1 { + t.Fatalf("started %v, want only the healthy job", runner.startedIDs()) + } + if sink.countOf(KindJobError) == 0 { + t.Error("expected an error event for the invalid job") + } + }) + } +} + +// --- failure isolation -------------------------------------------------------- + +func TestEngineSurvivesSourceError(t *testing.T) { + fc := newFakeClock(testStart()) + sink := &recordingSink{} + var calls int + var mu sync.Mutex + wantErr := errors.New("store unavailable") + src := sourceFunc(func(context.Context, time.Time) ([]Job, error) { + mu.Lock() + calls++ + n := calls + mu.Unlock() + if n == 1 { + return nil, wantErr + } + return []Job{{ID: "after-failure"}}, nil + }) + runner := newRecordingRunner() + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc, Events: sink}) + + tick(t, fc, e) + waitFor(t, "error event", func() bool { return sink.countOf(KindJobError) == 1 }) + tick(t, fc, e) + waitFor(t, "loop to keep ticking after a source error", func() bool { + return countOccurrences(runner.doneIDs(), "after-failure") == 1 + }) + + ev, ok := sink.firstOf(KindJobError) + if !ok { + t.Fatal("no error event recorded") + } + if got, _ := ev.data["phase"].(string); got != phaseSource { + t.Errorf("error phase = %q, want %q", got, phaseSource) + } +} + +func TestEngineSurvivesJobPanicAndError(t *testing.T) { + tests := []struct { + name string + run func(job Job) error + }{ + {name: "panic", run: func(Job) error { panic("boom") }}, + {name: "error", run: func(Job) error { return errors.New("job failed") }}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fc := newFakeClock(testStart()) + sink := &recordingSink{} + var mu sync.Mutex + var runs int + src := &staticSource{jobs: []Job{{ID: "unstable"}}} + runner := runnerFunc(func(_ context.Context, job Job) error { + mu.Lock() + runs++ + mu.Unlock() + return tc.run(job) + }) + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc, Events: sink}) + + tick(t, fc, e) + waitFor(t, "first run", func() bool { mu.Lock(); defer mu.Unlock(); return runs == 1 }) + waitFor(t, "error event", func() bool { return sink.countOf(KindJobError) == 1 }) + // The loop survives: the same job is dispatched again next tick. + tick(t, fc, e) + waitFor(t, "second run", func() bool { mu.Lock(); defer mu.Unlock(); return runs == 2 }) + + ev, _ := sink.firstOf(KindJobError) + if got, _ := ev.data["phase"].(string); got != phaseRun { + t.Errorf("error phase = %q, want %q", got, phaseRun) + } + }) + } +} + +func TestEngineBoundsRunWithJobTimeout(t *testing.T) { + fc := newFakeClock(testStart()) + sink := &recordingSink{} + src := &staticSource{jobs: []Job{{ID: "hangs"}}} + deadlines := make(chan error, 1) + runner := runnerFunc(func(ctx context.Context, _ Job) error { + <-ctx.Done() + deadlines <- ctx.Err() + return ctx.Err() + }) + // A real (short) timeout: the run's deadline is wall-clock, because a + // Runner's ctx must be honoured by code that knows nothing of our clock. + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, JobTimeout: 20 * time.Millisecond, Clock: fc, Events: sink}) + + tick(t, fc, e) + select { + case err := <-deadlines: + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("run ctx error = %v, want DeadlineExceeded", err) + } + case <-time.After(testTimeout): + t.Fatal("run was never cancelled by the job timeout") + } + waitFor(t, "timeout to be reported", func() bool { return sink.countOf(KindJobError) == 1 }) +} + +// --- lifecycle ---------------------------------------------------------------- + +func TestEngineStartTwiceIsAnError(t *testing.T) { + fc := newFakeClock(testStart()) + e := startEngine(t, &staticSource{}, newRecordingRunner(), EngineConfig{Interval: testInterval, Clock: fc}) + if err := e.Start(context.Background()); err == nil { + t.Fatal("expected an error starting an already-started engine") + } +} + +func TestEngineRestartAfterStopIsRefused(t *testing.T) { + fc := newFakeClock(testStart()) + e, err := NewEngine(&staticSource{}, newRecordingRunner(), EngineConfig{Interval: testInterval, Clock: fc}) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + if err := e.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + if err := e.Stop(ctx); err != nil { + t.Fatalf("Stop: %v", err) + } + if err := e.Start(context.Background()); err == nil { + t.Fatal("expected an error restarting a stopped engine") + } +} + +func TestEngineStopWithoutStartIsNoOp(t *testing.T) { + e, err := NewEngine(&staticSource{}, newRecordingRunner(), EngineConfig{}) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + if err := e.Stop(ctx); err != nil { + t.Fatalf("Stop before Start = %v, want nil", err) + } +} + +func TestEngineStopWaitsForInFlightRuns(t *testing.T) { + fc := newFakeClock(testStart()) + src := &staticSource{jobs: []Job{{ID: "par", Pool: PoolParallel}, {ID: "seq", Pool: PoolSequential}}} + runner := newRecordingRunner() + runner.gate("par") + runner.gate("seq") + e, err := NewEngine(src, runner, EngineConfig{Interval: testInterval, Clock: fc}) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + if err := e.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + + tick(t, fc, e) + waitFor(t, "both runs to start", func() bool { return len(runner.startedIDs()) == 2 }) + + stopped := make(chan error, 1) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + stopped <- e.Stop(ctx) + }() + + select { + case <-stopped: + t.Fatal("Stop returned while runs were still in flight") + case <-time.After(20 * pollGap): + } + + runner.release("par") + runner.release("seq") + select { + case err := <-stopped: + if err != nil { + t.Fatalf("Stop: %v", err) + } + case <-time.After(testTimeout): + t.Fatal("Stop did not return after runs finished") + } + if got := len(runner.doneIDs()); got != 2 { + t.Fatalf("finished runs = %d, want 2", got) + } +} + +func TestEngineStopIsBoundedByContext(t *testing.T) { + fc := newFakeClock(testStart()) + src := &staticSource{jobs: []Job{{ID: "uninterruptible"}}} + release := make(chan struct{}) + started := make(chan struct{}, 1) + runner := runnerFunc(func(context.Context, Job) error { + select { + case started <- struct{}{}: + default: + } + <-release // deliberately ignores cancellation + return nil + }) + e, err := NewEngine(src, runner, EngineConfig{Interval: testInterval, Clock: fc}) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + if err := e.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + defer close(release) + + tick(t, fc, e) + <-started + + ctx, cancel := context.WithTimeout(context.Background(), 20*pollGap) + defer cancel() + if err := e.Stop(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Stop = %v, want a deadline-exceeded error", err) + } +} + +func TestEngineStopStopsTicking(t *testing.T) { + fc := newFakeClock(testStart()) + src := &staticSource{jobs: []Job{{ID: "job"}}} + runner := newRecordingRunner() + e, err := NewEngine(src, runner, EngineConfig{Interval: testInterval, Clock: fc}) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + if err := e.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + tick(t, fc, e) + waitFor(t, "first run", func() bool { return len(runner.doneIDs()) == 1 }) + + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + if err := e.Stop(ctx); err != nil { + t.Fatalf("Stop: %v", err) + } + before := src.callCount() + fc.Advance(testInterval * 10) + time.Sleep(20 * pollGap) + if got := src.callCount(); got != before { + t.Fatalf("source polled %d times after Stop (was %d)", got, before) + } +} + +// --- events ------------------------------------------------------------------- + +func TestEngineEmitsRunEvents(t *testing.T) { + fc := newFakeClock(testStart()) + sink := &recordingSink{} + const detail = "daily status summary" + src := &staticSource{jobs: []Job{{ID: "summary", Pool: PoolParallel, Detail: detail}}} + e := startEngine(t, src, newRecordingRunner(), EngineConfig{Interval: testInterval, Clock: fc, Events: sink}) + + tick(t, fc, e) + waitFor(t, "run event", func() bool { return sink.countOf(KindJobRun) == 1 }) + + ev, _ := sink.firstOf(KindJobRun) + if ev.detail != detail { + t.Errorf("event detail = %q, want %q", ev.detail, detail) + } + if got, _ := ev.data["job"].(string); got != "summary" { + t.Errorf("event job = %q, want %q", got, "summary") + } + if got, _ := ev.data["pool"].(string); got != string(PoolParallel) { + t.Errorf("event pool = %q, want %q", got, PoolParallel) + } +} + +func TestEngineWithoutSinkDoesNotPanic(t *testing.T) { + fc := newFakeClock(testStart()) + src := &staticSource{jobs: []Job{{ID: "bad-pool", Pool: Pool("nowhere")}, {ID: "fine"}}} + runner := newRecordingRunner() + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc}) + + tick(t, fc, e) + waitFor(t, "valid job to run", func() bool { return countOccurrences(runner.doneIDs(), "fine") == 1 }) +} + +// --- contract ----------------------------------------------------------------- + +func TestPoolValidate(t *testing.T) { + tests := []struct { + pool Pool + wantErr bool + }{ + {pool: PoolParallel}, + {pool: PoolSequential}, + {pool: ""}, + {pool: Pool("PARALLEL"), wantErr: true}, + {pool: Pool("other"), wantErr: true}, + } + for _, tc := range tests { + t.Run(string(tc.pool), func(t *testing.T) { + if err := tc.pool.Validate(); (err != nil) != tc.wantErr { + t.Fatalf("Validate(%q) = %v, wantErr %v", tc.pool, err, tc.wantErr) + } + }) + } +} + +func TestPoolResolveDefaultsToSequential(t *testing.T) { + tests := []struct { + pool Pool + want Pool + }{ + {pool: "", want: PoolSequential}, + {pool: PoolSequential, want: PoolSequential}, + {pool: PoolParallel, want: PoolParallel}, + } + for _, tc := range tests { + t.Run(string(tc.pool), func(t *testing.T) { + if got := tc.pool.resolve(); got != tc.want { + t.Fatalf("resolve(%q) = %q, want %q", tc.pool, got, tc.want) + } + }) + } +} + +func TestJobValidate(t *testing.T) { + tests := []struct { + name string + job Job + wantErr bool + }{ + {name: "valid", job: Job{ID: "a", Pool: PoolParallel}}, + {name: "valid default pool", job: Job{ID: "a"}}, + {name: "empty id", job: Job{ID: ""}, wantErr: true}, + {name: "blank id", job: Job{ID: "\t "}, wantErr: true}, + {name: "unknown pool", job: Job{ID: "a", Pool: Pool("x")}, wantErr: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if err := tc.job.Validate(); (err != nil) != tc.wantErr { + t.Fatalf("Validate() = %v, wantErr %v", err, tc.wantErr) + } + }) + } +} diff --git a/internal/events/events.go b/internal/events/events.go index df21803fb..817c6bf94 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -48,6 +48,14 @@ const ( KindCuratorAction = "curator_action" // data: curator, type, detail, reason KindCuratorError = "curator_error" // data: curator, phase, err + // Scheduled-job activity from the ticker engine + // (internal/domain/scheduling). A scheduled run has no task and no chat + // turn behind it, so without these a job that fired, stalled or failed + // is invisible everywhere. data: job, pool, duration_ms (run) and job, + // pool, phase, err (error). + KindJobRun = "job_run" + KindJobError = "job_error" + // KindTurnCompleted marks one completed primary chat turn // (archie-core-035). Input-driven curators wake on it. Curator output // never produces this kind, so derived work cannot feed its own diff --git a/internal/gateway/stream.go b/internal/gateway/stream.go index 6278f24df..b0579bb4d 100644 --- a/internal/gateway/stream.go +++ b/internal/gateway/stream.go @@ -97,10 +97,7 @@ func toolPreview(name, output string) string { } return "completed with no printable preview" } - remaining := countNonEmptyOrContentLines(content) - 1 - skipped - if remaining < 0 { - remaining = 0 - } + remaining := max(countNonEmptyOrContentLines(content)-1-skipped, 0) if remaining > 0 { return fmt.Sprintf("%s\n… %d more lines", preview, remaining) } @@ -110,7 +107,7 @@ func toolPreview(name, output string) string { func compactToolPreview(output string) (string, int) { skipped := 0 - for _, line := range strings.Split(output, "\n") { + for line := range strings.SplitSeq(output, "\n") { line = strings.TrimSpace(line) if line == "" { continue