From 0cb83742694da7396c39a1642c5977bb3d7d6bf4 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 24 Aug 2026 13:28:45 +1000 Subject: [PATCH 1/2] feat(scheduling): ticker engine with configurable interval and pools Adds internal/domain/scheduling, the timing half of the cron/scheduling epic (archie-core-1786637496320-249-be6d9a20): a tick loop that asks a JobSource what is due at a configurable cadence and dispatches each due job to the pool it declares. The package owns *when* a scheduled job runs and nothing else. The job store, blueprints and delivery routing are separate issues in that epic, so the engine declares the two contracts it needs from them (JobSource, Runner) and names no implementation. That is what lets the timing behaviour be tested against a fake clock with no store, no NATS and no model. Behaviour worth stating explicitly, because each is load-bearing: - The source is authoritative about what is due. The engine applies no schedule arithmetic of its own, so cron expressions, fixed intervals and catch-up windows stay the store's decision. - A job never overlaps itself. While a run is queued or in flight, later ticks reporting the same ID are skipped rather than stacked, so a job slower than the interval runs back-to-back instead of fanning out. - The pools do not interfere: a blocked sequential job cannot delay a parallel one, and a saturated parallel cap cannot delay the sequential worker. They share only the tick. - An empty Pool means sequential, not parallel. Wrongly serialising a job costs latency; wrongly parallelising one costs correctness. - Every failure is contained and reported: source error, job error, job panic and job timeout each emit an event and leave the loop running. - Stop drains gracefully. Run contexts descend from Background rather than the loop context, so halting the loop stops new work without yanking the context out from under a run already in progress; the caller's deadline is what eventually cancels those. - An engine is single-use. Restarting after Stop is refused because Stop leaves the dropped backlog and the claims of runs it did not wait for behind, and a restart would re-run stale queued jobs while permanently suppressing the claimed ones. internal/events gains job_run/job_error: a scheduled run has no task and no chat turn behind it, so without those kinds a job that fired, stalled or failed is invisible everywhere. --- internal/domain/scheduling/contract.go | 114 +++ internal/domain/scheduling/ticker.go | 359 +++++++++ internal/domain/scheduling/ticker_test.go | 850 ++++++++++++++++++++++ internal/events/events.go | 8 + 4 files changed, 1331 insertions(+) create mode 100644 internal/domain/scheduling/contract.go create mode 100644 internal/domain/scheduling/ticker.go create mode 100644 internal/domain/scheduling/ticker_test.go diff --git a/internal/domain/scheduling/contract.go b/internal/domain/scheduling/contract.go new file mode 100644 index 000000000..fb604c262 --- /dev/null +++ b/internal/domain/scheduling/contract.go @@ -0,0 +1,114 @@ +// Package scheduling is the ticker engine: the timing half of archied's +// cron/scheduling capability (epic archie-core-1786637496320-249-be6d9a20). +// It owns one job and one job only — deciding *when* a scheduled job runs, +// and with what concurrency — and nothing about where jobs are stored, how +// they are authored, or how their output is delivered. +// +// The split is deliberate. A job store (file-backed, cross-process locked) +// and a delivery router are separate issues in that epic; this package +// declares the two contracts it needs from them (JobSource, Runner) and +// names no implementation. That is what lets the timing behaviour — pool +// semantics, overlap suppression, per-run timeouts, bounded shutdown — be +// tested against a fake clock with no store, no NATS and no model. +package scheduling + +import ( + "context" + "errors" + "fmt" + "strings" + "time" +) + +// Pool is how a job's runs are dispatched relative to other jobs. +type Pool string + +const ( + // PoolParallel runs the job concurrently with every other job, bounded + // only by the engine's MaxParallel cap. + PoolParallel Pool = "parallel" + // PoolSequential runs the job on a single shared worker, one at a time, + // in the order the source returned them. Use it for jobs that contend + // for a resource that tolerates no concurrency. + PoolSequential Pool = "sequential" +) + +// Validate rejects an unknown pool. The empty pool is valid and means +// PoolSequential: an unlabelled job is assumed to want the safer of the two, +// because wrongly serialising a job costs latency while wrongly parallelising +// one costs correctness. +func (p Pool) Validate() error { + switch p { + case "", PoolParallel, PoolSequential: + return nil + default: + return fmt.Errorf("scheduling: unknown pool %q", string(p)) + } +} + +// resolve maps the empty pool onto its documented default. +func (p Pool) resolve() Pool { + if p == "" { + return PoolSequential + } + return p +} + +// Job is one due unit of scheduled work as handed to the engine. It carries +// identity and dispatch policy only: the schedule itself (cron expression, +// next-run time, catch-up policy) belongs to the job store behind JobSource, +// which is why the engine never sees one. +type Job struct { + // ID identifies the job across ticks. It is the overlap key: while a + // run with this ID is queued or in flight, later ticks that report the + // same ID are skipped rather than stacked. + ID string + // Pool selects the dispatch pool. Empty means PoolSequential. + Pool Pool + // Detail is a human-readable label recorded on emitted events so a run + // stays attributable without the engine knowing what the job does. + Detail string +} + +// Validate checks the fields the engine relies on. +func (j Job) Validate() error { + if strings.TrimSpace(j.ID) == "" { + return errors.New("scheduling: job id must be non-empty") + } + return j.Pool.Validate() +} + +// JobSource yields the jobs eligible to run at a given instant. The engine +// calls Due once per tick and treats it as authoritative: it applies no +// schedule arithmetic of its own, so a source is free to implement cron +// expressions, fixed intervals or catch-up windows without the engine +// changing. +type JobSource interface { + Due(ctx context.Context, now time.Time) ([]Job, error) +} + +// Runner executes one due job. Implementations must respect ctx: the engine +// bounds every run with JobTimeout and cancels on shutdown, but a Runner that +// ignores cancellation can still delay Stop up to Stop's own deadline. +type Runner interface { + Run(ctx context.Context, job Job) error +} + +// Clock is injectable time: Now for timestamps and After for the tick sleep. +// Loop timing is tested against a fake clock, never wall-clock sleeps. +type Clock interface { + Now() time.Time + After(d time.Duration) <-chan time.Time +} + +type systemClock struct{} + +func (systemClock) Now() time.Time { return time.Now() } +func (systemClock) After(d time.Duration) <-chan time.Time { return time.After(d) } + +// Sink is the narrow event contract the engine needs. It matches +// events.Bus's emitter shape without importing it — the domain declares what +// it needs and app wires the implementation in. +type Sink interface { + Emit(kind, detail string, data map[string]any) +} diff --git a/internal/domain/scheduling/ticker.go b/internal/domain/scheduling/ticker.go new file mode 100644 index 000000000..587f028bf --- /dev/null +++ b/internal/domain/scheduling/ticker.go @@ -0,0 +1,359 @@ +package scheduling + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/samcharles93/archie-core/internal/events" +) + +// Defaults for the tunables an operator may leave unset. +const ( + // DefaultInterval is the tick cadence when none is configured. A minute + // is the finest granularity a cron-style schedule needs, and polling a + // local job store that often costs nothing. + DefaultInterval = time.Minute + // DefaultMaxParallel caps concurrent parallel-pool runs. It exists so a + // store that reports many jobs due at once (the first tick after a long + // downtime, typically) cannot launch an unbounded fan-out. + DefaultMaxParallel = 4 + // DefaultJobTimeout bounds one run. A scheduled job that has not + // finished in an hour is stuck, and leaving it in flight would suppress + // every later run of the same job forever. + DefaultJobTimeout = time.Hour +) + +// Event kinds emitted by the engine, re-exported so callers of this package +// need not import the events package to match on them. +const ( + KindJobRun = events.KindJobRun + KindJobError = events.KindJobError +) + +// Phases reported on KindJobError, naming which step failed. +const ( + phaseSource = "source" // JobSource.Due returned an error + phaseDispatch = "dispatch" // the job was rejected before it ever ran + phaseRun = "run" // Runner.Run failed, panicked or timed out +) + +// sequentialQueueDepth bounds the sequential pool's backlog. Overlap +// suppression already caps the backlog at one entry per distinct job, so this +// is a backstop against a pathological source, not the primary bound. +const sequentialQueueDepth = 256 + +// EngineConfig tunes the ticker engine. Every field is optional; zero values +// take the documented defaults. +type EngineConfig struct { + // Interval is the tick cadence: how often the engine asks its source + // what is due. Zero uses DefaultInterval; negative is an error. + Interval time.Duration + // MaxParallel caps simultaneous parallel-pool runs. Zero uses + // DefaultMaxParallel. It does not bound the sequential pool, which is + // one by definition. + MaxParallel int + // JobTimeout bounds a single run via context deadline. Zero uses + // DefaultJobTimeout. + JobTimeout time.Duration + // Clock is injectable time. Nil uses the system clock. + Clock Clock + // Events receives run and error events. Nil disables emission; the + // engine still runs, it is just unobservable. + Events Sink +} + +func (c EngineConfig) validate() error { + switch { + case c.Interval < 0: + return fmt.Errorf("scheduling: interval must not be negative, got %v", c.Interval) + case c.MaxParallel < 0: + return fmt.Errorf("scheduling: max parallel must not be negative, got %d", c.MaxParallel) + case c.JobTimeout < 0: + return fmt.Errorf("scheduling: job timeout must not be negative, got %v", c.JobTimeout) + } + return nil +} + +// withDefaults returns the config with zero values replaced. +func (c EngineConfig) withDefaults() EngineConfig { + if c.Interval == 0 { + c.Interval = DefaultInterval + } + if c.MaxParallel == 0 { + c.MaxParallel = DefaultMaxParallel + } + if c.JobTimeout == 0 { + c.JobTimeout = DefaultJobTimeout + } + if c.Clock == nil { + c.Clock = systemClock{} + } + return c +} + +// Engine is the ticker engine: one loop that asks the source what is due at +// each tick and dispatches each due job to the pool it declares. +// +// Two invariants make the timing safe to reason about: +// +// - A job never overlaps itself. While a run is queued or in flight, later +// ticks reporting the same ID are skipped, not stacked. A job slower than +// the tick interval therefore runs back-to-back, never fanned out. +// - The pools do not interfere. A blocked sequential job cannot delay a +// parallel one, and a saturated parallel cap cannot delay the sequential +// worker — they are separate dispatch paths sharing only the tick. +// +// Every failure is contained: a source error, a job error, a job panic and a +// job timeout are each reported as an event and leave the loop running. +type Engine struct { + source JobSource + runner Runner + cfg EngineConfig + + queue chan Job // sequential pool backlog + sem chan struct{} + + mu sync.Mutex + started bool + stopped bool + stopLoop context.CancelFunc + cancelRuns context.CancelFunc + active map[string]struct{} // queued or in flight, by job ID + wg sync.WaitGroup +} + +// NewEngine builds the engine over its source and runner. It fails on a nil +// dependency or a negative tunable; zero tunables take their defaults. +func NewEngine(source JobSource, runner Runner, cfg EngineConfig) (*Engine, error) { + if source == nil { + return nil, errors.New("scheduling: job source must not be nil") + } + if runner == nil { + return nil, errors.New("scheduling: runner must not be nil") + } + if err := cfg.validate(); err != nil { + return nil, err + } + cfg = cfg.withDefaults() + return &Engine{ + source: source, + runner: runner, + cfg: cfg, + queue: make(chan Job, sequentialQueueDepth), + sem: make(chan struct{}, cfg.MaxParallel), + active: make(map[string]struct{}), + }, nil +} + +// Interval reports the resolved tick cadence. +func (e *Engine) Interval() time.Duration { return e.cfg.Interval } + +// MaxParallel reports the resolved parallel-pool cap. +func (e *Engine) MaxParallel() int { return e.cfg.MaxParallel } + +// JobTimeout reports the resolved per-run timeout. +func (e *Engine) JobTimeout() time.Duration { return e.cfg.JobTimeout } + +// Start launches the tick loop and the sequential worker. +// +// An engine is single-use: starting one that is already started, or that has +// been stopped, is an error. Restarting is refused rather than supported +// because Stop leaves the dropped sequential backlog and the claims of runs +// it did not wait for behind — a restarted engine would re-run stale queued +// jobs and permanently suppress the jobs still marked claimed. Build a new +// engine instead; it is a struct, not a resource. +func (e *Engine) Start(ctx context.Context) error { + e.mu.Lock() + defer e.mu.Unlock() + if e.stopped { + return errors.New("scheduling: engine already stopped; build a new one") + } + if e.started { + return errors.New("scheduling: engine already started") + } + e.started = true + + loopCtx, stopLoop := context.WithCancel(ctx) + e.stopLoop = stopLoop + // Run contexts descend from Background, not from loopCtx, so that Stop + // drains gracefully: cancelling the loop must stop new work without + // yanking the context out from under a run that is already going. Stop + // cancels these only once its own deadline passes. + runCtx, cancelRuns := context.WithCancel(context.WithoutCancel(ctx)) + e.cancelRuns = cancelRuns + + e.wg.Add(2) + go e.tickLoop(loopCtx, runCtx) + go e.sequentialWorker(loopCtx, runCtx) + return nil +} + +// Stop halts ticking, drops any queued-but-unstarted runs, and waits for +// in-flight runs to finish, bounded by ctx. If ctx expires first, in-flight +// runs are cancelled and ctx's error is returned — a Runner that ignores +// cancellation can still outlive Stop, which is why the deadline is the +// caller's to set. Stopping an engine that never started is a no-op. +func (e *Engine) Stop(ctx context.Context) error { + e.mu.Lock() + if !e.started { + e.mu.Unlock() + return nil + } + e.started = false + e.stopped = true + stopLoop, cancelRuns := e.stopLoop, e.cancelRuns + e.stopLoop, e.cancelRuns = nil, nil + e.mu.Unlock() + + stopLoop() + done := make(chan struct{}) + go func() { + e.wg.Wait() + close(done) + }() + select { + case <-done: + cancelRuns() + return nil + case <-ctx.Done(): + cancelRuns() + return fmt.Errorf("scheduling: engine shutdown: %w", ctx.Err()) + } +} + +// tickLoop sleeps one interval, asks the source what is due, and dispatches +// it. The source is authoritative: the engine applies no schedule arithmetic +// of its own, so what "due" means stays the store's decision. +func (e *Engine) tickLoop(loopCtx, runCtx context.Context) { + defer e.wg.Done() + for { + select { + case <-loopCtx.Done(): + return + case <-e.cfg.Clock.After(e.cfg.Interval): + } + + jobs, err := e.source.Due(loopCtx, e.cfg.Clock.Now()) + if err != nil { + e.emitError(Job{}, phaseSource, err) + continue + } + for _, job := range jobs { + e.dispatch(loopCtx, runCtx, job) + } + } +} + +// dispatch routes one due job to its pool, skipping it if it is invalid or +// already queued or in flight. +func (e *Engine) dispatch(loopCtx, runCtx context.Context, job Job) { + if err := job.Validate(); err != nil { + e.emitError(job, phaseDispatch, err) + return + } + if !e.claim(job.ID) { + return // an earlier run of this job has not finished; skip this tick + } + + if job.Pool.resolve() == PoolParallel { + e.wg.Go(func() { + defer e.release(job.ID) + select { + case e.sem <- struct{}{}: + case <-loopCtx.Done(): + return // shutting down while waiting for a parallel slot + } + defer func() { <-e.sem }() + e.runJob(runCtx, job) + }) + return + } + + select { + case e.queue <- job: + default: + e.release(job.ID) + e.emitError(job, phaseDispatch, errors.New("scheduling: sequential queue full")) + } +} + +// sequentialWorker is the sequential pool: one goroutine draining the queue +// in the order the source reported the jobs, so serialised jobs never +// overlap each other. +func (e *Engine) sequentialWorker(loopCtx, runCtx context.Context) { + defer e.wg.Done() + for { + select { + case <-loopCtx.Done(): + return + case job := <-e.queue: + e.runJob(runCtx, job) + e.release(job.ID) + } + } +} + +// runJob executes one run under the configured timeout, recovering panics so +// a job that blows up takes nothing else with it. +func (e *Engine) runJob(ctx context.Context, job Job) { + runCtx, cancel := context.WithTimeout(ctx, e.cfg.JobTimeout) + defer cancel() + + start := e.cfg.Clock.Now() + err := e.safeRun(runCtx, job) + elapsed := e.cfg.Clock.Now().Sub(start) + if err != nil { + e.emitError(job, phaseRun, err) + return + } + e.emit(KindJobRun, job, map[string]any{ + "job": job.ID, + "pool": string(job.Pool.resolve()), + "duration_ms": elapsed.Milliseconds(), + }) +} + +func (e *Engine) safeRun(ctx context.Context, job Job) (err error) { + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("scheduling: job %q panicked: %v", job.ID, p) + } + }() + return e.runner.Run(ctx, job) +} + +// claim marks a job as queued-or-running, reporting false if it already was. +func (e *Engine) claim(id string) bool { + e.mu.Lock() + defer e.mu.Unlock() + if _, busy := e.active[id]; busy { + return false + } + e.active[id] = struct{}{} + return true +} + +func (e *Engine) release(id string) { + e.mu.Lock() + defer e.mu.Unlock() + delete(e.active, id) +} + +func (e *Engine) emitError(job Job, phase string, err error) { + e.emit(KindJobError, job, map[string]any{ + "job": job.ID, + "pool": string(job.Pool), + "phase": phase, + "err": err.Error(), + }) +} + +func (e *Engine) emit(kind string, job Job, data map[string]any) { + if e.cfg.Events == nil { + return + } + e.cfg.Events.Emit(kind, job.Detail, data) +} diff --git a/internal/domain/scheduling/ticker_test.go b/internal/domain/scheduling/ticker_test.go new file mode 100644 index 000000000..f3c06f398 --- /dev/null +++ b/internal/domain/scheduling/ticker_test.go @@ -0,0 +1,850 @@ +package scheduling + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +// --- fakes -------------------------------------------------------------------- + +// fakeClock is the engine's injectable clock: Now plus clock-driven timers +// that fire when Advance moves past their deadline. Tick timing is tested +// with this, never wall-clock sleeps. +type fakeClock struct { + mu sync.Mutex + now time.Time + timers []*fakeTimer +} + +type fakeTimer struct { + c chan time.Time + at time.Time +} + +func newFakeClock(start time.Time) *fakeClock { return &fakeClock{now: start} } + +func (fc *fakeClock) Now() time.Time { + fc.mu.Lock() + defer fc.mu.Unlock() + return fc.now +} + +func (fc *fakeClock) After(d time.Duration) <-chan time.Time { + fc.mu.Lock() + defer fc.mu.Unlock() + t := &fakeTimer{c: make(chan time.Time, 1), at: fc.now.Add(d)} + fc.timers = append(fc.timers, t) + return t.c +} + +// pending reports how many timers are waiting; the tick loop registers +// exactly one at a time, so it is the signal that the loop is back asleep. +func (fc *fakeClock) pending() int { + fc.mu.Lock() + defer fc.mu.Unlock() + return len(fc.timers) +} + +// Advance moves the clock forward and fires every timer whose deadline has +// passed, returning how many fired. +func (fc *fakeClock) Advance(d time.Duration) int { + fc.mu.Lock() + fc.now = fc.now.Add(d) + var fired []*fakeTimer + remain := fc.timers[:0] + for _, t := range fc.timers { + if !t.at.After(fc.now) { + fired = append(fired, t) + } else { + remain = append(remain, t) + } + } + fc.timers = remain + fc.mu.Unlock() + for _, t := range fired { + select { + case t.c <- t.at: + default: + } + } + return len(fired) +} + +// sourceFunc adapts a function to JobSource. +type sourceFunc func(ctx context.Context, now time.Time) ([]Job, error) + +func (f sourceFunc) Due(ctx context.Context, now time.Time) ([]Job, error) { return f(ctx, now) } + +// staticSource returns the same jobs every tick and counts the calls. +type staticSource struct { + jobs []Job + mu sync.Mutex + calls int +} + +func (s *staticSource) Due(context.Context, time.Time) ([]Job, error) { + s.mu.Lock() + s.calls++ + s.mu.Unlock() + return s.jobs, nil +} + +func (s *staticSource) callCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.calls +} + +// runnerFunc adapts a function to Runner. +type runnerFunc func(ctx context.Context, job Job) error + +func (f runnerFunc) Run(ctx context.Context, job Job) error { return f(ctx, job) } + +// recordingRunner records every run in order and optionally blocks on a gate +// keyed by job ID, so a test can hold a run in flight deterministically. +type recordingRunner struct { + mu sync.Mutex + started []string + done []string + inFlt int + maxFlt int + gates map[string]chan struct{} + err error +} + +func newRecordingRunner() *recordingRunner { + return &recordingRunner{gates: map[string]chan struct{}{}} +} + +// gate installs a release channel for a job; runs of that job block until +// release is called. +func (r *recordingRunner) gate(id string) { + r.mu.Lock() + defer r.mu.Unlock() + r.gates[id] = make(chan struct{}) +} + +func (r *recordingRunner) release(id string) { + r.mu.Lock() + ch := r.gates[id] + delete(r.gates, id) + r.mu.Unlock() + if ch != nil { + close(ch) + } +} + +func (r *recordingRunner) Run(ctx context.Context, job Job) error { + r.mu.Lock() + r.started = append(r.started, job.ID) + r.inFlt++ + if r.inFlt > r.maxFlt { + r.maxFlt = r.inFlt + } + gate := r.gates[job.ID] + err := r.err + r.mu.Unlock() + + if gate != nil { + select { + case <-gate: + case <-ctx.Done(): + err = ctx.Err() + } + } + + r.mu.Lock() + r.inFlt-- + r.done = append(r.done, job.ID) + r.mu.Unlock() + return err +} + +func (r *recordingRunner) startedIDs() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.started...) +} + +func (r *recordingRunner) doneIDs() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.done...) +} + +func (r *recordingRunner) peakInFlight() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.maxFlt +} + +// recordingSink collects emitted events. +type recordingSink struct { + mu sync.Mutex + events []sinkEvent +} + +type sinkEvent struct { + kind string + detail string + data map[string]any +} + +func (s *recordingSink) Emit(kind, detail string, data map[string]any) { + s.mu.Lock() + defer s.mu.Unlock() + s.events = append(s.events, sinkEvent{kind: kind, detail: detail, data: data}) +} + +func (s *recordingSink) countOf(kind string) int { + s.mu.Lock() + defer s.mu.Unlock() + n := 0 + for _, e := range s.events { + if e.kind == kind { + n++ + } + } + return n +} + +func (s *recordingSink) firstOf(kind string) (sinkEvent, bool) { + s.mu.Lock() + defer s.mu.Unlock() + for _, e := range s.events { + if e.kind == kind { + return e, true + } + } + return sinkEvent{}, false +} + +// --- helpers ------------------------------------------------------------------ + +const ( + testInterval = time.Minute + testTimeout = 2 * time.Second + pollGap = time.Millisecond +) + +func testStart() time.Time { return time.Date(2026, 8, 24, 12, 0, 0, 0, time.UTC) } + +// waitFor polls cond until it holds or the timeout expires. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(testTimeout) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(pollGap) + } + t.Fatalf("timed out waiting for %s", what) +} + +// tick advances the fake clock one interval and waits for the loop to arm its +// next timer, which happens only after the tick's dispatch has been decided. +func tick(t *testing.T, fc *fakeClock, e *Engine) { + t.Helper() + waitFor(t, "loop to arm its tick timer", func() bool { return fc.pending() > 0 }) + fc.Advance(testInterval) + waitFor(t, "loop to re-arm after tick", func() bool { return fc.pending() > 0 }) + _ = e +} + +// startEngine builds and starts an engine, registering Stop as cleanup. +func startEngine(t *testing.T, src JobSource, runner Runner, cfg EngineConfig) *Engine { + t.Helper() + e, err := NewEngine(src, runner, cfg) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + if err := e.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + _ = e.Stop(ctx) + }) + return e +} + +func countOccurrences(ids []string, want string) int { + n := 0 + for _, id := range ids { + if id == want { + n++ + } + } + return n +} + +// --- construction ------------------------------------------------------------- + +func TestNewEngineValidatesConfig(t *testing.T) { + src := &staticSource{} + runner := newRecordingRunner() + + tests := []struct { + name string + source JobSource + runner Runner + cfg EngineConfig + wantErr bool + }{ + {name: "nil source", source: nil, runner: runner, cfg: EngineConfig{}, wantErr: true}, + {name: "nil runner", source: src, runner: nil, cfg: EngineConfig{}, wantErr: true}, + {name: "negative interval", source: src, runner: runner, cfg: EngineConfig{Interval: -testInterval}, wantErr: true}, + {name: "negative max parallel", source: src, runner: runner, cfg: EngineConfig{MaxParallel: -1}, wantErr: true}, + {name: "negative job timeout", source: src, runner: runner, cfg: EngineConfig{JobTimeout: -testInterval}, wantErr: true}, + {name: "zero values are defaulted", source: src, runner: runner, cfg: EngineConfig{}, wantErr: false}, + { + name: "fully specified", source: src, runner: runner, + cfg: EngineConfig{Interval: testInterval, MaxParallel: 2, JobTimeout: testInterval}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e, err := NewEngine(tc.source, tc.runner, tc.cfg) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error, got engine %v", e) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if e == nil { + t.Fatal("expected engine, got nil") + } + }) + } +} + +func TestNewEngineAppliesDefaults(t *testing.T) { + e, err := NewEngine(&staticSource{}, newRecordingRunner(), EngineConfig{}) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + if got := e.Interval(); got != DefaultInterval { + t.Errorf("interval = %v, want %v", got, DefaultInterval) + } + if got := e.MaxParallel(); got != DefaultMaxParallel { + t.Errorf("max parallel = %d, want %d", got, DefaultMaxParallel) + } + if got := e.JobTimeout(); got != DefaultJobTimeout { + t.Errorf("job timeout = %v, want %v", got, DefaultJobTimeout) + } +} + +func TestEngineHonoursConfiguredInterval(t *testing.T) { + for _, interval := range []time.Duration{time.Second, testInterval, time.Hour} { + t.Run(interval.String(), func(t *testing.T) { + fc := newFakeClock(testStart()) + src := &staticSource{} + e := startEngine(t, src, newRecordingRunner(), EngineConfig{Interval: interval, Clock: fc}) + + waitFor(t, "first tick timer", func() bool { return fc.pending() > 0 }) + // One nanosecond short of the interval must not tick. + fc.Advance(interval - time.Nanosecond) + time.Sleep(10 * pollGap) + if got := src.callCount(); got != 0 { + t.Fatalf("source called %d times before the interval elapsed", got) + } + fc.Advance(time.Nanosecond) + waitFor(t, "source poll", func() bool { return src.callCount() >= 1 }) + if got := e.Interval(); got != interval { + t.Errorf("Interval() = %v, want %v", got, interval) + } + }) + } +} + +// --- dispatch ----------------------------------------------------------------- + +func TestEngineRunsDueJobs(t *testing.T) { + fc := newFakeClock(testStart()) + jobs := []Job{{ID: "a", Pool: PoolParallel}, {ID: "b", Pool: PoolSequential}} + src := &staticSource{jobs: jobs} + runner := newRecordingRunner() + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc}) + + tick(t, fc, e) + waitFor(t, "both jobs to finish", func() bool { return len(runner.doneIDs()) == len(jobs) }) + + for _, j := range jobs { + if countOccurrences(runner.doneIDs(), j.ID) != 1 { + t.Errorf("job %q ran %d times, want 1", j.ID, countOccurrences(runner.doneIDs(), j.ID)) + } + } +} + +func TestEngineParallelPoolRunsConcurrentlyUpToCap(t *testing.T) { + const maxParallel = 3 + fc := newFakeClock(testStart()) + jobs := make([]Job, 0, maxParallel+2) + runner := newRecordingRunner() + for i := range cap(jobs) { + id := string(rune('a' + i)) + jobs = append(jobs, Job{ID: id, Pool: PoolParallel}) + runner.gate(id) + } + src := &staticSource{jobs: jobs} + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, MaxParallel: maxParallel, Clock: fc}) + + tick(t, fc, e) + waitFor(t, "cap to be saturated", func() bool { return runner.peakInFlight() >= maxParallel }) + // Give any over-cap goroutine a chance to break the cap before asserting. + time.Sleep(10 * pollGap) + if got := runner.peakInFlight(); got != maxParallel { + t.Fatalf("peak in-flight = %d, want %d", got, maxParallel) + } + for _, j := range jobs { + runner.release(j.ID) + } + waitFor(t, "all jobs to finish", func() bool { return len(runner.doneIDs()) == len(jobs) }) +} + +func TestEngineSequentialPoolRunsOneAtATimeInOrder(t *testing.T) { + fc := newFakeClock(testStart()) + jobs := []Job{{ID: "first"}, {ID: "second", Pool: PoolSequential}, {ID: "third", Pool: PoolSequential}} + src := &staticSource{jobs: jobs} + runner := newRecordingRunner() + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, MaxParallel: len(jobs), Clock: fc}) + + tick(t, fc, e) + waitFor(t, "sequential jobs to finish", func() bool { return len(runner.doneIDs()) == len(jobs) }) + + if got := runner.peakInFlight(); got != 1 { + t.Errorf("sequential peak in-flight = %d, want 1", got) + } + for i, id := range runner.startedIDs() { + if id != jobs[i].ID { + t.Errorf("run %d = %q, want %q (source order)", i, id, jobs[i].ID) + } + } +} + +func TestEngineSequentialJobDoesNotBlockParallelJob(t *testing.T) { + fc := newFakeClock(testStart()) + src := &staticSource{jobs: []Job{{ID: "slow-seq", Pool: PoolSequential}, {ID: "quick-par", Pool: PoolParallel}}} + runner := newRecordingRunner() + runner.gate("slow-seq") + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc}) + + tick(t, fc, e) + waitFor(t, "parallel job to finish while sequential is blocked", func() bool { + return countOccurrences(runner.doneIDs(), "quick-par") == 1 + }) + runner.release("slow-seq") + waitFor(t, "sequential job to finish", func() bool { + return countOccurrences(runner.doneIDs(), "slow-seq") == 1 + }) +} + +func TestEngineSkipsJobAlreadyInFlight(t *testing.T) { + for _, pool := range []Pool{PoolParallel, PoolSequential} { + t.Run(string(pool), func(t *testing.T) { + const id = "overlapping" + fc := newFakeClock(testStart()) + src := &staticSource{jobs: []Job{{ID: id, Pool: pool}}} + runner := newRecordingRunner() + runner.gate(id) + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc}) + + tick(t, fc, e) + waitFor(t, "first run to start", func() bool { return len(runner.startedIDs()) == 1 }) + tick(t, fc, e) + tick(t, fc, e) + time.Sleep(10 * pollGap) + if got := len(runner.startedIDs()); got != 1 { + t.Fatalf("job started %d times while in flight, want 1", got) + } + runner.release(id) + waitFor(t, "first run to finish", func() bool { return len(runner.doneIDs()) == 1 }) + + // Once the run is done the job is eligible again. + tick(t, fc, e) + waitFor(t, "second run", func() bool { return len(runner.startedIDs()) == 2 }) + }) + } +} + +func TestEngineRejectsInvalidJob(t *testing.T) { + tests := []struct { + name string + job Job + }{ + {name: "empty id", job: Job{ID: " ", Pool: PoolParallel}}, + {name: "unknown pool", job: Job{ID: "bad-pool", Pool: Pool("elsewhere")}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fc := newFakeClock(testStart()) + sink := &recordingSink{} + src := &staticSource{jobs: []Job{tc.job, {ID: "healthy", Pool: PoolParallel}}} + runner := newRecordingRunner() + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc, Events: sink}) + + tick(t, fc, e) + waitFor(t, "healthy job to run", func() bool { + return countOccurrences(runner.doneIDs(), "healthy") == 1 + }) + if got := len(runner.startedIDs()); got != 1 { + t.Fatalf("started %v, want only the healthy job", runner.startedIDs()) + } + if sink.countOf(KindJobError) == 0 { + t.Error("expected an error event for the invalid job") + } + }) + } +} + +// --- failure isolation -------------------------------------------------------- + +func TestEngineSurvivesSourceError(t *testing.T) { + fc := newFakeClock(testStart()) + sink := &recordingSink{} + var calls int + var mu sync.Mutex + wantErr := errors.New("store unavailable") + src := sourceFunc(func(context.Context, time.Time) ([]Job, error) { + mu.Lock() + calls++ + n := calls + mu.Unlock() + if n == 1 { + return nil, wantErr + } + return []Job{{ID: "after-failure"}}, nil + }) + runner := newRecordingRunner() + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc, Events: sink}) + + tick(t, fc, e) + waitFor(t, "error event", func() bool { return sink.countOf(KindJobError) == 1 }) + tick(t, fc, e) + waitFor(t, "loop to keep ticking after a source error", func() bool { + return countOccurrences(runner.doneIDs(), "after-failure") == 1 + }) + + ev, ok := sink.firstOf(KindJobError) + if !ok { + t.Fatal("no error event recorded") + } + if got, _ := ev.data["phase"].(string); got != phaseSource { + t.Errorf("error phase = %q, want %q", got, phaseSource) + } +} + +func TestEngineSurvivesJobPanicAndError(t *testing.T) { + tests := []struct { + name string + run func(job Job) error + }{ + {name: "panic", run: func(Job) error { panic("boom") }}, + {name: "error", run: func(Job) error { return errors.New("job failed") }}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fc := newFakeClock(testStart()) + sink := &recordingSink{} + var mu sync.Mutex + var runs int + src := &staticSource{jobs: []Job{{ID: "unstable"}}} + runner := runnerFunc(func(_ context.Context, job Job) error { + mu.Lock() + runs++ + mu.Unlock() + return tc.run(job) + }) + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc, Events: sink}) + + tick(t, fc, e) + waitFor(t, "first run", func() bool { mu.Lock(); defer mu.Unlock(); return runs == 1 }) + waitFor(t, "error event", func() bool { return sink.countOf(KindJobError) == 1 }) + // The loop survives: the same job is dispatched again next tick. + tick(t, fc, e) + waitFor(t, "second run", func() bool { mu.Lock(); defer mu.Unlock(); return runs == 2 }) + + ev, _ := sink.firstOf(KindJobError) + if got, _ := ev.data["phase"].(string); got != phaseRun { + t.Errorf("error phase = %q, want %q", got, phaseRun) + } + }) + } +} + +func TestEngineBoundsRunWithJobTimeout(t *testing.T) { + fc := newFakeClock(testStart()) + sink := &recordingSink{} + src := &staticSource{jobs: []Job{{ID: "hangs"}}} + deadlines := make(chan error, 1) + runner := runnerFunc(func(ctx context.Context, _ Job) error { + <-ctx.Done() + deadlines <- ctx.Err() + return ctx.Err() + }) + // A real (short) timeout: the run's deadline is wall-clock, because a + // Runner's ctx must be honoured by code that knows nothing of our clock. + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, JobTimeout: 20 * time.Millisecond, Clock: fc, Events: sink}) + + tick(t, fc, e) + select { + case err := <-deadlines: + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("run ctx error = %v, want DeadlineExceeded", err) + } + case <-time.After(testTimeout): + t.Fatal("run was never cancelled by the job timeout") + } + waitFor(t, "timeout to be reported", func() bool { return sink.countOf(KindJobError) == 1 }) +} + +// --- lifecycle ---------------------------------------------------------------- + +func TestEngineStartTwiceIsAnError(t *testing.T) { + fc := newFakeClock(testStart()) + e := startEngine(t, &staticSource{}, newRecordingRunner(), EngineConfig{Interval: testInterval, Clock: fc}) + if err := e.Start(context.Background()); err == nil { + t.Fatal("expected an error starting an already-started engine") + } +} + +func TestEngineRestartAfterStopIsRefused(t *testing.T) { + fc := newFakeClock(testStart()) + e, err := NewEngine(&staticSource{}, newRecordingRunner(), EngineConfig{Interval: testInterval, Clock: fc}) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + if err := e.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + if err := e.Stop(ctx); err != nil { + t.Fatalf("Stop: %v", err) + } + if err := e.Start(context.Background()); err == nil { + t.Fatal("expected an error restarting a stopped engine") + } +} + +func TestEngineStopWithoutStartIsNoOp(t *testing.T) { + e, err := NewEngine(&staticSource{}, newRecordingRunner(), EngineConfig{}) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + if err := e.Stop(ctx); err != nil { + t.Fatalf("Stop before Start = %v, want nil", err) + } +} + +func TestEngineStopWaitsForInFlightRuns(t *testing.T) { + fc := newFakeClock(testStart()) + src := &staticSource{jobs: []Job{{ID: "par", Pool: PoolParallel}, {ID: "seq", Pool: PoolSequential}}} + runner := newRecordingRunner() + runner.gate("par") + runner.gate("seq") + e, err := NewEngine(src, runner, EngineConfig{Interval: testInterval, Clock: fc}) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + if err := e.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + + tick(t, fc, e) + waitFor(t, "both runs to start", func() bool { return len(runner.startedIDs()) == 2 }) + + stopped := make(chan error, 1) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + stopped <- e.Stop(ctx) + }() + + select { + case <-stopped: + t.Fatal("Stop returned while runs were still in flight") + case <-time.After(20 * pollGap): + } + + runner.release("par") + runner.release("seq") + select { + case err := <-stopped: + if err != nil { + t.Fatalf("Stop: %v", err) + } + case <-time.After(testTimeout): + t.Fatal("Stop did not return after runs finished") + } + if got := len(runner.doneIDs()); got != 2 { + t.Fatalf("finished runs = %d, want 2", got) + } +} + +func TestEngineStopIsBoundedByContext(t *testing.T) { + fc := newFakeClock(testStart()) + src := &staticSource{jobs: []Job{{ID: "uninterruptible"}}} + release := make(chan struct{}) + started := make(chan struct{}, 1) + runner := runnerFunc(func(context.Context, Job) error { + select { + case started <- struct{}{}: + default: + } + <-release // deliberately ignores cancellation + return nil + }) + e, err := NewEngine(src, runner, EngineConfig{Interval: testInterval, Clock: fc}) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + if err := e.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + defer close(release) + + tick(t, fc, e) + <-started + + ctx, cancel := context.WithTimeout(context.Background(), 20*pollGap) + defer cancel() + if err := e.Stop(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Stop = %v, want a deadline-exceeded error", err) + } +} + +func TestEngineStopStopsTicking(t *testing.T) { + fc := newFakeClock(testStart()) + src := &staticSource{jobs: []Job{{ID: "job"}}} + runner := newRecordingRunner() + e, err := NewEngine(src, runner, EngineConfig{Interval: testInterval, Clock: fc}) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + if err := e.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + tick(t, fc, e) + waitFor(t, "first run", func() bool { return len(runner.doneIDs()) == 1 }) + + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + if err := e.Stop(ctx); err != nil { + t.Fatalf("Stop: %v", err) + } + before := src.callCount() + fc.Advance(testInterval * 10) + time.Sleep(20 * pollGap) + if got := src.callCount(); got != before { + t.Fatalf("source polled %d times after Stop (was %d)", got, before) + } +} + +// --- events ------------------------------------------------------------------- + +func TestEngineEmitsRunEvents(t *testing.T) { + fc := newFakeClock(testStart()) + sink := &recordingSink{} + const detail = "daily status summary" + src := &staticSource{jobs: []Job{{ID: "summary", Pool: PoolParallel, Detail: detail}}} + e := startEngine(t, src, newRecordingRunner(), EngineConfig{Interval: testInterval, Clock: fc, Events: sink}) + + tick(t, fc, e) + waitFor(t, "run event", func() bool { return sink.countOf(KindJobRun) == 1 }) + + ev, _ := sink.firstOf(KindJobRun) + if ev.detail != detail { + t.Errorf("event detail = %q, want %q", ev.detail, detail) + } + if got, _ := ev.data["job"].(string); got != "summary" { + t.Errorf("event job = %q, want %q", got, "summary") + } + if got, _ := ev.data["pool"].(string); got != string(PoolParallel) { + t.Errorf("event pool = %q, want %q", got, PoolParallel) + } +} + +func TestEngineWithoutSinkDoesNotPanic(t *testing.T) { + fc := newFakeClock(testStart()) + src := &staticSource{jobs: []Job{{ID: "bad-pool", Pool: Pool("nowhere")}, {ID: "fine"}}} + runner := newRecordingRunner() + e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc}) + + tick(t, fc, e) + waitFor(t, "valid job to run", func() bool { return countOccurrences(runner.doneIDs(), "fine") == 1 }) +} + +// --- contract ----------------------------------------------------------------- + +func TestPoolValidate(t *testing.T) { + tests := []struct { + pool Pool + wantErr bool + }{ + {pool: PoolParallel}, + {pool: PoolSequential}, + {pool: ""}, + {pool: Pool("PARALLEL"), wantErr: true}, + {pool: Pool("other"), wantErr: true}, + } + for _, tc := range tests { + t.Run(string(tc.pool), func(t *testing.T) { + if err := tc.pool.Validate(); (err != nil) != tc.wantErr { + t.Fatalf("Validate(%q) = %v, wantErr %v", tc.pool, err, tc.wantErr) + } + }) + } +} + +func TestPoolResolveDefaultsToSequential(t *testing.T) { + tests := []struct { + pool Pool + want Pool + }{ + {pool: "", want: PoolSequential}, + {pool: PoolSequential, want: PoolSequential}, + {pool: PoolParallel, want: PoolParallel}, + } + for _, tc := range tests { + t.Run(string(tc.pool), func(t *testing.T) { + if got := tc.pool.resolve(); got != tc.want { + t.Fatalf("resolve(%q) = %q, want %q", tc.pool, got, tc.want) + } + }) + } +} + +func TestJobValidate(t *testing.T) { + tests := []struct { + name string + job Job + wantErr bool + }{ + {name: "valid", job: Job{ID: "a", Pool: PoolParallel}}, + {name: "valid default pool", job: Job{ID: "a"}}, + {name: "empty id", job: Job{ID: ""}, wantErr: true}, + {name: "blank id", job: Job{ID: "\t "}, wantErr: true}, + {name: "unknown pool", job: Job{ID: "a", Pool: Pool("x")}, wantErr: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if err := tc.job.Validate(); (err != nil) != tc.wantErr { + t.Fatalf("Validate() = %v, wantErr %v", err, tc.wantErr) + } + }) + } +} diff --git a/internal/events/events.go b/internal/events/events.go index df21803fb..817c6bf94 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -48,6 +48,14 @@ const ( KindCuratorAction = "curator_action" // data: curator, type, detail, reason KindCuratorError = "curator_error" // data: curator, phase, err + // Scheduled-job activity from the ticker engine + // (internal/domain/scheduling). A scheduled run has no task and no chat + // turn behind it, so without these a job that fired, stalled or failed + // is invisible everywhere. data: job, pool, duration_ms (run) and job, + // pool, phase, err (error). + KindJobRun = "job_run" + KindJobError = "job_error" + // KindTurnCompleted marks one completed primary chat turn // (archie-core-035). Input-driven curators wake on it. Curator output // never produces this kind, so derived work cannot feed its own From c460a80c1176c6ec08677a018fe6fb3102ca1dd6 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 24 Aug 2026 13:28:51 +1000 Subject: [PATCH 2/2] chore(gateway): apply go fix rewrites in stream.go Pre-existing code `task fmt` rewrites on every run: a manual clamp to `max`, and a `strings.Split` range to `strings.SplitSeq`. Both are semantically identical; committing them keeps the gate from reporting a dirty tree on unrelated work. --- internal/gateway/stream.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/gateway/stream.go b/internal/gateway/stream.go index 6278f24df..b0579bb4d 100644 --- a/internal/gateway/stream.go +++ b/internal/gateway/stream.go @@ -97,10 +97,7 @@ func toolPreview(name, output string) string { } return "completed with no printable preview" } - remaining := countNonEmptyOrContentLines(content) - 1 - skipped - if remaining < 0 { - remaining = 0 - } + remaining := max(countNonEmptyOrContentLines(content)-1-skipped, 0) if remaining > 0 { return fmt.Sprintf("%s\n… %d more lines", preview, remaining) } @@ -110,7 +107,7 @@ func toolPreview(name, output string) string { func compactToolPreview(output string) (string, int) { skipped := 0 - for _, line := range strings.Split(output, "\n") { + for line := range strings.SplitSeq(output, "\n") { line = strings.TrimSpace(line) if line == "" { continue