Merge pull request #614 from samcharles93/feat/275-ticker-engine

Feat/275 ticker engine
This commit is contained in:
sam authored and GitHub committed 2026-08-24 14:37:21 +10:00
commit eb12a02442
5 files changed
+1333 -5

No files matched your search

+114
View File
@@ -0,0 +1,114 @@
// Package scheduling is the ticker engine: the timing half of archied's
// cron/scheduling capability (epic archie-core-1786637496320-249-be6d9a20).
// It owns one job and one job only — deciding *when* a scheduled job runs,
// and with what concurrency — and nothing about where jobs are stored, how
// they are authored, or how their output is delivered.
//
// The split is deliberate. A job store (file-backed, cross-process locked)
// and a delivery router are separate issues in that epic; this package
// declares the two contracts it needs from them (JobSource, Runner) and
// names no implementation. That is what lets the timing behaviour — pool
// semantics, overlap suppression, per-run timeouts, bounded shutdown — be
// tested against a fake clock with no store, no NATS and no model.
package scheduling
import (
"context"
"errors"
"fmt"
"strings"
"time"
)
// Pool is how a job's runs are dispatched relative to other jobs.
type Pool string
const (
// PoolParallel runs the job concurrently with every other job, bounded
// only by the engine's MaxParallel cap.
PoolParallel Pool = "parallel"
// PoolSequential runs the job on a single shared worker, one at a time,
// in the order the source returned them. Use it for jobs that contend
// for a resource that tolerates no concurrency.
PoolSequential Pool = "sequential"
)
// Validate rejects an unknown pool. The empty pool is valid and means
// PoolSequential: an unlabelled job is assumed to want the safer of the two,
// because wrongly serialising a job costs latency while wrongly parallelising
// one costs correctness.
func (p Pool) Validate() error {
switch p {
case "", PoolParallel, PoolSequential:
return nil
default:
return fmt.Errorf("scheduling: unknown pool %q", string(p))
}
}
// resolve maps the empty pool onto its documented default.
func (p Pool) resolve() Pool {
if p == "" {
return PoolSequential
}
return p
}
// Job is one due unit of scheduled work as handed to the engine. It carries
// identity and dispatch policy only: the schedule itself (cron expression,
// next-run time, catch-up policy) belongs to the job store behind JobSource,
// which is why the engine never sees one.
type Job struct {
// ID identifies the job across ticks. It is the overlap key: while a
// run with this ID is queued or in flight, later ticks that report the
// same ID are skipped rather than stacked.
ID string
// Pool selects the dispatch pool. Empty means PoolSequential.
Pool Pool
// Detail is a human-readable label recorded on emitted events so a run
// stays attributable without the engine knowing what the job does.
Detail string
}
// Validate checks the fields the engine relies on.
func (j Job) Validate() error {
if strings.TrimSpace(j.ID) == "" {
return errors.New("scheduling: job id must be non-empty")
}
return j.Pool.Validate()
}
// JobSource yields the jobs eligible to run at a given instant. The engine
// calls Due once per tick and treats it as authoritative: it applies no
// schedule arithmetic of its own, so a source is free to implement cron
// expressions, fixed intervals or catch-up windows without the engine
// changing.
type JobSource interface {
Due(ctx context.Context, now time.Time) ([]Job, error)
}
// Runner executes one due job. Implementations must respect ctx: the engine
// bounds every run with JobTimeout and cancels on shutdown, but a Runner that
// ignores cancellation can still delay Stop up to Stop's own deadline.
type Runner interface {
Run(ctx context.Context, job Job) error
}
// Clock is injectable time: Now for timestamps and After for the tick sleep.
// Loop timing is tested against a fake clock, never wall-clock sleeps.
type Clock interface {
Now() time.Time
After(d time.Duration) <-chan time.Time
}
type systemClock struct{}
func (systemClock) Now() time.Time { return time.Now() }
func (systemClock) After(d time.Duration) <-chan time.Time { return time.After(d) }
// Sink is the narrow event contract the engine needs. It matches
// events.Bus's emitter shape without importing it — the domain declares what
// it needs and app wires the implementation in.
type Sink interface {
Emit(kind, detail string, data map[string]any)
}
+359
View File
@@ -0,0 +1,359 @@
package scheduling
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/samcharles93/archie-core/internal/events"
)
// Defaults for the tunables an operator may leave unset.
const (
// DefaultInterval is the tick cadence when none is configured. A minute
// is the finest granularity a cron-style schedule needs, and polling a
// local job store that often costs nothing.
DefaultInterval = time.Minute
// DefaultMaxParallel caps concurrent parallel-pool runs. It exists so a
// store that reports many jobs due at once (the first tick after a long
// downtime, typically) cannot launch an unbounded fan-out.
DefaultMaxParallel = 4
// DefaultJobTimeout bounds one run. A scheduled job that has not
// finished in an hour is stuck, and leaving it in flight would suppress
// every later run of the same job forever.
DefaultJobTimeout = time.Hour
)
// Event kinds emitted by the engine, re-exported so callers of this package
// need not import the events package to match on them.
const (
KindJobRun = events.KindJobRun
KindJobError = events.KindJobError
)
// Phases reported on KindJobError, naming which step failed.
const (
phaseSource = "source" // JobSource.Due returned an error
phaseDispatch = "dispatch" // the job was rejected before it ever ran
phaseRun = "run" // Runner.Run failed, panicked or timed out
)
// sequentialQueueDepth bounds the sequential pool's backlog. Overlap
// suppression already caps the backlog at one entry per distinct job, so this
// is a backstop against a pathological source, not the primary bound.
const sequentialQueueDepth = 256
// EngineConfig tunes the ticker engine. Every field is optional; zero values
// take the documented defaults.
type EngineConfig struct {
// Interval is the tick cadence: how often the engine asks its source
// what is due. Zero uses DefaultInterval; negative is an error.
Interval time.Duration
// MaxParallel caps simultaneous parallel-pool runs. Zero uses
// DefaultMaxParallel. It does not bound the sequential pool, which is
// one by definition.
MaxParallel int
// JobTimeout bounds a single run via context deadline. Zero uses
// DefaultJobTimeout.
JobTimeout time.Duration
// Clock is injectable time. Nil uses the system clock.
Clock Clock
// Events receives run and error events. Nil disables emission; the
// engine still runs, it is just unobservable.
Events Sink
}
func (c EngineConfig) validate() error {
switch {
case c.Interval < 0:
return fmt.Errorf("scheduling: interval must not be negative, got %v", c.Interval)
case c.MaxParallel < 0:
return fmt.Errorf("scheduling: max parallel must not be negative, got %d", c.MaxParallel)
case c.JobTimeout < 0:
return fmt.Errorf("scheduling: job timeout must not be negative, got %v", c.JobTimeout)
}
return nil
}
// withDefaults returns the config with zero values replaced.
func (c EngineConfig) withDefaults() EngineConfig {
if c.Interval == 0 {
c.Interval = DefaultInterval
}
if c.MaxParallel == 0 {
c.MaxParallel = DefaultMaxParallel
}
if c.JobTimeout == 0 {
c.JobTimeout = DefaultJobTimeout
}
if c.Clock == nil {
c.Clock = systemClock{}
}
return c
}
// Engine is the ticker engine: one loop that asks the source what is due at
// each tick and dispatches each due job to the pool it declares.
//
// Two invariants make the timing safe to reason about:
//
// - A job never overlaps itself. While a run is queued or in flight, later
// ticks reporting the same ID are skipped, not stacked. A job slower than
// the tick interval therefore runs back-to-back, never fanned out.
// - The pools do not interfere. A blocked sequential job cannot delay a
// parallel one, and a saturated parallel cap cannot delay the sequential
// worker — they are separate dispatch paths sharing only the tick.
//
// Every failure is contained: a source error, a job error, a job panic and a
// job timeout are each reported as an event and leave the loop running.
type Engine struct {
source JobSource
runner Runner
cfg EngineConfig
queue chan Job // sequential pool backlog
sem chan struct{}
mu sync.Mutex
started bool
stopped bool
stopLoop context.CancelFunc
cancelRuns context.CancelFunc
active map[string]struct{} // queued or in flight, by job ID
wg sync.WaitGroup
}
// NewEngine builds the engine over its source and runner. It fails on a nil
// dependency or a negative tunable; zero tunables take their defaults.
func NewEngine(source JobSource, runner Runner, cfg EngineConfig) (*Engine, error) {
if source == nil {
return nil, errors.New("scheduling: job source must not be nil")
}
if runner == nil {
return nil, errors.New("scheduling: runner must not be nil")
}
if err := cfg.validate(); err != nil {
return nil, err
}
cfg = cfg.withDefaults()
return &Engine{
source: source,
runner: runner,
cfg: cfg,
queue: make(chan Job, sequentialQueueDepth),
sem: make(chan struct{}, cfg.MaxParallel),
active: make(map[string]struct{}),
}, nil
}
// Interval reports the resolved tick cadence.
func (e *Engine) Interval() time.Duration { return e.cfg.Interval }
// MaxParallel reports the resolved parallel-pool cap.
func (e *Engine) MaxParallel() int { return e.cfg.MaxParallel }
// JobTimeout reports the resolved per-run timeout.
func (e *Engine) JobTimeout() time.Duration { return e.cfg.JobTimeout }
// Start launches the tick loop and the sequential worker.
//
// An engine is single-use: starting one that is already started, or that has
// been stopped, is an error. Restarting is refused rather than supported
// because Stop leaves the dropped sequential backlog and the claims of runs
// it did not wait for behind — a restarted engine would re-run stale queued
// jobs and permanently suppress the jobs still marked claimed. Build a new
// engine instead; it is a struct, not a resource.
func (e *Engine) Start(ctx context.Context) error {
e.mu.Lock()
defer e.mu.Unlock()
if e.stopped {
return errors.New("scheduling: engine already stopped; build a new one")
}
if e.started {
return errors.New("scheduling: engine already started")
}
e.started = true
loopCtx, stopLoop := context.WithCancel(ctx)
e.stopLoop = stopLoop
// Run contexts descend from Background, not from loopCtx, so that Stop
// drains gracefully: cancelling the loop must stop new work without
// yanking the context out from under a run that is already going. Stop
// cancels these only once its own deadline passes.
runCtx, cancelRuns := context.WithCancel(context.WithoutCancel(ctx))
e.cancelRuns = cancelRuns
e.wg.Add(2)
go e.tickLoop(loopCtx, runCtx)
go e.sequentialWorker(loopCtx, runCtx)
return nil
}
// Stop halts ticking, drops any queued-but-unstarted runs, and waits for
// in-flight runs to finish, bounded by ctx. If ctx expires first, in-flight
// runs are cancelled and ctx's error is returned — a Runner that ignores
// cancellation can still outlive Stop, which is why the deadline is the
// caller's to set. Stopping an engine that never started is a no-op.
func (e *Engine) Stop(ctx context.Context) error {
e.mu.Lock()
if !e.started {
e.mu.Unlock()
return nil
}
e.started = false
e.stopped = true
stopLoop, cancelRuns := e.stopLoop, e.cancelRuns
e.stopLoop, e.cancelRuns = nil, nil
e.mu.Unlock()
stopLoop()
done := make(chan struct{})
go func() {
e.wg.Wait()
close(done)
}()
select {
case <-done:
cancelRuns()
return nil
case <-ctx.Done():
cancelRuns()
return fmt.Errorf("scheduling: engine shutdown: %w", ctx.Err())
}
}
// tickLoop sleeps one interval, asks the source what is due, and dispatches
// it. The source is authoritative: the engine applies no schedule arithmetic
// of its own, so what "due" means stays the store's decision.
func (e *Engine) tickLoop(loopCtx, runCtx context.Context) {
defer e.wg.Done()
for {
select {
case <-loopCtx.Done():
return
case <-e.cfg.Clock.After(e.cfg.Interval):
}
jobs, err := e.source.Due(loopCtx, e.cfg.Clock.Now())
if err != nil {
e.emitError(Job{}, phaseSource, err)
continue
}
for _, job := range jobs {
e.dispatch(loopCtx, runCtx, job)
}
}
}
// dispatch routes one due job to its pool, skipping it if it is invalid or
// already queued or in flight.
func (e *Engine) dispatch(loopCtx, runCtx context.Context, job Job) {
if err := job.Validate(); err != nil {
e.emitError(job, phaseDispatch, err)
return
}
if !e.claim(job.ID) {
return // an earlier run of this job has not finished; skip this tick
}
if job.Pool.resolve() == PoolParallel {
e.wg.Go(func() {
defer e.release(job.ID)
select {
case e.sem <- struct{}{}:
case <-loopCtx.Done():
return // shutting down while waiting for a parallel slot
}
defer func() { <-e.sem }()
e.runJob(runCtx, job)
})
return
}
select {
case e.queue <- job:
default:
e.release(job.ID)
e.emitError(job, phaseDispatch, errors.New("scheduling: sequential queue full"))
}
}
// sequentialWorker is the sequential pool: one goroutine draining the queue
// in the order the source reported the jobs, so serialised jobs never
// overlap each other.
func (e *Engine) sequentialWorker(loopCtx, runCtx context.Context) {
defer e.wg.Done()
for {
select {
case <-loopCtx.Done():
return
case job := <-e.queue:
e.runJob(runCtx, job)
e.release(job.ID)
}
}
}
// runJob executes one run under the configured timeout, recovering panics so
// a job that blows up takes nothing else with it.
func (e *Engine) runJob(ctx context.Context, job Job) {
runCtx, cancel := context.WithTimeout(ctx, e.cfg.JobTimeout)
defer cancel()
start := e.cfg.Clock.Now()
err := e.safeRun(runCtx, job)
elapsed := e.cfg.Clock.Now().Sub(start)
if err != nil {
e.emitError(job, phaseRun, err)
return
}
e.emit(KindJobRun, job, map[string]any{
"job": job.ID,
"pool": string(job.Pool.resolve()),
"duration_ms": elapsed.Milliseconds(),
})
}
func (e *Engine) safeRun(ctx context.Context, job Job) (err error) {
defer func() {
if p := recover(); p != nil {
err = fmt.Errorf("scheduling: job %q panicked: %v", job.ID, p)
}
}()
return e.runner.Run(ctx, job)
}
// claim marks a job as queued-or-running, reporting false if it already was.
func (e *Engine) claim(id string) bool {
e.mu.Lock()
defer e.mu.Unlock()
if _, busy := e.active[id]; busy {
return false
}
e.active[id] = struct{}{}
return true
}
func (e *Engine) release(id string) {
e.mu.Lock()
defer e.mu.Unlock()
delete(e.active, id)
}
func (e *Engine) emitError(job Job, phase string, err error) {
e.emit(KindJobError, job, map[string]any{
"job": job.ID,
"pool": string(job.Pool),
"phase": phase,
"err": err.Error(),
})
}
func (e *Engine) emit(kind string, job Job, data map[string]any) {
if e.cfg.Events == nil {
return
}
e.cfg.Events.Emit(kind, job.Detail, data)
}
+850
View File
@@ -0,0 +1,850 @@
package scheduling
import (
"context"
"errors"
"sync"
"testing"
"time"
)
// --- fakes --------------------------------------------------------------------
// fakeClock is the engine's injectable clock: Now plus clock-driven timers
// that fire when Advance moves past their deadline. Tick timing is tested
// with this, never wall-clock sleeps.
type fakeClock struct {
mu sync.Mutex
now time.Time
timers []*fakeTimer
}
type fakeTimer struct {
c chan time.Time
at time.Time
}
func newFakeClock(start time.Time) *fakeClock { return &fakeClock{now: start} }
func (fc *fakeClock) Now() time.Time {
fc.mu.Lock()
defer fc.mu.Unlock()
return fc.now
}
func (fc *fakeClock) After(d time.Duration) <-chan time.Time {
fc.mu.Lock()
defer fc.mu.Unlock()
t := &fakeTimer{c: make(chan time.Time, 1), at: fc.now.Add(d)}
fc.timers = append(fc.timers, t)
return t.c
}
// pending reports how many timers are waiting; the tick loop registers
// exactly one at a time, so it is the signal that the loop is back asleep.
func (fc *fakeClock) pending() int {
fc.mu.Lock()
defer fc.mu.Unlock()
return len(fc.timers)
}
// Advance moves the clock forward and fires every timer whose deadline has
// passed, returning how many fired.
func (fc *fakeClock) Advance(d time.Duration) int {
fc.mu.Lock()
fc.now = fc.now.Add(d)
var fired []*fakeTimer
remain := fc.timers[:0]
for _, t := range fc.timers {
if !t.at.After(fc.now) {
fired = append(fired, t)
} else {
remain = append(remain, t)
}
}
fc.timers = remain
fc.mu.Unlock()
for _, t := range fired {
select {
case t.c <- t.at:
default:
}
}
return len(fired)
}
// sourceFunc adapts a function to JobSource.
type sourceFunc func(ctx context.Context, now time.Time) ([]Job, error)
func (f sourceFunc) Due(ctx context.Context, now time.Time) ([]Job, error) { return f(ctx, now) }
// staticSource returns the same jobs every tick and counts the calls.
type staticSource struct {
jobs []Job
mu sync.Mutex
calls int
}
func (s *staticSource) Due(context.Context, time.Time) ([]Job, error) {
s.mu.Lock()
s.calls++
s.mu.Unlock()
return s.jobs, nil
}
func (s *staticSource) callCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.calls
}
// runnerFunc adapts a function to Runner.
type runnerFunc func(ctx context.Context, job Job) error
func (f runnerFunc) Run(ctx context.Context, job Job) error { return f(ctx, job) }
// recordingRunner records every run in order and optionally blocks on a gate
// keyed by job ID, so a test can hold a run in flight deterministically.
type recordingRunner struct {
mu sync.Mutex
started []string
done []string
inFlt int
maxFlt int
gates map[string]chan struct{}
err error
}
func newRecordingRunner() *recordingRunner {
return &recordingRunner{gates: map[string]chan struct{}{}}
}
// gate installs a release channel for a job; runs of that job block until
// release is called.
func (r *recordingRunner) gate(id string) {
r.mu.Lock()
defer r.mu.Unlock()
r.gates[id] = make(chan struct{})
}
func (r *recordingRunner) release(id string) {
r.mu.Lock()
ch := r.gates[id]
delete(r.gates, id)
r.mu.Unlock()
if ch != nil {
close(ch)
}
}
func (r *recordingRunner) Run(ctx context.Context, job Job) error {
r.mu.Lock()
r.started = append(r.started, job.ID)
r.inFlt++
if r.inFlt > r.maxFlt {
r.maxFlt = r.inFlt
}
gate := r.gates[job.ID]
err := r.err
r.mu.Unlock()
if gate != nil {
select {
case <-gate:
case <-ctx.Done():
err = ctx.Err()
}
}
r.mu.Lock()
r.inFlt--
r.done = append(r.done, job.ID)
r.mu.Unlock()
return err
}
func (r *recordingRunner) startedIDs() []string {
r.mu.Lock()
defer r.mu.Unlock()
return append([]string(nil), r.started...)
}
func (r *recordingRunner) doneIDs() []string {
r.mu.Lock()
defer r.mu.Unlock()
return append([]string(nil), r.done...)
}
func (r *recordingRunner) peakInFlight() int {
r.mu.Lock()
defer r.mu.Unlock()
return r.maxFlt
}
// recordingSink collects emitted events.
type recordingSink struct {
mu sync.Mutex
events []sinkEvent
}
type sinkEvent struct {
kind string
detail string
data map[string]any
}
func (s *recordingSink) Emit(kind, detail string, data map[string]any) {
s.mu.Lock()
defer s.mu.Unlock()
s.events = append(s.events, sinkEvent{kind: kind, detail: detail, data: data})
}
func (s *recordingSink) countOf(kind string) int {
s.mu.Lock()
defer s.mu.Unlock()
n := 0
for _, e := range s.events {
if e.kind == kind {
n++
}
}
return n
}
func (s *recordingSink) firstOf(kind string) (sinkEvent, bool) {
s.mu.Lock()
defer s.mu.Unlock()
for _, e := range s.events {
if e.kind == kind {
return e, true
}
}
return sinkEvent{}, false
}
// --- helpers ------------------------------------------------------------------
const (
testInterval = time.Minute
testTimeout = 2 * time.Second
pollGap = time.Millisecond
)
func testStart() time.Time { return time.Date(2026, 8, 24, 12, 0, 0, 0, time.UTC) }
// waitFor polls cond until it holds or the timeout expires.
func waitFor(t *testing.T, what string, cond func() bool) {
t.Helper()
deadline := time.Now().Add(testTimeout)
for time.Now().Before(deadline) {
if cond() {
return
}
time.Sleep(pollGap)
}
t.Fatalf("timed out waiting for %s", what)
}
// tick advances the fake clock one interval and waits for the loop to arm its
// next timer, which happens only after the tick's dispatch has been decided.
func tick(t *testing.T, fc *fakeClock, e *Engine) {
t.Helper()
waitFor(t, "loop to arm its tick timer", func() bool { return fc.pending() > 0 })
fc.Advance(testInterval)
waitFor(t, "loop to re-arm after tick", func() bool { return fc.pending() > 0 })
_ = e
}
// startEngine builds and starts an engine, registering Stop as cleanup.
func startEngine(t *testing.T, src JobSource, runner Runner, cfg EngineConfig) *Engine {
t.Helper()
e, err := NewEngine(src, runner, cfg)
if err != nil {
t.Fatalf("NewEngine: %v", err)
}
if err := e.Start(context.Background()); err != nil {
t.Fatalf("Start: %v", err)
}
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
defer cancel()
_ = e.Stop(ctx)
})
return e
}
func countOccurrences(ids []string, want string) int {
n := 0
for _, id := range ids {
if id == want {
n++
}
}
return n
}
// --- construction -------------------------------------------------------------
func TestNewEngineValidatesConfig(t *testing.T) {
src := &staticSource{}
runner := newRecordingRunner()
tests := []struct {
name string
source JobSource
runner Runner
cfg EngineConfig
wantErr bool
}{
{name: "nil source", source: nil, runner: runner, cfg: EngineConfig{}, wantErr: true},
{name: "nil runner", source: src, runner: nil, cfg: EngineConfig{}, wantErr: true},
{name: "negative interval", source: src, runner: runner, cfg: EngineConfig{Interval: -testInterval}, wantErr: true},
{name: "negative max parallel", source: src, runner: runner, cfg: EngineConfig{MaxParallel: -1}, wantErr: true},
{name: "negative job timeout", source: src, runner: runner, cfg: EngineConfig{JobTimeout: -testInterval}, wantErr: true},
{name: "zero values are defaulted", source: src, runner: runner, cfg: EngineConfig{}, wantErr: false},
{
name: "fully specified", source: src, runner: runner,
cfg: EngineConfig{Interval: testInterval, MaxParallel: 2, JobTimeout: testInterval},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
e, err := NewEngine(tc.source, tc.runner, tc.cfg)
if tc.wantErr {
if err == nil {
t.Fatalf("expected error, got engine %v", e)
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if e == nil {
t.Fatal("expected engine, got nil")
}
})
}
}
func TestNewEngineAppliesDefaults(t *testing.T) {
e, err := NewEngine(&staticSource{}, newRecordingRunner(), EngineConfig{})
if err != nil {
t.Fatalf("NewEngine: %v", err)
}
if got := e.Interval(); got != DefaultInterval {
t.Errorf("interval = %v, want %v", got, DefaultInterval)
}
if got := e.MaxParallel(); got != DefaultMaxParallel {
t.Errorf("max parallel = %d, want %d", got, DefaultMaxParallel)
}
if got := e.JobTimeout(); got != DefaultJobTimeout {
t.Errorf("job timeout = %v, want %v", got, DefaultJobTimeout)
}
}
func TestEngineHonoursConfiguredInterval(t *testing.T) {
for _, interval := range []time.Duration{time.Second, testInterval, time.Hour} {
t.Run(interval.String(), func(t *testing.T) {
fc := newFakeClock(testStart())
src := &staticSource{}
e := startEngine(t, src, newRecordingRunner(), EngineConfig{Interval: interval, Clock: fc})
waitFor(t, "first tick timer", func() bool { return fc.pending() > 0 })
// One nanosecond short of the interval must not tick.
fc.Advance(interval - time.Nanosecond)
time.Sleep(10 * pollGap)
if got := src.callCount(); got != 0 {
t.Fatalf("source called %d times before the interval elapsed", got)
}
fc.Advance(time.Nanosecond)
waitFor(t, "source poll", func() bool { return src.callCount() >= 1 })
if got := e.Interval(); got != interval {
t.Errorf("Interval() = %v, want %v", got, interval)
}
})
}
}
// --- dispatch -----------------------------------------------------------------
func TestEngineRunsDueJobs(t *testing.T) {
fc := newFakeClock(testStart())
jobs := []Job{{ID: "a", Pool: PoolParallel}, {ID: "b", Pool: PoolSequential}}
src := &staticSource{jobs: jobs}
runner := newRecordingRunner()
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc})
tick(t, fc, e)
waitFor(t, "both jobs to finish", func() bool { return len(runner.doneIDs()) == len(jobs) })
for _, j := range jobs {
if countOccurrences(runner.doneIDs(), j.ID) != 1 {
t.Errorf("job %q ran %d times, want 1", j.ID, countOccurrences(runner.doneIDs(), j.ID))
}
}
}
func TestEngineParallelPoolRunsConcurrentlyUpToCap(t *testing.T) {
const maxParallel = 3
fc := newFakeClock(testStart())
jobs := make([]Job, 0, maxParallel+2)
runner := newRecordingRunner()
for i := range cap(jobs) {
id := string(rune('a' + i))
jobs = append(jobs, Job{ID: id, Pool: PoolParallel})
runner.gate(id)
}
src := &staticSource{jobs: jobs}
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, MaxParallel: maxParallel, Clock: fc})
tick(t, fc, e)
waitFor(t, "cap to be saturated", func() bool { return runner.peakInFlight() >= maxParallel })
// Give any over-cap goroutine a chance to break the cap before asserting.
time.Sleep(10 * pollGap)
if got := runner.peakInFlight(); got != maxParallel {
t.Fatalf("peak in-flight = %d, want %d", got, maxParallel)
}
for _, j := range jobs {
runner.release(j.ID)
}
waitFor(t, "all jobs to finish", func() bool { return len(runner.doneIDs()) == len(jobs) })
}
func TestEngineSequentialPoolRunsOneAtATimeInOrder(t *testing.T) {
fc := newFakeClock(testStart())
jobs := []Job{{ID: "first"}, {ID: "second", Pool: PoolSequential}, {ID: "third", Pool: PoolSequential}}
src := &staticSource{jobs: jobs}
runner := newRecordingRunner()
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, MaxParallel: len(jobs), Clock: fc})
tick(t, fc, e)
waitFor(t, "sequential jobs to finish", func() bool { return len(runner.doneIDs()) == len(jobs) })
if got := runner.peakInFlight(); got != 1 {
t.Errorf("sequential peak in-flight = %d, want 1", got)
}
for i, id := range runner.startedIDs() {
if id != jobs[i].ID {
t.Errorf("run %d = %q, want %q (source order)", i, id, jobs[i].ID)
}
}
}
func TestEngineSequentialJobDoesNotBlockParallelJob(t *testing.T) {
fc := newFakeClock(testStart())
src := &staticSource{jobs: []Job{{ID: "slow-seq", Pool: PoolSequential}, {ID: "quick-par", Pool: PoolParallel}}}
runner := newRecordingRunner()
runner.gate("slow-seq")
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc})
tick(t, fc, e)
waitFor(t, "parallel job to finish while sequential is blocked", func() bool {
return countOccurrences(runner.doneIDs(), "quick-par") == 1
})
runner.release("slow-seq")
waitFor(t, "sequential job to finish", func() bool {
return countOccurrences(runner.doneIDs(), "slow-seq") == 1
})
}
func TestEngineSkipsJobAlreadyInFlight(t *testing.T) {
for _, pool := range []Pool{PoolParallel, PoolSequential} {
t.Run(string(pool), func(t *testing.T) {
const id = "overlapping"
fc := newFakeClock(testStart())
src := &staticSource{jobs: []Job{{ID: id, Pool: pool}}}
runner := newRecordingRunner()
runner.gate(id)
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc})
tick(t, fc, e)
waitFor(t, "first run to start", func() bool { return len(runner.startedIDs()) == 1 })
tick(t, fc, e)
tick(t, fc, e)
time.Sleep(10 * pollGap)
if got := len(runner.startedIDs()); got != 1 {
t.Fatalf("job started %d times while in flight, want 1", got)
}
runner.release(id)
waitFor(t, "first run to finish", func() bool { return len(runner.doneIDs()) == 1 })
// Once the run is done the job is eligible again.
tick(t, fc, e)
waitFor(t, "second run", func() bool { return len(runner.startedIDs()) == 2 })
})
}
}
func TestEngineRejectsInvalidJob(t *testing.T) {
tests := []struct {
name string
job Job
}{
{name: "empty id", job: Job{ID: " ", Pool: PoolParallel}},
{name: "unknown pool", job: Job{ID: "bad-pool", Pool: Pool("elsewhere")}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fc := newFakeClock(testStart())
sink := &recordingSink{}
src := &staticSource{jobs: []Job{tc.job, {ID: "healthy", Pool: PoolParallel}}}
runner := newRecordingRunner()
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc, Events: sink})
tick(t, fc, e)
waitFor(t, "healthy job to run", func() bool {
return countOccurrences(runner.doneIDs(), "healthy") == 1
})
if got := len(runner.startedIDs()); got != 1 {
t.Fatalf("started %v, want only the healthy job", runner.startedIDs())
}
if sink.countOf(KindJobError) == 0 {
t.Error("expected an error event for the invalid job")
}
})
}
}
// --- failure isolation --------------------------------------------------------
func TestEngineSurvivesSourceError(t *testing.T) {
fc := newFakeClock(testStart())
sink := &recordingSink{}
var calls int
var mu sync.Mutex
wantErr := errors.New("store unavailable")
src := sourceFunc(func(context.Context, time.Time) ([]Job, error) {
mu.Lock()
calls++
n := calls
mu.Unlock()
if n == 1 {
return nil, wantErr
}
return []Job{{ID: "after-failure"}}, nil
})
runner := newRecordingRunner()
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc, Events: sink})
tick(t, fc, e)
waitFor(t, "error event", func() bool { return sink.countOf(KindJobError) == 1 })
tick(t, fc, e)
waitFor(t, "loop to keep ticking after a source error", func() bool {
return countOccurrences(runner.doneIDs(), "after-failure") == 1
})
ev, ok := sink.firstOf(KindJobError)
if !ok {
t.Fatal("no error event recorded")
}
if got, _ := ev.data["phase"].(string); got != phaseSource {
t.Errorf("error phase = %q, want %q", got, phaseSource)
}
}
func TestEngineSurvivesJobPanicAndError(t *testing.T) {
tests := []struct {
name string
run func(job Job) error
}{
{name: "panic", run: func(Job) error { panic("boom") }},
{name: "error", run: func(Job) error { return errors.New("job failed") }},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fc := newFakeClock(testStart())
sink := &recordingSink{}
var mu sync.Mutex
var runs int
src := &staticSource{jobs: []Job{{ID: "unstable"}}}
runner := runnerFunc(func(_ context.Context, job Job) error {
mu.Lock()
runs++
mu.Unlock()
return tc.run(job)
})
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc, Events: sink})
tick(t, fc, e)
waitFor(t, "first run", func() bool { mu.Lock(); defer mu.Unlock(); return runs == 1 })
waitFor(t, "error event", func() bool { return sink.countOf(KindJobError) == 1 })
// The loop survives: the same job is dispatched again next tick.
tick(t, fc, e)
waitFor(t, "second run", func() bool { mu.Lock(); defer mu.Unlock(); return runs == 2 })
ev, _ := sink.firstOf(KindJobError)
if got, _ := ev.data["phase"].(string); got != phaseRun {
t.Errorf("error phase = %q, want %q", got, phaseRun)
}
})
}
}
func TestEngineBoundsRunWithJobTimeout(t *testing.T) {
fc := newFakeClock(testStart())
sink := &recordingSink{}
src := &staticSource{jobs: []Job{{ID: "hangs"}}}
deadlines := make(chan error, 1)
runner := runnerFunc(func(ctx context.Context, _ Job) error {
<-ctx.Done()
deadlines <- ctx.Err()
return ctx.Err()
})
// A real (short) timeout: the run's deadline is wall-clock, because a
// Runner's ctx must be honoured by code that knows nothing of our clock.
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, JobTimeout: 20 * time.Millisecond, Clock: fc, Events: sink})
tick(t, fc, e)
select {
case err := <-deadlines:
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("run ctx error = %v, want DeadlineExceeded", err)
}
case <-time.After(testTimeout):
t.Fatal("run was never cancelled by the job timeout")
}
waitFor(t, "timeout to be reported", func() bool { return sink.countOf(KindJobError) == 1 })
}
// --- lifecycle ----------------------------------------------------------------
func TestEngineStartTwiceIsAnError(t *testing.T) {
fc := newFakeClock(testStart())
e := startEngine(t, &staticSource{}, newRecordingRunner(), EngineConfig{Interval: testInterval, Clock: fc})
if err := e.Start(context.Background()); err == nil {
t.Fatal("expected an error starting an already-started engine")
}
}
func TestEngineRestartAfterStopIsRefused(t *testing.T) {
fc := newFakeClock(testStart())
e, err := NewEngine(&staticSource{}, newRecordingRunner(), EngineConfig{Interval: testInterval, Clock: fc})
if err != nil {
t.Fatalf("NewEngine: %v", err)
}
if err := e.Start(context.Background()); err != nil {
t.Fatalf("Start: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
defer cancel()
if err := e.Stop(ctx); err != nil {
t.Fatalf("Stop: %v", err)
}
if err := e.Start(context.Background()); err == nil {
t.Fatal("expected an error restarting a stopped engine")
}
}
func TestEngineStopWithoutStartIsNoOp(t *testing.T) {
e, err := NewEngine(&staticSource{}, newRecordingRunner(), EngineConfig{})
if err != nil {
t.Fatalf("NewEngine: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
defer cancel()
if err := e.Stop(ctx); err != nil {
t.Fatalf("Stop before Start = %v, want nil", err)
}
}
func TestEngineStopWaitsForInFlightRuns(t *testing.T) {
fc := newFakeClock(testStart())
src := &staticSource{jobs: []Job{{ID: "par", Pool: PoolParallel}, {ID: "seq", Pool: PoolSequential}}}
runner := newRecordingRunner()
runner.gate("par")
runner.gate("seq")
e, err := NewEngine(src, runner, EngineConfig{Interval: testInterval, Clock: fc})
if err != nil {
t.Fatalf("NewEngine: %v", err)
}
if err := e.Start(context.Background()); err != nil {
t.Fatalf("Start: %v", err)
}
tick(t, fc, e)
waitFor(t, "both runs to start", func() bool { return len(runner.startedIDs()) == 2 })
stopped := make(chan error, 1)
go func() {
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
defer cancel()
stopped <- e.Stop(ctx)
}()
select {
case <-stopped:
t.Fatal("Stop returned while runs were still in flight")
case <-time.After(20 * pollGap):
}
runner.release("par")
runner.release("seq")
select {
case err := <-stopped:
if err != nil {
t.Fatalf("Stop: %v", err)
}
case <-time.After(testTimeout):
t.Fatal("Stop did not return after runs finished")
}
if got := len(runner.doneIDs()); got != 2 {
t.Fatalf("finished runs = %d, want 2", got)
}
}
func TestEngineStopIsBoundedByContext(t *testing.T) {
fc := newFakeClock(testStart())
src := &staticSource{jobs: []Job{{ID: "uninterruptible"}}}
release := make(chan struct{})
started := make(chan struct{}, 1)
runner := runnerFunc(func(context.Context, Job) error {
select {
case started <- struct{}{}:
default:
}
<-release // deliberately ignores cancellation
return nil
})
e, err := NewEngine(src, runner, EngineConfig{Interval: testInterval, Clock: fc})
if err != nil {
t.Fatalf("NewEngine: %v", err)
}
if err := e.Start(context.Background()); err != nil {
t.Fatalf("Start: %v", err)
}
defer close(release)
tick(t, fc, e)
<-started
ctx, cancel := context.WithTimeout(context.Background(), 20*pollGap)
defer cancel()
if err := e.Stop(ctx); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("Stop = %v, want a deadline-exceeded error", err)
}
}
func TestEngineStopStopsTicking(t *testing.T) {
fc := newFakeClock(testStart())
src := &staticSource{jobs: []Job{{ID: "job"}}}
runner := newRecordingRunner()
e, err := NewEngine(src, runner, EngineConfig{Interval: testInterval, Clock: fc})
if err != nil {
t.Fatalf("NewEngine: %v", err)
}
if err := e.Start(context.Background()); err != nil {
t.Fatalf("Start: %v", err)
}
tick(t, fc, e)
waitFor(t, "first run", func() bool { return len(runner.doneIDs()) == 1 })
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
defer cancel()
if err := e.Stop(ctx); err != nil {
t.Fatalf("Stop: %v", err)
}
before := src.callCount()
fc.Advance(testInterval * 10)
time.Sleep(20 * pollGap)
if got := src.callCount(); got != before {
t.Fatalf("source polled %d times after Stop (was %d)", got, before)
}
}
// --- events -------------------------------------------------------------------
func TestEngineEmitsRunEvents(t *testing.T) {
fc := newFakeClock(testStart())
sink := &recordingSink{}
const detail = "daily status summary"
src := &staticSource{jobs: []Job{{ID: "summary", Pool: PoolParallel, Detail: detail}}}
e := startEngine(t, src, newRecordingRunner(), EngineConfig{Interval: testInterval, Clock: fc, Events: sink})
tick(t, fc, e)
waitFor(t, "run event", func() bool { return sink.countOf(KindJobRun) == 1 })
ev, _ := sink.firstOf(KindJobRun)
if ev.detail != detail {
t.Errorf("event detail = %q, want %q", ev.detail, detail)
}
if got, _ := ev.data["job"].(string); got != "summary" {
t.Errorf("event job = %q, want %q", got, "summary")
}
if got, _ := ev.data["pool"].(string); got != string(PoolParallel) {
t.Errorf("event pool = %q, want %q", got, PoolParallel)
}
}
func TestEngineWithoutSinkDoesNotPanic(t *testing.T) {
fc := newFakeClock(testStart())
src := &staticSource{jobs: []Job{{ID: "bad-pool", Pool: Pool("nowhere")}, {ID: "fine"}}}
runner := newRecordingRunner()
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc})
tick(t, fc, e)
waitFor(t, "valid job to run", func() bool { return countOccurrences(runner.doneIDs(), "fine") == 1 })
}
// --- contract -----------------------------------------------------------------
func TestPoolValidate(t *testing.T) {
tests := []struct {
pool Pool
wantErr bool
}{
{pool: PoolParallel},
{pool: PoolSequential},
{pool: ""},
{pool: Pool("PARALLEL"), wantErr: true},
{pool: Pool("other"), wantErr: true},
}
for _, tc := range tests {
t.Run(string(tc.pool), func(t *testing.T) {
if err := tc.pool.Validate(); (err != nil) != tc.wantErr {
t.Fatalf("Validate(%q) = %v, wantErr %v", tc.pool, err, tc.wantErr)
}
})
}
}
func TestPoolResolveDefaultsToSequential(t *testing.T) {
tests := []struct {
pool Pool
want Pool
}{
{pool: "", want: PoolSequential},
{pool: PoolSequential, want: PoolSequential},
{pool: PoolParallel, want: PoolParallel},
}
for _, tc := range tests {
t.Run(string(tc.pool), func(t *testing.T) {
if got := tc.pool.resolve(); got != tc.want {
t.Fatalf("resolve(%q) = %q, want %q", tc.pool, got, tc.want)
}
})
}
}
func TestJobValidate(t *testing.T) {
tests := []struct {
name string
job Job
wantErr bool
}{
{name: "valid", job: Job{ID: "a", Pool: PoolParallel}},
{name: "valid default pool", job: Job{ID: "a"}},
{name: "empty id", job: Job{ID: ""}, wantErr: true},
{name: "blank id", job: Job{ID: "\t "}, wantErr: true},
{name: "unknown pool", job: Job{ID: "a", Pool: Pool("x")}, wantErr: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if err := tc.job.Validate(); (err != nil) != tc.wantErr {
t.Fatalf("Validate() = %v, wantErr %v", err, tc.wantErr)
}
})
}
}
+8
View File
@@ -48,6 +48,14 @@ const (
KindCuratorAction = "curator_action" // data: curator, type, detail, reason
KindCuratorError = "curator_error" // data: curator, phase, err
// Scheduled-job activity from the ticker engine
// (internal/domain/scheduling). A scheduled run has no task and no chat
// turn behind it, so without these a job that fired, stalled or failed
// is invisible everywhere. data: job, pool, duration_ms (run) and job,
// pool, phase, err (error).
KindJobRun = "job_run"
KindJobError = "job_error"
// KindTurnCompleted marks one completed primary chat turn
// (archie-core-035). Input-driven curators wake on it. Curator output
// never produces this kind, so derived work cannot feed its own
+2 -5
View File
@@ -97,10 +97,7 @@ func toolPreview(name, output string) string {
}
return "completed with no printable preview"
}
remaining := countNonEmptyOrContentLines(content) - 1 - skipped
if remaining < 0 {
remaining = 0
}
remaining := max(countNonEmptyOrContentLines(content)-1-skipped, 0)
if remaining > 0 {
return fmt.Sprintf("%s\n… %d more lines", preview, remaining)
}
@@ -110,7 +107,7 @@ func toolPreview(name, output string) string {
func compactToolPreview(output string) (string, int) {
skipped := 0
for _, line := range strings.Split(output, "\n") {
for line := range strings.SplitSeq(output, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue