Merge pull request #614 from samcharles93/feat/275-ticker-engine
Feat/275 ticker engine
This commit is contained in:
commit
eb12a02442
5 files changed
+1333
-5
No files matched your search
@@ -0,0 +1,114 @@
|
||||
// Package scheduling is the ticker engine: the timing half of archied's
|
||||
// cron/scheduling capability (epic archie-core-1786637496320-249-be6d9a20).
|
||||
// It owns one job and one job only — deciding *when* a scheduled job runs,
|
||||
// and with what concurrency — and nothing about where jobs are stored, how
|
||||
// they are authored, or how their output is delivered.
|
||||
//
|
||||
// The split is deliberate. A job store (file-backed, cross-process locked)
|
||||
// and a delivery router are separate issues in that epic; this package
|
||||
// declares the two contracts it needs from them (JobSource, Runner) and
|
||||
// names no implementation. That is what lets the timing behaviour — pool
|
||||
// semantics, overlap suppression, per-run timeouts, bounded shutdown — be
|
||||
// tested against a fake clock with no store, no NATS and no model.
|
||||
package scheduling
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Pool is how a job's runs are dispatched relative to other jobs.
|
||||
type Pool string
|
||||
|
||||
const (
|
||||
// PoolParallel runs the job concurrently with every other job, bounded
|
||||
// only by the engine's MaxParallel cap.
|
||||
PoolParallel Pool = "parallel"
|
||||
// PoolSequential runs the job on a single shared worker, one at a time,
|
||||
// in the order the source returned them. Use it for jobs that contend
|
||||
// for a resource that tolerates no concurrency.
|
||||
PoolSequential Pool = "sequential"
|
||||
)
|
||||
|
||||
// Validate rejects an unknown pool. The empty pool is valid and means
|
||||
// PoolSequential: an unlabelled job is assumed to want the safer of the two,
|
||||
// because wrongly serialising a job costs latency while wrongly parallelising
|
||||
// one costs correctness.
|
||||
func (p Pool) Validate() error {
|
||||
switch p {
|
||||
case "", PoolParallel, PoolSequential:
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("scheduling: unknown pool %q", string(p))
|
||||
}
|
||||
}
|
||||
|
||||
// resolve maps the empty pool onto its documented default.
|
||||
func (p Pool) resolve() Pool {
|
||||
if p == "" {
|
||||
return PoolSequential
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Job is one due unit of scheduled work as handed to the engine. It carries
|
||||
// identity and dispatch policy only: the schedule itself (cron expression,
|
||||
// next-run time, catch-up policy) belongs to the job store behind JobSource,
|
||||
// which is why the engine never sees one.
|
||||
type Job struct {
|
||||
// ID identifies the job across ticks. It is the overlap key: while a
|
||||
// run with this ID is queued or in flight, later ticks that report the
|
||||
// same ID are skipped rather than stacked.
|
||||
ID string
|
||||
// Pool selects the dispatch pool. Empty means PoolSequential.
|
||||
Pool Pool
|
||||
// Detail is a human-readable label recorded on emitted events so a run
|
||||
// stays attributable without the engine knowing what the job does.
|
||||
Detail string
|
||||
}
|
||||
|
||||
// Validate checks the fields the engine relies on.
|
||||
func (j Job) Validate() error {
|
||||
if strings.TrimSpace(j.ID) == "" {
|
||||
return errors.New("scheduling: job id must be non-empty")
|
||||
}
|
||||
return j.Pool.Validate()
|
||||
}
|
||||
|
||||
// JobSource yields the jobs eligible to run at a given instant. The engine
|
||||
// calls Due once per tick and treats it as authoritative: it applies no
|
||||
// schedule arithmetic of its own, so a source is free to implement cron
|
||||
// expressions, fixed intervals or catch-up windows without the engine
|
||||
// changing.
|
||||
type JobSource interface {
|
||||
Due(ctx context.Context, now time.Time) ([]Job, error)
|
||||
}
|
||||
|
||||
// Runner executes one due job. Implementations must respect ctx: the engine
|
||||
// bounds every run with JobTimeout and cancels on shutdown, but a Runner that
|
||||
// ignores cancellation can still delay Stop up to Stop's own deadline.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, job Job) error
|
||||
}
|
||||
|
||||
// Clock is injectable time: Now for timestamps and After for the tick sleep.
|
||||
// Loop timing is tested against a fake clock, never wall-clock sleeps.
|
||||
type Clock interface {
|
||||
Now() time.Time
|
||||
After(d time.Duration) <-chan time.Time
|
||||
}
|
||||
|
||||
type systemClock struct{}
|
||||
|
||||
func (systemClock) Now() time.Time { return time.Now() }
|
||||
func (systemClock) After(d time.Duration) <-chan time.Time { return time.After(d) }
|
||||
|
||||
// Sink is the narrow event contract the engine needs. It matches
|
||||
// events.Bus's emitter shape without importing it — the domain declares what
|
||||
// it needs and app wires the implementation in.
|
||||
type Sink interface {
|
||||
Emit(kind, detail string, data map[string]any)
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
package scheduling
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
)
|
||||
|
||||
// Defaults for the tunables an operator may leave unset.
|
||||
const (
|
||||
// DefaultInterval is the tick cadence when none is configured. A minute
|
||||
// is the finest granularity a cron-style schedule needs, and polling a
|
||||
// local job store that often costs nothing.
|
||||
DefaultInterval = time.Minute
|
||||
// DefaultMaxParallel caps concurrent parallel-pool runs. It exists so a
|
||||
// store that reports many jobs due at once (the first tick after a long
|
||||
// downtime, typically) cannot launch an unbounded fan-out.
|
||||
DefaultMaxParallel = 4
|
||||
// DefaultJobTimeout bounds one run. A scheduled job that has not
|
||||
// finished in an hour is stuck, and leaving it in flight would suppress
|
||||
// every later run of the same job forever.
|
||||
DefaultJobTimeout = time.Hour
|
||||
)
|
||||
|
||||
// Event kinds emitted by the engine, re-exported so callers of this package
|
||||
// need not import the events package to match on them.
|
||||
const (
|
||||
KindJobRun = events.KindJobRun
|
||||
KindJobError = events.KindJobError
|
||||
)
|
||||
|
||||
// Phases reported on KindJobError, naming which step failed.
|
||||
const (
|
||||
phaseSource = "source" // JobSource.Due returned an error
|
||||
phaseDispatch = "dispatch" // the job was rejected before it ever ran
|
||||
phaseRun = "run" // Runner.Run failed, panicked or timed out
|
||||
)
|
||||
|
||||
// sequentialQueueDepth bounds the sequential pool's backlog. Overlap
|
||||
// suppression already caps the backlog at one entry per distinct job, so this
|
||||
// is a backstop against a pathological source, not the primary bound.
|
||||
const sequentialQueueDepth = 256
|
||||
|
||||
// EngineConfig tunes the ticker engine. Every field is optional; zero values
|
||||
// take the documented defaults.
|
||||
type EngineConfig struct {
|
||||
// Interval is the tick cadence: how often the engine asks its source
|
||||
// what is due. Zero uses DefaultInterval; negative is an error.
|
||||
Interval time.Duration
|
||||
// MaxParallel caps simultaneous parallel-pool runs. Zero uses
|
||||
// DefaultMaxParallel. It does not bound the sequential pool, which is
|
||||
// one by definition.
|
||||
MaxParallel int
|
||||
// JobTimeout bounds a single run via context deadline. Zero uses
|
||||
// DefaultJobTimeout.
|
||||
JobTimeout time.Duration
|
||||
// Clock is injectable time. Nil uses the system clock.
|
||||
Clock Clock
|
||||
// Events receives run and error events. Nil disables emission; the
|
||||
// engine still runs, it is just unobservable.
|
||||
Events Sink
|
||||
}
|
||||
|
||||
func (c EngineConfig) validate() error {
|
||||
switch {
|
||||
case c.Interval < 0:
|
||||
return fmt.Errorf("scheduling: interval must not be negative, got %v", c.Interval)
|
||||
case c.MaxParallel < 0:
|
||||
return fmt.Errorf("scheduling: max parallel must not be negative, got %d", c.MaxParallel)
|
||||
case c.JobTimeout < 0:
|
||||
return fmt.Errorf("scheduling: job timeout must not be negative, got %v", c.JobTimeout)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// withDefaults returns the config with zero values replaced.
|
||||
func (c EngineConfig) withDefaults() EngineConfig {
|
||||
if c.Interval == 0 {
|
||||
c.Interval = DefaultInterval
|
||||
}
|
||||
if c.MaxParallel == 0 {
|
||||
c.MaxParallel = DefaultMaxParallel
|
||||
}
|
||||
if c.JobTimeout == 0 {
|
||||
c.JobTimeout = DefaultJobTimeout
|
||||
}
|
||||
if c.Clock == nil {
|
||||
c.Clock = systemClock{}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// Engine is the ticker engine: one loop that asks the source what is due at
|
||||
// each tick and dispatches each due job to the pool it declares.
|
||||
//
|
||||
// Two invariants make the timing safe to reason about:
|
||||
//
|
||||
// - A job never overlaps itself. While a run is queued or in flight, later
|
||||
// ticks reporting the same ID are skipped, not stacked. A job slower than
|
||||
// the tick interval therefore runs back-to-back, never fanned out.
|
||||
// - The pools do not interfere. A blocked sequential job cannot delay a
|
||||
// parallel one, and a saturated parallel cap cannot delay the sequential
|
||||
// worker — they are separate dispatch paths sharing only the tick.
|
||||
//
|
||||
// Every failure is contained: a source error, a job error, a job panic and a
|
||||
// job timeout are each reported as an event and leave the loop running.
|
||||
type Engine struct {
|
||||
source JobSource
|
||||
runner Runner
|
||||
cfg EngineConfig
|
||||
|
||||
queue chan Job // sequential pool backlog
|
||||
sem chan struct{}
|
||||
|
||||
mu sync.Mutex
|
||||
started bool
|
||||
stopped bool
|
||||
stopLoop context.CancelFunc
|
||||
cancelRuns context.CancelFunc
|
||||
active map[string]struct{} // queued or in flight, by job ID
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewEngine builds the engine over its source and runner. It fails on a nil
|
||||
// dependency or a negative tunable; zero tunables take their defaults.
|
||||
func NewEngine(source JobSource, runner Runner, cfg EngineConfig) (*Engine, error) {
|
||||
if source == nil {
|
||||
return nil, errors.New("scheduling: job source must not be nil")
|
||||
}
|
||||
if runner == nil {
|
||||
return nil, errors.New("scheduling: runner must not be nil")
|
||||
}
|
||||
if err := cfg.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg = cfg.withDefaults()
|
||||
return &Engine{
|
||||
source: source,
|
||||
runner: runner,
|
||||
cfg: cfg,
|
||||
queue: make(chan Job, sequentialQueueDepth),
|
||||
sem: make(chan struct{}, cfg.MaxParallel),
|
||||
active: make(map[string]struct{}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Interval reports the resolved tick cadence.
|
||||
func (e *Engine) Interval() time.Duration { return e.cfg.Interval }
|
||||
|
||||
// MaxParallel reports the resolved parallel-pool cap.
|
||||
func (e *Engine) MaxParallel() int { return e.cfg.MaxParallel }
|
||||
|
||||
// JobTimeout reports the resolved per-run timeout.
|
||||
func (e *Engine) JobTimeout() time.Duration { return e.cfg.JobTimeout }
|
||||
|
||||
// Start launches the tick loop and the sequential worker.
|
||||
//
|
||||
// An engine is single-use: starting one that is already started, or that has
|
||||
// been stopped, is an error. Restarting is refused rather than supported
|
||||
// because Stop leaves the dropped sequential backlog and the claims of runs
|
||||
// it did not wait for behind — a restarted engine would re-run stale queued
|
||||
// jobs and permanently suppress the jobs still marked claimed. Build a new
|
||||
// engine instead; it is a struct, not a resource.
|
||||
func (e *Engine) Start(ctx context.Context) error {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if e.stopped {
|
||||
return errors.New("scheduling: engine already stopped; build a new one")
|
||||
}
|
||||
if e.started {
|
||||
return errors.New("scheduling: engine already started")
|
||||
}
|
||||
e.started = true
|
||||
|
||||
loopCtx, stopLoop := context.WithCancel(ctx)
|
||||
e.stopLoop = stopLoop
|
||||
// Run contexts descend from Background, not from loopCtx, so that Stop
|
||||
// drains gracefully: cancelling the loop must stop new work without
|
||||
// yanking the context out from under a run that is already going. Stop
|
||||
// cancels these only once its own deadline passes.
|
||||
runCtx, cancelRuns := context.WithCancel(context.WithoutCancel(ctx))
|
||||
e.cancelRuns = cancelRuns
|
||||
|
||||
e.wg.Add(2)
|
||||
go e.tickLoop(loopCtx, runCtx)
|
||||
go e.sequentialWorker(loopCtx, runCtx)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop halts ticking, drops any queued-but-unstarted runs, and waits for
|
||||
// in-flight runs to finish, bounded by ctx. If ctx expires first, in-flight
|
||||
// runs are cancelled and ctx's error is returned — a Runner that ignores
|
||||
// cancellation can still outlive Stop, which is why the deadline is the
|
||||
// caller's to set. Stopping an engine that never started is a no-op.
|
||||
func (e *Engine) Stop(ctx context.Context) error {
|
||||
e.mu.Lock()
|
||||
if !e.started {
|
||||
e.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
e.started = false
|
||||
e.stopped = true
|
||||
stopLoop, cancelRuns := e.stopLoop, e.cancelRuns
|
||||
e.stopLoop, e.cancelRuns = nil, nil
|
||||
e.mu.Unlock()
|
||||
|
||||
stopLoop()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
e.wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
cancelRuns()
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
cancelRuns()
|
||||
return fmt.Errorf("scheduling: engine shutdown: %w", ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
// tickLoop sleeps one interval, asks the source what is due, and dispatches
|
||||
// it. The source is authoritative: the engine applies no schedule arithmetic
|
||||
// of its own, so what "due" means stays the store's decision.
|
||||
func (e *Engine) tickLoop(loopCtx, runCtx context.Context) {
|
||||
defer e.wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-loopCtx.Done():
|
||||
return
|
||||
case <-e.cfg.Clock.After(e.cfg.Interval):
|
||||
}
|
||||
|
||||
jobs, err := e.source.Due(loopCtx, e.cfg.Clock.Now())
|
||||
if err != nil {
|
||||
e.emitError(Job{}, phaseSource, err)
|
||||
continue
|
||||
}
|
||||
for _, job := range jobs {
|
||||
e.dispatch(loopCtx, runCtx, job)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dispatch routes one due job to its pool, skipping it if it is invalid or
|
||||
// already queued or in flight.
|
||||
func (e *Engine) dispatch(loopCtx, runCtx context.Context, job Job) {
|
||||
if err := job.Validate(); err != nil {
|
||||
e.emitError(job, phaseDispatch, err)
|
||||
return
|
||||
}
|
||||
if !e.claim(job.ID) {
|
||||
return // an earlier run of this job has not finished; skip this tick
|
||||
}
|
||||
|
||||
if job.Pool.resolve() == PoolParallel {
|
||||
e.wg.Go(func() {
|
||||
defer e.release(job.ID)
|
||||
select {
|
||||
case e.sem <- struct{}{}:
|
||||
case <-loopCtx.Done():
|
||||
return // shutting down while waiting for a parallel slot
|
||||
}
|
||||
defer func() { <-e.sem }()
|
||||
e.runJob(runCtx, job)
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case e.queue <- job:
|
||||
default:
|
||||
e.release(job.ID)
|
||||
e.emitError(job, phaseDispatch, errors.New("scheduling: sequential queue full"))
|
||||
}
|
||||
}
|
||||
|
||||
// sequentialWorker is the sequential pool: one goroutine draining the queue
|
||||
// in the order the source reported the jobs, so serialised jobs never
|
||||
// overlap each other.
|
||||
func (e *Engine) sequentialWorker(loopCtx, runCtx context.Context) {
|
||||
defer e.wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-loopCtx.Done():
|
||||
return
|
||||
case job := <-e.queue:
|
||||
e.runJob(runCtx, job)
|
||||
e.release(job.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runJob executes one run under the configured timeout, recovering panics so
|
||||
// a job that blows up takes nothing else with it.
|
||||
func (e *Engine) runJob(ctx context.Context, job Job) {
|
||||
runCtx, cancel := context.WithTimeout(ctx, e.cfg.JobTimeout)
|
||||
defer cancel()
|
||||
|
||||
start := e.cfg.Clock.Now()
|
||||
err := e.safeRun(runCtx, job)
|
||||
elapsed := e.cfg.Clock.Now().Sub(start)
|
||||
if err != nil {
|
||||
e.emitError(job, phaseRun, err)
|
||||
return
|
||||
}
|
||||
e.emit(KindJobRun, job, map[string]any{
|
||||
"job": job.ID,
|
||||
"pool": string(job.Pool.resolve()),
|
||||
"duration_ms": elapsed.Milliseconds(),
|
||||
})
|
||||
}
|
||||
|
||||
func (e *Engine) safeRun(ctx context.Context, job Job) (err error) {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
err = fmt.Errorf("scheduling: job %q panicked: %v", job.ID, p)
|
||||
}
|
||||
}()
|
||||
return e.runner.Run(ctx, job)
|
||||
}
|
||||
|
||||
// claim marks a job as queued-or-running, reporting false if it already was.
|
||||
func (e *Engine) claim(id string) bool {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if _, busy := e.active[id]; busy {
|
||||
return false
|
||||
}
|
||||
e.active[id] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
func (e *Engine) release(id string) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
delete(e.active, id)
|
||||
}
|
||||
|
||||
func (e *Engine) emitError(job Job, phase string, err error) {
|
||||
e.emit(KindJobError, job, map[string]any{
|
||||
"job": job.ID,
|
||||
"pool": string(job.Pool),
|
||||
"phase": phase,
|
||||
"err": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
func (e *Engine) emit(kind string, job Job, data map[string]any) {
|
||||
if e.cfg.Events == nil {
|
||||
return
|
||||
}
|
||||
e.cfg.Events.Emit(kind, job.Detail, data)
|
||||
}
|
||||
@@ -0,0 +1,850 @@
|
||||
package scheduling
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// --- fakes --------------------------------------------------------------------
|
||||
|
||||
// fakeClock is the engine's injectable clock: Now plus clock-driven timers
|
||||
// that fire when Advance moves past their deadline. Tick timing is tested
|
||||
// with this, never wall-clock sleeps.
|
||||
type fakeClock struct {
|
||||
mu sync.Mutex
|
||||
now time.Time
|
||||
timers []*fakeTimer
|
||||
}
|
||||
|
||||
type fakeTimer struct {
|
||||
c chan time.Time
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func newFakeClock(start time.Time) *fakeClock { return &fakeClock{now: start} }
|
||||
|
||||
func (fc *fakeClock) Now() time.Time {
|
||||
fc.mu.Lock()
|
||||
defer fc.mu.Unlock()
|
||||
return fc.now
|
||||
}
|
||||
|
||||
func (fc *fakeClock) After(d time.Duration) <-chan time.Time {
|
||||
fc.mu.Lock()
|
||||
defer fc.mu.Unlock()
|
||||
t := &fakeTimer{c: make(chan time.Time, 1), at: fc.now.Add(d)}
|
||||
fc.timers = append(fc.timers, t)
|
||||
return t.c
|
||||
}
|
||||
|
||||
// pending reports how many timers are waiting; the tick loop registers
|
||||
// exactly one at a time, so it is the signal that the loop is back asleep.
|
||||
func (fc *fakeClock) pending() int {
|
||||
fc.mu.Lock()
|
||||
defer fc.mu.Unlock()
|
||||
return len(fc.timers)
|
||||
}
|
||||
|
||||
// Advance moves the clock forward and fires every timer whose deadline has
|
||||
// passed, returning how many fired.
|
||||
func (fc *fakeClock) Advance(d time.Duration) int {
|
||||
fc.mu.Lock()
|
||||
fc.now = fc.now.Add(d)
|
||||
var fired []*fakeTimer
|
||||
remain := fc.timers[:0]
|
||||
for _, t := range fc.timers {
|
||||
if !t.at.After(fc.now) {
|
||||
fired = append(fired, t)
|
||||
} else {
|
||||
remain = append(remain, t)
|
||||
}
|
||||
}
|
||||
fc.timers = remain
|
||||
fc.mu.Unlock()
|
||||
for _, t := range fired {
|
||||
select {
|
||||
case t.c <- t.at:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return len(fired)
|
||||
}
|
||||
|
||||
// sourceFunc adapts a function to JobSource.
|
||||
type sourceFunc func(ctx context.Context, now time.Time) ([]Job, error)
|
||||
|
||||
func (f sourceFunc) Due(ctx context.Context, now time.Time) ([]Job, error) { return f(ctx, now) }
|
||||
|
||||
// staticSource returns the same jobs every tick and counts the calls.
|
||||
type staticSource struct {
|
||||
jobs []Job
|
||||
mu sync.Mutex
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *staticSource) Due(context.Context, time.Time) ([]Job, error) {
|
||||
s.mu.Lock()
|
||||
s.calls++
|
||||
s.mu.Unlock()
|
||||
return s.jobs, nil
|
||||
}
|
||||
|
||||
func (s *staticSource) callCount() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.calls
|
||||
}
|
||||
|
||||
// runnerFunc adapts a function to Runner.
|
||||
type runnerFunc func(ctx context.Context, job Job) error
|
||||
|
||||
func (f runnerFunc) Run(ctx context.Context, job Job) error { return f(ctx, job) }
|
||||
|
||||
// recordingRunner records every run in order and optionally blocks on a gate
|
||||
// keyed by job ID, so a test can hold a run in flight deterministically.
|
||||
type recordingRunner struct {
|
||||
mu sync.Mutex
|
||||
started []string
|
||||
done []string
|
||||
inFlt int
|
||||
maxFlt int
|
||||
gates map[string]chan struct{}
|
||||
err error
|
||||
}
|
||||
|
||||
func newRecordingRunner() *recordingRunner {
|
||||
return &recordingRunner{gates: map[string]chan struct{}{}}
|
||||
}
|
||||
|
||||
// gate installs a release channel for a job; runs of that job block until
|
||||
// release is called.
|
||||
func (r *recordingRunner) gate(id string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.gates[id] = make(chan struct{})
|
||||
}
|
||||
|
||||
func (r *recordingRunner) release(id string) {
|
||||
r.mu.Lock()
|
||||
ch := r.gates[id]
|
||||
delete(r.gates, id)
|
||||
r.mu.Unlock()
|
||||
if ch != nil {
|
||||
close(ch)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *recordingRunner) Run(ctx context.Context, job Job) error {
|
||||
r.mu.Lock()
|
||||
r.started = append(r.started, job.ID)
|
||||
r.inFlt++
|
||||
if r.inFlt > r.maxFlt {
|
||||
r.maxFlt = r.inFlt
|
||||
}
|
||||
gate := r.gates[job.ID]
|
||||
err := r.err
|
||||
r.mu.Unlock()
|
||||
|
||||
if gate != nil {
|
||||
select {
|
||||
case <-gate:
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
r.mu.Lock()
|
||||
r.inFlt--
|
||||
r.done = append(r.done, job.ID)
|
||||
r.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *recordingRunner) startedIDs() []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return append([]string(nil), r.started...)
|
||||
}
|
||||
|
||||
func (r *recordingRunner) doneIDs() []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return append([]string(nil), r.done...)
|
||||
}
|
||||
|
||||
func (r *recordingRunner) peakInFlight() int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.maxFlt
|
||||
}
|
||||
|
||||
// recordingSink collects emitted events.
|
||||
type recordingSink struct {
|
||||
mu sync.Mutex
|
||||
events []sinkEvent
|
||||
}
|
||||
|
||||
type sinkEvent struct {
|
||||
kind string
|
||||
detail string
|
||||
data map[string]any
|
||||
}
|
||||
|
||||
func (s *recordingSink) Emit(kind, detail string, data map[string]any) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.events = append(s.events, sinkEvent{kind: kind, detail: detail, data: data})
|
||||
}
|
||||
|
||||
func (s *recordingSink) countOf(kind string) int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
n := 0
|
||||
for _, e := range s.events {
|
||||
if e.kind == kind {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (s *recordingSink) firstOf(kind string) (sinkEvent, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, e := range s.events {
|
||||
if e.kind == kind {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return sinkEvent{}, false
|
||||
}
|
||||
|
||||
// --- helpers ------------------------------------------------------------------
|
||||
|
||||
const (
|
||||
testInterval = time.Minute
|
||||
testTimeout = 2 * time.Second
|
||||
pollGap = time.Millisecond
|
||||
)
|
||||
|
||||
func testStart() time.Time { return time.Date(2026, 8, 24, 12, 0, 0, 0, time.UTC) }
|
||||
|
||||
// waitFor polls cond until it holds or the timeout expires.
|
||||
func waitFor(t *testing.T, what string, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(testTimeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if cond() {
|
||||
return
|
||||
}
|
||||
time.Sleep(pollGap)
|
||||
}
|
||||
t.Fatalf("timed out waiting for %s", what)
|
||||
}
|
||||
|
||||
// tick advances the fake clock one interval and waits for the loop to arm its
|
||||
// next timer, which happens only after the tick's dispatch has been decided.
|
||||
func tick(t *testing.T, fc *fakeClock, e *Engine) {
|
||||
t.Helper()
|
||||
waitFor(t, "loop to arm its tick timer", func() bool { return fc.pending() > 0 })
|
||||
fc.Advance(testInterval)
|
||||
waitFor(t, "loop to re-arm after tick", func() bool { return fc.pending() > 0 })
|
||||
_ = e
|
||||
}
|
||||
|
||||
// startEngine builds and starts an engine, registering Stop as cleanup.
|
||||
func startEngine(t *testing.T, src JobSource, runner Runner, cfg EngineConfig) *Engine {
|
||||
t.Helper()
|
||||
e, err := NewEngine(src, runner, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
if err := e.Start(context.Background()); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
|
||||
defer cancel()
|
||||
_ = e.Stop(ctx)
|
||||
})
|
||||
return e
|
||||
}
|
||||
|
||||
func countOccurrences(ids []string, want string) int {
|
||||
n := 0
|
||||
for _, id := range ids {
|
||||
if id == want {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// --- construction -------------------------------------------------------------
|
||||
|
||||
func TestNewEngineValidatesConfig(t *testing.T) {
|
||||
src := &staticSource{}
|
||||
runner := newRecordingRunner()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
source JobSource
|
||||
runner Runner
|
||||
cfg EngineConfig
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "nil source", source: nil, runner: runner, cfg: EngineConfig{}, wantErr: true},
|
||||
{name: "nil runner", source: src, runner: nil, cfg: EngineConfig{}, wantErr: true},
|
||||
{name: "negative interval", source: src, runner: runner, cfg: EngineConfig{Interval: -testInterval}, wantErr: true},
|
||||
{name: "negative max parallel", source: src, runner: runner, cfg: EngineConfig{MaxParallel: -1}, wantErr: true},
|
||||
{name: "negative job timeout", source: src, runner: runner, cfg: EngineConfig{JobTimeout: -testInterval}, wantErr: true},
|
||||
{name: "zero values are defaulted", source: src, runner: runner, cfg: EngineConfig{}, wantErr: false},
|
||||
{
|
||||
name: "fully specified", source: src, runner: runner,
|
||||
cfg: EngineConfig{Interval: testInterval, MaxParallel: 2, JobTimeout: testInterval},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
e, err := NewEngine(tc.source, tc.runner, tc.cfg)
|
||||
if tc.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error, got engine %v", e)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if e == nil {
|
||||
t.Fatal("expected engine, got nil")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewEngineAppliesDefaults(t *testing.T) {
|
||||
e, err := NewEngine(&staticSource{}, newRecordingRunner(), EngineConfig{})
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
if got := e.Interval(); got != DefaultInterval {
|
||||
t.Errorf("interval = %v, want %v", got, DefaultInterval)
|
||||
}
|
||||
if got := e.MaxParallel(); got != DefaultMaxParallel {
|
||||
t.Errorf("max parallel = %d, want %d", got, DefaultMaxParallel)
|
||||
}
|
||||
if got := e.JobTimeout(); got != DefaultJobTimeout {
|
||||
t.Errorf("job timeout = %v, want %v", got, DefaultJobTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineHonoursConfiguredInterval(t *testing.T) {
|
||||
for _, interval := range []time.Duration{time.Second, testInterval, time.Hour} {
|
||||
t.Run(interval.String(), func(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
src := &staticSource{}
|
||||
e := startEngine(t, src, newRecordingRunner(), EngineConfig{Interval: interval, Clock: fc})
|
||||
|
||||
waitFor(t, "first tick timer", func() bool { return fc.pending() > 0 })
|
||||
// One nanosecond short of the interval must not tick.
|
||||
fc.Advance(interval - time.Nanosecond)
|
||||
time.Sleep(10 * pollGap)
|
||||
if got := src.callCount(); got != 0 {
|
||||
t.Fatalf("source called %d times before the interval elapsed", got)
|
||||
}
|
||||
fc.Advance(time.Nanosecond)
|
||||
waitFor(t, "source poll", func() bool { return src.callCount() >= 1 })
|
||||
if got := e.Interval(); got != interval {
|
||||
t.Errorf("Interval() = %v, want %v", got, interval)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --- dispatch -----------------------------------------------------------------
|
||||
|
||||
func TestEngineRunsDueJobs(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
jobs := []Job{{ID: "a", Pool: PoolParallel}, {ID: "b", Pool: PoolSequential}}
|
||||
src := &staticSource{jobs: jobs}
|
||||
runner := newRecordingRunner()
|
||||
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc})
|
||||
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "both jobs to finish", func() bool { return len(runner.doneIDs()) == len(jobs) })
|
||||
|
||||
for _, j := range jobs {
|
||||
if countOccurrences(runner.doneIDs(), j.ID) != 1 {
|
||||
t.Errorf("job %q ran %d times, want 1", j.ID, countOccurrences(runner.doneIDs(), j.ID))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineParallelPoolRunsConcurrentlyUpToCap(t *testing.T) {
|
||||
const maxParallel = 3
|
||||
fc := newFakeClock(testStart())
|
||||
jobs := make([]Job, 0, maxParallel+2)
|
||||
runner := newRecordingRunner()
|
||||
for i := range cap(jobs) {
|
||||
id := string(rune('a' + i))
|
||||
jobs = append(jobs, Job{ID: id, Pool: PoolParallel})
|
||||
runner.gate(id)
|
||||
}
|
||||
src := &staticSource{jobs: jobs}
|
||||
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, MaxParallel: maxParallel, Clock: fc})
|
||||
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "cap to be saturated", func() bool { return runner.peakInFlight() >= maxParallel })
|
||||
// Give any over-cap goroutine a chance to break the cap before asserting.
|
||||
time.Sleep(10 * pollGap)
|
||||
if got := runner.peakInFlight(); got != maxParallel {
|
||||
t.Fatalf("peak in-flight = %d, want %d", got, maxParallel)
|
||||
}
|
||||
for _, j := range jobs {
|
||||
runner.release(j.ID)
|
||||
}
|
||||
waitFor(t, "all jobs to finish", func() bool { return len(runner.doneIDs()) == len(jobs) })
|
||||
}
|
||||
|
||||
func TestEngineSequentialPoolRunsOneAtATimeInOrder(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
jobs := []Job{{ID: "first"}, {ID: "second", Pool: PoolSequential}, {ID: "third", Pool: PoolSequential}}
|
||||
src := &staticSource{jobs: jobs}
|
||||
runner := newRecordingRunner()
|
||||
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, MaxParallel: len(jobs), Clock: fc})
|
||||
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "sequential jobs to finish", func() bool { return len(runner.doneIDs()) == len(jobs) })
|
||||
|
||||
if got := runner.peakInFlight(); got != 1 {
|
||||
t.Errorf("sequential peak in-flight = %d, want 1", got)
|
||||
}
|
||||
for i, id := range runner.startedIDs() {
|
||||
if id != jobs[i].ID {
|
||||
t.Errorf("run %d = %q, want %q (source order)", i, id, jobs[i].ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineSequentialJobDoesNotBlockParallelJob(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
src := &staticSource{jobs: []Job{{ID: "slow-seq", Pool: PoolSequential}, {ID: "quick-par", Pool: PoolParallel}}}
|
||||
runner := newRecordingRunner()
|
||||
runner.gate("slow-seq")
|
||||
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc})
|
||||
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "parallel job to finish while sequential is blocked", func() bool {
|
||||
return countOccurrences(runner.doneIDs(), "quick-par") == 1
|
||||
})
|
||||
runner.release("slow-seq")
|
||||
waitFor(t, "sequential job to finish", func() bool {
|
||||
return countOccurrences(runner.doneIDs(), "slow-seq") == 1
|
||||
})
|
||||
}
|
||||
|
||||
func TestEngineSkipsJobAlreadyInFlight(t *testing.T) {
|
||||
for _, pool := range []Pool{PoolParallel, PoolSequential} {
|
||||
t.Run(string(pool), func(t *testing.T) {
|
||||
const id = "overlapping"
|
||||
fc := newFakeClock(testStart())
|
||||
src := &staticSource{jobs: []Job{{ID: id, Pool: pool}}}
|
||||
runner := newRecordingRunner()
|
||||
runner.gate(id)
|
||||
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc})
|
||||
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "first run to start", func() bool { return len(runner.startedIDs()) == 1 })
|
||||
tick(t, fc, e)
|
||||
tick(t, fc, e)
|
||||
time.Sleep(10 * pollGap)
|
||||
if got := len(runner.startedIDs()); got != 1 {
|
||||
t.Fatalf("job started %d times while in flight, want 1", got)
|
||||
}
|
||||
runner.release(id)
|
||||
waitFor(t, "first run to finish", func() bool { return len(runner.doneIDs()) == 1 })
|
||||
|
||||
// Once the run is done the job is eligible again.
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "second run", func() bool { return len(runner.startedIDs()) == 2 })
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineRejectsInvalidJob(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
job Job
|
||||
}{
|
||||
{name: "empty id", job: Job{ID: " ", Pool: PoolParallel}},
|
||||
{name: "unknown pool", job: Job{ID: "bad-pool", Pool: Pool("elsewhere")}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
sink := &recordingSink{}
|
||||
src := &staticSource{jobs: []Job{tc.job, {ID: "healthy", Pool: PoolParallel}}}
|
||||
runner := newRecordingRunner()
|
||||
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc, Events: sink})
|
||||
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "healthy job to run", func() bool {
|
||||
return countOccurrences(runner.doneIDs(), "healthy") == 1
|
||||
})
|
||||
if got := len(runner.startedIDs()); got != 1 {
|
||||
t.Fatalf("started %v, want only the healthy job", runner.startedIDs())
|
||||
}
|
||||
if sink.countOf(KindJobError) == 0 {
|
||||
t.Error("expected an error event for the invalid job")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --- failure isolation --------------------------------------------------------
|
||||
|
||||
func TestEngineSurvivesSourceError(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
sink := &recordingSink{}
|
||||
var calls int
|
||||
var mu sync.Mutex
|
||||
wantErr := errors.New("store unavailable")
|
||||
src := sourceFunc(func(context.Context, time.Time) ([]Job, error) {
|
||||
mu.Lock()
|
||||
calls++
|
||||
n := calls
|
||||
mu.Unlock()
|
||||
if n == 1 {
|
||||
return nil, wantErr
|
||||
}
|
||||
return []Job{{ID: "after-failure"}}, nil
|
||||
})
|
||||
runner := newRecordingRunner()
|
||||
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc, Events: sink})
|
||||
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "error event", func() bool { return sink.countOf(KindJobError) == 1 })
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "loop to keep ticking after a source error", func() bool {
|
||||
return countOccurrences(runner.doneIDs(), "after-failure") == 1
|
||||
})
|
||||
|
||||
ev, ok := sink.firstOf(KindJobError)
|
||||
if !ok {
|
||||
t.Fatal("no error event recorded")
|
||||
}
|
||||
if got, _ := ev.data["phase"].(string); got != phaseSource {
|
||||
t.Errorf("error phase = %q, want %q", got, phaseSource)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineSurvivesJobPanicAndError(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
run func(job Job) error
|
||||
}{
|
||||
{name: "panic", run: func(Job) error { panic("boom") }},
|
||||
{name: "error", run: func(Job) error { return errors.New("job failed") }},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
sink := &recordingSink{}
|
||||
var mu sync.Mutex
|
||||
var runs int
|
||||
src := &staticSource{jobs: []Job{{ID: "unstable"}}}
|
||||
runner := runnerFunc(func(_ context.Context, job Job) error {
|
||||
mu.Lock()
|
||||
runs++
|
||||
mu.Unlock()
|
||||
return tc.run(job)
|
||||
})
|
||||
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc, Events: sink})
|
||||
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "first run", func() bool { mu.Lock(); defer mu.Unlock(); return runs == 1 })
|
||||
waitFor(t, "error event", func() bool { return sink.countOf(KindJobError) == 1 })
|
||||
// The loop survives: the same job is dispatched again next tick.
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "second run", func() bool { mu.Lock(); defer mu.Unlock(); return runs == 2 })
|
||||
|
||||
ev, _ := sink.firstOf(KindJobError)
|
||||
if got, _ := ev.data["phase"].(string); got != phaseRun {
|
||||
t.Errorf("error phase = %q, want %q", got, phaseRun)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineBoundsRunWithJobTimeout(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
sink := &recordingSink{}
|
||||
src := &staticSource{jobs: []Job{{ID: "hangs"}}}
|
||||
deadlines := make(chan error, 1)
|
||||
runner := runnerFunc(func(ctx context.Context, _ Job) error {
|
||||
<-ctx.Done()
|
||||
deadlines <- ctx.Err()
|
||||
return ctx.Err()
|
||||
})
|
||||
// A real (short) timeout: the run's deadline is wall-clock, because a
|
||||
// Runner's ctx must be honoured by code that knows nothing of our clock.
|
||||
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, JobTimeout: 20 * time.Millisecond, Clock: fc, Events: sink})
|
||||
|
||||
tick(t, fc, e)
|
||||
select {
|
||||
case err := <-deadlines:
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("run ctx error = %v, want DeadlineExceeded", err)
|
||||
}
|
||||
case <-time.After(testTimeout):
|
||||
t.Fatal("run was never cancelled by the job timeout")
|
||||
}
|
||||
waitFor(t, "timeout to be reported", func() bool { return sink.countOf(KindJobError) == 1 })
|
||||
}
|
||||
|
||||
// --- lifecycle ----------------------------------------------------------------
|
||||
|
||||
func TestEngineStartTwiceIsAnError(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
e := startEngine(t, &staticSource{}, newRecordingRunner(), EngineConfig{Interval: testInterval, Clock: fc})
|
||||
if err := e.Start(context.Background()); err == nil {
|
||||
t.Fatal("expected an error starting an already-started engine")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineRestartAfterStopIsRefused(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
e, err := NewEngine(&staticSource{}, newRecordingRunner(), EngineConfig{Interval: testInterval, Clock: fc})
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
if err := e.Start(context.Background()); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
|
||||
defer cancel()
|
||||
if err := e.Stop(ctx); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
if err := e.Start(context.Background()); err == nil {
|
||||
t.Fatal("expected an error restarting a stopped engine")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineStopWithoutStartIsNoOp(t *testing.T) {
|
||||
e, err := NewEngine(&staticSource{}, newRecordingRunner(), EngineConfig{})
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
|
||||
defer cancel()
|
||||
if err := e.Stop(ctx); err != nil {
|
||||
t.Fatalf("Stop before Start = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineStopWaitsForInFlightRuns(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
src := &staticSource{jobs: []Job{{ID: "par", Pool: PoolParallel}, {ID: "seq", Pool: PoolSequential}}}
|
||||
runner := newRecordingRunner()
|
||||
runner.gate("par")
|
||||
runner.gate("seq")
|
||||
e, err := NewEngine(src, runner, EngineConfig{Interval: testInterval, Clock: fc})
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
if err := e.Start(context.Background()); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "both runs to start", func() bool { return len(runner.startedIDs()) == 2 })
|
||||
|
||||
stopped := make(chan error, 1)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
|
||||
defer cancel()
|
||||
stopped <- e.Stop(ctx)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-stopped:
|
||||
t.Fatal("Stop returned while runs were still in flight")
|
||||
case <-time.After(20 * pollGap):
|
||||
}
|
||||
|
||||
runner.release("par")
|
||||
runner.release("seq")
|
||||
select {
|
||||
case err := <-stopped:
|
||||
if err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
case <-time.After(testTimeout):
|
||||
t.Fatal("Stop did not return after runs finished")
|
||||
}
|
||||
if got := len(runner.doneIDs()); got != 2 {
|
||||
t.Fatalf("finished runs = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineStopIsBoundedByContext(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
src := &staticSource{jobs: []Job{{ID: "uninterruptible"}}}
|
||||
release := make(chan struct{})
|
||||
started := make(chan struct{}, 1)
|
||||
runner := runnerFunc(func(context.Context, Job) error {
|
||||
select {
|
||||
case started <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
<-release // deliberately ignores cancellation
|
||||
return nil
|
||||
})
|
||||
e, err := NewEngine(src, runner, EngineConfig{Interval: testInterval, Clock: fc})
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
if err := e.Start(context.Background()); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
defer close(release)
|
||||
|
||||
tick(t, fc, e)
|
||||
<-started
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*pollGap)
|
||||
defer cancel()
|
||||
if err := e.Stop(ctx); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("Stop = %v, want a deadline-exceeded error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineStopStopsTicking(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
src := &staticSource{jobs: []Job{{ID: "job"}}}
|
||||
runner := newRecordingRunner()
|
||||
e, err := NewEngine(src, runner, EngineConfig{Interval: testInterval, Clock: fc})
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
if err := e.Start(context.Background()); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "first run", func() bool { return len(runner.doneIDs()) == 1 })
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
|
||||
defer cancel()
|
||||
if err := e.Stop(ctx); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
before := src.callCount()
|
||||
fc.Advance(testInterval * 10)
|
||||
time.Sleep(20 * pollGap)
|
||||
if got := src.callCount(); got != before {
|
||||
t.Fatalf("source polled %d times after Stop (was %d)", got, before)
|
||||
}
|
||||
}
|
||||
|
||||
// --- events -------------------------------------------------------------------
|
||||
|
||||
func TestEngineEmitsRunEvents(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
sink := &recordingSink{}
|
||||
const detail = "daily status summary"
|
||||
src := &staticSource{jobs: []Job{{ID: "summary", Pool: PoolParallel, Detail: detail}}}
|
||||
e := startEngine(t, src, newRecordingRunner(), EngineConfig{Interval: testInterval, Clock: fc, Events: sink})
|
||||
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "run event", func() bool { return sink.countOf(KindJobRun) == 1 })
|
||||
|
||||
ev, _ := sink.firstOf(KindJobRun)
|
||||
if ev.detail != detail {
|
||||
t.Errorf("event detail = %q, want %q", ev.detail, detail)
|
||||
}
|
||||
if got, _ := ev.data["job"].(string); got != "summary" {
|
||||
t.Errorf("event job = %q, want %q", got, "summary")
|
||||
}
|
||||
if got, _ := ev.data["pool"].(string); got != string(PoolParallel) {
|
||||
t.Errorf("event pool = %q, want %q", got, PoolParallel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineWithoutSinkDoesNotPanic(t *testing.T) {
|
||||
fc := newFakeClock(testStart())
|
||||
src := &staticSource{jobs: []Job{{ID: "bad-pool", Pool: Pool("nowhere")}, {ID: "fine"}}}
|
||||
runner := newRecordingRunner()
|
||||
e := startEngine(t, src, runner, EngineConfig{Interval: testInterval, Clock: fc})
|
||||
|
||||
tick(t, fc, e)
|
||||
waitFor(t, "valid job to run", func() bool { return countOccurrences(runner.doneIDs(), "fine") == 1 })
|
||||
}
|
||||
|
||||
// --- contract -----------------------------------------------------------------
|
||||
|
||||
func TestPoolValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
pool Pool
|
||||
wantErr bool
|
||||
}{
|
||||
{pool: PoolParallel},
|
||||
{pool: PoolSequential},
|
||||
{pool: ""},
|
||||
{pool: Pool("PARALLEL"), wantErr: true},
|
||||
{pool: Pool("other"), wantErr: true},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(string(tc.pool), func(t *testing.T) {
|
||||
if err := tc.pool.Validate(); (err != nil) != tc.wantErr {
|
||||
t.Fatalf("Validate(%q) = %v, wantErr %v", tc.pool, err, tc.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolResolveDefaultsToSequential(t *testing.T) {
|
||||
tests := []struct {
|
||||
pool Pool
|
||||
want Pool
|
||||
}{
|
||||
{pool: "", want: PoolSequential},
|
||||
{pool: PoolSequential, want: PoolSequential},
|
||||
{pool: PoolParallel, want: PoolParallel},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(string(tc.pool), func(t *testing.T) {
|
||||
if got := tc.pool.resolve(); got != tc.want {
|
||||
t.Fatalf("resolve(%q) = %q, want %q", tc.pool, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
job Job
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "valid", job: Job{ID: "a", Pool: PoolParallel}},
|
||||
{name: "valid default pool", job: Job{ID: "a"}},
|
||||
{name: "empty id", job: Job{ID: ""}, wantErr: true},
|
||||
{name: "blank id", job: Job{ID: "\t "}, wantErr: true},
|
||||
{name: "unknown pool", job: Job{ID: "a", Pool: Pool("x")}, wantErr: true},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if err := tc.job.Validate(); (err != nil) != tc.wantErr {
|
||||
t.Fatalf("Validate() = %v, wantErr %v", err, tc.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -48,6 +48,14 @@ const (
|
||||
KindCuratorAction = "curator_action" // data: curator, type, detail, reason
|
||||
KindCuratorError = "curator_error" // data: curator, phase, err
|
||||
|
||||
// Scheduled-job activity from the ticker engine
|
||||
// (internal/domain/scheduling). A scheduled run has no task and no chat
|
||||
// turn behind it, so without these a job that fired, stalled or failed
|
||||
// is invisible everywhere. data: job, pool, duration_ms (run) and job,
|
||||
// pool, phase, err (error).
|
||||
KindJobRun = "job_run"
|
||||
KindJobError = "job_error"
|
||||
|
||||
// KindTurnCompleted marks one completed primary chat turn
|
||||
// (archie-core-035). Input-driven curators wake on it. Curator output
|
||||
// never produces this kind, so derived work cannot feed its own
|
||||
|
||||
@@ -97,10 +97,7 @@ func toolPreview(name, output string) string {
|
||||
}
|
||||
return "completed with no printable preview"
|
||||
}
|
||||
remaining := countNonEmptyOrContentLines(content) - 1 - skipped
|
||||
if remaining < 0 {
|
||||
remaining = 0
|
||||
}
|
||||
remaining := max(countNonEmptyOrContentLines(content)-1-skipped, 0)
|
||||
if remaining > 0 {
|
||||
return fmt.Sprintf("%s\n… %d more lines", preview, remaining)
|
||||
}
|
||||
@@ -110,7 +107,7 @@ func toolPreview(name, output string) string {
|
||||
|
||||
func compactToolPreview(output string) (string, int) {
|
||||
skipped := 0
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
for line := range strings.SplitSeq(output, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user