Files
archie/internal/domain/scheduling/ticker.go
T

343 lines
9.2 KiB
Go

package scheduling
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/samcharles93/archie-core/internal/events"
)
const (
defaultInterval = time.Minute
defaultMaxParallel = 4
defaultJobTimeout = time.Hour
)
// Event kinds emitted by the engine, re-exported so callers of this package
// need not import the events package to match on them.
const (
KindJobRun = events.KindJobRun
KindJobError = events.KindJobError
)
// Phases reported on KindJobError, naming which step failed.
const (
phaseSource = "source" // JobSource.Due returned an error
phaseDispatch = "dispatch" // the job was rejected before it ever ran
phaseRun = "run" // Runner.Run failed, panicked or timed out
)
// sequentialQueueDepth bounds the sequential pool's backlog. Overlap
// suppression already caps the backlog at one entry per distinct job, so this
// is a backstop against a pathological source, not the primary bound.
const sequentialQueueDepth = 256
// EngineConfig tunes the ticker engine. Every field is optional; zero values
// take the documented defaults.
type EngineConfig struct {
// Interval is the tick cadence: how often the engine asks its source
// what is due. Zero uses the value from DefaultEngineConfig; negative
// is an error.
Interval time.Duration
// MaxParallel caps simultaneous parallel-pool runs. Zero uses
// the value from DefaultEngineConfig. It does not bound the sequential
// pool, which is one by definition.
MaxParallel int
// JobTimeout bounds a single run via context deadline. Zero uses
// the value from DefaultEngineConfig.
JobTimeout time.Duration
// Clock is injectable time. Nil uses the system clock.
Clock Clock
// Events receives run and error events. Nil disables emission; the
// engine still runs, it is just unobservable.
Events Sink
}
// DefaultEngineConfig returns the domain-owned defaults for the engine's
// operator-selectable runtime settings.
func DefaultEngineConfig() EngineConfig {
return EngineConfig{
Interval: defaultInterval,
MaxParallel: defaultMaxParallel,
JobTimeout: defaultJobTimeout,
}
}
// Validate rejects unusable engine settings.
func (c EngineConfig) Validate() error {
switch {
case c.Interval < 0:
return fmt.Errorf("scheduling: interval must not be negative, got %v", c.Interval)
case c.MaxParallel < 0:
return fmt.Errorf("scheduling: max parallel must not be negative, got %d", c.MaxParallel)
case c.JobTimeout < 0:
return fmt.Errorf("scheduling: job timeout must not be negative, got %v", c.JobTimeout)
}
return nil
}
// withDefaults returns the config with zero values replaced.
func (c EngineConfig) withDefaults() EngineConfig {
d := DefaultEngineConfig()
if c.Interval == 0 {
c.Interval = d.Interval
}
if c.MaxParallel == 0 {
c.MaxParallel = d.MaxParallel
}
if c.JobTimeout == 0 {
c.JobTimeout = d.JobTimeout
}
if c.Clock == nil {
c.Clock = systemClock{}
}
return c
}
// Engine asks the source what is due each tick and dispatches each job to
// its pool. A job never overlaps itself, and the sequential and parallel
// pools are independent. Errors, panics and timeouts are reported as events.
type Engine struct {
source JobSource
runner Runner
cfg EngineConfig
queue chan Job // sequential pool backlog
sem chan struct{}
mu sync.Mutex
started bool
stopped bool
stopLoop context.CancelFunc
cancelRuns context.CancelFunc
active map[string]struct{} // queued or in flight, by job ID
wg sync.WaitGroup
}
// NewEngine builds the engine over its source and runner. It fails on a nil
// dependency or a negative tunable; zero tunables take their defaults.
func NewEngine(source JobSource, runner Runner, cfg EngineConfig) (*Engine, error) {
if source == nil {
return nil, errors.New("scheduling: job source must not be nil")
}
if runner == nil {
return nil, errors.New("scheduling: runner must not be nil")
}
if err := cfg.Validate(); err != nil {
return nil, err
}
cfg = cfg.withDefaults()
return &Engine{
source: source,
runner: runner,
cfg: cfg,
queue: make(chan Job, sequentialQueueDepth),
sem: make(chan struct{}, cfg.MaxParallel),
active: make(map[string]struct{}),
}, nil
}
// Interval reports the resolved tick cadence.
func (e *Engine) Interval() time.Duration { return e.cfg.Interval }
// MaxParallel reports the resolved parallel-pool cap.
func (e *Engine) MaxParallel() int { return e.cfg.MaxParallel }
// JobTimeout reports the resolved per-run timeout.
func (e *Engine) JobTimeout() time.Duration { return e.cfg.JobTimeout }
// Start launches the tick loop and the sequential worker. An engine can be
// started only once.
func (e *Engine) Start(ctx context.Context) error {
e.mu.Lock()
defer e.mu.Unlock()
if e.stopped {
return errors.New("scheduling: engine already stopped; build a new one")
}
if e.started {
return errors.New("scheduling: engine already started")
}
e.started = true
loopCtx, stopLoop := context.WithCancel(ctx)
e.stopLoop = stopLoop
// Run contexts descend from Background, not from loopCtx, so that Stop
// drains gracefully: cancelling the loop must stop new work without
// yanking the context out from under a run that is already going. Stop
// cancels these only once its own deadline passes.
runCtx, cancelRuns := context.WithCancel(context.WithoutCancel(ctx))
e.cancelRuns = cancelRuns
e.wg.Add(2)
go e.tickLoop(loopCtx, runCtx)
go e.sequentialWorker(loopCtx, runCtx)
return nil
}
// Stop halts ticking, drops queued runs and waits for running ones within
// ctx, then cancels them. Stopping an unstarted engine is a no-op.
func (e *Engine) Stop(ctx context.Context) error {
e.mu.Lock()
if !e.started {
e.mu.Unlock()
return nil
}
e.started = false
e.stopped = true
stopLoop, cancelRuns := e.stopLoop, e.cancelRuns
e.stopLoop, e.cancelRuns = nil, nil
e.mu.Unlock()
stopLoop()
done := make(chan struct{})
go func() {
e.wg.Wait()
close(done)
}()
select {
case <-done:
cancelRuns()
return nil
case <-ctx.Done():
cancelRuns()
return fmt.Errorf("scheduling: engine shutdown: %w", ctx.Err())
}
}
// tickLoop sleeps one interval, asks the source what is due, and dispatches
// it. The source is authoritative: the engine applies no schedule arithmetic
// of its own, so what "due" means stays the store's decision.
func (e *Engine) tickLoop(loopCtx, runCtx context.Context) {
defer e.wg.Done()
for {
select {
case <-loopCtx.Done():
return
case <-e.cfg.Clock.After(e.cfg.Interval):
}
jobs, err := e.source.Due(loopCtx, e.cfg.Clock.Now())
if err != nil {
e.emitError(Job{}, phaseSource, err)
continue
}
for _, job := range jobs {
e.dispatch(loopCtx, runCtx, job)
}
}
}
// dispatch routes one due job to its pool, skipping it if it is invalid or
// already queued or in flight.
func (e *Engine) dispatch(loopCtx, runCtx context.Context, job Job) {
if err := job.Validate(); err != nil {
e.emitError(job, phaseDispatch, err)
return
}
if !e.claim(job.ID) {
return // an earlier run of this job has not finished; skip this tick
}
if job.Pool.resolve() == PoolParallel {
e.wg.Go(func() {
defer e.release(job.ID)
select {
case e.sem <- struct{}{}:
case <-loopCtx.Done():
return // shutting down while waiting for a parallel slot
}
defer func() { <-e.sem }()
e.runJob(runCtx, job)
})
return
}
select {
case e.queue <- job:
default:
e.release(job.ID)
e.emitError(job, phaseDispatch, errors.New("scheduling: sequential queue full"))
}
}
// sequentialWorker is the sequential pool: one goroutine draining the queue
// in the order the source reported the jobs, so serialised jobs never
// overlap each other.
func (e *Engine) sequentialWorker(loopCtx, runCtx context.Context) {
defer e.wg.Done()
for {
select {
case <-loopCtx.Done():
return
case job := <-e.queue:
e.runJob(runCtx, job)
e.release(job.ID)
}
}
}
// runJob executes one run under the configured timeout, recovering panics so
// a job that blows up takes nothing else with it.
func (e *Engine) runJob(ctx context.Context, job Job) {
runCtx, cancel := context.WithTimeout(ctx, e.cfg.JobTimeout)
defer cancel()
start := e.cfg.Clock.Now()
err := e.safeRun(runCtx, job)
elapsed := e.cfg.Clock.Now().Sub(start)
if err != nil {
e.emitError(job, phaseRun, err)
return
}
e.emit(KindJobRun, job, map[string]any{
"job": job.ID,
"pool": string(job.Pool.resolve()),
"duration_ms": elapsed.Milliseconds(),
})
}
func (e *Engine) safeRun(ctx context.Context, job Job) (err error) {
defer func() {
if p := recover(); p != nil {
err = fmt.Errorf("scheduling: job %q panicked: %v", job.ID, p)
}
}()
return e.runner.Run(ctx, job)
}
// claim marks a job as queued-or-running, reporting false if it already was.
func (e *Engine) claim(id string) bool {
e.mu.Lock()
defer e.mu.Unlock()
if _, busy := e.active[id]; busy {
return false
}
e.active[id] = struct{}{}
return true
}
func (e *Engine) release(id string) {
e.mu.Lock()
defer e.mu.Unlock()
delete(e.active, id)
}
func (e *Engine) emitError(job Job, phase string, err error) {
e.emit(KindJobError, job, map[string]any{
"job": job.ID,
"pool": string(job.Pool),
"phase": phase,
"err": err.Error(),
})
}
func (e *Engine) emit(kind string, job Job, data map[string]any) {
if e.cfg.Events == nil {
return
}
e.cfg.Events.Emit(kind, job.Detail, data)
}