343 lines
9.2 KiB
Go
343 lines
9.2 KiB
Go
package scheduling
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
)
|
|
|
|
const (
|
|
defaultInterval = time.Minute
|
|
defaultMaxParallel = 4
|
|
defaultJobTimeout = time.Hour
|
|
)
|
|
|
|
// Event kinds emitted by the engine, re-exported so callers of this package
|
|
// need not import the events package to match on them.
|
|
const (
|
|
KindJobRun = events.KindJobRun
|
|
KindJobError = events.KindJobError
|
|
)
|
|
|
|
// Phases reported on KindJobError, naming which step failed.
|
|
const (
|
|
phaseSource = "source" // JobSource.Due returned an error
|
|
phaseDispatch = "dispatch" // the job was rejected before it ever ran
|
|
phaseRun = "run" // Runner.Run failed, panicked or timed out
|
|
)
|
|
|
|
// sequentialQueueDepth bounds the sequential pool's backlog. Overlap
|
|
// suppression already caps the backlog at one entry per distinct job, so this
|
|
// is a backstop against a pathological source, not the primary bound.
|
|
const sequentialQueueDepth = 256
|
|
|
|
// EngineConfig tunes the ticker engine. Every field is optional; zero values
|
|
// take the documented defaults.
|
|
type EngineConfig struct {
|
|
// Interval is the tick cadence: how often the engine asks its source
|
|
// what is due. Zero uses the value from DefaultEngineConfig; negative
|
|
// is an error.
|
|
Interval time.Duration
|
|
// MaxParallel caps simultaneous parallel-pool runs. Zero uses
|
|
// the value from DefaultEngineConfig. It does not bound the sequential
|
|
// pool, which is one by definition.
|
|
MaxParallel int
|
|
// JobTimeout bounds a single run via context deadline. Zero uses
|
|
// the value from DefaultEngineConfig.
|
|
JobTimeout time.Duration
|
|
// Clock is injectable time. Nil uses the system clock.
|
|
Clock Clock
|
|
// Events receives run and error events. Nil disables emission; the
|
|
// engine still runs, it is just unobservable.
|
|
Events Sink
|
|
}
|
|
|
|
// DefaultEngineConfig returns the domain-owned defaults for the engine's
|
|
// operator-selectable runtime settings.
|
|
func DefaultEngineConfig() EngineConfig {
|
|
return EngineConfig{
|
|
Interval: defaultInterval,
|
|
MaxParallel: defaultMaxParallel,
|
|
JobTimeout: defaultJobTimeout,
|
|
}
|
|
}
|
|
|
|
// Validate rejects unusable engine settings.
|
|
func (c EngineConfig) Validate() error {
|
|
switch {
|
|
case c.Interval < 0:
|
|
return fmt.Errorf("scheduling: interval must not be negative, got %v", c.Interval)
|
|
case c.MaxParallel < 0:
|
|
return fmt.Errorf("scheduling: max parallel must not be negative, got %d", c.MaxParallel)
|
|
case c.JobTimeout < 0:
|
|
return fmt.Errorf("scheduling: job timeout must not be negative, got %v", c.JobTimeout)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// withDefaults returns the config with zero values replaced.
|
|
func (c EngineConfig) withDefaults() EngineConfig {
|
|
d := DefaultEngineConfig()
|
|
if c.Interval == 0 {
|
|
c.Interval = d.Interval
|
|
}
|
|
if c.MaxParallel == 0 {
|
|
c.MaxParallel = d.MaxParallel
|
|
}
|
|
if c.JobTimeout == 0 {
|
|
c.JobTimeout = d.JobTimeout
|
|
}
|
|
if c.Clock == nil {
|
|
c.Clock = systemClock{}
|
|
}
|
|
return c
|
|
}
|
|
|
|
// Engine asks the source what is due each tick and dispatches each job to
|
|
// its pool. A job never overlaps itself, and the sequential and parallel
|
|
// pools are independent. Errors, panics and timeouts are reported as events.
|
|
type Engine struct {
|
|
source JobSource
|
|
runner Runner
|
|
cfg EngineConfig
|
|
|
|
queue chan Job // sequential pool backlog
|
|
sem chan struct{}
|
|
|
|
mu sync.Mutex
|
|
started bool
|
|
stopped bool
|
|
stopLoop context.CancelFunc
|
|
cancelRuns context.CancelFunc
|
|
active map[string]struct{} // queued or in flight, by job ID
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
// NewEngine builds the engine over its source and runner. It fails on a nil
|
|
// dependency or a negative tunable; zero tunables take their defaults.
|
|
func NewEngine(source JobSource, runner Runner, cfg EngineConfig) (*Engine, error) {
|
|
if source == nil {
|
|
return nil, errors.New("scheduling: job source must not be nil")
|
|
}
|
|
if runner == nil {
|
|
return nil, errors.New("scheduling: runner must not be nil")
|
|
}
|
|
if err := cfg.Validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
cfg = cfg.withDefaults()
|
|
return &Engine{
|
|
source: source,
|
|
runner: runner,
|
|
cfg: cfg,
|
|
queue: make(chan Job, sequentialQueueDepth),
|
|
sem: make(chan struct{}, cfg.MaxParallel),
|
|
active: make(map[string]struct{}),
|
|
}, nil
|
|
}
|
|
|
|
// Interval reports the resolved tick cadence.
|
|
func (e *Engine) Interval() time.Duration { return e.cfg.Interval }
|
|
|
|
// MaxParallel reports the resolved parallel-pool cap.
|
|
func (e *Engine) MaxParallel() int { return e.cfg.MaxParallel }
|
|
|
|
// JobTimeout reports the resolved per-run timeout.
|
|
func (e *Engine) JobTimeout() time.Duration { return e.cfg.JobTimeout }
|
|
|
|
// Start launches the tick loop and the sequential worker. An engine can be
|
|
// started only once.
|
|
func (e *Engine) Start(ctx context.Context) error {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
if e.stopped {
|
|
return errors.New("scheduling: engine already stopped; build a new one")
|
|
}
|
|
if e.started {
|
|
return errors.New("scheduling: engine already started")
|
|
}
|
|
e.started = true
|
|
|
|
loopCtx, stopLoop := context.WithCancel(ctx)
|
|
e.stopLoop = stopLoop
|
|
// Run contexts descend from Background, not from loopCtx, so that Stop
|
|
// drains gracefully: cancelling the loop must stop new work without
|
|
// yanking the context out from under a run that is already going. Stop
|
|
// cancels these only once its own deadline passes.
|
|
runCtx, cancelRuns := context.WithCancel(context.WithoutCancel(ctx))
|
|
e.cancelRuns = cancelRuns
|
|
|
|
e.wg.Add(2)
|
|
go e.tickLoop(loopCtx, runCtx)
|
|
go e.sequentialWorker(loopCtx, runCtx)
|
|
return nil
|
|
}
|
|
|
|
// Stop halts ticking, drops queued runs and waits for running ones within
|
|
// ctx, then cancels them. Stopping an unstarted engine is a no-op.
|
|
func (e *Engine) Stop(ctx context.Context) error {
|
|
e.mu.Lock()
|
|
if !e.started {
|
|
e.mu.Unlock()
|
|
return nil
|
|
}
|
|
e.started = false
|
|
e.stopped = true
|
|
stopLoop, cancelRuns := e.stopLoop, e.cancelRuns
|
|
e.stopLoop, e.cancelRuns = nil, nil
|
|
e.mu.Unlock()
|
|
|
|
stopLoop()
|
|
done := make(chan struct{})
|
|
go func() {
|
|
e.wg.Wait()
|
|
close(done)
|
|
}()
|
|
select {
|
|
case <-done:
|
|
cancelRuns()
|
|
return nil
|
|
case <-ctx.Done():
|
|
cancelRuns()
|
|
return fmt.Errorf("scheduling: engine shutdown: %w", ctx.Err())
|
|
}
|
|
}
|
|
|
|
// tickLoop sleeps one interval, asks the source what is due, and dispatches
|
|
// it. The source is authoritative: the engine applies no schedule arithmetic
|
|
// of its own, so what "due" means stays the store's decision.
|
|
func (e *Engine) tickLoop(loopCtx, runCtx context.Context) {
|
|
defer e.wg.Done()
|
|
for {
|
|
select {
|
|
case <-loopCtx.Done():
|
|
return
|
|
case <-e.cfg.Clock.After(e.cfg.Interval):
|
|
}
|
|
|
|
jobs, err := e.source.Due(loopCtx, e.cfg.Clock.Now())
|
|
if err != nil {
|
|
e.emitError(Job{}, phaseSource, err)
|
|
continue
|
|
}
|
|
for _, job := range jobs {
|
|
e.dispatch(loopCtx, runCtx, job)
|
|
}
|
|
}
|
|
}
|
|
|
|
// dispatch routes one due job to its pool, skipping it if it is invalid or
|
|
// already queued or in flight.
|
|
func (e *Engine) dispatch(loopCtx, runCtx context.Context, job Job) {
|
|
if err := job.Validate(); err != nil {
|
|
e.emitError(job, phaseDispatch, err)
|
|
return
|
|
}
|
|
if !e.claim(job.ID) {
|
|
return // an earlier run of this job has not finished; skip this tick
|
|
}
|
|
|
|
if job.Pool.resolve() == PoolParallel {
|
|
e.wg.Go(func() {
|
|
defer e.release(job.ID)
|
|
select {
|
|
case e.sem <- struct{}{}:
|
|
case <-loopCtx.Done():
|
|
return // shutting down while waiting for a parallel slot
|
|
}
|
|
defer func() { <-e.sem }()
|
|
e.runJob(runCtx, job)
|
|
})
|
|
return
|
|
}
|
|
|
|
select {
|
|
case e.queue <- job:
|
|
default:
|
|
e.release(job.ID)
|
|
e.emitError(job, phaseDispatch, errors.New("scheduling: sequential queue full"))
|
|
}
|
|
}
|
|
|
|
// sequentialWorker is the sequential pool: one goroutine draining the queue
|
|
// in the order the source reported the jobs, so serialised jobs never
|
|
// overlap each other.
|
|
func (e *Engine) sequentialWorker(loopCtx, runCtx context.Context) {
|
|
defer e.wg.Done()
|
|
for {
|
|
select {
|
|
case <-loopCtx.Done():
|
|
return
|
|
case job := <-e.queue:
|
|
e.runJob(runCtx, job)
|
|
e.release(job.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
// runJob executes one run under the configured timeout, recovering panics so
|
|
// a job that blows up takes nothing else with it.
|
|
func (e *Engine) runJob(ctx context.Context, job Job) {
|
|
runCtx, cancel := context.WithTimeout(ctx, e.cfg.JobTimeout)
|
|
defer cancel()
|
|
|
|
start := e.cfg.Clock.Now()
|
|
err := e.safeRun(runCtx, job)
|
|
elapsed := e.cfg.Clock.Now().Sub(start)
|
|
if err != nil {
|
|
e.emitError(job, phaseRun, err)
|
|
return
|
|
}
|
|
e.emit(KindJobRun, job, map[string]any{
|
|
"job": job.ID,
|
|
"pool": string(job.Pool.resolve()),
|
|
"duration_ms": elapsed.Milliseconds(),
|
|
})
|
|
}
|
|
|
|
func (e *Engine) safeRun(ctx context.Context, job Job) (err error) {
|
|
defer func() {
|
|
if p := recover(); p != nil {
|
|
err = fmt.Errorf("scheduling: job %q panicked: %v", job.ID, p)
|
|
}
|
|
}()
|
|
return e.runner.Run(ctx, job)
|
|
}
|
|
|
|
// claim marks a job as queued-or-running, reporting false if it already was.
|
|
func (e *Engine) claim(id string) bool {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
if _, busy := e.active[id]; busy {
|
|
return false
|
|
}
|
|
e.active[id] = struct{}{}
|
|
return true
|
|
}
|
|
|
|
func (e *Engine) release(id string) {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
delete(e.active, id)
|
|
}
|
|
|
|
func (e *Engine) emitError(job Job, phase string, err error) {
|
|
e.emit(KindJobError, job, map[string]any{
|
|
"job": job.ID,
|
|
"pool": string(job.Pool),
|
|
"phase": phase,
|
|
"err": err.Error(),
|
|
})
|
|
}
|
|
|
|
func (e *Engine) emit(kind string, job Job, data map[string]any) {
|
|
if e.cfg.Events == nil {
|
|
return
|
|
}
|
|
e.cfg.Events.Emit(kind, job.Detail, data)
|
|
}
|