Files
archie/internal/app/archied/scheduling.go
T

216 lines
7.5 KiB
Go

package archied
import (
"context"
"errors"
"fmt"
"log/slog"
"time"
"github.com/samcharles93/archie-core/internal/domain/alert"
"github.com/samcharles93/archie-core/internal/domain/org"
"github.com/samcharles93/archie-core/internal/domain/scheduling"
"github.com/samcharles93/archie-core/internal/domain/source"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/cronevents"
"github.com/samcharles93/archie-core/internal/infrastructure/schedulecapture"
)
// scheduleFailureAlertStreak is how many failed runs in a row raise an alert.
const scheduleFailureAlertStreak = 3
// schedulingConfig translates external input into the domain-owned engine
// configuration and connects the selected event infrastructure.
func schedulingConfig(input configuration.SchedulingInput, bus *events.Bus) (scheduling.EngineConfig, error) {
cfg := scheduling.DefaultEngineConfig()
if input.Interval != 0 {
cfg.Interval = input.Interval.Std()
}
if input.MaxParallel != 0 {
cfg.MaxParallel = input.MaxParallel
}
if input.JobTimeout != 0 {
cfg.JobTimeout = input.JobTimeout.Std()
}
switch input.EventsSink {
case "", configuration.EventsSinkNone:
case configuration.EventsSinkBus:
cfg.Events = cronevents.New(bus)
default:
return scheduling.EngineConfig{}, fmt.Errorf("%w: scheduling.events_sink %q (want %q or %q)",
configuration.ErrInvalidInput, input.EventsSink, configuration.EventsSinkBus, configuration.EventsSinkNone)
}
if err := cfg.Validate(); err != nil {
return scheduling.EngineConfig{}, fmt.Errorf("%w: %w", configuration.ErrInvalidInput, err)
}
return cfg, nil
}
// scheduleRuns is the State Store's schedule-run surface, as
// *staterpc.Client serves it.
type scheduleRuns interface {
ClaimDueRuns(ctx context.Context, now time.Time, stale time.Duration) ([]scheduling.Claim, error)
FinishScheduleRun(ctx context.Context, runID int64, result scheduling.RunResult, errText string) (int, error)
SetScheduleRunCapture(ctx context.Context, runID int64, captureID string) error
EnsureBuiltinSchedules(ctx context.Context, builtins []scheduling.Builtin) error
}
// setupScheduling builds the scheduler: claimed runs of a built-in schedule
// go to its in-process job, every other run records a schedule-fired capture
// that bindings start workflows from. The engine is started by startServices.
func (b *boot) setupScheduling() error {
runs, ok := b.stateStore.(scheduleRuns)
if !ok {
return nil
}
cfg, err := schedulingConfig(b.doc.Scheduling, b.bus)
if err != nil {
return fmt.Errorf("scheduling: %w", err)
}
jobs := b.d.BuiltinJobs()
builtins := make(map[string]scheduling.Runner, len(jobs))
seeds := make([]scheduling.Builtin, 0, len(jobs))
for _, job := range jobs {
builtins[job.ID] = builtinJob(job.Run)
seeds = append(seeds, scheduling.Builtin{ID: job.ID, Name: job.Name, Interval: job.DefaultInterval})
}
src := &scheduleSource{runs: runs, stale: 2 * cfg.JobTimeout, log: b.log, builtins: seeds}
if alerts, ok := b.stateStore.(alert.Store); ok {
src.alerts = alert.Reporter{Raiser: alerts, Log: b.log, By: "archied"}
}
var fire scheduling.Runner
if captures, ok := b.stateStore.(storecontract.CaptureStore); ok {
fire = scheduleFirer{runs: runs, firer: schedulecapture.Firer{
Captures: captures, Retention: b.cfg.Capture.Retention.Std(), MaxEvents: b.cfg.Capture.MaxEvents,
}}
}
engine, err := scheduling.NewEngine(src, scheduleRunner{Builtins: builtins, Fire: fire}, cfg)
if err != nil {
return fmt.Errorf("scheduling: build engine: %w", err)
}
b.schedulingEngine = engine
b.addCleanup(shutdownSchedulingEngine(engine, b.log))
return nil
}
// scheduleSource claims runs from the State Store and records their results,
// raising an alert while a schedule keeps failing.
type scheduleSource struct {
runs scheduleRuns
stale time.Duration
alerts alert.Reporter
log *slog.Logger
// builtins are seeded before the first claim, and again on a later tick
// if the State Store was unavailable.
builtins []scheduling.Builtin
seeded bool
}
func (s *scheduleSource) Due(ctx context.Context, now time.Time) ([]scheduling.Job, error) {
if !s.seeded {
if err := s.runs.EnsureBuiltinSchedules(ctx, s.builtins); err != nil {
return nil, fmt.Errorf("seed built-in schedules: %w", err)
}
s.seeded = true
}
claims, err := s.runs.ClaimDueRuns(ctx, now, s.stale)
if err != nil {
return nil, err
}
jobs := make([]scheduling.Job, 0, len(claims))
for _, c := range claims {
// Built-in jobs share the daemon's state, so they run one at a time.
pool := scheduling.PoolParallel
if c.BuiltIn {
pool = scheduling.PoolSequential
}
jobs = append(jobs, scheduling.Job{
ID: c.ScheduleID, RunID: c.ID, Pool: pool, Detail: c.Name,
ScheduledFor: c.ScheduledFor, BuiltIn: c.BuiltIn,
})
}
return jobs, nil
}
func (s *scheduleSource) Finish(ctx context.Context, job scheduling.Job, result scheduling.RunResult, runErr error) error {
errText := ""
if runErr != nil {
errText = runErr.Error()
}
streak, err := s.runs.FinishScheduleRun(ctx, job.RunID, result, errText)
if err != nil {
return err
}
if result == scheduling.RunSkipped || s.alerts.Raiser == nil {
return nil
}
s.alerts.Sync(ctx, streak >= scheduleFailureAlertStreak, alert.Input{
Org: org.DefaultOrgID, Kind: alert.KindScheduleFailing, Subject: job.ID,
Severity: alert.SeverityWarning,
Title: fmt.Sprintf("Schedule %q failed %d times in a row", job.Detail, streak),
Body: errText,
Link: "/org/schedules",
Condition: true,
})
return nil
}
// builtinJob runs a daemon maintenance job as a schedule's run.
type builtinJob func(ctx context.Context) error
func (run builtinJob) Run(ctx context.Context, _ scheduling.Job) error { return run(ctx) }
// scheduleRunner sends a built-in schedule's run to its job and any other
// run to Fire.
type scheduleRunner struct {
Builtins map[string]scheduling.Runner
Fire scheduling.Runner
}
func (r scheduleRunner) Run(ctx context.Context, job scheduling.Job) error {
if job.BuiltIn {
if run := r.Builtins[job.ID]; run != nil {
return run.Run(ctx, job)
}
return fmt.Errorf("built-in schedule %q has no job in this archied", job.ID)
}
if r.Fire == nil {
return errors.New("schedules cannot fire: the State Store offers no capture store")
}
return r.Fire.Run(ctx, job)
}
// scheduleFirer records a schedule-fired capture for the run and links the
// run to it.
type scheduleFirer struct {
runs scheduleRuns
firer schedulecapture.Firer
}
func (f scheduleFirer) Run(ctx context.Context, job scheduling.Job) error {
captureID, err := f.firer.Fire(ctx, source.ScheduleFired{
ScheduleID: job.ID, Name: job.Detail, ScheduledFor: job.ScheduledFor, FiredAt: time.Now(),
})
if err != nil {
return err
}
return f.runs.SetScheduleRunCapture(ctx, job.RunID, captureID)
}
// shutdownSchedulingEngine returns a cleanup that stops the ticker engine.
// The stop path derives its own timeout budget from Background rather than
// the boot context, which is already cancelled by the time shutdown runs --
// the same reasoning as shutdownCuratorRuntime.
func shutdownSchedulingEngine(e *scheduling.Engine, log *slog.Logger) func() {
return func() {
stopCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := e.Stop(stopCtx); err != nil {
log.Error("scheduling engine shutdown", "err", err)
}
}
}