216 lines
7.5 KiB
Go
216 lines
7.5 KiB
Go
package archied
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/domain/alert"
|
|
"github.com/samcharles93/archie-core/internal/domain/org"
|
|
"github.com/samcharles93/archie-core/internal/domain/scheduling"
|
|
"github.com/samcharles93/archie-core/internal/domain/source"
|
|
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/cronevents"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/schedulecapture"
|
|
)
|
|
|
|
// scheduleFailureAlertStreak is how many failed runs in a row raise an alert.
|
|
const scheduleFailureAlertStreak = 3
|
|
|
|
// schedulingConfig translates external input into the domain-owned engine
|
|
// configuration and connects the selected event infrastructure.
|
|
func schedulingConfig(input configuration.SchedulingInput, bus *events.Bus) (scheduling.EngineConfig, error) {
|
|
cfg := scheduling.DefaultEngineConfig()
|
|
if input.Interval != 0 {
|
|
cfg.Interval = input.Interval.Std()
|
|
}
|
|
if input.MaxParallel != 0 {
|
|
cfg.MaxParallel = input.MaxParallel
|
|
}
|
|
if input.JobTimeout != 0 {
|
|
cfg.JobTimeout = input.JobTimeout.Std()
|
|
}
|
|
|
|
switch input.EventsSink {
|
|
case "", configuration.EventsSinkNone:
|
|
case configuration.EventsSinkBus:
|
|
cfg.Events = cronevents.New(bus)
|
|
default:
|
|
return scheduling.EngineConfig{}, fmt.Errorf("%w: scheduling.events_sink %q (want %q or %q)",
|
|
configuration.ErrInvalidInput, input.EventsSink, configuration.EventsSinkBus, configuration.EventsSinkNone)
|
|
}
|
|
if err := cfg.Validate(); err != nil {
|
|
return scheduling.EngineConfig{}, fmt.Errorf("%w: %w", configuration.ErrInvalidInput, err)
|
|
}
|
|
return cfg, nil
|
|
}
|
|
|
|
// scheduleRuns is the State Store's schedule-run surface, as
|
|
// *staterpc.Client serves it.
|
|
type scheduleRuns interface {
|
|
ClaimDueRuns(ctx context.Context, now time.Time, stale time.Duration) ([]scheduling.Claim, error)
|
|
FinishScheduleRun(ctx context.Context, runID int64, result scheduling.RunResult, errText string) (int, error)
|
|
SetScheduleRunCapture(ctx context.Context, runID int64, captureID string) error
|
|
EnsureBuiltinSchedules(ctx context.Context, builtins []scheduling.Builtin) error
|
|
}
|
|
|
|
// setupScheduling builds the scheduler: claimed runs of a built-in schedule
|
|
// go to its in-process job, every other run records a schedule-fired capture
|
|
// that bindings start workflows from. The engine is started by startServices.
|
|
func (b *boot) setupScheduling() error {
|
|
runs, ok := b.stateStore.(scheduleRuns)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
cfg, err := schedulingConfig(b.doc.Scheduling, b.bus)
|
|
if err != nil {
|
|
return fmt.Errorf("scheduling: %w", err)
|
|
}
|
|
jobs := b.d.BuiltinJobs()
|
|
builtins := make(map[string]scheduling.Runner, len(jobs))
|
|
seeds := make([]scheduling.Builtin, 0, len(jobs))
|
|
for _, job := range jobs {
|
|
builtins[job.ID] = builtinJob(job.Run)
|
|
seeds = append(seeds, scheduling.Builtin{ID: job.ID, Name: job.Name, Interval: job.DefaultInterval})
|
|
}
|
|
src := &scheduleSource{runs: runs, stale: 2 * cfg.JobTimeout, log: b.log, builtins: seeds}
|
|
if alerts, ok := b.stateStore.(alert.Store); ok {
|
|
src.alerts = alert.Reporter{Raiser: alerts, Log: b.log, By: "archied"}
|
|
}
|
|
var fire scheduling.Runner
|
|
if captures, ok := b.stateStore.(storecontract.CaptureStore); ok {
|
|
fire = scheduleFirer{runs: runs, firer: schedulecapture.Firer{
|
|
Captures: captures, Retention: b.cfg.Capture.Retention.Std(), MaxEvents: b.cfg.Capture.MaxEvents,
|
|
}}
|
|
}
|
|
engine, err := scheduling.NewEngine(src, scheduleRunner{Builtins: builtins, Fire: fire}, cfg)
|
|
if err != nil {
|
|
return fmt.Errorf("scheduling: build engine: %w", err)
|
|
}
|
|
b.schedulingEngine = engine
|
|
b.addCleanup(shutdownSchedulingEngine(engine, b.log))
|
|
return nil
|
|
}
|
|
|
|
// scheduleSource claims runs from the State Store and records their results,
|
|
// raising an alert while a schedule keeps failing.
|
|
type scheduleSource struct {
|
|
runs scheduleRuns
|
|
stale time.Duration
|
|
alerts alert.Reporter
|
|
log *slog.Logger
|
|
// builtins are seeded before the first claim, and again on a later tick
|
|
// if the State Store was unavailable.
|
|
builtins []scheduling.Builtin
|
|
seeded bool
|
|
}
|
|
|
|
func (s *scheduleSource) Due(ctx context.Context, now time.Time) ([]scheduling.Job, error) {
|
|
if !s.seeded {
|
|
if err := s.runs.EnsureBuiltinSchedules(ctx, s.builtins); err != nil {
|
|
return nil, fmt.Errorf("seed built-in schedules: %w", err)
|
|
}
|
|
s.seeded = true
|
|
}
|
|
claims, err := s.runs.ClaimDueRuns(ctx, now, s.stale)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
jobs := make([]scheduling.Job, 0, len(claims))
|
|
for _, c := range claims {
|
|
// Built-in jobs share the daemon's state, so they run one at a time.
|
|
pool := scheduling.PoolParallel
|
|
if c.BuiltIn {
|
|
pool = scheduling.PoolSequential
|
|
}
|
|
jobs = append(jobs, scheduling.Job{
|
|
ID: c.ScheduleID, RunID: c.ID, Pool: pool, Detail: c.Name,
|
|
ScheduledFor: c.ScheduledFor, BuiltIn: c.BuiltIn,
|
|
})
|
|
}
|
|
return jobs, nil
|
|
}
|
|
|
|
func (s *scheduleSource) Finish(ctx context.Context, job scheduling.Job, result scheduling.RunResult, runErr error) error {
|
|
errText := ""
|
|
if runErr != nil {
|
|
errText = runErr.Error()
|
|
}
|
|
streak, err := s.runs.FinishScheduleRun(ctx, job.RunID, result, errText)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if result == scheduling.RunSkipped || s.alerts.Raiser == nil {
|
|
return nil
|
|
}
|
|
s.alerts.Sync(ctx, streak >= scheduleFailureAlertStreak, alert.Input{
|
|
Org: org.DefaultOrgID, Kind: alert.KindScheduleFailing, Subject: job.ID,
|
|
Severity: alert.SeverityWarning,
|
|
Title: fmt.Sprintf("Schedule %q failed %d times in a row", job.Detail, streak),
|
|
Body: errText,
|
|
Link: "/org/schedules",
|
|
Condition: true,
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// builtinJob runs a daemon maintenance job as a schedule's run.
|
|
type builtinJob func(ctx context.Context) error
|
|
|
|
func (run builtinJob) Run(ctx context.Context, _ scheduling.Job) error { return run(ctx) }
|
|
|
|
// scheduleRunner sends a built-in schedule's run to its job and any other
|
|
// run to Fire.
|
|
type scheduleRunner struct {
|
|
Builtins map[string]scheduling.Runner
|
|
Fire scheduling.Runner
|
|
}
|
|
|
|
func (r scheduleRunner) Run(ctx context.Context, job scheduling.Job) error {
|
|
if job.BuiltIn {
|
|
if run := r.Builtins[job.ID]; run != nil {
|
|
return run.Run(ctx, job)
|
|
}
|
|
return fmt.Errorf("built-in schedule %q has no job in this archied", job.ID)
|
|
}
|
|
if r.Fire == nil {
|
|
return errors.New("schedules cannot fire: the State Store offers no capture store")
|
|
}
|
|
return r.Fire.Run(ctx, job)
|
|
}
|
|
|
|
// scheduleFirer records a schedule-fired capture for the run and links the
|
|
// run to it.
|
|
type scheduleFirer struct {
|
|
runs scheduleRuns
|
|
firer schedulecapture.Firer
|
|
}
|
|
|
|
func (f scheduleFirer) Run(ctx context.Context, job scheduling.Job) error {
|
|
captureID, err := f.firer.Fire(ctx, source.ScheduleFired{
|
|
ScheduleID: job.ID, Name: job.Detail, ScheduledFor: job.ScheduledFor, FiredAt: time.Now(),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return f.runs.SetScheduleRunCapture(ctx, job.RunID, captureID)
|
|
}
|
|
|
|
// shutdownSchedulingEngine returns a cleanup that stops the ticker engine.
|
|
// The stop path derives its own timeout budget from Background rather than
|
|
// the boot context, which is already cancelled by the time shutdown runs --
|
|
// the same reasoning as shutdownCuratorRuntime.
|
|
func shutdownSchedulingEngine(e *scheduling.Engine, log *slog.Logger) func() {
|
|
return func() {
|
|
stopCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if err := e.Stop(stopCtx); err != nil {
|
|
log.Error("scheduling engine shutdown", "err", err)
|
|
}
|
|
}
|
|
}
|