Schedules leave the schedules control-plane resource for State Store tables behind a ScheduleService contract. A run is claimed per schedule slot, missed slots fire once, the next run stays on the schedule's grid, cron is read in the schedule-settings time zone, and failed runs are recorded and advance. Three failures in a row raise a schedule.failing alert. A fired run records a schedule-fired capture. Run history is pruned by age and count. Removes the chat and spawn-task runners, the dangerous-payload guard and the old schedules settings page.
349 lines
11 KiB
Go
349 lines
11 KiB
Go
// Schedules: the Postgres implementation of internal/domain/scheduling.Store.
|
|
package postgres
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
|
|
"github.com/samcharles93/archie-core/internal/domain/scheduling"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb"
|
|
)
|
|
|
|
var (
|
|
_ scheduling.Store = (*Store)(nil)
|
|
_ scheduling.RunStore = (*Store)(nil)
|
|
)
|
|
|
|
// failureStreakWindow bounds how many finished runs FinishRun reads to count
|
|
// a failure streak.
|
|
const failureStreakWindow = 20
|
|
|
|
func optionalTime(t time.Time) pgtype.Timestamptz {
|
|
return pgtype.Timestamptz{Time: t, Valid: !t.IsZero()}
|
|
}
|
|
|
|
func scheduleFromRow(r postgresdb.Schedule) scheduling.Schedule {
|
|
s := scheduling.Schedule{
|
|
ID: r.ID, Name: r.Name,
|
|
Owner: scheduling.Owner{Kind: scheduling.OwnerKind(r.OwnerKind), ID: r.OwnerID},
|
|
Enabled: r.Enabled, BuiltIn: r.BuiltIn,
|
|
Timing: scheduling.Timing{
|
|
Kind: r.TimingKind,
|
|
Interval: scheduling.Duration(time.Duration(r.IntervalSeconds) * time.Second),
|
|
Cron: r.Cron,
|
|
Start: r.Anchor.UTC(),
|
|
},
|
|
Created: r.CreatedAt, Updated: r.UpdatedAt,
|
|
}
|
|
if r.RunAt.Valid {
|
|
at := r.RunAt.Time.UTC()
|
|
s.Timing.At = &at
|
|
}
|
|
if r.NextRun.Valid {
|
|
s.NextRun = r.NextRun.Time.UTC()
|
|
}
|
|
return s
|
|
}
|
|
|
|
func runFromRow(r postgresdb.ScheduleRun) scheduling.Run {
|
|
return scheduling.Run{
|
|
ID: r.ID, ScheduleID: r.ScheduleID, ScheduledFor: r.ScheduledFor.UTC(),
|
|
Started: r.StartedAt.Time, Finished: r.FinishedAt.Time,
|
|
Result: scheduling.RunResult(r.Result), Error: r.Error, CaptureID: r.CaptureID,
|
|
}
|
|
}
|
|
|
|
// nextRun is the first fire time strictly after now, unset when the timing
|
|
// never fires again.
|
|
func nextRun(t scheduling.Timing, now time.Time, loc *time.Location) (pgtype.Timestamptz, error) {
|
|
next, ok, err := t.Next(now, loc)
|
|
if err != nil || !ok {
|
|
return pgtype.Timestamptz{}, err
|
|
}
|
|
return optionalTime(next), nil
|
|
}
|
|
|
|
func (s *Store) ListSchedules(ctx context.Context) ([]scheduling.Schedule, error) {
|
|
q := s.queries()
|
|
rows, err := q.ListSchedules(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("postgres: list schedules: %w", err)
|
|
}
|
|
latest, err := q.LatestScheduleRuns(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("postgres: latest schedule runs: %w", err)
|
|
}
|
|
last := make(map[string]scheduling.Run, len(latest))
|
|
for _, r := range latest {
|
|
last[r.ScheduleID] = runFromRow(r)
|
|
}
|
|
out := make([]scheduling.Schedule, 0, len(rows))
|
|
for _, r := range rows {
|
|
schedule := scheduleFromRow(r)
|
|
if run, ok := last[r.ID]; ok {
|
|
schedule.LastRun = &run
|
|
}
|
|
out = append(out, schedule)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (s *Store) GetSchedule(ctx context.Context, id string) (scheduling.Schedule, error) {
|
|
row, err := s.queries().GetSchedule(ctx, id)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return scheduling.Schedule{}, scheduling.ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return scheduling.Schedule{}, fmt.Errorf("postgres: get schedule: %w", err)
|
|
}
|
|
return scheduleFromRow(row), nil
|
|
}
|
|
|
|
// PutSchedule stores in. A built-in schedule takes only Enabled and Timing
|
|
// from in. A timing with no start is anchored at the time it is stored.
|
|
func (s *Store) PutSchedule(ctx context.Context, in scheduling.Schedule, loc *time.Location) (scheduling.Schedule, error) {
|
|
var stored scheduling.Schedule
|
|
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
|
q := postgresdb.New(tx)
|
|
existing, err := q.LockSchedule(ctx, in.ID)
|
|
switch {
|
|
case errors.Is(err, pgx.ErrNoRows):
|
|
if in.BuiltIn {
|
|
return fmt.Errorf("%w: built-in schedules are defined in code", scheduling.ErrInvalidSpec)
|
|
}
|
|
case err != nil:
|
|
return err
|
|
case existing.BuiltIn:
|
|
current := scheduleFromRow(existing)
|
|
current.Enabled, current.Timing = in.Enabled, in.Timing
|
|
in = current
|
|
default:
|
|
in.BuiltIn = false
|
|
}
|
|
if err := in.Validate(); err != nil {
|
|
return err
|
|
}
|
|
now := time.Now().UTC()
|
|
if in.Timing.Start.IsZero() {
|
|
in.Timing.Start = now
|
|
}
|
|
next, err := nextRun(in.Timing, now, loc)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
row, err := q.UpsertSchedule(ctx, postgresdb.UpsertScheduleParams{
|
|
ID: in.ID, OwnerKind: string(in.Owner.Kind), OwnerID: in.Owner.ID, Name: in.Name,
|
|
Enabled: in.Enabled, BuiltIn: in.BuiltIn, TimingKind: in.Timing.Kind,
|
|
IntervalSeconds: int64(in.Timing.Interval.Std() / time.Second), Cron: in.Timing.Cron,
|
|
RunAt: timingAt(in.Timing), Anchor: in.Timing.Start, NextRun: next, Zone: loc.String(),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
stored = scheduleFromRow(row)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return scheduling.Schedule{}, fmt.Errorf("postgres: put schedule: %w", err)
|
|
}
|
|
return stored, nil
|
|
}
|
|
|
|
func timingAt(t scheduling.Timing) pgtype.Timestamptz {
|
|
if t.At == nil {
|
|
return pgtype.Timestamptz{}
|
|
}
|
|
return optionalTime(*t.At)
|
|
}
|
|
|
|
func (s *Store) DeleteSchedule(ctx context.Context, id string) error {
|
|
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
|
q := postgresdb.New(tx)
|
|
existing, err := q.LockSchedule(ctx, id)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return scheduling.ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("postgres: delete schedule: %w", err)
|
|
}
|
|
if existing.BuiltIn {
|
|
return scheduling.ErrBuiltIn
|
|
}
|
|
_, err = q.DeleteSchedule(ctx, id)
|
|
return err
|
|
})
|
|
}
|
|
|
|
func (s *Store) EnsureBuiltins(ctx context.Context, builtins []scheduling.Builtin, loc *time.Location) error {
|
|
now := time.Now().UTC()
|
|
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
|
q := postgresdb.New(tx)
|
|
for _, b := range builtins {
|
|
timing := scheduling.Timing{Kind: scheduling.TimingInterval, Interval: scheduling.Duration(b.Interval), Start: now}
|
|
if err := (scheduling.Schedule{ID: b.ID, Name: b.Name, Owner: scheduling.Owner{Kind: scheduling.OwnerInstance}, Timing: timing}).Validate(); err != nil {
|
|
return err
|
|
}
|
|
next, err := nextRun(timing, now, loc)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := q.InsertBuiltinSchedule(ctx, postgresdb.InsertBuiltinScheduleParams{
|
|
ID: b.ID, Name: b.Name, IntervalSeconds: int64(b.Interval / time.Second),
|
|
Anchor: now, NextRun: next, Zone: loc.String(),
|
|
}); err != nil {
|
|
return fmt.Errorf("postgres: seed built-in schedule %q: %w", b.ID, err)
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// ClaimDue runs in one transaction: stale runs fail, requested runs start, a
|
|
// cron schedule computed in another zone is recomputed, and each due schedule
|
|
// gets one run for its slot and a next run past now. However many slots were
|
|
// missed, the schedule fires once. Due rows are locked SKIP LOCKED and the run
|
|
// is unique per slot, so a second scheduler claims nothing twice.
|
|
func (s *Store) ClaimDue(ctx context.Context, now time.Time, stale time.Duration, loc *time.Location) ([]scheduling.Claim, error) {
|
|
var claims []scheduling.Claim
|
|
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
|
q := postgresdb.New(tx)
|
|
at := optionalTime(now)
|
|
if err := q.FailStaleRuns(ctx, postgresdb.FailStaleRunsParams{FinishedAt: at, StartedAt: optionalTime(now.Add(-stale))}); err != nil {
|
|
return err
|
|
}
|
|
requested, err := q.StartRequestedRuns(ctx, at)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, r := range requested {
|
|
claims = append(claims, scheduling.Claim{
|
|
ID: r.ID, ScheduleID: r.ScheduleID, ScheduledFor: r.ScheduledFor, Started: now, Result: scheduling.RunRunning,
|
|
Name: r.Name, BuiltIn: r.BuiltIn,
|
|
})
|
|
}
|
|
rezoned, err := q.LockStaleZoneCronSchedules(ctx, loc.String())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, row := range rezoned {
|
|
if err := advance(ctx, q, scheduleFromRow(row), now, loc); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
due, err := q.LockDueSchedules(ctx, at)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, row := range due {
|
|
schedule := scheduleFromRow(row)
|
|
run, err := q.InsertScheduleRun(ctx, postgresdb.InsertScheduleRunParams{
|
|
ScheduleID: schedule.ID, ScheduledFor: schedule.NextRun, StartedAt: at, Result: string(scheduling.RunRunning),
|
|
})
|
|
switch {
|
|
case errors.Is(err, pgx.ErrNoRows):
|
|
// The slot already has a run; only the next run moves.
|
|
case err != nil:
|
|
return err
|
|
default:
|
|
claims = append(claims, scheduling.Claim{Run: runFromRow(run), Name: schedule.Name, BuiltIn: schedule.BuiltIn})
|
|
}
|
|
if err := advance(ctx, q, schedule, now, loc); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("postgres: claim due schedules: %w", err)
|
|
}
|
|
return claims, nil
|
|
}
|
|
|
|
// advance moves a schedule's next run to its first slot after now.
|
|
func advance(ctx context.Context, q *postgresdb.Queries, schedule scheduling.Schedule, now time.Time, loc *time.Location) error {
|
|
next, err := nextRun(schedule.Timing, now, loc)
|
|
if err != nil {
|
|
return fmt.Errorf("schedule %q: %w", schedule.ID, err)
|
|
}
|
|
return q.SetScheduleNextRun(ctx, postgresdb.SetScheduleNextRunParams{ID: schedule.ID, NextRun: next, Zone: loc.String()})
|
|
}
|
|
|
|
func (s *Store) RequestRun(ctx context.Context, id string, now time.Time) (scheduling.Run, error) {
|
|
run, err := s.queries().InsertScheduleRun(ctx, postgresdb.InsertScheduleRunParams{
|
|
ScheduleID: id, ScheduledFor: now.UTC(), Result: string(scheduling.RunPending),
|
|
})
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return scheduling.Run{}, fmt.Errorf("%w: a run of %q was already requested at %s", scheduling.ErrInvalidSpec, id, now.UTC().Format(time.RFC3339Nano))
|
|
}
|
|
if isForeignKeyViolation(err) {
|
|
return scheduling.Run{}, scheduling.ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return scheduling.Run{}, fmt.Errorf("postgres: request schedule run: %w", err)
|
|
}
|
|
return runFromRow(run), nil
|
|
}
|
|
|
|
// FinishRun records a running run's result and counts the schedule's latest
|
|
// consecutive failures. A run that is no longer running is ErrNotFound.
|
|
func (s *Store) FinishRun(ctx context.Context, runID int64, result scheduling.RunResult, errText string, at time.Time) (int, error) {
|
|
q := s.queries()
|
|
scheduleID, err := q.FinishScheduleRun(ctx, postgresdb.FinishScheduleRunParams{
|
|
ID: runID, Result: string(result), Error: errText, FinishedAt: optionalTime(at),
|
|
})
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return 0, scheduling.ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return 0, fmt.Errorf("postgres: finish schedule run: %w", err)
|
|
}
|
|
results, err := q.RecentFinishedResults(ctx, postgresdb.RecentFinishedResultsParams{ScheduleID: scheduleID, Limit: failureStreakWindow})
|
|
if err != nil {
|
|
return 0, fmt.Errorf("postgres: schedule failure streak: %w", err)
|
|
}
|
|
streak := 0
|
|
for _, r := range results {
|
|
if r != string(scheduling.RunFailed) {
|
|
break
|
|
}
|
|
streak++
|
|
}
|
|
return streak, nil
|
|
}
|
|
|
|
func (s *Store) SetRunCapture(ctx context.Context, runID int64, captureID string) error {
|
|
n, err := s.queries().SetScheduleRunCapture(ctx, postgresdb.SetScheduleRunCaptureParams{ID: runID, CaptureID: captureID})
|
|
if err != nil {
|
|
return fmt.Errorf("postgres: set schedule run capture: %w", err)
|
|
}
|
|
if n == 0 {
|
|
return scheduling.ErrNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) ListRuns(ctx context.Context, scheduleID string, limit int) ([]scheduling.Run, error) {
|
|
rows, err := s.queries().ListScheduleRuns(ctx, postgresdb.ListScheduleRunsParams{ScheduleID: scheduleID, Limit: int32(min(max(limit, 1), 500))})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("postgres: list schedule runs: %w", err)
|
|
}
|
|
out := make([]scheduling.Run, 0, len(rows))
|
|
for _, r := range rows {
|
|
out = append(out, runFromRow(r))
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (s *Store) PruneRuns(ctx context.Context, before time.Time, keep int) (int64, error) {
|
|
n, err := s.queries().PruneScheduleRuns(ctx, postgresdb.PruneScheduleRunsParams{Before: before, Keep: int64(keep)})
|
|
if err != nil {
|
|
return 0, fmt.Errorf("postgres: prune schedule runs: %w", err)
|
|
}
|
|
return n, nil
|
|
}
|