// Schedules: the Postgres implementation of internal/domain/scheduling.Store. package postgres import ( "context" "errors" "fmt" "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/samcharles93/archie-core/internal/domain/scheduling" "github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb" ) var ( _ scheduling.Store = (*Store)(nil) _ scheduling.RunStore = (*Store)(nil) ) // failureStreakWindow bounds how many finished runs FinishRun reads to count // a failure streak. const failureStreakWindow = 20 func optionalTime(t time.Time) pgtype.Timestamptz { return pgtype.Timestamptz{Time: t, Valid: !t.IsZero()} } func scheduleFromRow(r postgresdb.Schedule) scheduling.Schedule { s := scheduling.Schedule{ ID: r.ID, Name: r.Name, Owner: scheduling.Owner{Kind: scheduling.OwnerKind(r.OwnerKind), ID: r.OwnerID}, Enabled: r.Enabled, BuiltIn: r.BuiltIn, Timing: scheduling.Timing{ Kind: r.TimingKind, Interval: scheduling.Duration(time.Duration(r.IntervalSeconds) * time.Second), Cron: r.Cron, Start: r.Anchor.UTC(), }, Created: r.CreatedAt, Updated: r.UpdatedAt, } if r.RunAt.Valid { at := r.RunAt.Time.UTC() s.Timing.At = &at } if r.NextRun.Valid { s.NextRun = r.NextRun.Time.UTC() } return s } func runFromRow(r postgresdb.ScheduleRun) scheduling.Run { return scheduling.Run{ ID: r.ID, ScheduleID: r.ScheduleID, ScheduledFor: r.ScheduledFor.UTC(), Started: r.StartedAt.Time, Finished: r.FinishedAt.Time, Result: scheduling.RunResult(r.Result), Error: r.Error, CaptureID: r.CaptureID, } } // nextRun is the first fire time strictly after now, unset when the timing // never fires again. func nextRun(t scheduling.Timing, now time.Time, loc *time.Location) (pgtype.Timestamptz, error) { next, ok, err := t.Next(now, loc) if err != nil || !ok { return pgtype.Timestamptz{}, err } return optionalTime(next), nil } func (s *Store) ListSchedules(ctx context.Context) ([]scheduling.Schedule, error) { q := s.queries() rows, err := q.ListSchedules(ctx) if err != nil { return nil, fmt.Errorf("postgres: list schedules: %w", err) } latest, err := q.LatestScheduleRuns(ctx) if err != nil { return nil, fmt.Errorf("postgres: latest schedule runs: %w", err) } last := make(map[string]scheduling.Run, len(latest)) for _, r := range latest { last[r.ScheduleID] = runFromRow(r) } out := make([]scheduling.Schedule, 0, len(rows)) for _, r := range rows { schedule := scheduleFromRow(r) if run, ok := last[r.ID]; ok { schedule.LastRun = &run } out = append(out, schedule) } return out, nil } func (s *Store) GetSchedule(ctx context.Context, id string) (scheduling.Schedule, error) { row, err := s.queries().GetSchedule(ctx, id) if errors.Is(err, pgx.ErrNoRows) { return scheduling.Schedule{}, scheduling.ErrNotFound } if err != nil { return scheduling.Schedule{}, fmt.Errorf("postgres: get schedule: %w", err) } return scheduleFromRow(row), nil } // PutSchedule stores in. A built-in schedule takes only Enabled and Timing // from in. A timing with no start is anchored at the time it is stored. func (s *Store) PutSchedule(ctx context.Context, in scheduling.Schedule, loc *time.Location) (scheduling.Schedule, error) { var stored scheduling.Schedule err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { q := postgresdb.New(tx) existing, err := q.LockSchedule(ctx, in.ID) switch { case errors.Is(err, pgx.ErrNoRows): if in.BuiltIn { return fmt.Errorf("%w: built-in schedules are defined in code", scheduling.ErrInvalidSpec) } case err != nil: return err case existing.BuiltIn: current := scheduleFromRow(existing) current.Enabled, current.Timing = in.Enabled, in.Timing in = current default: in.BuiltIn = false } if err := in.Validate(); err != nil { return err } now := time.Now().UTC() if in.Timing.Start.IsZero() { in.Timing.Start = now } next, err := nextRun(in.Timing, now, loc) if err != nil { return err } row, err := q.UpsertSchedule(ctx, postgresdb.UpsertScheduleParams{ ID: in.ID, OwnerKind: string(in.Owner.Kind), OwnerID: in.Owner.ID, Name: in.Name, Enabled: in.Enabled, BuiltIn: in.BuiltIn, TimingKind: in.Timing.Kind, IntervalSeconds: int64(in.Timing.Interval.Std() / time.Second), Cron: in.Timing.Cron, RunAt: timingAt(in.Timing), Anchor: in.Timing.Start, NextRun: next, Zone: loc.String(), }) if err != nil { return err } stored = scheduleFromRow(row) return nil }) if err != nil { return scheduling.Schedule{}, fmt.Errorf("postgres: put schedule: %w", err) } return stored, nil } func timingAt(t scheduling.Timing) pgtype.Timestamptz { if t.At == nil { return pgtype.Timestamptz{} } return optionalTime(*t.At) } func (s *Store) DeleteSchedule(ctx context.Context, id string) error { return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { q := postgresdb.New(tx) existing, err := q.LockSchedule(ctx, id) if errors.Is(err, pgx.ErrNoRows) { return scheduling.ErrNotFound } if err != nil { return fmt.Errorf("postgres: delete schedule: %w", err) } if existing.BuiltIn { return scheduling.ErrBuiltIn } _, err = q.DeleteSchedule(ctx, id) return err }) } func (s *Store) EnsureBuiltins(ctx context.Context, builtins []scheduling.Builtin, loc *time.Location) error { now := time.Now().UTC() return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { q := postgresdb.New(tx) for _, b := range builtins { timing := scheduling.Timing{Kind: scheduling.TimingInterval, Interval: scheduling.Duration(b.Interval), Start: now} if err := (scheduling.Schedule{ID: b.ID, Name: b.Name, Owner: scheduling.Owner{Kind: scheduling.OwnerInstance}, Timing: timing}).Validate(); err != nil { return err } next, err := nextRun(timing, now, loc) if err != nil { return err } if err := q.InsertBuiltinSchedule(ctx, postgresdb.InsertBuiltinScheduleParams{ ID: b.ID, Name: b.Name, IntervalSeconds: int64(b.Interval / time.Second), Anchor: now, NextRun: next, Zone: loc.String(), }); err != nil { return fmt.Errorf("postgres: seed built-in schedule %q: %w", b.ID, err) } } return nil }) } // ClaimDue runs in one transaction: stale runs fail, requested runs start, a // cron schedule computed in another zone is recomputed, and each due schedule // gets one run for its slot and a next run past now. However many slots were // missed, the schedule fires once. Due rows are locked SKIP LOCKED and the run // is unique per slot, so a second scheduler claims nothing twice. func (s *Store) ClaimDue(ctx context.Context, now time.Time, stale time.Duration, loc *time.Location) ([]scheduling.Claim, error) { var claims []scheduling.Claim err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { q := postgresdb.New(tx) at := optionalTime(now) if err := q.FailStaleRuns(ctx, postgresdb.FailStaleRunsParams{FinishedAt: at, StartedAt: optionalTime(now.Add(-stale))}); err != nil { return err } requested, err := q.StartRequestedRuns(ctx, at) if err != nil { return err } for _, r := range requested { claims = append(claims, scheduling.Claim{ ID: r.ID, ScheduleID: r.ScheduleID, ScheduledFor: r.ScheduledFor, Started: now, Result: scheduling.RunRunning, Name: r.Name, BuiltIn: r.BuiltIn, }) } rezoned, err := q.LockStaleZoneCronSchedules(ctx, loc.String()) if err != nil { return err } for _, row := range rezoned { if err := advance(ctx, q, scheduleFromRow(row), now, loc); err != nil { return err } } due, err := q.LockDueSchedules(ctx, at) if err != nil { return err } for _, row := range due { schedule := scheduleFromRow(row) run, err := q.InsertScheduleRun(ctx, postgresdb.InsertScheduleRunParams{ ScheduleID: schedule.ID, ScheduledFor: schedule.NextRun, StartedAt: at, Result: string(scheduling.RunRunning), }) switch { case errors.Is(err, pgx.ErrNoRows): // The slot already has a run; only the next run moves. case err != nil: return err default: claims = append(claims, scheduling.Claim{Run: runFromRow(run), Name: schedule.Name, BuiltIn: schedule.BuiltIn}) } if err := advance(ctx, q, schedule, now, loc); err != nil { return err } } return nil }) if err != nil { return nil, fmt.Errorf("postgres: claim due schedules: %w", err) } return claims, nil } // advance moves a schedule's next run to its first slot after now. func advance(ctx context.Context, q *postgresdb.Queries, schedule scheduling.Schedule, now time.Time, loc *time.Location) error { next, err := nextRun(schedule.Timing, now, loc) if err != nil { return fmt.Errorf("schedule %q: %w", schedule.ID, err) } return q.SetScheduleNextRun(ctx, postgresdb.SetScheduleNextRunParams{ID: schedule.ID, NextRun: next, Zone: loc.String()}) } func (s *Store) RequestRun(ctx context.Context, id string, now time.Time) (scheduling.Run, error) { run, err := s.queries().InsertScheduleRun(ctx, postgresdb.InsertScheduleRunParams{ ScheduleID: id, ScheduledFor: now.UTC(), Result: string(scheduling.RunPending), }) if errors.Is(err, pgx.ErrNoRows) { return scheduling.Run{}, fmt.Errorf("%w: a run of %q was already requested at %s", scheduling.ErrInvalidSpec, id, now.UTC().Format(time.RFC3339Nano)) } if isForeignKeyViolation(err) { return scheduling.Run{}, scheduling.ErrNotFound } if err != nil { return scheduling.Run{}, fmt.Errorf("postgres: request schedule run: %w", err) } return runFromRow(run), nil } // FinishRun records a running run's result and counts the schedule's latest // consecutive failures. A run that is no longer running is ErrNotFound. func (s *Store) FinishRun(ctx context.Context, runID int64, result scheduling.RunResult, errText string, at time.Time) (int, error) { q := s.queries() scheduleID, err := q.FinishScheduleRun(ctx, postgresdb.FinishScheduleRunParams{ ID: runID, Result: string(result), Error: errText, FinishedAt: optionalTime(at), }) if errors.Is(err, pgx.ErrNoRows) { return 0, scheduling.ErrNotFound } if err != nil { return 0, fmt.Errorf("postgres: finish schedule run: %w", err) } results, err := q.RecentFinishedResults(ctx, postgresdb.RecentFinishedResultsParams{ScheduleID: scheduleID, Limit: failureStreakWindow}) if err != nil { return 0, fmt.Errorf("postgres: schedule failure streak: %w", err) } streak := 0 for _, r := range results { if r != string(scheduling.RunFailed) { break } streak++ } return streak, nil } func (s *Store) SetRunCapture(ctx context.Context, runID int64, captureID string) error { n, err := s.queries().SetScheduleRunCapture(ctx, postgresdb.SetScheduleRunCaptureParams{ID: runID, CaptureID: captureID}) if err != nil { return fmt.Errorf("postgres: set schedule run capture: %w", err) } if n == 0 { return scheduling.ErrNotFound } return nil } func (s *Store) ListRuns(ctx context.Context, scheduleID string, limit int) ([]scheduling.Run, error) { rows, err := s.queries().ListScheduleRuns(ctx, postgresdb.ListScheduleRunsParams{ScheduleID: scheduleID, Limit: int32(min(max(limit, 1), 500))}) if err != nil { return nil, fmt.Errorf("postgres: list schedule runs: %w", err) } out := make([]scheduling.Run, 0, len(rows)) for _, r := range rows { out = append(out, runFromRow(r)) } return out, nil } func (s *Store) PruneRuns(ctx context.Context, before time.Time, keep int) (int64, error) { n, err := s.queries().PruneScheduleRuns(ctx, postgresdb.PruneScheduleRunsParams{Before: before, Keep: int64(keep)}) if err != nil { return 0, fmt.Errorf("postgres: prune schedule runs: %w", err) } return n, nil }