Files
archie/modules/runs/internal/store/store.go
T

552 lines
20 KiB
Go

// Package store holds the runs module's SQL. Every write a worker makes is
// fenced by its lease token, so a worker that lost its lease writes nothing.
package store
import (
"context"
"database/sql"
"encoding/json/jsontext"
"errors"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
)
// Run states.
const (
StateQueued = "queued"
StateRunning = "running"
StateWaiting = "waiting"
StateSucceeded = "succeeded"
StateFailed = "failed"
StateCancelled = "cancelled"
// StateParked is a run stopped for an operator, by a failed step or a
// workflow.finish step. It is retried from a step.
StateParked = "parked"
// StateWaitingApproval and StateDeclined are the other endings a
// workflow.finish step chooses.
StateWaitingApproval = "waiting_approval"
StateDeclined = "declined"
)
// MarkerBase is where a run's markers are numbered, far above any step a
// workflow takes, so a marker never shifts the position a replay depends on.
const MarkerBase = 1 << 20
// Step outcomes.
const (
OutcomeDone = "done"
OutcomeFailed = "failed"
OutcomeUnknown = "unknown"
)
var (
// ErrLeaseLost means the writer no longer holds the run's lease.
ErrLeaseLost = errors.New("lease lost")
// ErrStepExists means the step was already recorded, by this run's replay
// or by a previous lease holder.
ErrStepExists = errors.New("step already recorded")
// ErrNoRun means the organisation has no such run to retry.
ErrNoRun = errors.New("no such run")
// ErrNotParked means the run is not parked, so there is nothing to retry.
ErrNotParked = errors.New("run is not parked")
// ErrNoStep means the run never reached the step it was to retry from.
ErrNoStep = errors.New("run did not reach that step")
)
// Run is a claimed run.
type Run struct {
ID uuid.UUID `db:"id"`
OrgID string `db:"org_id"`
Workflow string `db:"workflow"`
// Version is the workflow version the run started under.
Version int `db:"workflow_version"`
Input jsontext.Value `db:"input"`
Attempts int `db:"attempts"`
// Dry marks a dry run, and Overlay is the document and policy it runs
// under; a dry run pins no stored version.
Dry bool `db:"dry"`
// Scan into []byte because live runs have a SQL NULL overlay.
Overlay []byte `db:"overlay"`
// CallDepth is how many workflow.call steps lie between this run and the
// run that started the chain.
CallDepth int `db:"call_depth"`
}
// Step is one recorded step of a run.
type Step struct {
RunID uuid.UUID `db:"run_id"`
Seq int `db:"seq"`
Name string `db:"name"`
Outcome string `db:"outcome"`
Output jsontext.Value `db:"output"`
Error sql.NullString `db:"error"`
}
// Store is the runs module's Postgres store.
type Store struct {
db *sqlx.DB
}
// New returns a store on db.
func New(db *sqlx.DB) *Store {
return &Store{db: db}
}
// NewRun is a run to queue.
type NewRun struct {
OrgID string
Workflow string
Version int
StartKey string
Input jsontext.Value
Dry bool
Overlay jsontext.Value
// CallDepth is the depth of a run started by a call.
CallDepth int
// ParentSignal is the signal key to send the run that called this one
// when this run ends; empty for a run nothing waits on.
ParentSignal string
}
// Start inserts a queued run. A run with the same organisation and start key
// is not inserted again; its id is returned instead.
func (s *Store) Start(ctx context.Context, n NewRun) (uuid.UUID, error) {
key := sql.NullString{String: n.StartKey, Valid: n.StartKey != ""}
var id uuid.UUID
err := s.db.GetContext(ctx, &id, `
INSERT INTO runs.runs (id, org_id, workflow, workflow_version, start_key, input, state, dry, overlay)
VALUES ($1, $2, $3, $4, $5, $6, 'queued', $7, $8)
ON CONFLICT (org_id, start_key) DO NOTHING
RETURNING id`, uuid.New(), n.OrgID, n.Workflow, n.Version, key, n.Input, n.Dry, nullJSON(n.Overlay))
if errors.Is(err, sql.ErrNoRows) && key.Valid {
err = s.db.GetContext(ctx, &id, `SELECT id FROM runs.runs WHERE org_id = $1 AND start_key = $2`, n.OrgID, n.StartKey)
}
if err != nil {
return uuid.Nil, fmt.Errorf("start run: %w", err)
}
return id, nil
}
// StartFor inserts a queued run on behalf of a run whose lease the token
// holds. The start key makes it idempotent, so a replayed step or a restarted
// worker gets the run already started. A non-empty ParentSignal links the new
// run to the parent: its ending signals the parent under that key.
func (s *Store) StartFor(ctx context.Context, token, parent uuid.UUID, n NewRun) (uuid.UUID, error) {
if n.StartKey == "" {
return uuid.Nil, errors.New("start run: a run started for another needs a start key")
}
var link uuid.NullUUID
if n.ParentSignal != "" {
link = uuid.NullUUID{UUID: parent, Valid: true}
}
var id uuid.UUID
err := s.db.GetContext(ctx, &id, `
INSERT INTO runs.runs (id, org_id, workflow, workflow_version, start_key, input, state, parent_run_id, parent_signal, call_depth)
SELECT $1, $2, $3, $4, $5, $6, 'queued', $7, NULLIF($8, ''), $9
WHERE EXISTS (SELECT 1 FROM runs.runs WHERE id = $10 AND lease_token = $11)
ON CONFLICT (org_id, start_key) DO NOTHING
RETURNING id`, uuid.New(), n.OrgID, n.Workflow, n.Version, n.StartKey, n.Input, link, n.ParentSignal, n.CallDepth, parent, token)
if errors.Is(err, sql.ErrNoRows) {
err = s.db.GetContext(ctx, &id, `SELECT id FROM runs.runs WHERE org_id = $1 AND start_key = $2`, n.OrgID, n.StartKey)
if errors.Is(err, sql.ErrNoRows) {
return uuid.Nil, ErrLeaseLost
}
}
if err != nil {
return uuid.Nil, fmt.Errorf("start run: %w", err)
}
return id, nil
}
// Claim takes the oldest run that is queued, due to wake, or whose lease has
// expired. SKIP LOCKED only keeps claimers from waiting on each other;
// correctness rests on the lease token every later write is fenced by.
func (s *Store) Claim(ctx context.Context, owner string, token uuid.UUID, ttl time.Duration) (Run, bool, error) {
var run Run
err := s.db.GetContext(ctx, &run, `
UPDATE runs.runs r
SET state = 'running', lease_token = $1, lease_owner = $2,
lease_expires_at = now() + make_interval(secs => $3),
attempts = attempts + 1, updated_at = now()
WHERE r.id = (
SELECT id FROM runs.runs
WHERE state = 'queued'
OR (state = 'waiting' AND wake_at <= now())
OR (state = 'running' AND lease_expires_at < now())
ORDER BY created_at
FOR UPDATE SKIP LOCKED
LIMIT 1)
RETURNING r.id, r.org_id, r.workflow, r.workflow_version, r.input, r.attempts, r.dry, r.overlay, r.call_depth`, token, owner, ttl.Seconds())
if errors.Is(err, sql.ErrNoRows) {
return Run{}, false, nil
}
if err != nil {
return Run{}, false, fmt.Errorf("claim run: %w", err)
}
return run, true, nil
}
// RenewLease extends the lease while the token still holds it.
func (s *Store) RenewLease(ctx context.Context, id, token uuid.UUID, ttl time.Duration) (bool, error) {
res, err := s.db.ExecContext(ctx, `
UPDATE runs.runs SET lease_expires_at = now() + make_interval(secs => $3), updated_at = now()
WHERE id = $1 AND lease_token = $2`, id, token, ttl.Seconds())
if err != nil {
return false, fmt.Errorf("renew lease: %w", err)
}
return affected(res) == 1, nil
}
// Steps returns a run's recorded steps in order.
func (s *Store) Steps(ctx context.Context, id uuid.UUID) ([]Step, error) {
var steps []Step
if err := s.db.SelectContext(ctx, &steps, `
SELECT run_id, seq, name, outcome, coalesce(output, 'null') AS output, error FROM runs.steps
WHERE run_id = $1 ORDER BY seq`, id); err != nil {
return nil, fmt.Errorf("load steps: %w", err)
}
return steps, nil
}
// RecordStep writes a step if the token holds the lease and the step is new.
func (s *Store) RecordStep(ctx context.Context, token uuid.UUID, step Step) error {
res, err := s.db.ExecContext(ctx, `
INSERT INTO runs.steps (run_id, seq, name, outcome, output, error)
SELECT $1, $2, $3, $4, $5, $6
WHERE EXISTS (SELECT 1 FROM runs.runs WHERE id = $1 AND lease_token = $7)
ON CONFLICT (run_id, seq) DO NOTHING`,
step.RunID, step.Seq, step.Name, step.Outcome, nullJSON(step.Output), step.Error, token)
if err != nil {
return fmt.Errorf("record step: %w", err)
}
if affected(res) == 1 {
return nil
}
return s.whyNotWritten(ctx, step.RunID, token)
}
// Suspend releases the lease and parks the run until wakeAt or a signal for
// waitingFor. A signal that arrived before the park queues the run at once.
func (s *Store) Suspend(ctx context.Context, id, token uuid.UUID, wakeAt *time.Time, waitingFor string) error {
res, err := s.db.ExecContext(ctx, `
UPDATE runs.runs r
SET state = CASE WHEN $4 <> '' AND EXISTS (
SELECT 1 FROM runs.signals s WHERE s.run_id = r.id AND s.key = $4)
THEN 'queued' ELSE 'waiting' END,
wake_at = $3, waiting_for = NULLIF($4, ''),
lease_token = NULL, lease_owner = NULL, lease_expires_at = NULL, updated_at = now()
WHERE id = $1 AND lease_token = $2`, id, token, wakeAt, waitingFor)
if err != nil {
return fmt.Errorf("suspend run: %w", err)
}
if affected(res) == 0 {
return ErrLeaseLost
}
return nil
}
// Ended describes how a run ended. Error is the failure text, and Detail is
// the ending's explanation, such as a workflow.finish step's detail.
type Ended struct {
State string
Result jsontext.Value
Error string
Detail string
}
// Finish records how a run ended while the token holds the lease, and tells
// the run waiting on it, if any, in the same transaction so that a crash
// cannot leave the parent waiting for a run that has ended.
func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, e Ended) error {
tx, err := s.db.BeginTxx(ctx, nil)
if err != nil {
return fmt.Errorf("finish run: %w", err)
}
defer func() { _ = tx.Rollback() }()
res, err := tx.ExecContext(ctx, `
UPDATE runs.runs
SET state = $3, result = $4, error = NULLIF($5, ''), detail = NULLIF($6, ''),
finished_at = now(), updated_at = now(),
lease_token = NULL, lease_owner = NULL, lease_expires_at = NULL
WHERE id = $1 AND lease_token = $2`, id, token, e.State, nullJSON(e.Result), e.Error, e.Detail)
if err != nil {
return fmt.Errorf("finish run: %w", err)
}
if affected(res) == 0 {
return ErrLeaseLost
}
if err := signalParent(ctx, tx, id); err != nil {
return err
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("finish run: %w", err)
}
return nil
}
type execer interface {
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
}
// signalParent tells the run that called this one how it ended, and wakes it
// if it is waiting. Only an ending an operator or the caller must act on
// counts: a run waiting for approval has not ended, and its caller keeps
// waiting. The first ending wins, so a later retry of a parked child does not
// change what the parent was told.
func signalParent(ctx context.Context, q execer, id uuid.UUID) error {
if _, err := q.ExecContext(ctx, `
WITH ended AS (
SELECT parent_run_id, parent_signal,
jsonb_build_object('state', state, 'detail', coalesce(detail, ''),
'error', coalesce(error, ''), 'result', coalesce(result, 'null'::jsonb)) AS payload
FROM runs.runs
WHERE id = $1 AND parent_signal IS NOT NULL
AND state IN ('succeeded', 'failed', 'cancelled', 'parked', 'declined')),
sent AS (
INSERT INTO runs.signals (run_id, key, payload)
SELECT parent_run_id, parent_signal, payload FROM ended
ON CONFLICT (run_id, key) DO NOTHING)
UPDATE runs.runs r SET state = 'queued', wake_at = NULL, updated_at = now()
FROM ended e
WHERE r.id = e.parent_run_id AND r.state = 'waiting' AND r.waiting_for = e.parent_signal`, id); err != nil {
return fmt.Errorf("signal parent run: %w", err)
}
return nil
}
// Retry queues a parked run to run again from a step, keeping what the steps
// before it recorded: the run replays them without repeating their effects.
// from names the step; empty means the step the run last reached. What that
// step and every later one recorded is forgotten, including the markers that
// stop an effect starting twice, so the step runs afresh.
func (s *Store) Retry(ctx context.Context, orgID string, id uuid.UUID, from string) error {
tx, err := s.db.BeginTxx(ctx, nil)
if err != nil {
return fmt.Errorf("retry run: %w", err)
}
defer func() { _ = tx.Rollback() }()
var state string
err = tx.GetContext(ctx, &state, `SELECT state FROM runs.runs WHERE org_id = $1 AND id = $2 AND NOT dry FOR UPDATE`, orgID, id)
switch {
case errors.Is(err, sql.ErrNoRows):
return ErrNoRun
case err != nil:
return fmt.Errorf("retry run: %w", err)
case state != StateParked:
return ErrNotParked
}
var at struct {
Marker int `db:"marker"`
FirstSeq int `db:"first_seq"`
}
err = tx.GetContext(ctx, &at, `
SELECT seq AS marker, (output #>> '{}')::int AS first_seq FROM runs.steps
WHERE run_id = $1 AND name LIKE 'at:%' AND ($2 = '' OR name = 'at:' || $2)
ORDER BY seq DESC LIMIT 1`, id, from)
if errors.Is(err, sql.ErrNoRows) {
return ErrNoStep
}
if err != nil {
return fmt.Errorf("find step: %w", err)
}
// A call being retried starts a fresh child: the old one is detached, so
// its ending no longer reaches this run, and the signal it already sent is
// forgotten.
if _, err := tx.ExecContext(ctx, `
WITH retried AS (
SELECT 'call:' || substr(name, length('call-start:') + 1) AS signal FROM runs.steps
WHERE run_id = $1 AND name LIKE 'call-start:%' AND ((seq >= $2 AND seq < $3) OR seq >= $4)),
gone AS (
DELETE FROM runs.signals WHERE run_id = $1 AND key IN (SELECT signal FROM retried))
UPDATE runs.runs SET parent_run_id = NULL, parent_signal = NULL, start_key = NULL
WHERE parent_run_id = $1 AND parent_signal IN (SELECT signal FROM retried)`,
id, at.FirstSeq, MarkerBase, at.Marker); err != nil {
return fmt.Errorf("detach calls: %w", err)
}
if _, err := tx.ExecContext(ctx, `
DELETE FROM runs.steps WHERE run_id = $1 AND ((seq >= $2 AND seq < $3) OR seq >= $4)`,
id, at.FirstSeq, MarkerBase, at.Marker); err != nil {
return fmt.Errorf("forget steps: %w", err)
}
if _, err := tx.ExecContext(ctx, `
UPDATE runs.runs
SET state = 'queued', result = NULL, error = NULL, detail = NULL, finished_at = NULL,
wake_at = NULL, waiting_for = NULL, updated_at = now()
WHERE id = $1`, id); err != nil {
return fmt.Errorf("queue run: %w", err)
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("retry run: %w", err)
}
return nil
}
// Signal stores a signal for a run and queues the run if it is waiting for it.
func (s *Store) Signal(ctx context.Context, id uuid.UUID, key string, payload jsontext.Value) error {
tx, err := s.db.BeginTxx(ctx, nil)
if err != nil {
return fmt.Errorf("signal run: %w", err)
}
defer func() { _ = tx.Rollback() }()
if _, err := tx.ExecContext(ctx, `
INSERT INTO runs.signals (run_id, key, payload) VALUES ($1, $2, $3)
ON CONFLICT (run_id, key) DO NOTHING`, id, key, payload); err != nil {
return fmt.Errorf("store signal: %w", err)
}
if _, err := tx.ExecContext(ctx, `
UPDATE runs.runs SET state = 'queued', wake_at = NULL, updated_at = now()
WHERE id = $1 AND state = 'waiting' AND waiting_for = $2`, id, key); err != nil {
return fmt.Errorf("wake run: %w", err)
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("signal run: %w", err)
}
return nil
}
// Signalled returns the payload of a run's signal, if one arrived.
func (s *Store) Signalled(ctx context.Context, id uuid.UUID, key string) (jsontext.Value, bool, error) {
var payload jsontext.Value
err := s.db.GetContext(ctx, &payload, `SELECT payload FROM runs.signals WHERE run_id = $1 AND key = $2`, id, key)
if errors.Is(err, sql.ErrNoRows) {
return nil, false, nil
}
if err != nil {
return nil, false, fmt.Errorf("read signal: %w", err)
}
return payload, true, nil
}
// Cancel asks a run to stop. A run not currently held ends at once and tells
// the run waiting on it; a running one sees the request at its next step.
func (s *Store) Cancel(ctx context.Context, id uuid.UUID) error {
tx, err := s.db.BeginTxx(ctx, nil)
if err != nil {
return fmt.Errorf("cancel run: %w", err)
}
defer func() { _ = tx.Rollback() }()
if _, err := tx.ExecContext(ctx, `
UPDATE runs.runs
SET cancel_requested = true, updated_at = now(),
state = CASE WHEN state IN ('queued', 'waiting', 'parked', 'waiting_approval') THEN 'cancelled' ELSE state END,
finished_at = CASE WHEN state IN ('queued', 'waiting', 'parked', 'waiting_approval') THEN now() ELSE finished_at END
WHERE id = $1 AND state NOT IN ('succeeded', 'failed', 'cancelled', 'declined')`, id); err != nil {
return fmt.Errorf("cancel run: %w", err)
}
if err := signalParent(ctx, tx, id); err != nil {
return err
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("cancel run: %w", err)
}
return nil
}
// CancelRequested reports whether a run has been asked to stop.
func (s *Store) CancelRequested(ctx context.Context, id uuid.UUID) (bool, error) {
var requested bool
if err := s.db.GetContext(ctx, &requested, `SELECT cancel_requested FROM runs.runs WHERE id = $1`, id); err != nil {
return false, fmt.Errorf("read cancel request: %w", err)
}
return requested, nil
}
// Summary is a run as a list shows it.
type Summary struct {
ID uuid.UUID `db:"id"`
Workflow string `db:"workflow"`
Version int `db:"workflow_version"`
State string `db:"state"`
Attempts int `db:"attempts"`
Error sql.NullString `db:"error"`
Detail sql.NullString `db:"detail"`
CreatedAt time.Time `db:"created_at"`
FinishedAt sql.NullTime `db:"finished_at"`
Dry bool `db:"dry"`
}
// Detail is a run with its input and result. A run that failed has no result,
// which the row holds as NULL and every reader gets as JSON null, as a step
// with no output does.
type Detail struct {
Summary
Input jsontext.Value `db:"input"`
Result jsontext.Value `db:"result"`
}
// Recorded is a step with the time it was recorded.
type Recorded struct {
Step
RecordedAt time.Time `db:"recorded_at"`
}
// Recent returns an organisation's latest runs, newest first.
func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary, error) {
var out []Summary
if err := s.db.SelectContext(ctx, &out, `
SELECT id, workflow, workflow_version, state, attempts, error, detail, created_at, finished_at, dry FROM runs.runs
WHERE org_id = $1 AND NOT dry ORDER BY created_at DESC LIMIT $2`, orgID, limit); err != nil {
return nil, fmt.Errorf("list runs: %w", err)
}
return out, nil
}
// Detail returns one of an organisation's runs; ok is false when it has none
// with that id.
func (s *Store) Detail(ctx context.Context, orgID string, id uuid.UUID) (Detail, bool, error) {
var d Detail
err := s.db.GetContext(ctx, &d, `
SELECT id, workflow, workflow_version, state, attempts, error, detail, created_at, finished_at, dry, input,
coalesce(result, 'null') AS result
FROM runs.runs WHERE org_id = $1 AND id = $2`, orgID, id)
if errors.Is(err, sql.ErrNoRows) {
return Detail{}, false, nil
}
if err != nil {
return Detail{}, false, fmt.Errorf("read run: %w", err)
}
return d, true, nil
}
// RecordedSteps returns a run's steps in order, with when each was recorded.
func (s *Store) RecordedSteps(ctx context.Context, id uuid.UUID) ([]Recorded, error) {
var out []Recorded
if err := s.db.SelectContext(ctx, &out, `
SELECT run_id, seq, name, outcome, coalesce(output, 'null') AS output, error, recorded_at FROM runs.steps
WHERE run_id = $1 ORDER BY seq`, id); err != nil {
return nil, fmt.Errorf("load steps: %w", err)
}
return out, nil
}
// whyNotWritten tells a lost lease from a step that already exists.
func (s *Store) whyNotWritten(ctx context.Context, id, token uuid.UUID) error {
var holds bool
if err := s.db.GetContext(ctx, &holds, `
SELECT COALESCE(lease_token = $2, false) FROM runs.runs WHERE id = $1`, id, token); err != nil {
return fmt.Errorf("check lease: %w", err)
}
if !holds {
return ErrLeaseLost
}
return ErrStepExists
}
func affected(res sql.Result) int64 {
n, err := res.RowsAffected()
if err != nil {
return 0
}
return n
}
func nullJSON(v jsontext.Value) any {
if len(v) == 0 {
return nil
}
return []byte(v)
}