Also fix repository-free message.send validation (#2046) and add a single-binary development smoke path (#1977). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
364 lines
12 KiB
Go
364 lines
12 KiB
Go
// Package store holds the runs module's SQL. Every write a worker makes is
|
|
// fenced by its lease token, so a worker that lost its lease writes nothing.
|
|
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json/jsontext"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jmoiron/sqlx"
|
|
)
|
|
|
|
// Run states.
|
|
const (
|
|
StateQueued = "queued"
|
|
StateRunning = "running"
|
|
StateWaiting = "waiting"
|
|
StateSucceeded = "succeeded"
|
|
StateFailed = "failed"
|
|
StateCancelled = "cancelled"
|
|
)
|
|
|
|
// Step outcomes.
|
|
const (
|
|
OutcomeDone = "done"
|
|
OutcomeFailed = "failed"
|
|
OutcomeUnknown = "unknown"
|
|
)
|
|
|
|
var (
|
|
// ErrLeaseLost means the writer no longer holds the run's lease.
|
|
ErrLeaseLost = errors.New("lease lost")
|
|
// ErrStepExists means the step was already recorded, by this run's replay
|
|
// or by a previous lease holder.
|
|
ErrStepExists = errors.New("step already recorded")
|
|
)
|
|
|
|
// Run is a claimed run.
|
|
type Run struct {
|
|
ID uuid.UUID `db:"id"`
|
|
OrgID string `db:"org_id"`
|
|
Workflow string `db:"workflow"`
|
|
// Version is the workflow version the run started under.
|
|
Version int `db:"workflow_version"`
|
|
Input jsontext.Value `db:"input"`
|
|
Attempts int `db:"attempts"`
|
|
// Dry marks a dry run, and Overlay is the document and policy it runs
|
|
// under; a dry run pins no stored version.
|
|
Dry bool `db:"dry"`
|
|
// Scan into []byte because live runs have a SQL NULL overlay.
|
|
Overlay []byte `db:"overlay"`
|
|
}
|
|
|
|
// Step is one recorded step of a run.
|
|
type Step struct {
|
|
RunID uuid.UUID `db:"run_id"`
|
|
Seq int `db:"seq"`
|
|
Name string `db:"name"`
|
|
Outcome string `db:"outcome"`
|
|
Output jsontext.Value `db:"output"`
|
|
Error sql.NullString `db:"error"`
|
|
}
|
|
|
|
// Store is the runs module's Postgres store.
|
|
type Store struct {
|
|
db *sqlx.DB
|
|
}
|
|
|
|
// New returns a store on db.
|
|
func New(db *sqlx.DB) *Store {
|
|
return &Store{db: db}
|
|
}
|
|
|
|
// NewRun is a run to queue.
|
|
type NewRun struct {
|
|
OrgID string
|
|
Workflow string
|
|
Version int
|
|
StartKey string
|
|
Input jsontext.Value
|
|
Dry bool
|
|
Overlay jsontext.Value
|
|
}
|
|
|
|
// Start inserts a queued run. A run with the same organisation and start key
|
|
// is not inserted again; its id is returned instead.
|
|
func (s *Store) Start(ctx context.Context, n NewRun) (uuid.UUID, error) {
|
|
key := sql.NullString{String: n.StartKey, Valid: n.StartKey != ""}
|
|
var id uuid.UUID
|
|
err := s.db.GetContext(ctx, &id, `
|
|
INSERT INTO runs.runs (id, org_id, workflow, workflow_version, start_key, input, state, dry, overlay)
|
|
VALUES ($1, $2, $3, $4, $5, $6, 'queued', $7, $8)
|
|
ON CONFLICT (org_id, start_key) DO NOTHING
|
|
RETURNING id`, uuid.New(), n.OrgID, n.Workflow, n.Version, key, n.Input, n.Dry, nullJSON(n.Overlay))
|
|
if errors.Is(err, sql.ErrNoRows) && key.Valid {
|
|
err = s.db.GetContext(ctx, &id, `SELECT id FROM runs.runs WHERE org_id = $1 AND start_key = $2`, n.OrgID, n.StartKey)
|
|
}
|
|
if err != nil {
|
|
return uuid.Nil, fmt.Errorf("start run: %w", err)
|
|
}
|
|
return id, nil
|
|
}
|
|
|
|
// Claim takes the oldest run that is queued, due to wake, or whose lease has
|
|
// expired. SKIP LOCKED only keeps claimers from waiting on each other;
|
|
// correctness rests on the lease token every later write is fenced by.
|
|
func (s *Store) Claim(ctx context.Context, owner string, token uuid.UUID, ttl time.Duration) (Run, bool, error) {
|
|
var run Run
|
|
err := s.db.GetContext(ctx, &run, `
|
|
UPDATE runs.runs r
|
|
SET state = 'running', lease_token = $1, lease_owner = $2,
|
|
lease_expires_at = now() + make_interval(secs => $3),
|
|
attempts = attempts + 1, updated_at = now()
|
|
WHERE r.id = (
|
|
SELECT id FROM runs.runs
|
|
WHERE state = 'queued'
|
|
OR (state = 'waiting' AND wake_at <= now())
|
|
OR (state = 'running' AND lease_expires_at < now())
|
|
ORDER BY created_at
|
|
FOR UPDATE SKIP LOCKED
|
|
LIMIT 1)
|
|
RETURNING r.id, r.org_id, r.workflow, r.workflow_version, r.input, r.attempts, r.dry, r.overlay`, token, owner, ttl.Seconds())
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return Run{}, false, nil
|
|
}
|
|
if err != nil {
|
|
return Run{}, false, fmt.Errorf("claim run: %w", err)
|
|
}
|
|
return run, true, nil
|
|
}
|
|
|
|
// RenewLease extends the lease while the token still holds it.
|
|
func (s *Store) RenewLease(ctx context.Context, id, token uuid.UUID, ttl time.Duration) (bool, error) {
|
|
res, err := s.db.ExecContext(ctx, `
|
|
UPDATE runs.runs SET lease_expires_at = now() + make_interval(secs => $3), updated_at = now()
|
|
WHERE id = $1 AND lease_token = $2`, id, token, ttl.Seconds())
|
|
if err != nil {
|
|
return false, fmt.Errorf("renew lease: %w", err)
|
|
}
|
|
return affected(res) == 1, nil
|
|
}
|
|
|
|
// Steps returns a run's recorded steps in order.
|
|
func (s *Store) Steps(ctx context.Context, id uuid.UUID) ([]Step, error) {
|
|
var steps []Step
|
|
if err := s.db.SelectContext(ctx, &steps, `
|
|
SELECT run_id, seq, name, outcome, coalesce(output, 'null') AS output, error FROM runs.steps
|
|
WHERE run_id = $1 ORDER BY seq`, id); err != nil {
|
|
return nil, fmt.Errorf("load steps: %w", err)
|
|
}
|
|
return steps, nil
|
|
}
|
|
|
|
// RecordStep writes a step if the token holds the lease and the step is new.
|
|
func (s *Store) RecordStep(ctx context.Context, token uuid.UUID, step Step) error {
|
|
res, err := s.db.ExecContext(ctx, `
|
|
INSERT INTO runs.steps (run_id, seq, name, outcome, output, error)
|
|
SELECT $1, $2, $3, $4, $5, $6
|
|
WHERE EXISTS (SELECT 1 FROM runs.runs WHERE id = $1 AND lease_token = $7)
|
|
ON CONFLICT (run_id, seq) DO NOTHING`,
|
|
step.RunID, step.Seq, step.Name, step.Outcome, nullJSON(step.Output), step.Error, token)
|
|
if err != nil {
|
|
return fmt.Errorf("record step: %w", err)
|
|
}
|
|
if affected(res) == 1 {
|
|
return nil
|
|
}
|
|
return s.whyNotWritten(ctx, step.RunID, token)
|
|
}
|
|
|
|
// Suspend releases the lease and parks the run until wakeAt or a signal for
|
|
// waitingFor. A signal that arrived before the park queues the run at once.
|
|
func (s *Store) Suspend(ctx context.Context, id, token uuid.UUID, wakeAt *time.Time, waitingFor string) error {
|
|
res, err := s.db.ExecContext(ctx, `
|
|
UPDATE runs.runs r
|
|
SET state = CASE WHEN $4 <> '' AND EXISTS (
|
|
SELECT 1 FROM runs.signals s WHERE s.run_id = r.id AND s.key = $4)
|
|
THEN 'queued' ELSE 'waiting' END,
|
|
wake_at = $3, waiting_for = NULLIF($4, ''),
|
|
lease_token = NULL, lease_owner = NULL, lease_expires_at = NULL, updated_at = now()
|
|
WHERE id = $1 AND lease_token = $2`, id, token, wakeAt, waitingFor)
|
|
if err != nil {
|
|
return fmt.Errorf("suspend run: %w", err)
|
|
}
|
|
if affected(res) == 0 {
|
|
return ErrLeaseLost
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Finish records a terminal state while the token holds the lease.
|
|
func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, state string, result jsontext.Value, errText string) error {
|
|
res, err := s.db.ExecContext(ctx, `
|
|
UPDATE runs.runs
|
|
SET state = $3, result = $4, error = NULLIF($5, ''), finished_at = now(), updated_at = now(),
|
|
lease_token = NULL, lease_owner = NULL, lease_expires_at = NULL
|
|
WHERE id = $1 AND lease_token = $2`, id, token, state, nullJSON(result), errText)
|
|
if err != nil {
|
|
return fmt.Errorf("finish run: %w", err)
|
|
}
|
|
if affected(res) == 0 {
|
|
return ErrLeaseLost
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Signal stores a signal for a run and queues the run if it is waiting for it.
|
|
func (s *Store) Signal(ctx context.Context, id uuid.UUID, key string, payload jsontext.Value) error {
|
|
tx, err := s.db.BeginTxx(ctx, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("signal run: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback() }()
|
|
if _, err := tx.ExecContext(ctx, `
|
|
INSERT INTO runs.signals (run_id, key, payload) VALUES ($1, $2, $3)
|
|
ON CONFLICT (run_id, key) DO NOTHING`, id, key, payload); err != nil {
|
|
return fmt.Errorf("store signal: %w", err)
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `
|
|
UPDATE runs.runs SET state = 'queued', wake_at = NULL, updated_at = now()
|
|
WHERE id = $1 AND state = 'waiting' AND waiting_for = $2`, id, key); err != nil {
|
|
return fmt.Errorf("wake run: %w", err)
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return fmt.Errorf("signal run: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Signalled returns the payload of a run's signal, if one arrived.
|
|
func (s *Store) Signalled(ctx context.Context, id uuid.UUID, key string) (jsontext.Value, bool, error) {
|
|
var payload jsontext.Value
|
|
err := s.db.GetContext(ctx, &payload, `SELECT payload FROM runs.signals WHERE run_id = $1 AND key = $2`, id, key)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return nil, false, nil
|
|
}
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("read signal: %w", err)
|
|
}
|
|
return payload, true, nil
|
|
}
|
|
|
|
// Cancel asks a run to stop. A run not currently held ends at once; a running
|
|
// one sees the request at its next step.
|
|
func (s *Store) Cancel(ctx context.Context, id uuid.UUID) error {
|
|
if _, err := s.db.ExecContext(ctx, `
|
|
UPDATE runs.runs
|
|
SET cancel_requested = true, updated_at = now(),
|
|
state = CASE WHEN state IN ('queued', 'waiting') THEN 'cancelled' ELSE state END,
|
|
finished_at = CASE WHEN state IN ('queued', 'waiting') THEN now() ELSE finished_at END
|
|
WHERE id = $1 AND state NOT IN ('succeeded', 'failed', 'cancelled')`, id); err != nil {
|
|
return fmt.Errorf("cancel run: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// CancelRequested reports whether a run has been asked to stop.
|
|
func (s *Store) CancelRequested(ctx context.Context, id uuid.UUID) (bool, error) {
|
|
var requested bool
|
|
if err := s.db.GetContext(ctx, &requested, `SELECT cancel_requested FROM runs.runs WHERE id = $1`, id); err != nil {
|
|
return false, fmt.Errorf("read cancel request: %w", err)
|
|
}
|
|
return requested, nil
|
|
}
|
|
|
|
// Summary is a run as a list shows it.
|
|
type Summary struct {
|
|
ID uuid.UUID `db:"id"`
|
|
Workflow string `db:"workflow"`
|
|
Version int `db:"workflow_version"`
|
|
State string `db:"state"`
|
|
Attempts int `db:"attempts"`
|
|
Error sql.NullString `db:"error"`
|
|
CreatedAt time.Time `db:"created_at"`
|
|
FinishedAt sql.NullTime `db:"finished_at"`
|
|
Dry bool `db:"dry"`
|
|
}
|
|
|
|
// Detail is a run with its input and result. A run that failed has no result,
|
|
// which the row holds as NULL and every reader gets as JSON null, as a step
|
|
// with no output does.
|
|
type Detail struct {
|
|
Summary
|
|
Input jsontext.Value `db:"input"`
|
|
Result jsontext.Value `db:"result"`
|
|
}
|
|
|
|
// Recorded is a step with the time it was recorded.
|
|
type Recorded struct {
|
|
Step
|
|
RecordedAt time.Time `db:"recorded_at"`
|
|
}
|
|
|
|
// Recent returns an organisation's latest runs, newest first.
|
|
func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary, error) {
|
|
var out []Summary
|
|
if err := s.db.SelectContext(ctx, &out, `
|
|
SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, dry FROM runs.runs
|
|
WHERE org_id = $1 AND NOT dry ORDER BY created_at DESC LIMIT $2`, orgID, limit); err != nil {
|
|
return nil, fmt.Errorf("list runs: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Detail returns one of an organisation's runs; ok is false when it has none
|
|
// with that id.
|
|
func (s *Store) Detail(ctx context.Context, orgID string, id uuid.UUID) (Detail, bool, error) {
|
|
var d Detail
|
|
err := s.db.GetContext(ctx, &d, `
|
|
SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, dry, input,
|
|
coalesce(result, 'null') AS result
|
|
FROM runs.runs WHERE org_id = $1 AND id = $2`, orgID, id)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return Detail{}, false, nil
|
|
}
|
|
if err != nil {
|
|
return Detail{}, false, fmt.Errorf("read run: %w", err)
|
|
}
|
|
return d, true, nil
|
|
}
|
|
|
|
// RecordedSteps returns a run's steps in order, with when each was recorded.
|
|
func (s *Store) RecordedSteps(ctx context.Context, id uuid.UUID) ([]Recorded, error) {
|
|
var out []Recorded
|
|
if err := s.db.SelectContext(ctx, &out, `
|
|
SELECT run_id, seq, name, outcome, coalesce(output, 'null') AS output, error, recorded_at FROM runs.steps
|
|
WHERE run_id = $1 ORDER BY seq`, id); err != nil {
|
|
return nil, fmt.Errorf("load steps: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// whyNotWritten tells a lost lease from a step that already exists.
|
|
func (s *Store) whyNotWritten(ctx context.Context, id, token uuid.UUID) error {
|
|
var holds bool
|
|
if err := s.db.GetContext(ctx, &holds, `
|
|
SELECT COALESCE(lease_token = $2, false) FROM runs.runs WHERE id = $1`, id, token); err != nil {
|
|
return fmt.Errorf("check lease: %w", err)
|
|
}
|
|
if !holds {
|
|
return ErrLeaseLost
|
|
}
|
|
return ErrStepExists
|
|
}
|
|
|
|
func affected(res sql.Result) int64 {
|
|
n, err := res.RowsAffected()
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
return n
|
|
}
|
|
|
|
func nullJSON(v jsontext.Value) any {
|
|
if len(v) == 0 {
|
|
return nil
|
|
}
|
|
return []byte(v)
|
|
}
|