feat(events,approvals): triggers fire on an interval, and an expired approval is escalated on every channel (archie #1992)
This commit is contained in:
6 files changed
+187
-8
No files matched your search
@@ -0,0 +1,9 @@
|
|||||||
|
-- +goose Up
|
||||||
|
-- A trigger may fire on an interval instead of on an event. since is when the
|
||||||
|
-- schedule began, so a new schedule doesn't fire the slot it was created in.
|
||||||
|
ALTER TABLE events.triggers
|
||||||
|
ADD COLUMN schedule_every_s integer CHECK (schedule_every_s >= 60),
|
||||||
|
ADD COLUMN schedule_since timestamptz;
|
||||||
|
|
||||||
|
-- +goose Down
|
||||||
|
ALTER TABLE events.triggers DROP COLUMN schedule_since, DROP COLUMN schedule_every_s;
|
||||||
@@ -119,6 +119,7 @@ type Signaller interface {
|
|||||||
type Delivery interface {
|
type Delivery interface {
|
||||||
Ask(ctx context.Context, a deliveries.ApprovalRequest) error
|
Ask(ctx context.Context, a deliveries.ApprovalRequest) error
|
||||||
Told(ctx context.Context, a deliveries.ApprovalRequest, answer string) error
|
Told(ctx context.Context, a deliveries.ApprovalRequest, answer string) error
|
||||||
|
Notify(ctx context.Context, n deliveries.Notification) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Module owns one organisation's approval requests.
|
// Module owns one organisation's approval requests.
|
||||||
@@ -220,10 +221,19 @@ func (m *Module) Expire(ctx context.Context, id uuid.UUID) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
m.log.InfoContext(ctx, "approval request expired", "request", row.ID, "run", row.RunID)
|
m.log.InfoContext(ctx, "approval request expired", "request", row.ID, "run", row.RunID)
|
||||||
return m.tell.Told(ctx, deliveries.ApprovalRequest{
|
if err := m.tell.Told(ctx, deliveries.ApprovalRequest{
|
||||||
RunID: row.RunID, RequestID: row.ID, Key: row.Key + ":ask",
|
RunID: row.RunID, RequestID: row.ID, Key: row.Key + ":ask",
|
||||||
Title: "Approval expired", Body: "The request ran out of time, so nothing was run.",
|
Title: "Approval expired", Body: "The request ran out of time, so nothing was run.",
|
||||||
}, "Expired without an answer.")
|
}, "Expired without an answer."); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// The escalation goes to every channel. Its key is stable, so an Expire
|
||||||
|
// retried after a failed delivery sends it once.
|
||||||
|
return m.tell.Notify(ctx, deliveries.Notification{
|
||||||
|
RunID: row.RunID, Key: row.Key + ":expired",
|
||||||
|
Title: "Approval timed out: " + row.Connector + " " + row.Action,
|
||||||
|
Body: "Nobody answered before the deadline, so " + row.Connector + " " + row.Action + " did not run. " + row.Summary,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Pending returns the requests waiting for an answer, oldest first.
|
// Pending returns the requests waiting for an answer, oldest first.
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ package events
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
|
"database/sql"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json/jsontext"
|
"encoding/json/jsontext"
|
||||||
json "encoding/json/v2"
|
json "encoding/json/v2"
|
||||||
@@ -144,8 +145,14 @@ func (m *Module) Work(ctx context.Context) {
|
|||||||
var listeners sync.WaitGroup
|
var listeners sync.WaitGroup
|
||||||
defer listeners.Wait()
|
defer listeners.Wait()
|
||||||
listening := map[string]bool{}
|
listening := map[string]bool{}
|
||||||
nextCheck := time.Time{}
|
nextCheck, nextTick := time.Time{}, time.Time{}
|
||||||
for ctx.Err() == nil {
|
for ctx.Err() == nil {
|
||||||
|
if now := time.Now(); now.After(nextTick) {
|
||||||
|
if err := m.tick(ctx, now); err != nil && ctx.Err() == nil {
|
||||||
|
m.log.ErrorContext(ctx, "fire schedules", "error", err)
|
||||||
|
}
|
||||||
|
nextTick = now.Add(scheduleCheck)
|
||||||
|
}
|
||||||
if time.Now().After(nextCheck) {
|
if time.Now().After(nextCheck) {
|
||||||
m.startListeners(ctx, &listeners, listening)
|
m.startListeners(ctx, &listeners, listening)
|
||||||
nextCheck = time.Now().Add(listenRetry)
|
nextCheck = time.Now().Add(listenRetry)
|
||||||
@@ -342,6 +349,9 @@ type triggerRequest struct {
|
|||||||
Filter map[string]string `json:"filter"`
|
Filter map[string]string `json:"filter"`
|
||||||
Workflow string `json:"workflow"`
|
Workflow string `json:"workflow"`
|
||||||
Disabled bool `json:"disabled"`
|
Disabled bool `json:"disabled"`
|
||||||
|
// Every makes the trigger fire on an interval ("1h") instead of on
|
||||||
|
// events; source and event_type are then fixed to the schedule's own.
|
||||||
|
Every string `json:"every"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Module) putTrigger(w http.ResponseWriter, r *http.Request) {
|
func (m *Module) putTrigger(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -351,8 +361,21 @@ func (m *Module) putTrigger(w http.ResponseWriter, r *http.Request) {
|
|||||||
httpjson.Error(w, http.StatusBadRequest, err.Error())
|
httpjson.Error(w, http.StatusBadRequest, err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
var every sql.NullInt64
|
||||||
|
if req.Every != "" {
|
||||||
|
d, err := time.ParseDuration(req.Every)
|
||||||
|
if err != nil || d < minInterval || d%time.Second != 0 {
|
||||||
|
httpjson.Error(w, http.StatusBadRequest, "every is a whole number of seconds of at least one minute, such as 1h")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
every = sql.NullInt64{Int64: int64(d / time.Second), Valid: true}
|
||||||
|
req.Source, req.EventType = scheduleSource, scheduleType
|
||||||
|
// Every schedule's ticks share a source and type, so the filter keeps
|
||||||
|
// this trigger to its own.
|
||||||
|
req.Filter = map[string]string{"trigger": name}
|
||||||
|
}
|
||||||
if !validName.MatchString(name) || req.Source == "" || req.EventType == "" || req.Workflow == "" {
|
if !validName.MatchString(name) || req.Source == "" || req.EventType == "" || req.Workflow == "" {
|
||||||
httpjson.Error(w, http.StatusBadRequest, "a trigger needs a lowercase name, a source, an event type and a workflow")
|
httpjson.Error(w, http.StatusBadRequest, "a trigger needs a lowercase name, a workflow, and either a source and event type or an every interval")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
filter, err := json.Marshal(req.Filter)
|
filter, err := json.Marshal(req.Filter)
|
||||||
@@ -362,7 +385,7 @@ func (m *Module) putTrigger(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
if err := m.store.UpsertTrigger(r.Context(), store.Trigger{
|
if err := m.store.UpsertTrigger(r.Context(), store.Trigger{
|
||||||
OrgID: m.orgID, Name: name, Source: req.Source, EventType: req.EventType,
|
OrgID: m.orgID, Name: name, Source: req.Source, EventType: req.EventType,
|
||||||
Filter: filter, Workflow: req.Workflow, Enabled: !req.Disabled,
|
Filter: filter, Workflow: req.Workflow, Enabled: !req.Disabled, ScheduleEverySeconds: every,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
httpjson.Error(w, http.StatusInternalServerError, err.Error())
|
httpjson.Error(w, http.StatusInternalServerError, err.Error())
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -54,6 +54,11 @@ type Trigger struct {
|
|||||||
Filter jsontext.Value `db:"filter"`
|
Filter jsontext.Value `db:"filter"`
|
||||||
Workflow string `db:"workflow"`
|
Workflow string `db:"workflow"`
|
||||||
Enabled bool `db:"enabled"`
|
Enabled bool `db:"enabled"`
|
||||||
|
// ScheduleEverySeconds is the interval a scheduled trigger fires on; it
|
||||||
|
// is null for a trigger that fires on events.
|
||||||
|
ScheduleEverySeconds sql.NullInt64 `db:"schedule_every_s"`
|
||||||
|
// ScheduleSince is when the schedule began or last changed.
|
||||||
|
ScheduleSince sql.NullTime `db:"schedule_since"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store is the events module's Postgres store.
|
// Store is the events module's Postgres store.
|
||||||
@@ -212,16 +217,36 @@ func RecordDispatch(ctx context.Context, tx *sqlx.Tx, eventID uuid.UUID, trigger
|
|||||||
// UpsertTrigger creates or replaces a trigger.
|
// UpsertTrigger creates or replaces a trigger.
|
||||||
func (s *Store) UpsertTrigger(ctx context.Context, t Trigger) error {
|
func (s *Store) UpsertTrigger(ctx context.Context, t Trigger) error {
|
||||||
if _, err := s.db.NamedExecContext(ctx, `
|
if _, err := s.db.NamedExecContext(ctx, `
|
||||||
INSERT INTO events.triggers (org_id, name, source, event_type, filter, workflow, enabled)
|
INSERT INTO events.triggers (org_id, name, source, event_type, filter, workflow, enabled,
|
||||||
VALUES (:org_id, :name, :source, :event_type, :filter, :workflow, :enabled)
|
schedule_every_s, schedule_since)
|
||||||
|
VALUES (:org_id, :name, :source, :event_type, :filter, :workflow, :enabled,
|
||||||
|
CAST(:schedule_every_s AS integer), CASE WHEN CAST(:schedule_every_s AS integer) IS NULL THEN NULL ELSE now() END)
|
||||||
ON CONFLICT (org_id, name) DO UPDATE
|
ON CONFLICT (org_id, name) DO UPDATE
|
||||||
SET source = EXCLUDED.source, event_type = EXCLUDED.event_type, filter = EXCLUDED.filter,
|
SET source = EXCLUDED.source, event_type = EXCLUDED.event_type, filter = EXCLUDED.filter,
|
||||||
workflow = EXCLUDED.workflow, enabled = EXCLUDED.enabled, updated_at = now()`, t); err != nil {
|
workflow = EXCLUDED.workflow, enabled = EXCLUDED.enabled,
|
||||||
|
schedule_since = CASE
|
||||||
|
WHEN EXCLUDED.schedule_every_s IS NULL THEN NULL
|
||||||
|
WHEN events.triggers.schedule_every_s IS NOT DISTINCT FROM EXCLUDED.schedule_every_s
|
||||||
|
THEN events.triggers.schedule_since
|
||||||
|
ELSE now() END,
|
||||||
|
schedule_every_s = EXCLUDED.schedule_every_s, updated_at = now()`, t); err != nil {
|
||||||
return fmt.Errorf("save trigger: %w", err)
|
return fmt.Errorf("save trigger: %w", err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ScheduledTriggers returns the enabled triggers that fire on an interval.
|
||||||
|
func (s *Store) ScheduledTriggers(ctx context.Context, orgID string) ([]Trigger, error) {
|
||||||
|
var out []Trigger
|
||||||
|
if err := s.db.SelectContext(ctx, &out, `
|
||||||
|
SELECT org_id, name, source, event_type, filter, workflow, enabled, schedule_every_s, schedule_since
|
||||||
|
FROM events.triggers
|
||||||
|
WHERE org_id = $1 AND enabled AND schedule_every_s IS NOT NULL ORDER BY name`, orgID); err != nil {
|
||||||
|
return nil, fmt.Errorf("read schedules: %w", err)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Recent returns an organisation's newest events.
|
// Recent returns an organisation's newest events.
|
||||||
func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Event, error) {
|
func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Event, error) {
|
||||||
var out []Event
|
var out []Event
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package events
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json/jsontext"
|
||||||
|
json "encoding/json/v2"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/modules/events/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The source and type of the event a schedule stores for each slot.
|
||||||
|
const (
|
||||||
|
scheduleSource = "schedule"
|
||||||
|
scheduleType = "schedule.tick"
|
||||||
|
)
|
||||||
|
|
||||||
|
// minInterval keeps a schedule from firing faster than the dispatcher can use.
|
||||||
|
const minInterval = time.Minute
|
||||||
|
|
||||||
|
// scheduleCheck is how often Work looks for due slots.
|
||||||
|
const scheduleCheck = 10 * time.Second
|
||||||
|
|
||||||
|
// dueSlot returns the start of the latest slot of length every that began
|
||||||
|
// after since and not after now. Slots are aligned to the clock, so every
|
||||||
|
// process computes the same ones; slots missed while the process was down
|
||||||
|
// collapse into this one.
|
||||||
|
func dueSlot(now, since time.Time, every time.Duration) (time.Time, bool) {
|
||||||
|
slot := now.Truncate(every)
|
||||||
|
return slot, slot.After(since)
|
||||||
|
}
|
||||||
|
|
||||||
|
// tick stores the event for each schedule whose slot is due. The slot's start
|
||||||
|
// is part of the source event id, so a restart or a second process storing the
|
||||||
|
// same slot conflicts instead of firing twice.
|
||||||
|
func (m *Module) tick(ctx context.Context, now time.Time) error {
|
||||||
|
triggers, err := m.store.ScheduledTriggers(ctx, m.orgID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, t := range triggers {
|
||||||
|
slot, due := dueSlot(now, t.ScheduleSince.Time, time.Duration(t.ScheduleEverySeconds.Int64)*time.Second)
|
||||||
|
if !due {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fields, err := json.Marshal(map[string]string{"trigger": t.Name, "slot": slot.UTC().Format(time.RFC3339)})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("encode tick: %w", err)
|
||||||
|
}
|
||||||
|
isNew, err := m.store.Insert(ctx, tickEvent(m.orgID, t.Name, slot, fields))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if isNew {
|
||||||
|
m.log.InfoContext(ctx, "schedule fired", "trigger", t.Name, "slot", slot)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func tickEvent(orgID, trigger string, slot time.Time, fields jsontext.Value) store.Event {
|
||||||
|
e := store.Event{
|
||||||
|
ID: uuid.New(), OrgID: orgID, Source: scheduleSource, Type: scheduleType,
|
||||||
|
SourceEventID: trigger + "@" + slot.UTC().Format(time.RFC3339), Fields: fields,
|
||||||
|
}
|
||||||
|
e.OccurredAt.Time, e.OccurredAt.Valid = slot, true
|
||||||
|
return e
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
package events
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestDueSlot guards the rule that a schedule fires once per slot: a slot
|
||||||
|
// missed while down collapses into one firing, and a new schedule doesn't fire
|
||||||
|
// the slot it was created in.
|
||||||
|
func TestDueSlot(t *testing.T) {
|
||||||
|
at := func(s string) time.Time {
|
||||||
|
v, err := time.Parse(time.RFC3339, s)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
now string
|
||||||
|
since string
|
||||||
|
every time.Duration
|
||||||
|
want string
|
||||||
|
wantFires bool
|
||||||
|
}{
|
||||||
|
{"created mid slot", "2026-01-01T10:45:00Z", "2026-01-01T10:30:00Z", time.Hour, "2026-01-01T10:00:00Z", false},
|
||||||
|
{"next slot begins", "2026-01-01T11:00:00Z", "2026-01-01T10:30:00Z", time.Hour, "2026-01-01T11:00:00Z", true},
|
||||||
|
{"down for hours fires the latest slot once", "2026-01-01T14:20:00Z", "2026-01-01T10:30:00Z", time.Hour, "2026-01-01T14:00:00Z", true},
|
||||||
|
{"same slot again is the same slot", "2026-01-01T14:59:00Z", "2026-01-01T10:30:00Z", time.Hour, "2026-01-01T14:00:00Z", true},
|
||||||
|
{"minute interval", "2026-01-01T10:31:10Z", "2026-01-01T10:30:20Z", time.Minute, "2026-01-01T10:31:00Z", true},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
slot, fires := dueSlot(at(tt.now), at(tt.since), tt.every)
|
||||||
|
if !slot.Equal(at(tt.want)) || fires != tt.wantFires {
|
||||||
|
t.Errorf("dueSlot = %v, %v; want %v, %v", slot, fires, tt.want, tt.wantFires)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user