feat(events,approvals): triggers fire on an interval, and an expired approval is escalated on every channel (archie #1992)
This commit is contained in:
6 files changed
+187
-8
No files matched your search
@@ -0,0 +1,9 @@
|
||||
-- +goose Up
|
||||
-- A trigger may fire on an interval instead of on an event. since is when the
|
||||
-- schedule began, so a new schedule doesn't fire the slot it was created in.
|
||||
ALTER TABLE events.triggers
|
||||
ADD COLUMN schedule_every_s integer CHECK (schedule_every_s >= 60),
|
||||
ADD COLUMN schedule_since timestamptz;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE events.triggers DROP COLUMN schedule_since, DROP COLUMN schedule_every_s;
|
||||
@@ -119,6 +119,7 @@ type Signaller interface {
|
||||
type Delivery interface {
|
||||
Ask(ctx context.Context, a deliveries.ApprovalRequest) error
|
||||
Told(ctx context.Context, a deliveries.ApprovalRequest, answer string) error
|
||||
Notify(ctx context.Context, n deliveries.Notification) error
|
||||
}
|
||||
|
||||
// Module owns one organisation's approval requests.
|
||||
@@ -220,10 +221,19 @@ func (m *Module) Expire(ctx context.Context, id uuid.UUID) error {
|
||||
return nil
|
||||
}
|
||||
m.log.InfoContext(ctx, "approval request expired", "request", row.ID, "run", row.RunID)
|
||||
return m.tell.Told(ctx, deliveries.ApprovalRequest{
|
||||
if err := m.tell.Told(ctx, deliveries.ApprovalRequest{
|
||||
RunID: row.RunID, RequestID: row.ID, Key: row.Key + ":ask",
|
||||
Title: "Approval expired", Body: "The request ran out of time, so nothing was run.",
|
||||
}, "Expired without an answer.")
|
||||
}, "Expired without an answer."); err != nil {
|
||||
return err
|
||||
}
|
||||
// The escalation goes to every channel. Its key is stable, so an Expire
|
||||
// retried after a failed delivery sends it once.
|
||||
return m.tell.Notify(ctx, deliveries.Notification{
|
||||
RunID: row.RunID, Key: row.Key + ":expired",
|
||||
Title: "Approval timed out: " + row.Connector + " " + row.Action,
|
||||
Body: "Nobody answered before the deadline, so " + row.Connector + " " + row.Action + " did not run. " + row.Summary,
|
||||
})
|
||||
}
|
||||
|
||||
// Pending returns the requests waiting for an answer, oldest first.
|
||||
|
||||
@@ -10,6 +10,7 @@ package events
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json/jsontext"
|
||||
json "encoding/json/v2"
|
||||
@@ -144,8 +145,14 @@ func (m *Module) Work(ctx context.Context) {
|
||||
var listeners sync.WaitGroup
|
||||
defer listeners.Wait()
|
||||
listening := map[string]bool{}
|
||||
nextCheck := time.Time{}
|
||||
nextCheck, nextTick := time.Time{}, time.Time{}
|
||||
for ctx.Err() == nil {
|
||||
if now := time.Now(); now.After(nextTick) {
|
||||
if err := m.tick(ctx, now); err != nil && ctx.Err() == nil {
|
||||
m.log.ErrorContext(ctx, "fire schedules", "error", err)
|
||||
}
|
||||
nextTick = now.Add(scheduleCheck)
|
||||
}
|
||||
if time.Now().After(nextCheck) {
|
||||
m.startListeners(ctx, &listeners, listening)
|
||||
nextCheck = time.Now().Add(listenRetry)
|
||||
@@ -342,6 +349,9 @@ type triggerRequest struct {
|
||||
Filter map[string]string `json:"filter"`
|
||||
Workflow string `json:"workflow"`
|
||||
Disabled bool `json:"disabled"`
|
||||
// Every makes the trigger fire on an interval ("1h") instead of on
|
||||
// events; source and event_type are then fixed to the schedule's own.
|
||||
Every string `json:"every"`
|
||||
}
|
||||
|
||||
func (m *Module) putTrigger(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -351,8 +361,21 @@ func (m *Module) putTrigger(w http.ResponseWriter, r *http.Request) {
|
||||
httpjson.Error(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
var every sql.NullInt64
|
||||
if req.Every != "" {
|
||||
d, err := time.ParseDuration(req.Every)
|
||||
if err != nil || d < minInterval || d%time.Second != 0 {
|
||||
httpjson.Error(w, http.StatusBadRequest, "every is a whole number of seconds of at least one minute, such as 1h")
|
||||
return
|
||||
}
|
||||
every = sql.NullInt64{Int64: int64(d / time.Second), Valid: true}
|
||||
req.Source, req.EventType = scheduleSource, scheduleType
|
||||
// Every schedule's ticks share a source and type, so the filter keeps
|
||||
// this trigger to its own.
|
||||
req.Filter = map[string]string{"trigger": name}
|
||||
}
|
||||
if !validName.MatchString(name) || req.Source == "" || req.EventType == "" || req.Workflow == "" {
|
||||
httpjson.Error(w, http.StatusBadRequest, "a trigger needs a lowercase name, a source, an event type and a workflow")
|
||||
httpjson.Error(w, http.StatusBadRequest, "a trigger needs a lowercase name, a workflow, and either a source and event type or an every interval")
|
||||
return
|
||||
}
|
||||
filter, err := json.Marshal(req.Filter)
|
||||
@@ -362,7 +385,7 @@ func (m *Module) putTrigger(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
if err := m.store.UpsertTrigger(r.Context(), store.Trigger{
|
||||
OrgID: m.orgID, Name: name, Source: req.Source, EventType: req.EventType,
|
||||
Filter: filter, Workflow: req.Workflow, Enabled: !req.Disabled,
|
||||
Filter: filter, Workflow: req.Workflow, Enabled: !req.Disabled, ScheduleEverySeconds: every,
|
||||
}); err != nil {
|
||||
httpjson.Error(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
|
||||
@@ -54,6 +54,11 @@ type Trigger struct {
|
||||
Filter jsontext.Value `db:"filter"`
|
||||
Workflow string `db:"workflow"`
|
||||
Enabled bool `db:"enabled"`
|
||||
// ScheduleEverySeconds is the interval a scheduled trigger fires on; it
|
||||
// is null for a trigger that fires on events.
|
||||
ScheduleEverySeconds sql.NullInt64 `db:"schedule_every_s"`
|
||||
// ScheduleSince is when the schedule began or last changed.
|
||||
ScheduleSince sql.NullTime `db:"schedule_since"`
|
||||
}
|
||||
|
||||
// Store is the events module's Postgres store.
|
||||
@@ -212,16 +217,36 @@ func RecordDispatch(ctx context.Context, tx *sqlx.Tx, eventID uuid.UUID, trigger
|
||||
// UpsertTrigger creates or replaces a trigger.
|
||||
func (s *Store) UpsertTrigger(ctx context.Context, t Trigger) error {
|
||||
if _, err := s.db.NamedExecContext(ctx, `
|
||||
INSERT INTO events.triggers (org_id, name, source, event_type, filter, workflow, enabled)
|
||||
VALUES (:org_id, :name, :source, :event_type, :filter, :workflow, :enabled)
|
||||
INSERT INTO events.triggers (org_id, name, source, event_type, filter, workflow, enabled,
|
||||
schedule_every_s, schedule_since)
|
||||
VALUES (:org_id, :name, :source, :event_type, :filter, :workflow, :enabled,
|
||||
CAST(:schedule_every_s AS integer), CASE WHEN CAST(:schedule_every_s AS integer) IS NULL THEN NULL ELSE now() END)
|
||||
ON CONFLICT (org_id, name) DO UPDATE
|
||||
SET source = EXCLUDED.source, event_type = EXCLUDED.event_type, filter = EXCLUDED.filter,
|
||||
workflow = EXCLUDED.workflow, enabled = EXCLUDED.enabled, updated_at = now()`, t); err != nil {
|
||||
workflow = EXCLUDED.workflow, enabled = EXCLUDED.enabled,
|
||||
schedule_since = CASE
|
||||
WHEN EXCLUDED.schedule_every_s IS NULL THEN NULL
|
||||
WHEN events.triggers.schedule_every_s IS NOT DISTINCT FROM EXCLUDED.schedule_every_s
|
||||
THEN events.triggers.schedule_since
|
||||
ELSE now() END,
|
||||
schedule_every_s = EXCLUDED.schedule_every_s, updated_at = now()`, t); err != nil {
|
||||
return fmt.Errorf("save trigger: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ScheduledTriggers returns the enabled triggers that fire on an interval.
|
||||
func (s *Store) ScheduledTriggers(ctx context.Context, orgID string) ([]Trigger, error) {
|
||||
var out []Trigger
|
||||
if err := s.db.SelectContext(ctx, &out, `
|
||||
SELECT org_id, name, source, event_type, filter, workflow, enabled, schedule_every_s, schedule_since
|
||||
FROM events.triggers
|
||||
WHERE org_id = $1 AND enabled AND schedule_every_s IS NOT NULL ORDER BY name`, orgID); err != nil {
|
||||
return nil, fmt.Errorf("read schedules: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Recent returns an organisation's newest events.
|
||||
func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Event, error) {
|
||||
var out []Event
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json/jsontext"
|
||||
json "encoding/json/v2"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/samcharles93/archie-core/modules/events/internal/store"
|
||||
)
|
||||
|
||||
// The source and type of the event a schedule stores for each slot.
|
||||
const (
|
||||
scheduleSource = "schedule"
|
||||
scheduleType = "schedule.tick"
|
||||
)
|
||||
|
||||
// minInterval keeps a schedule from firing faster than the dispatcher can use.
|
||||
const minInterval = time.Minute
|
||||
|
||||
// scheduleCheck is how often Work looks for due slots.
|
||||
const scheduleCheck = 10 * time.Second
|
||||
|
||||
// dueSlot returns the start of the latest slot of length every that began
|
||||
// after since and not after now. Slots are aligned to the clock, so every
|
||||
// process computes the same ones; slots missed while the process was down
|
||||
// collapse into this one.
|
||||
func dueSlot(now, since time.Time, every time.Duration) (time.Time, bool) {
|
||||
slot := now.Truncate(every)
|
||||
return slot, slot.After(since)
|
||||
}
|
||||
|
||||
// tick stores the event for each schedule whose slot is due. The slot's start
|
||||
// is part of the source event id, so a restart or a second process storing the
|
||||
// same slot conflicts instead of firing twice.
|
||||
func (m *Module) tick(ctx context.Context, now time.Time) error {
|
||||
triggers, err := m.store.ScheduledTriggers(ctx, m.orgID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, t := range triggers {
|
||||
slot, due := dueSlot(now, t.ScheduleSince.Time, time.Duration(t.ScheduleEverySeconds.Int64)*time.Second)
|
||||
if !due {
|
||||
continue
|
||||
}
|
||||
fields, err := json.Marshal(map[string]string{"trigger": t.Name, "slot": slot.UTC().Format(time.RFC3339)})
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode tick: %w", err)
|
||||
}
|
||||
isNew, err := m.store.Insert(ctx, tickEvent(m.orgID, t.Name, slot, fields))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isNew {
|
||||
m.log.InfoContext(ctx, "schedule fired", "trigger", t.Name, "slot", slot)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func tickEvent(orgID, trigger string, slot time.Time, fields jsontext.Value) store.Event {
|
||||
e := store.Event{
|
||||
ID: uuid.New(), OrgID: orgID, Source: scheduleSource, Type: scheduleType,
|
||||
SourceEventID: trigger + "@" + slot.UTC().Format(time.RFC3339), Fields: fields,
|
||||
}
|
||||
e.OccurredAt.Time, e.OccurredAt.Valid = slot, true
|
||||
return e
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestDueSlot guards the rule that a schedule fires once per slot: a slot
|
||||
// missed while down collapses into one firing, and a new schedule doesn't fire
|
||||
// the slot it was created in.
|
||||
func TestDueSlot(t *testing.T) {
|
||||
at := func(s string) time.Time {
|
||||
v, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
now string
|
||||
since string
|
||||
every time.Duration
|
||||
want string
|
||||
wantFires bool
|
||||
}{
|
||||
{"created mid slot", "2026-01-01T10:45:00Z", "2026-01-01T10:30:00Z", time.Hour, "2026-01-01T10:00:00Z", false},
|
||||
{"next slot begins", "2026-01-01T11:00:00Z", "2026-01-01T10:30:00Z", time.Hour, "2026-01-01T11:00:00Z", true},
|
||||
{"down for hours fires the latest slot once", "2026-01-01T14:20:00Z", "2026-01-01T10:30:00Z", time.Hour, "2026-01-01T14:00:00Z", true},
|
||||
{"same slot again is the same slot", "2026-01-01T14:59:00Z", "2026-01-01T10:30:00Z", time.Hour, "2026-01-01T14:00:00Z", true},
|
||||
{"minute interval", "2026-01-01T10:31:10Z", "2026-01-01T10:30:20Z", time.Minute, "2026-01-01T10:31:00Z", true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
slot, fires := dueSlot(at(tt.now), at(tt.since), tt.every)
|
||||
if !slot.Equal(at(tt.want)) || fires != tt.wantFires {
|
||||
t.Errorf("dueSlot = %v, %v; want %v, %v", slot, fires, tt.want, tt.wantFires)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user