feat(runs): authorize runs, deliveries and intake as the workload principal (#2078)

Trigger dispatch, run start, every run claim, every delivery attempt and
webhook intake now ask the policy chain. Background work acts as the
workload principal; a refusal is final and recorded on the run or the
delivery, and an undecidable chain leaves the work queued for another
attempt. Intake applies the per-source delivery policy after source
verification.
This commit is contained in:
sam committed 2026-11-22 02:15:05 +11:00
1 parent 6c59d6b742
commit 098d8cc328
12 files changed
+639 -42

No files matched your search

+10 -3
View File
@@ -55,11 +55,15 @@ func (o *operations) Start(ctx context.Context) error {
if err != nil {
return err
}
// Every entry point asks the same chain, and each refusal is stored as a
// denial.
checked := governance.Recording(access, o.settings, o.log.With("module", "governance"))
// A workflow takes its context from the run handle, as a handler takes it
// from the request.
o.engine.Use(runs.Deps{
Settings: o.settings, Toolbox: o.conns, Judge: o.judge, Approver: o.allow,
Notifier: o.tell, Messenger: o.tell, Memory: o.remember, Capabilities: o.conns,
Access: checked,
})
if err := installStarter(ctx, o.engine); err != nil {
return err
@@ -76,9 +80,12 @@ func (o *operations) Start(ctx context.Context) error {
// Nothing below can fail, so a retry cannot launch a connector or a loop twice.
o.checks.Register("connectors", o.conns.Ready)
// Administrative writes are checked against the same live policy chain as
// every other request.
o.settings.UseAuthorizer(access)
// Administrative writes, and every API request, are checked against the same
// live policy chain as chat and background work. The webhook intake applies
// each source's delivery policy from it.
o.settings.UseAuthorizer(checked)
o.tell.UseAccess(checked)
o.hear.UseDeliveryPolicy(access)
o.talk.Use(chat.Deps{
People: o.settings, Access: access, Settings: o.settings, Toolbox: o.conns, Judge: o.judge,
Approver: o.allow, Memory: o.remember, Runs: o.engine, Notifier: o.tell,
+107
View File
@@ -0,0 +1,107 @@
package deliveries
import (
"context"
"log/slog"
"slices"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/deliveries/internal/store"
"github.com/samcharles93/archie-core/modules/governance"
)
// sequencedAccess answers each Authorize with the next decision, the last one
// repeating, and records what was asked.
type sequencedAccess struct {
decisions []governance.Decision
asked []string
}
func (s *sequencedAccess) Authorize(_ context.Context, p governance.Principal, a governance.Action, r governance.Resource, _ governance.AccessContext) governance.Decision {
s.asked = append(s.asked, string(p.IdentityID)+" "+string(a)+" "+string(r.Kind)+" "+r.ID)
d := s.decisions[0]
if len(s.decisions) > 1 {
s.decisions = s.decisions[1:]
}
return d
}
// ledger records each attempt's outcome by kind.
type ledger struct{ outcomes []string }
func (l *ledger) Sent(context.Context, int64, int, string, time.Duration) error {
l.outcomes = append(l.outcomes, "sent")
return nil
}
func (l *ledger) Failed(_ context.Context, _ int64, _ int, reason string, _ time.Duration) error {
l.outcomes = append(l.outcomes, "failed: "+reason)
return nil
}
func (l *ledger) GaveUp(_ context.Context, _ int64, _ int, reason string, _ time.Duration) error {
l.outcomes = append(l.outcomes, "gave up: "+reason)
return nil
}
// channelLookups counts the sends a worker tried: it has no connector running,
// so a send that was attempted fails and a send that was refused never looks.
type channelLookups struct{ looked int }
func (c *channelLookups) Delivering(string) []*connectors.Connector { return nil }
func (c *channelLookups) Get(string) (*connectors.Connector, bool) {
c.looked++
return nil, false
}
// TestDeliveryIsAuthorizedOnEveryAttempt guards that a queued message is sent
// only while the workload principal may still deliver on its channel. Each
// attempt, including a retry after a backoff or a restart, asks again: a
// revoked grant ends the message as failed, and a chain that can't decide
// leaves it queued for its next attempt instead of dropping it.
func TestDeliveryIsAuthorizedOnEveryAttempt(t *testing.T) {
allowed, revoked, unavailable := governance.Allowed(), governance.DeniedAt(governance.LevelOrg, []string{"org-workload"}), governance.Unavailable()
tests := []struct {
name string
decisions []governance.Decision
unwired bool
attempts int
wantSends int
wantOutcomes []string
}{
{name: "an allowed attempt is sent", decisions: []governance.Decision{allowed}, attempts: 1, wantSends: 1, wantOutcomes: []string{"failed: connector \"slack\" isn't running"}},
{name: "a revoked grant ends the message", decisions: []governance.Decision{revoked}, attempts: 1, wantOutcomes: []string{"gave up: " + connectors.ErrRejected.Error() + ": " + governance.ErrDenied.Error() + ": deliver delivery slack"}},
{name: "a retry after revocation is refused", decisions: []governance.Decision{allowed, revoked}, attempts: 2, wantSends: 1, wantOutcomes: []string{"failed: connector \"slack\" isn't running", "gave up: " + connectors.ErrRejected.Error() + ": " + governance.ErrDenied.Error() + ": deliver delivery slack"}},
{name: "an undecidable chain leaves the message queued", decisions: []governance.Decision{unavailable}, attempts: 3},
{name: "no policy wired leaves the message queued", unwired: true, attempts: 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
access, out, channels := &sequencedAccess{decisions: tt.decisions}, &ledger{}, &channelLookups{}
m := &Module{outcomes: out, channels: channels, log: slog.New(slog.DiscardHandler)}
if !tt.unwired {
m.UseAccess(access)
}
for range tt.attempts {
m.deliver(t.Context(), store.Message{Seq: 1, ID: uuid.New(), Channel: "slack", Attempts: 1})
}
if channels.looked != tt.wantSends {
t.Fatalf("sends tried = %d, want %d", channels.looked, tt.wantSends)
}
if !slices.Equal(out.outcomes, tt.wantOutcomes) {
t.Fatalf("outcomes = %q, want %q", out.outcomes, tt.wantOutcomes)
}
for _, call := range access.asked {
if !strings.HasPrefix(call, "workload deliver delivery ") {
t.Fatalf("asked %q, want the workload principal to deliver", call)
}
}
})
}
}
+39 -7
View File
@@ -28,6 +28,7 @@ import (
"github.com/samcharles93/archie-core/internal/platform/hub"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/deliveries/internal/store"
"github.com/samcharles93/archie-core/modules/governance"
)
const (
@@ -111,16 +112,31 @@ type Channels interface {
type Module struct {
orgID string
store *store.Store
outcomes outcomes
access governance.Authorizer
hub *hub.Hub
channels Channels
log *slog.Logger
poll time.Duration
}
// outcomes records how each attempt to deliver a message ended.
type outcomes interface {
Sent(ctx context.Context, seq int64, attempt int, externalID string, took time.Duration) error
Failed(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error
GaveUp(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error
}
// UseAccess gives the module the policy chain that decides, each time a
// worker claims a message, whether it may go out. Until it is set nothing is
// delivered.
func (m *Module) UseAccess(a governance.Authorizer) { m.access = a }
// NewModule returns the deliveries module for orgID, announcing new messages
// on h and sending them on every connector in channels that delivers them.
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, h *hub.Hub, channels Channels) *Module {
return &Module{orgID: orgID, store: store.New(db), hub: h, channels: channels, log: log, poll: time.Second}
st := store.New(db)
return &Module{orgID: orgID, store: st, outcomes: st, hub: h, channels: channels, log: log, poll: time.Second}
}
// Notify delivers n on every channel. Delivering the same key again changes
@@ -231,7 +247,23 @@ func (m *Module) Work(ctx context.Context) {
}
}
// deliver sends a claimed message when the workload principal may still deliver
// on its channel. Each attempt is decided afresh, so a retry after a backoff or
// a restart honours a revocation made meanwhile. A refusal ends the message as
// failed. When the policy chain can't decide, the message stays queued for its
// next attempt.
func (m *Module) deliver(ctx context.Context, msg store.Message) {
err := governance.RequireWorkload(ctx, m.access, governance.ActionDeliver, governance.Resource{
Kind: governance.KindDelivery, ID: msg.Channel,
})
switch {
case errors.Is(err, governance.ErrDenied):
m.recordFailure(ctx, msg, fmt.Errorf("%w: %w", connectors.ErrRejected, err), 0)
return
case err != nil:
m.log.WarnContext(ctx, "delivery waits for access policies", "channel", msg.Channel, "message", msg.ID, "error", err)
return
}
start := time.Now()
external, err := m.sendTo(ctx, msg)
took := time.Since(start)
@@ -240,7 +272,7 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) {
m.recordFailure(ctx, msg, err, took)
return
}
if err := m.store.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil {
if err := m.outcomes.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil {
m.log.ErrorContext(ctx, "record delivery", "error", err)
}
}
@@ -248,13 +280,13 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) {
// recordFailure records why an attempt failed. It marks the delivery failed
// when the channel rejected it permanently or the retry limit is reached.
func (m *Module) recordFailure(ctx context.Context, msg store.Message, cause error, took time.Duration) {
record := m.store.Failed
record := m.outcomes.Failed
reason := cause.Error()
switch {
case errors.Is(cause, connectors.ErrRejected):
record = m.store.GaveUp
record = m.outcomes.GaveUp
case msg.Attempts >= maxAttempts:
record = m.store.GaveUp
record = m.outcomes.GaveUp
reason = fmt.Sprintf("gave up after %d attempts: %s", msg.Attempts, reason)
}
if err := record(ctx, msg.Seq, msg.Attempts, reason, took); err != nil {
@@ -292,11 +324,11 @@ func backoff(attempts int) time.Duration {
// Register adds the composer API.
func (m *Module) Register(api huma.API) {
tags := []string{"Composer"}
huma.Register(api, huma.Operation{
httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindDelivery), huma.Operation{
OperationID: "list-composer-messages", Method: http.MethodGet, Path: "/api/composer/messages",
Summary: "List composer messages with their delivery attempts", Tags: tags,
}, m.messages)
huma.Register(api, huma.Operation{
httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindDelivery), huma.Operation{
OperationID: "stream-composer", Method: http.MethodGet, Path: "/api/composer/stream",
Summary: "Stream the seq of each new composer message", Tags: tags,
Responses: map[string]*huma.Response{"200": {
+163
View File
@@ -0,0 +1,163 @@
package events
import (
"context"
"errors"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/internal/platform/httpjson"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/events/internal/store"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/runs"
)
// deliveryPolicy answers every delivery with one decision and records who
// asked.
type deliveryPolicy struct {
decision governance.Decision
asked []string
}
func (d *deliveryPolicy) AuthorizeDelivery(_ context.Context, org governance.OrgID, source, addr string) governance.Decision {
d.asked = append(d.asked, string(org)+" "+source+" "+addr)
return d.decision
}
// onceInserter stores an event once per source event id, as the table's
// uniqueness constraint does.
type onceInserter struct{ stored map[string]bool }
func (o *onceInserter) Insert(_ context.Context, e store.Event) (bool, error) {
if o.stored[e.SourceEventID] {
return false, nil
}
o.stored[e.SourceEventID] = true
return true, nil
}
type noEcho struct{}
func (noEcho) SentBy(context.Context, string, ...string) (uuid.UUID, bool, error) {
return uuid.Nil, false, nil
}
// TestIntakeAppliesTheDeliveryPolicyBeforeStoring guards the webhook entry
// point: a delivery that passes source verification is still refused when the
// per-source, per-address delivery policy denies it or can't be read, and
// nothing is parsed or stored then. A redelivery of a stored event id is
// counted as a duplicate and stored once, so it can't dispatch again.
func TestIntakeAppliesTheDeliveryPolicyBeforeStoring(t *testing.T) {
const token = "not-a-real-token" //nolint:gosec // fixture bearer token the verifier must accept
denied := governance.DeniedAt(governance.LevelObject, []string{"hook-allowlist"})
tests := []struct {
name string
bearer string
policy governance.Decision
unwired bool
deliveries int
want int
wantBody string
wantStored int
wantAsked []string
}{
{name: "an allowed delivery is stored", bearer: token, policy: governance.Allowed(), deliveries: 1, want: http.StatusAccepted, wantBody: `"stored":1`, wantStored: 1, wantAsked: []string{"org-sys hook 203.0.113.9"}},
{name: "a redelivered event id is counted, not stored again", bearer: token, policy: governance.Allowed(), deliveries: 2, want: http.StatusAccepted, wantBody: `"duplicates":1`, wantStored: 1, wantAsked: []string{"org-sys hook 203.0.113.9", "org-sys hook 203.0.113.9"}},
{name: "a valid token with a denying policy", bearer: token, policy: denied, deliveries: 1, want: http.StatusForbidden, wantAsked: []string{"org-sys hook 203.0.113.9"}},
{name: "a valid token while the chain is unavailable", bearer: token, policy: governance.Unavailable(), deliveries: 1, want: http.StatusServiceUnavailable, wantAsked: []string{"org-sys hook 203.0.113.9"}},
{name: "a valid token with no policy wired", bearer: token, unwired: true, deliveries: 1, want: http.StatusServiceUnavailable},
{name: "a wrong token never reaches the policy", bearer: "other", policy: governance.Allowed(), deliveries: 1, want: http.StatusUnauthorized},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
policy := &deliveryPolicy{decision: tt.policy}
inserter := &onceInserter{stored: map[string]bool{}}
m := &Module{
orgID: "default", log: slog.New(slog.DiscardHandler), inserter: inserter, deliveries: noEcho{},
reader: fakeSources{store.Source{OrgID: "default", Name: "hook", Connector: "c", Verification: store.Token, TokenSHA256: httpjson.TokenHash(token)}},
parse: func(context.Context, string, []byte, map[string]string) ([]connectors.Event, error) {
return []connectors.Event{{Type: "alert", SourceEventID: "evt-1"}}, nil
},
}
if !tt.unwired {
m.UseDeliveryPolicy(policy)
}
var rec *httptest.ResponseRecorder
for range tt.deliveries {
req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/hooks/hook", strings.NewReader(`{}`))
req.SetPathValue("source", "hook")
req.RemoteAddr = "203.0.113.9:4711"
req.Header.Set("Authorization", "Bearer "+tt.bearer)
rec = httptest.NewRecorder()
m.intake(rec, req)
}
body, _ := io.ReadAll(rec.Body)
if rec.Code != tt.want || !strings.Contains(string(body), tt.wantBody) {
t.Fatalf("status %d body %s, want %d containing %q", rec.Code, body, tt.want, tt.wantBody)
}
if len(inserter.stored) != tt.wantStored {
t.Fatalf("events stored = %d, want %d", len(inserter.stored), tt.wantStored)
}
if !slices.Equal(policy.asked, tt.wantAsked) {
t.Fatalf("delivery policy asked %v, want %v", policy.asked, tt.wantAsked)
}
})
}
}
// startRecorder is the run starter a trigger calls: it records the principal
// each start carried and answers with a fixed error.
type startRecorder struct {
err error
principals []governance.IdentityID
}
func (s *startRecorder) Start(ctx context.Context, _ runs.StartRequest) (uuid.UUID, error) {
p, _ := governance.PrincipalFromContext(ctx)
s.principals = append(s.principals, p.IdentityID)
return uuid.New(), s.err
}
func (*startRecorder) CheckTrigger(context.Context, string, string) error { return nil }
// TestTriggersStartRunsAsTheWorkload guards that dispatching an event never
// starts a run without a principal: the workload principal acts, a refusal
// skips the trigger for good, and a chain that can't decide fails the dispatch
// so the event stays queued instead of being dropped.
func TestTriggersStartRunsAsTheWorkload(t *testing.T) {
tests := []struct {
name string
err error
wantStarted bool
wantErr error
}{
{name: "an allowed start", wantStarted: true},
{name: "a refused start is final", err: governance.ErrDenied},
{name: "an undecidable start is retried", err: governance.ErrChainUnavailable, wantErr: governance.ErrChainUnavailable},
{name: "another failure is retried", err: errors.New("database down"), wantErr: errors.New("database down")},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
starts := &startRecorder{err: tt.err}
m := &Module{runs: starts, log: slog.New(slog.DiscardHandler)}
_, started, err := m.startRun(t.Context(), store.Event{ID: uuid.New(), OrgID: "default"}, store.Trigger{Name: "t", Workflow: "triage"})
if started != tt.wantStarted {
t.Fatalf("started = %v, want %v", started, tt.wantStarted)
}
if (err == nil) != (tt.wantErr == nil) || (tt.wantErr != nil && !strings.Contains(err.Error(), tt.wantErr.Error())) {
t.Fatalf("error = %v, want %v", err, tt.wantErr)
}
if !slices.Equal(starts.principals, []governance.IdentityID{governance.WorkloadID}) {
t.Fatalf("start acted as %v, want the workload principal", starts.principals)
}
})
}
}
+94 -18
View File
@@ -16,6 +16,7 @@ import (
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"regexp"
"sync"
@@ -28,6 +29,7 @@ import (
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/events/internal/match"
"github.com/samcharles93/archie-core/modules/events/internal/store"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/runs"
)
@@ -71,6 +73,8 @@ type Module struct {
orgID string
store *store.Store
reader sourceReader
inserter eventInserter
delivery governance.DeliveryAuthorizer
sealer Sealer
parse func(ctx context.Context, connector string, body []byte, headers map[string]string) ([]connectors.Event, error)
connectors Connectors
@@ -84,7 +88,7 @@ type Module struct {
// NewModule returns the events module for orgID.
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, sealer Sealer, c Connectors, r Runs, a Answers, d Deliveries) *Module {
st := store.New(db)
m := &Module{orgID: orgID, store: st, reader: st, sealer: sealer, connectors: c, runs: r, answers: a, deliveries: d, log: log, poll: time.Second}
m := &Module{orgID: orgID, store: st, reader: st, inserter: st, sealer: sealer, connectors: c, runs: r, answers: a, deliveries: d, log: log, poll: time.Second}
m.parse = m.parseWith
return m
}
@@ -94,15 +98,26 @@ type sourceReader interface {
Source(ctx context.Context, orgID, name string) (store.Source, error)
}
// eventInserter stores an event, and reports false when its source event id
// was stored before.
type eventInserter interface {
Insert(ctx context.Context, e store.Event) (isNew bool, err error)
}
// UseDeliveryPolicy gives intake the policy that decides, per source and
// sender address, who may deliver. Until it is set every delivery is refused.
func (m *Module) UseDeliveryPolicy(d governance.DeliveryAuthorizer) { m.delivery = d }
// Routes adds the webhook intake. It stays outside the API because each
// source verifies its own deliveries.
func (m *Module) Routes(mux *http.ServeMux) {
mux.HandleFunc("POST /hooks/{source}", m.intake)
}
// intake verifies a delivery, parses it with the source's connector and stores
// each event before responding. A redelivered event violates a uniqueness
// constraint and is counted as a duplicate.
// intake verifies a delivery, checks the source's delivery policy, parses it
// with the source's connector and stores each event before responding. A
// redelivered event violates a uniqueness constraint, is counted as a
// duplicate and is not dispatched again.
func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
src, parsed, body, status, err := m.accept(r)
if err != nil {
@@ -151,6 +166,9 @@ func (m *Module) accept(r *http.Request) (src store.Source, parsed []connectors.
return m.refuse(ctx, src, err)
}
}
if status, err := m.deliveryAllowed(ctx, src, r); err != nil {
return src, nil, nil, status, err
}
parsed, err = m.parse(ctx, src.Connector, body, map[string]string{"Content-Type": r.Header.Get("Content-Type")})
if errors.Is(err, errConnectorDown) {
return src, nil, nil, http.StatusServiceUnavailable, err
@@ -176,6 +194,36 @@ func (m *Module) refuse(ctx context.Context, src store.Source, err error) (store
return src, nil, nil, http.StatusServiceUnavailable, errors.New("couldn't verify the delivery")
}
var (
errDeliveryDenied = errors.New("delivery not permitted")
errDeliveryUnavailable = errors.New("delivery policy unavailable")
)
// deliveryAllowed applies the source's delivery policy to the sender's address.
// A verified delivery still needs it. When the policy chain can't decide the
// delivery is refused with 503, so the sender retries and nothing is stored.
func (m *Module) deliveryAllowed(ctx context.Context, src store.Source, r *http.Request) (int, error) {
if m.delivery == nil {
return http.StatusServiceUnavailable, errDeliveryUnavailable
}
addr, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
addr = r.RemoteAddr
}
// Delivery policies belong to the operating organisation, whatever the
// source's own org id says.
d := m.delivery.AuthorizeDelivery(ctx, governance.DefaultOrgID, src.Name, addr)
switch {
case d.Allowed:
return 0, nil
case errors.Is(d.Err, governance.ErrChainUnavailable):
m.log.WarnContext(ctx, "delivery policy unavailable", "source", src.Name)
return http.StatusServiceUnavailable, errDeliveryUnavailable
}
m.log.WarnContext(ctx, "refused webhook delivery", "source", src.Name, "reason", "delivery policy", "level", d.Level, "policies", d.Policies)
return http.StatusForbidden, errDeliveryDenied
}
// parseWith has a running connector parse a delivery.
func (m *Module) parseWith(ctx context.Context, connector string, body []byte, headers map[string]string) ([]connectors.Event, error) {
conn, ok := m.connectors.Get(connector)
@@ -193,7 +241,7 @@ func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors
if err != nil {
return stored, duplicates, fmt.Errorf("check delivery provenance: %w", err)
}
isNew, err := m.store.Insert(ctx, newRow(m.orgID, src, e, body, own, run))
isNew, err := m.inserter.Insert(ctx, newRow(m.orgID, src, e, body, own, run))
if err != nil {
return stored, duplicates, err
}
@@ -332,7 +380,7 @@ func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) {
if err != nil {
return false, err
}
matched := false
matched, refused := false, false
for _, t := range triggers {
var filter map[string]string
if err := json.Unmarshal(t.Filter, &filter); err != nil {
@@ -341,11 +389,13 @@ func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) {
if !match.Fields(e.Fields, filter) {
continue
}
runID, err := m.runs.Start(ctx, runs.StartRequest{
OrgID: e.OrgID, Workflow: t.Workflow, Key: e.ID.String() + ":" + t.Name, Input: runInput(e),
})
runID, started, err := m.startRun(ctx, e, t)
if err != nil {
return false, fmt.Errorf("start run for trigger %q: %w", t.Name, err)
return false, err
}
if !started {
refused = true
continue
}
if err := store.RecordDispatch(ctx, tx, e.ID, t.Name, runID); err != nil {
return false, err
@@ -353,18 +403,44 @@ func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) {
m.log.InfoContext(ctx, "trigger started run", "event", e.ID, "trigger", t.Name, "run", runID)
matched = true
}
if !matched {
created, err := store.NoteUnmatched(ctx, tx, e)
if err != nil {
return false, err
}
if created {
m.log.InfoContext(ctx, "unmatched events started a group", "source", e.Source, "type", e.Type)
}
if !matched && !refused {
return false, m.noteUnmatched(ctx, tx, e)
}
return matched, nil
}
// noteUnmatched files an event no trigger took into a group for review.
func (m *Module) noteUnmatched(ctx context.Context, tx *sqlx.Tx, e store.Event) error {
created, err := store.NoteUnmatched(ctx, tx, e)
if err != nil {
return err
}
if created {
m.log.InfoContext(ctx, "unmatched events started a group", "source", e.Source, "type", e.Type)
}
return nil
}
// startRun starts the run trigger t asks for, as the workload principal, so
// the start is decided by the same policy chain as any other caller. A refusal
// is final: it reports no run and no error, and the trigger is skipped. A policy
// chain that can't decide returns an error instead, which rolls the dispatch
// back so the event stays queued and is dispatched again.
func (m *Module) startRun(ctx context.Context, e store.Event, t store.Trigger) (runID uuid.UUID, started bool, err error) {
ctx = governance.WithPrincipal(ctx, governance.WorkloadPrincipal(governance.DefaultOrgID))
runID, err = m.runs.Start(ctx, runs.StartRequest{
OrgID: e.OrgID, Workflow: t.Workflow, Key: e.ID.String() + ":" + t.Name, Input: runInput(e),
})
if errors.Is(err, governance.ErrDenied) {
m.log.WarnContext(ctx, "trigger not permitted to start its workflow", "event", e.ID, "trigger", t.Name, "error", err)
return uuid.Nil, false, nil
}
if err != nil {
return uuid.Nil, false, fmt.Errorf("start run for trigger %q: %w", t.Name, err)
}
return runID, true, nil
}
// Origin returns the event and trigger that started a run.
func (m *Module) Origin(ctx context.Context, runID uuid.UUID) (runs.Origin, bool, error) {
o, ok, err := m.store.OriginOf(ctx, m.orgID, runID)
+1 -1
View File
@@ -50,7 +50,7 @@ func (m *Module) tick(ctx context.Context, now time.Time) error {
if err != nil {
return fmt.Errorf("encode tick: %w", err)
}
isNew, err := m.store.Insert(ctx, tickEvent(m.orgID, t.Name, slot, fields))
isNew, err := m.inserter.Insert(ctx, tickEvent(m.orgID, t.Name, slot, fields))
if err != nil {
return err
}
+9 -1
View File
@@ -16,6 +16,7 @@ import (
"github.com/samcharles93/archie-core/internal/platform/seal"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/events/internal/store"
"github.com/samcharles93/archie-core/modules/governance"
)
type fakeSources struct{ src store.Source }
@@ -71,7 +72,7 @@ func TestHMACSourceVerifiesBeforeParsing(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
parsed := false
m := &Module{
orgID: "org", reader: fakeSources{src}, sealer: sealer,
orgID: "org", reader: fakeSources{src}, sealer: sealer, delivery: permitAll{},
log: slog.New(slog.DiscardHandler),
parse: func(context.Context, string, []byte, map[string]string) ([]connectors.Event, error) {
parsed = true
@@ -93,3 +94,10 @@ func TestHMACSourceVerifiesBeforeParsing(t *testing.T) {
})
}
}
// permitAll accepts a delivery from any address.
type permitAll struct{}
func (permitAll) AuthorizeDelivery(context.Context, governance.OrgID, string, string) governance.Decision {
return governance.Allowed()
}
+133
View File
@@ -0,0 +1,133 @@
package runs
import (
"context"
"encoding/json/jsontext"
"errors"
"log/slog"
"slices"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
// asked is one decision the engine requested.
type asked struct {
principal governance.IdentityID
action governance.Action
kind governance.ResourceKind
id string
}
// recordingAccess answers every Authorize with one fixed decision and records
// what was asked.
type recordingAccess struct {
decision governance.Decision
calls []asked
}
func (r *recordingAccess) Authorize(_ context.Context, p governance.Principal, a governance.Action, res governance.Resource, _ governance.AccessContext) governance.Decision {
r.calls = append(r.calls, asked{p.IdentityID, a, res.Kind, res.ID})
return r.decision
}
// queueingStore holds the runs the engine queues.
type queueingStore struct {
fakeWork
queued []store.NewRun
}
func (q *queueingStore) Start(_ context.Context, n store.NewRun) (uuid.UUID, error) {
q.queued = append(q.queued, n)
return uuid.New(), nil
}
// claimsOne hands out one queued run to the worker, as the store does after a
// restart or a retry.
type claimsOne struct {
*queueingStore
run store.Run
}
func (c claimsOne) Claim(context.Context, string, uuid.UUID, time.Duration) (store.Run, bool, error) {
return c.run, true, nil
}
// TestRunsAreAuthorizedAtStartAndAtEveryClaim guards that a run started by a
// trigger and the same run claimed again after a restart or a retry are decided
// by the same live check, as the workload principal, and that a refusal stops
// the work. A policy chain that can't decide starts nothing, finishes nothing
// and leaves the run for another attempt. Without it a revoked grant would
// keep running queued work.
func TestRunsAreAuthorizedAtStartAndAtEveryClaim(t *testing.T) {
workload := asked{governance.WorkloadID, governance.ActionRun, governance.KindWorkflow, "triage"}
tests := []struct {
name string
decision governance.Decision
noPrincipal bool
claim bool
wantCalls []asked
wantQueued int
wantRan bool
wantFinished []string
wantErr error
}{
{name: "a trigger starts its run", decision: governance.Allowed(), wantCalls: []asked{workload}, wantQueued: 1},
{name: "a start with no principal", decision: governance.Allowed(), noPrincipal: true, wantErr: governance.ErrDenied},
{name: "a revoked start", decision: governance.DeniedAt(governance.LevelOrg, []string{"org-workload"}), wantCalls: []asked{workload}, wantErr: governance.ErrDenied},
{name: "a start the chain can't decide", decision: governance.Unavailable(), wantCalls: []asked{workload}, wantErr: governance.ErrChainUnavailable},
{name: "a claim after a restart", decision: governance.Allowed(), claim: true, wantCalls: []asked{workload}, wantRan: true, wantFinished: []string{store.StateSucceeded}},
{name: "a claim after revocation", decision: governance.DeniedAt(governance.LevelOrg, []string{"org-workload"}), claim: true, wantCalls: []asked{workload}, wantFinished: []string{store.StateFailed}},
{name: "a claim the chain can't decide", decision: governance.Unavailable(), claim: true, wantCalls: []asked{workload}, wantErr: governance.ErrChainUnavailable},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
work := &queueingStore{}
access := &recordingAccess{decision: tt.decision}
engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{})
engine.queue = work
engine.Use(Deps{Access: access})
ran := false
useWorkflow(engine, func(*Context, jsontext.Value) (jsontext.Value, error) {
ran = true
return jsontext.Value(`"done"`), nil
})
var err error
if tt.claim {
engine.work = claimsOne{work, store.Run{ID: uuid.New(), OrgID: "org", Workflow: "triage", Version: 1, Input: jsontext.Value(`null`)}}
_, err = engine.claimAndExecute(t.Context())
} else {
ctx := t.Context()
if !tt.noPrincipal {
ctx = governance.WithPrincipal(ctx, governance.WorkloadPrincipal(governance.DefaultOrgID))
}
_, err = engine.Start(ctx, StartRequest{OrgID: "org", Workflow: "triage", Key: "event:trigger"})
}
if !errors.Is(err, tt.wantErr) {
t.Fatalf("error = %v, want %v", err, tt.wantErr)
}
if !slices.Equal(access.calls, tt.wantCalls) {
t.Fatalf("decisions asked = %v, want %v", access.calls, tt.wantCalls)
}
if len(work.queued) != tt.wantQueued {
t.Fatalf("runs queued = %d, want %d", len(work.queued), tt.wantQueued)
}
if ran != tt.wantRan {
t.Fatalf("workflow ran = %v, want %v", ran, tt.wantRan)
}
if !slices.Equal(work.finished, tt.wantFinished) {
t.Fatalf("finished = %v, want %v", work.finished, tt.wantFinished)
}
if slices.Contains(tt.wantFinished, store.StateFailed) && !strings.Contains(work.errors[0], "not permitted") {
t.Fatalf("the run's trail says %q, want the refusal", work.errors[0])
}
})
}
}
+4 -1
View File
@@ -73,6 +73,9 @@ type DryRequest struct {
// StartDry queues a dry run of a document and returns its run's id. A document
// that doesn't validate starts nothing.
func (e *Engine) StartDry(ctx context.Context, req DryRequest) (uuid.UUID, error) {
if err := e.mayStart(ctx, req.Workflow); err != nil {
return uuid.Nil, err
}
doc, err := e.validateDocument(ctx, req.Document)
if err != nil {
return uuid.Nil, err
@@ -93,7 +96,7 @@ func (e *Engine) StartDry(ctx context.Context, req DryRequest) (uuid.UUID, error
if err := doc.checkInputs(decodeInputs(input)); err != nil {
return uuid.Nil, fmt.Errorf("workflow %q: %w", req.Workflow, err)
}
return e.db.Start(ctx, store.NewRun{
return e.queue.Start(ctx, store.NewRun{
OrgID: req.OrgID, Workflow: req.Workflow, Input: input, Dry: true, Overlay: overlay,
})
}
+4 -1
View File
@@ -10,6 +10,7 @@ import (
"github.com/google/uuid"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/judgement"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
@@ -48,6 +49,7 @@ func (anyBox) Tool(string) (connectors.Tool, bool) { return nil, true }
// is queued.
func TestStartDryRefusesWhatCannotStart(t *testing.T) {
engine := newEngine(nil, &fakeWork{}, slog.New(slog.DiscardHandler), Options{})
engine.Use(Deps{Access: &recordingAccess{decision: governance.Allowed()}})
tests := []struct {
name string
doc string
@@ -71,7 +73,8 @@ func TestStartDryRefusesWhatCannotStart(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := engine.StartDry(context.Background(), DryRequest{
ctx := governance.WithPrincipal(context.Background(), governance.SharedTokenOwner())
_, err := engine.StartDry(ctx, DryRequest{
OrgID: "org", Workflow: "draft", Document: tt.doc,
})
if err == nil || !strings.Contains(err.Error(), tt.want) {
+4
View File
@@ -103,6 +103,10 @@ type Deps struct {
// Capabilities is what saving a workflow checks connector references
// against.
Capabilities Capabilities
// Access decides whether a run may start and whether a claimed run may
// execute, for the principal acting or else the workload principal. With
// none, no run starts or executes.
Access governance.Authorizer
}
// Workflow is a document ready to run: the code that executes its steps for
+71 -10
View File
@@ -64,12 +64,14 @@ type runStarter interface {
// Engine starts, executes and resumes runs.
type Engine struct {
db *store.Store
work runWork
log *slog.Logger
opts Options
mu sync.RWMutex
deps *Deps
db *store.Store
// queue starts runs; it is db except in tests.
queue runQueue
work runWork
log *slog.Logger
opts Options
mu sync.RWMutex
deps *Deps
// load turns a claimed run into the code that executes it.
load func(ctx context.Context, run store.Run) (Workflow, error)
}
@@ -93,7 +95,7 @@ func newEngine(db *store.Store, work runWork, log *slog.Logger, opts Options) *E
if opts.Owner == "" {
opts.Owner = "archie"
}
e := &Engine{db: db, work: work, log: log, opts: opts}
e := &Engine{db: db, queue: db, work: work, log: log, opts: opts}
e.load = e.loadDocument
return e
}
@@ -185,14 +187,42 @@ func (e *Engine) Has(ctx context.Context, orgID, name string) (bool, error) {
return err == nil, err
}
// runQueue reads the workflow a run starts under and queues the run.
type runQueue interface {
workflowFinder
Start(ctx context.Context, n store.NewRun) (uuid.UUID, error)
}
// Start queues a run under the latest stored version of its workflow and
// returns its id.
// returns its id. The principal in ctx must be allowed to run the workflow;
// work that starts runs on its own, such as a trigger, passes the workload
// principal. A start the policy chain can't decide queues nothing, and the
// error wraps governance.ErrChainUnavailable so the caller tries again.
func (e *Engine) Start(ctx context.Context, req StartRequest) (uuid.UUID, error) {
n, err := newRun(ctx, e.db, req)
if err := e.mayStart(ctx, req.Workflow); err != nil {
return uuid.Nil, err
}
n, err := newRun(ctx, e.queue, req)
if err != nil {
return uuid.Nil, err
}
return e.db.Start(ctx, n)
return e.queue.Start(ctx, n)
}
// mayStart refuses a start by a caller who may not run workflow, or whose
// right can't be checked.
func (e *Engine) mayStart(ctx context.Context, workflow string) error {
p, ok := governance.PrincipalFromContext(ctx)
if !ok {
return fmt.Errorf("%w: no principal starts the run", governance.ErrDenied)
}
deps, err := e.steps()
if err != nil {
return fmt.Errorf("%w: %w", governance.ErrChainUnavailable, err)
}
return governance.Require(ctx, deps.Access, p, governance.ActionRun, governance.Resource{
Kind: governance.KindWorkflow, ID: workflow, Org: p.Org,
})
}
// workflowFinder reads stored workflow versions.
@@ -320,10 +350,41 @@ func (e *Engine) claimAndExecute(ctx context.Context) (bool, error) {
if err != nil || !ok {
return false, err
}
if err := e.mayExecute(ctx, run); err != nil {
return e.refuseClaim(ctx, run, token, err)
}
e.execute(ctx, run, token)
return true, nil
}
// mayExecute rechecks, each time a worker claims a run, that the workload
// principal may still run its workflow. A new run, a retry and a resume after
// a crash or a wake-up all pass through here.
func (e *Engine) mayExecute(ctx context.Context, run store.Run) error {
deps, err := e.steps()
if err != nil {
return fmt.Errorf("%w: %w", governance.ErrChainUnavailable, err)
}
return governance.RequireWorkload(ctx, deps.Access, governance.ActionRun, governance.Resource{
Kind: governance.KindWorkflow, ID: run.Workflow,
})
}
// refuseClaim handles a claimed run that may not execute. A policy chain that
// can't decide leaves the run unfinished, so its lease lapses and a worker
// claims it again. A refusal ends the run as failed, and its trail says why.
func (e *Engine) refuseClaim(ctx context.Context, run store.Run, token uuid.UUID, cause error) (bool, error) {
if !errors.Is(cause, governance.ErrDenied) {
return false, fmt.Errorf("run %s waits for access policies: %w", run.ID, cause)
}
ended := store.Ended{State: store.StateFailed, Error: cause.Error()}
if err := e.work.Finish(ctx, run.ID, token, ended); err != nil {
return false, fmt.Errorf("finish refused run %s: %w", run.ID, err)
}
e.log.WarnContext(ctx, "run refused", "run", run.ID, "workflow", run.Workflow, "error", cause)
return true, nil
}
// execute runs one claimed run to its next suspension or end. The run's
// context ends if the lease can't be renewed, so a stale worker stops.
func (e *Engine) execute(ctx context.Context, run store.Run, token uuid.UUID) {