From 098d8cc3286839cf874fdfd906c699bdd320982d Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sun, 22 Nov 2026 02:15:05 +1100 Subject: [PATCH] feat(runs): authorize runs, deliveries and intake as the workload principal (#2078) Trigger dispatch, run start, every run claim, every delivery attempt and webhook intake now ask the policy chain. Background work acts as the workload principal; a refusal is final and recorded on the run or the delivery, and an undecidable chain leaves the work queued for another attempt. Intake applies the per-source delivery policy after source verification. --- internal/app/archie/operations.go | 13 ++- modules/deliveries/access_test.go | 107 ++++++++++++++++++++ modules/deliveries/deliveries.go | 46 +++++++-- modules/events/access_test.go | 163 ++++++++++++++++++++++++++++++ modules/events/events.go | 112 ++++++++++++++++---- modules/events/schedule.go | 2 +- modules/events/verify_test.go | 10 +- modules/runs/access_test.go | 133 ++++++++++++++++++++++++ modules/runs/dry.go | 5 +- modules/runs/dry_test.go | 5 +- modules/runs/interpreter.go | 4 + modules/runs/runs.go | 81 +++++++++++++-- 12 files changed, 639 insertions(+), 42 deletions(-) create mode 100644 modules/deliveries/access_test.go create mode 100644 modules/events/access_test.go create mode 100644 modules/runs/access_test.go diff --git a/internal/app/archie/operations.go b/internal/app/archie/operations.go index 6bb080635..cfd2d1f66 100644 --- a/internal/app/archie/operations.go +++ b/internal/app/archie/operations.go @@ -55,11 +55,15 @@ func (o *operations) Start(ctx context.Context) error { if err != nil { return err } + // Every entry point asks the same chain, and each refusal is stored as a + // denial. + checked := governance.Recording(access, o.settings, o.log.With("module", "governance")) // A workflow takes its context from the run handle, as a handler takes it // from the request. o.engine.Use(runs.Deps{ Settings: o.settings, Toolbox: o.conns, Judge: o.judge, Approver: o.allow, Notifier: o.tell, Messenger: o.tell, Memory: o.remember, Capabilities: o.conns, + Access: checked, }) if err := installStarter(ctx, o.engine); err != nil { return err @@ -76,9 +80,12 @@ func (o *operations) Start(ctx context.Context) error { // Nothing below can fail, so a retry cannot launch a connector or a loop twice. o.checks.Register("connectors", o.conns.Ready) - // Administrative writes are checked against the same live policy chain as - // every other request. - o.settings.UseAuthorizer(access) + // Administrative writes, and every API request, are checked against the same + // live policy chain as chat and background work. The webhook intake applies + // each source's delivery policy from it. + o.settings.UseAuthorizer(checked) + o.tell.UseAccess(checked) + o.hear.UseDeliveryPolicy(access) o.talk.Use(chat.Deps{ People: o.settings, Access: access, Settings: o.settings, Toolbox: o.conns, Judge: o.judge, Approver: o.allow, Memory: o.remember, Runs: o.engine, Notifier: o.tell, diff --git a/modules/deliveries/access_test.go b/modules/deliveries/access_test.go new file mode 100644 index 000000000..4b7f7ef43 --- /dev/null +++ b/modules/deliveries/access_test.go @@ -0,0 +1,107 @@ +package deliveries + +import ( + "context" + "log/slog" + "slices" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/samcharles93/archie-core/modules/connectors" + "github.com/samcharles93/archie-core/modules/deliveries/internal/store" + "github.com/samcharles93/archie-core/modules/governance" +) + +// sequencedAccess answers each Authorize with the next decision, the last one +// repeating, and records what was asked. +type sequencedAccess struct { + decisions []governance.Decision + asked []string +} + +func (s *sequencedAccess) Authorize(_ context.Context, p governance.Principal, a governance.Action, r governance.Resource, _ governance.AccessContext) governance.Decision { + s.asked = append(s.asked, string(p.IdentityID)+" "+string(a)+" "+string(r.Kind)+" "+r.ID) + d := s.decisions[0] + if len(s.decisions) > 1 { + s.decisions = s.decisions[1:] + } + return d +} + +// ledger records each attempt's outcome by kind. +type ledger struct{ outcomes []string } + +func (l *ledger) Sent(context.Context, int64, int, string, time.Duration) error { + l.outcomes = append(l.outcomes, "sent") + return nil +} + +func (l *ledger) Failed(_ context.Context, _ int64, _ int, reason string, _ time.Duration) error { + l.outcomes = append(l.outcomes, "failed: "+reason) + return nil +} + +func (l *ledger) GaveUp(_ context.Context, _ int64, _ int, reason string, _ time.Duration) error { + l.outcomes = append(l.outcomes, "gave up: "+reason) + return nil +} + +// channelLookups counts the sends a worker tried: it has no connector running, +// so a send that was attempted fails and a send that was refused never looks. +type channelLookups struct{ looked int } + +func (c *channelLookups) Delivering(string) []*connectors.Connector { return nil } + +func (c *channelLookups) Get(string) (*connectors.Connector, bool) { + c.looked++ + return nil, false +} + +// TestDeliveryIsAuthorizedOnEveryAttempt guards that a queued message is sent +// only while the workload principal may still deliver on its channel. Each +// attempt, including a retry after a backoff or a restart, asks again: a +// revoked grant ends the message as failed, and a chain that can't decide +// leaves it queued for its next attempt instead of dropping it. +func TestDeliveryIsAuthorizedOnEveryAttempt(t *testing.T) { + allowed, revoked, unavailable := governance.Allowed(), governance.DeniedAt(governance.LevelOrg, []string{"org-workload"}), governance.Unavailable() + tests := []struct { + name string + decisions []governance.Decision + unwired bool + attempts int + wantSends int + wantOutcomes []string + }{ + {name: "an allowed attempt is sent", decisions: []governance.Decision{allowed}, attempts: 1, wantSends: 1, wantOutcomes: []string{"failed: connector \"slack\" isn't running"}}, + {name: "a revoked grant ends the message", decisions: []governance.Decision{revoked}, attempts: 1, wantOutcomes: []string{"gave up: " + connectors.ErrRejected.Error() + ": " + governance.ErrDenied.Error() + ": deliver delivery slack"}}, + {name: "a retry after revocation is refused", decisions: []governance.Decision{allowed, revoked}, attempts: 2, wantSends: 1, wantOutcomes: []string{"failed: connector \"slack\" isn't running", "gave up: " + connectors.ErrRejected.Error() + ": " + governance.ErrDenied.Error() + ": deliver delivery slack"}}, + {name: "an undecidable chain leaves the message queued", decisions: []governance.Decision{unavailable}, attempts: 3}, + {name: "no policy wired leaves the message queued", unwired: true, attempts: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + access, out, channels := &sequencedAccess{decisions: tt.decisions}, &ledger{}, &channelLookups{} + m := &Module{outcomes: out, channels: channels, log: slog.New(slog.DiscardHandler)} + if !tt.unwired { + m.UseAccess(access) + } + for range tt.attempts { + m.deliver(t.Context(), store.Message{Seq: 1, ID: uuid.New(), Channel: "slack", Attempts: 1}) + } + if channels.looked != tt.wantSends { + t.Fatalf("sends tried = %d, want %d", channels.looked, tt.wantSends) + } + if !slices.Equal(out.outcomes, tt.wantOutcomes) { + t.Fatalf("outcomes = %q, want %q", out.outcomes, tt.wantOutcomes) + } + for _, call := range access.asked { + if !strings.HasPrefix(call, "workload deliver delivery ") { + t.Fatalf("asked %q, want the workload principal to deliver", call) + } + } + }) + } +} diff --git a/modules/deliveries/deliveries.go b/modules/deliveries/deliveries.go index 79f6edfc4..aeca627e3 100644 --- a/modules/deliveries/deliveries.go +++ b/modules/deliveries/deliveries.go @@ -28,6 +28,7 @@ import ( "github.com/samcharles93/archie-core/internal/platform/hub" "github.com/samcharles93/archie-core/modules/connectors" "github.com/samcharles93/archie-core/modules/deliveries/internal/store" + "github.com/samcharles93/archie-core/modules/governance" ) const ( @@ -111,16 +112,31 @@ type Channels interface { type Module struct { orgID string store *store.Store + outcomes outcomes + access governance.Authorizer hub *hub.Hub channels Channels log *slog.Logger poll time.Duration } +// outcomes records how each attempt to deliver a message ended. +type outcomes interface { + Sent(ctx context.Context, seq int64, attempt int, externalID string, took time.Duration) error + Failed(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error + GaveUp(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error +} + +// UseAccess gives the module the policy chain that decides, each time a +// worker claims a message, whether it may go out. Until it is set nothing is +// delivered. +func (m *Module) UseAccess(a governance.Authorizer) { m.access = a } + // NewModule returns the deliveries module for orgID, announcing new messages // on h and sending them on every connector in channels that delivers them. func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, h *hub.Hub, channels Channels) *Module { - return &Module{orgID: orgID, store: store.New(db), hub: h, channels: channels, log: log, poll: time.Second} + st := store.New(db) + return &Module{orgID: orgID, store: st, outcomes: st, hub: h, channels: channels, log: log, poll: time.Second} } // Notify delivers n on every channel. Delivering the same key again changes @@ -231,7 +247,23 @@ func (m *Module) Work(ctx context.Context) { } } +// deliver sends a claimed message when the workload principal may still deliver +// on its channel. Each attempt is decided afresh, so a retry after a backoff or +// a restart honours a revocation made meanwhile. A refusal ends the message as +// failed. When the policy chain can't decide, the message stays queued for its +// next attempt. func (m *Module) deliver(ctx context.Context, msg store.Message) { + err := governance.RequireWorkload(ctx, m.access, governance.ActionDeliver, governance.Resource{ + Kind: governance.KindDelivery, ID: msg.Channel, + }) + switch { + case errors.Is(err, governance.ErrDenied): + m.recordFailure(ctx, msg, fmt.Errorf("%w: %w", connectors.ErrRejected, err), 0) + return + case err != nil: + m.log.WarnContext(ctx, "delivery waits for access policies", "channel", msg.Channel, "message", msg.ID, "error", err) + return + } start := time.Now() external, err := m.sendTo(ctx, msg) took := time.Since(start) @@ -240,7 +272,7 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) { m.recordFailure(ctx, msg, err, took) return } - if err := m.store.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil { + if err := m.outcomes.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil { m.log.ErrorContext(ctx, "record delivery", "error", err) } } @@ -248,13 +280,13 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) { // recordFailure records why an attempt failed. It marks the delivery failed // when the channel rejected it permanently or the retry limit is reached. func (m *Module) recordFailure(ctx context.Context, msg store.Message, cause error, took time.Duration) { - record := m.store.Failed + record := m.outcomes.Failed reason := cause.Error() switch { case errors.Is(cause, connectors.ErrRejected): - record = m.store.GaveUp + record = m.outcomes.GaveUp case msg.Attempts >= maxAttempts: - record = m.store.GaveUp + record = m.outcomes.GaveUp reason = fmt.Sprintf("gave up after %d attempts: %s", msg.Attempts, reason) } if err := record(ctx, msg.Seq, msg.Attempts, reason, took); err != nil { @@ -292,11 +324,11 @@ func backoff(attempts int) time.Duration { // Register adds the composer API. func (m *Module) Register(api huma.API) { tags := []string{"Composer"} - huma.Register(api, huma.Operation{ + httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindDelivery), huma.Operation{ OperationID: "list-composer-messages", Method: http.MethodGet, Path: "/api/composer/messages", Summary: "List composer messages with their delivery attempts", Tags: tags, }, m.messages) - huma.Register(api, huma.Operation{ + httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindDelivery), huma.Operation{ OperationID: "stream-composer", Method: http.MethodGet, Path: "/api/composer/stream", Summary: "Stream the seq of each new composer message", Tags: tags, Responses: map[string]*huma.Response{"200": { diff --git a/modules/events/access_test.go b/modules/events/access_test.go new file mode 100644 index 000000000..64a15466d --- /dev/null +++ b/modules/events/access_test.go @@ -0,0 +1,163 @@ +package events + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + + "github.com/google/uuid" + + "github.com/samcharles93/archie-core/internal/platform/httpjson" + "github.com/samcharles93/archie-core/modules/connectors" + "github.com/samcharles93/archie-core/modules/events/internal/store" + "github.com/samcharles93/archie-core/modules/governance" + "github.com/samcharles93/archie-core/modules/runs" +) + +// deliveryPolicy answers every delivery with one decision and records who +// asked. +type deliveryPolicy struct { + decision governance.Decision + asked []string +} + +func (d *deliveryPolicy) AuthorizeDelivery(_ context.Context, org governance.OrgID, source, addr string) governance.Decision { + d.asked = append(d.asked, string(org)+" "+source+" "+addr) + return d.decision +} + +// onceInserter stores an event once per source event id, as the table's +// uniqueness constraint does. +type onceInserter struct{ stored map[string]bool } + +func (o *onceInserter) Insert(_ context.Context, e store.Event) (bool, error) { + if o.stored[e.SourceEventID] { + return false, nil + } + o.stored[e.SourceEventID] = true + return true, nil +} + +type noEcho struct{} + +func (noEcho) SentBy(context.Context, string, ...string) (uuid.UUID, bool, error) { + return uuid.Nil, false, nil +} + +// TestIntakeAppliesTheDeliveryPolicyBeforeStoring guards the webhook entry +// point: a delivery that passes source verification is still refused when the +// per-source, per-address delivery policy denies it or can't be read, and +// nothing is parsed or stored then. A redelivery of a stored event id is +// counted as a duplicate and stored once, so it can't dispatch again. +func TestIntakeAppliesTheDeliveryPolicyBeforeStoring(t *testing.T) { + const token = "not-a-real-token" //nolint:gosec // fixture bearer token the verifier must accept + denied := governance.DeniedAt(governance.LevelObject, []string{"hook-allowlist"}) + tests := []struct { + name string + bearer string + policy governance.Decision + unwired bool + deliveries int + want int + wantBody string + wantStored int + wantAsked []string + }{ + {name: "an allowed delivery is stored", bearer: token, policy: governance.Allowed(), deliveries: 1, want: http.StatusAccepted, wantBody: `"stored":1`, wantStored: 1, wantAsked: []string{"org-sys hook 203.0.113.9"}}, + {name: "a redelivered event id is counted, not stored again", bearer: token, policy: governance.Allowed(), deliveries: 2, want: http.StatusAccepted, wantBody: `"duplicates":1`, wantStored: 1, wantAsked: []string{"org-sys hook 203.0.113.9", "org-sys hook 203.0.113.9"}}, + {name: "a valid token with a denying policy", bearer: token, policy: denied, deliveries: 1, want: http.StatusForbidden, wantAsked: []string{"org-sys hook 203.0.113.9"}}, + {name: "a valid token while the chain is unavailable", bearer: token, policy: governance.Unavailable(), deliveries: 1, want: http.StatusServiceUnavailable, wantAsked: []string{"org-sys hook 203.0.113.9"}}, + {name: "a valid token with no policy wired", bearer: token, unwired: true, deliveries: 1, want: http.StatusServiceUnavailable}, + {name: "a wrong token never reaches the policy", bearer: "other", policy: governance.Allowed(), deliveries: 1, want: http.StatusUnauthorized}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + policy := &deliveryPolicy{decision: tt.policy} + inserter := &onceInserter{stored: map[string]bool{}} + m := &Module{ + orgID: "default", log: slog.New(slog.DiscardHandler), inserter: inserter, deliveries: noEcho{}, + reader: fakeSources{store.Source{OrgID: "default", Name: "hook", Connector: "c", Verification: store.Token, TokenSHA256: httpjson.TokenHash(token)}}, + parse: func(context.Context, string, []byte, map[string]string) ([]connectors.Event, error) { + return []connectors.Event{{Type: "alert", SourceEventID: "evt-1"}}, nil + }, + } + if !tt.unwired { + m.UseDeliveryPolicy(policy) + } + var rec *httptest.ResponseRecorder + for range tt.deliveries { + req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/hooks/hook", strings.NewReader(`{}`)) + req.SetPathValue("source", "hook") + req.RemoteAddr = "203.0.113.9:4711" + req.Header.Set("Authorization", "Bearer "+tt.bearer) + rec = httptest.NewRecorder() + m.intake(rec, req) + } + body, _ := io.ReadAll(rec.Body) + if rec.Code != tt.want || !strings.Contains(string(body), tt.wantBody) { + t.Fatalf("status %d body %s, want %d containing %q", rec.Code, body, tt.want, tt.wantBody) + } + if len(inserter.stored) != tt.wantStored { + t.Fatalf("events stored = %d, want %d", len(inserter.stored), tt.wantStored) + } + if !slices.Equal(policy.asked, tt.wantAsked) { + t.Fatalf("delivery policy asked %v, want %v", policy.asked, tt.wantAsked) + } + }) + } +} + +// startRecorder is the run starter a trigger calls: it records the principal +// each start carried and answers with a fixed error. +type startRecorder struct { + err error + principals []governance.IdentityID +} + +func (s *startRecorder) Start(ctx context.Context, _ runs.StartRequest) (uuid.UUID, error) { + p, _ := governance.PrincipalFromContext(ctx) + s.principals = append(s.principals, p.IdentityID) + return uuid.New(), s.err +} + +func (*startRecorder) CheckTrigger(context.Context, string, string) error { return nil } + +// TestTriggersStartRunsAsTheWorkload guards that dispatching an event never +// starts a run without a principal: the workload principal acts, a refusal +// skips the trigger for good, and a chain that can't decide fails the dispatch +// so the event stays queued instead of being dropped. +func TestTriggersStartRunsAsTheWorkload(t *testing.T) { + tests := []struct { + name string + err error + wantStarted bool + wantErr error + }{ + {name: "an allowed start", wantStarted: true}, + {name: "a refused start is final", err: governance.ErrDenied}, + {name: "an undecidable start is retried", err: governance.ErrChainUnavailable, wantErr: governance.ErrChainUnavailable}, + {name: "another failure is retried", err: errors.New("database down"), wantErr: errors.New("database down")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + starts := &startRecorder{err: tt.err} + m := &Module{runs: starts, log: slog.New(slog.DiscardHandler)} + _, started, err := m.startRun(t.Context(), store.Event{ID: uuid.New(), OrgID: "default"}, store.Trigger{Name: "t", Workflow: "triage"}) + if started != tt.wantStarted { + t.Fatalf("started = %v, want %v", started, tt.wantStarted) + } + if (err == nil) != (tt.wantErr == nil) || (tt.wantErr != nil && !strings.Contains(err.Error(), tt.wantErr.Error())) { + t.Fatalf("error = %v, want %v", err, tt.wantErr) + } + if !slices.Equal(starts.principals, []governance.IdentityID{governance.WorkloadID}) { + t.Fatalf("start acted as %v, want the workload principal", starts.principals) + } + }) + } +} diff --git a/modules/events/events.go b/modules/events/events.go index 834cfb11f..8a07a448a 100644 --- a/modules/events/events.go +++ b/modules/events/events.go @@ -16,6 +16,7 @@ import ( "errors" "fmt" "log/slog" + "net" "net/http" "regexp" "sync" @@ -28,6 +29,7 @@ import ( "github.com/samcharles93/archie-core/modules/connectors" "github.com/samcharles93/archie-core/modules/events/internal/match" "github.com/samcharles93/archie-core/modules/events/internal/store" + "github.com/samcharles93/archie-core/modules/governance" "github.com/samcharles93/archie-core/modules/runs" ) @@ -71,6 +73,8 @@ type Module struct { orgID string store *store.Store reader sourceReader + inserter eventInserter + delivery governance.DeliveryAuthorizer sealer Sealer parse func(ctx context.Context, connector string, body []byte, headers map[string]string) ([]connectors.Event, error) connectors Connectors @@ -84,7 +88,7 @@ type Module struct { // NewModule returns the events module for orgID. func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, sealer Sealer, c Connectors, r Runs, a Answers, d Deliveries) *Module { st := store.New(db) - m := &Module{orgID: orgID, store: st, reader: st, sealer: sealer, connectors: c, runs: r, answers: a, deliveries: d, log: log, poll: time.Second} + m := &Module{orgID: orgID, store: st, reader: st, inserter: st, sealer: sealer, connectors: c, runs: r, answers: a, deliveries: d, log: log, poll: time.Second} m.parse = m.parseWith return m } @@ -94,15 +98,26 @@ type sourceReader interface { Source(ctx context.Context, orgID, name string) (store.Source, error) } +// eventInserter stores an event, and reports false when its source event id +// was stored before. +type eventInserter interface { + Insert(ctx context.Context, e store.Event) (isNew bool, err error) +} + +// UseDeliveryPolicy gives intake the policy that decides, per source and +// sender address, who may deliver. Until it is set every delivery is refused. +func (m *Module) UseDeliveryPolicy(d governance.DeliveryAuthorizer) { m.delivery = d } + // Routes adds the webhook intake. It stays outside the API because each // source verifies its own deliveries. func (m *Module) Routes(mux *http.ServeMux) { mux.HandleFunc("POST /hooks/{source}", m.intake) } -// intake verifies a delivery, parses it with the source's connector and stores -// each event before responding. A redelivered event violates a uniqueness -// constraint and is counted as a duplicate. +// intake verifies a delivery, checks the source's delivery policy, parses it +// with the source's connector and stores each event before responding. A +// redelivered event violates a uniqueness constraint, is counted as a +// duplicate and is not dispatched again. func (m *Module) intake(w http.ResponseWriter, r *http.Request) { src, parsed, body, status, err := m.accept(r) if err != nil { @@ -151,6 +166,9 @@ func (m *Module) accept(r *http.Request) (src store.Source, parsed []connectors. return m.refuse(ctx, src, err) } } + if status, err := m.deliveryAllowed(ctx, src, r); err != nil { + return src, nil, nil, status, err + } parsed, err = m.parse(ctx, src.Connector, body, map[string]string{"Content-Type": r.Header.Get("Content-Type")}) if errors.Is(err, errConnectorDown) { return src, nil, nil, http.StatusServiceUnavailable, err @@ -176,6 +194,36 @@ func (m *Module) refuse(ctx context.Context, src store.Source, err error) (store return src, nil, nil, http.StatusServiceUnavailable, errors.New("couldn't verify the delivery") } +var ( + errDeliveryDenied = errors.New("delivery not permitted") + errDeliveryUnavailable = errors.New("delivery policy unavailable") +) + +// deliveryAllowed applies the source's delivery policy to the sender's address. +// A verified delivery still needs it. When the policy chain can't decide the +// delivery is refused with 503, so the sender retries and nothing is stored. +func (m *Module) deliveryAllowed(ctx context.Context, src store.Source, r *http.Request) (int, error) { + if m.delivery == nil { + return http.StatusServiceUnavailable, errDeliveryUnavailable + } + addr, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + addr = r.RemoteAddr + } + // Delivery policies belong to the operating organisation, whatever the + // source's own org id says. + d := m.delivery.AuthorizeDelivery(ctx, governance.DefaultOrgID, src.Name, addr) + switch { + case d.Allowed: + return 0, nil + case errors.Is(d.Err, governance.ErrChainUnavailable): + m.log.WarnContext(ctx, "delivery policy unavailable", "source", src.Name) + return http.StatusServiceUnavailable, errDeliveryUnavailable + } + m.log.WarnContext(ctx, "refused webhook delivery", "source", src.Name, "reason", "delivery policy", "level", d.Level, "policies", d.Policies) + return http.StatusForbidden, errDeliveryDenied +} + // parseWith has a running connector parse a delivery. func (m *Module) parseWith(ctx context.Context, connector string, body []byte, headers map[string]string) ([]connectors.Event, error) { conn, ok := m.connectors.Get(connector) @@ -193,7 +241,7 @@ func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors if err != nil { return stored, duplicates, fmt.Errorf("check delivery provenance: %w", err) } - isNew, err := m.store.Insert(ctx, newRow(m.orgID, src, e, body, own, run)) + isNew, err := m.inserter.Insert(ctx, newRow(m.orgID, src, e, body, own, run)) if err != nil { return stored, duplicates, err } @@ -332,7 +380,7 @@ func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) { if err != nil { return false, err } - matched := false + matched, refused := false, false for _, t := range triggers { var filter map[string]string if err := json.Unmarshal(t.Filter, &filter); err != nil { @@ -341,11 +389,13 @@ func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) { if !match.Fields(e.Fields, filter) { continue } - runID, err := m.runs.Start(ctx, runs.StartRequest{ - OrgID: e.OrgID, Workflow: t.Workflow, Key: e.ID.String() + ":" + t.Name, Input: runInput(e), - }) + runID, started, err := m.startRun(ctx, e, t) if err != nil { - return false, fmt.Errorf("start run for trigger %q: %w", t.Name, err) + return false, err + } + if !started { + refused = true + continue } if err := store.RecordDispatch(ctx, tx, e.ID, t.Name, runID); err != nil { return false, err @@ -353,18 +403,44 @@ func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) { m.log.InfoContext(ctx, "trigger started run", "event", e.ID, "trigger", t.Name, "run", runID) matched = true } - if !matched { - created, err := store.NoteUnmatched(ctx, tx, e) - if err != nil { - return false, err - } - if created { - m.log.InfoContext(ctx, "unmatched events started a group", "source", e.Source, "type", e.Type) - } + if !matched && !refused { + return false, m.noteUnmatched(ctx, tx, e) } return matched, nil } +// noteUnmatched files an event no trigger took into a group for review. +func (m *Module) noteUnmatched(ctx context.Context, tx *sqlx.Tx, e store.Event) error { + created, err := store.NoteUnmatched(ctx, tx, e) + if err != nil { + return err + } + if created { + m.log.InfoContext(ctx, "unmatched events started a group", "source", e.Source, "type", e.Type) + } + return nil +} + +// startRun starts the run trigger t asks for, as the workload principal, so +// the start is decided by the same policy chain as any other caller. A refusal +// is final: it reports no run and no error, and the trigger is skipped. A policy +// chain that can't decide returns an error instead, which rolls the dispatch +// back so the event stays queued and is dispatched again. +func (m *Module) startRun(ctx context.Context, e store.Event, t store.Trigger) (runID uuid.UUID, started bool, err error) { + ctx = governance.WithPrincipal(ctx, governance.WorkloadPrincipal(governance.DefaultOrgID)) + runID, err = m.runs.Start(ctx, runs.StartRequest{ + OrgID: e.OrgID, Workflow: t.Workflow, Key: e.ID.String() + ":" + t.Name, Input: runInput(e), + }) + if errors.Is(err, governance.ErrDenied) { + m.log.WarnContext(ctx, "trigger not permitted to start its workflow", "event", e.ID, "trigger", t.Name, "error", err) + return uuid.Nil, false, nil + } + if err != nil { + return uuid.Nil, false, fmt.Errorf("start run for trigger %q: %w", t.Name, err) + } + return runID, true, nil +} + // Origin returns the event and trigger that started a run. func (m *Module) Origin(ctx context.Context, runID uuid.UUID) (runs.Origin, bool, error) { o, ok, err := m.store.OriginOf(ctx, m.orgID, runID) diff --git a/modules/events/schedule.go b/modules/events/schedule.go index ec13ebd56..1df8c2cc9 100644 --- a/modules/events/schedule.go +++ b/modules/events/schedule.go @@ -50,7 +50,7 @@ func (m *Module) tick(ctx context.Context, now time.Time) error { if err != nil { return fmt.Errorf("encode tick: %w", err) } - isNew, err := m.store.Insert(ctx, tickEvent(m.orgID, t.Name, slot, fields)) + isNew, err := m.inserter.Insert(ctx, tickEvent(m.orgID, t.Name, slot, fields)) if err != nil { return err } diff --git a/modules/events/verify_test.go b/modules/events/verify_test.go index 5d8a024ac..5568a95f6 100644 --- a/modules/events/verify_test.go +++ b/modules/events/verify_test.go @@ -16,6 +16,7 @@ import ( "github.com/samcharles93/archie-core/internal/platform/seal" "github.com/samcharles93/archie-core/modules/connectors" "github.com/samcharles93/archie-core/modules/events/internal/store" + "github.com/samcharles93/archie-core/modules/governance" ) type fakeSources struct{ src store.Source } @@ -71,7 +72,7 @@ func TestHMACSourceVerifiesBeforeParsing(t *testing.T) { t.Run(tt.name, func(t *testing.T) { parsed := false m := &Module{ - orgID: "org", reader: fakeSources{src}, sealer: sealer, + orgID: "org", reader: fakeSources{src}, sealer: sealer, delivery: permitAll{}, log: slog.New(slog.DiscardHandler), parse: func(context.Context, string, []byte, map[string]string) ([]connectors.Event, error) { parsed = true @@ -93,3 +94,10 @@ func TestHMACSourceVerifiesBeforeParsing(t *testing.T) { }) } } + +// permitAll accepts a delivery from any address. +type permitAll struct{} + +func (permitAll) AuthorizeDelivery(context.Context, governance.OrgID, string, string) governance.Decision { + return governance.Allowed() +} diff --git a/modules/runs/access_test.go b/modules/runs/access_test.go new file mode 100644 index 000000000..bb8259b7c --- /dev/null +++ b/modules/runs/access_test.go @@ -0,0 +1,133 @@ +package runs + +import ( + "context" + "encoding/json/jsontext" + "errors" + "log/slog" + "slices" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/samcharles93/archie-core/modules/governance" + "github.com/samcharles93/archie-core/modules/runs/internal/store" +) + +// asked is one decision the engine requested. +type asked struct { + principal governance.IdentityID + action governance.Action + kind governance.ResourceKind + id string +} + +// recordingAccess answers every Authorize with one fixed decision and records +// what was asked. +type recordingAccess struct { + decision governance.Decision + calls []asked +} + +func (r *recordingAccess) Authorize(_ context.Context, p governance.Principal, a governance.Action, res governance.Resource, _ governance.AccessContext) governance.Decision { + r.calls = append(r.calls, asked{p.IdentityID, a, res.Kind, res.ID}) + return r.decision +} + +// queueingStore holds the runs the engine queues. +type queueingStore struct { + fakeWork + queued []store.NewRun +} + +func (q *queueingStore) Start(_ context.Context, n store.NewRun) (uuid.UUID, error) { + q.queued = append(q.queued, n) + return uuid.New(), nil +} + +// claimsOne hands out one queued run to the worker, as the store does after a +// restart or a retry. +type claimsOne struct { + *queueingStore + run store.Run +} + +func (c claimsOne) Claim(context.Context, string, uuid.UUID, time.Duration) (store.Run, bool, error) { + return c.run, true, nil +} + +// TestRunsAreAuthorizedAtStartAndAtEveryClaim guards that a run started by a +// trigger and the same run claimed again after a restart or a retry are decided +// by the same live check, as the workload principal, and that a refusal stops +// the work. A policy chain that can't decide starts nothing, finishes nothing +// and leaves the run for another attempt. Without it a revoked grant would +// keep running queued work. +func TestRunsAreAuthorizedAtStartAndAtEveryClaim(t *testing.T) { + workload := asked{governance.WorkloadID, governance.ActionRun, governance.KindWorkflow, "triage"} + tests := []struct { + name string + decision governance.Decision + noPrincipal bool + claim bool + wantCalls []asked + wantQueued int + wantRan bool + wantFinished []string + wantErr error + }{ + {name: "a trigger starts its run", decision: governance.Allowed(), wantCalls: []asked{workload}, wantQueued: 1}, + {name: "a start with no principal", decision: governance.Allowed(), noPrincipal: true, wantErr: governance.ErrDenied}, + {name: "a revoked start", decision: governance.DeniedAt(governance.LevelOrg, []string{"org-workload"}), wantCalls: []asked{workload}, wantErr: governance.ErrDenied}, + {name: "a start the chain can't decide", decision: governance.Unavailable(), wantCalls: []asked{workload}, wantErr: governance.ErrChainUnavailable}, + {name: "a claim after a restart", decision: governance.Allowed(), claim: true, wantCalls: []asked{workload}, wantRan: true, wantFinished: []string{store.StateSucceeded}}, + {name: "a claim after revocation", decision: governance.DeniedAt(governance.LevelOrg, []string{"org-workload"}), claim: true, wantCalls: []asked{workload}, wantFinished: []string{store.StateFailed}}, + {name: "a claim the chain can't decide", decision: governance.Unavailable(), claim: true, wantCalls: []asked{workload}, wantErr: governance.ErrChainUnavailable}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + work := &queueingStore{} + access := &recordingAccess{decision: tt.decision} + engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{}) + engine.queue = work + engine.Use(Deps{Access: access}) + ran := false + useWorkflow(engine, func(*Context, jsontext.Value) (jsontext.Value, error) { + ran = true + return jsontext.Value(`"done"`), nil + }) + + var err error + if tt.claim { + engine.work = claimsOne{work, store.Run{ID: uuid.New(), OrgID: "org", Workflow: "triage", Version: 1, Input: jsontext.Value(`null`)}} + _, err = engine.claimAndExecute(t.Context()) + } else { + ctx := t.Context() + if !tt.noPrincipal { + ctx = governance.WithPrincipal(ctx, governance.WorkloadPrincipal(governance.DefaultOrgID)) + } + _, err = engine.Start(ctx, StartRequest{OrgID: "org", Workflow: "triage", Key: "event:trigger"}) + } + + if !errors.Is(err, tt.wantErr) { + t.Fatalf("error = %v, want %v", err, tt.wantErr) + } + if !slices.Equal(access.calls, tt.wantCalls) { + t.Fatalf("decisions asked = %v, want %v", access.calls, tt.wantCalls) + } + if len(work.queued) != tt.wantQueued { + t.Fatalf("runs queued = %d, want %d", len(work.queued), tt.wantQueued) + } + if ran != tt.wantRan { + t.Fatalf("workflow ran = %v, want %v", ran, tt.wantRan) + } + if !slices.Equal(work.finished, tt.wantFinished) { + t.Fatalf("finished = %v, want %v", work.finished, tt.wantFinished) + } + if slices.Contains(tt.wantFinished, store.StateFailed) && !strings.Contains(work.errors[0], "not permitted") { + t.Fatalf("the run's trail says %q, want the refusal", work.errors[0]) + } + }) + } +} diff --git a/modules/runs/dry.go b/modules/runs/dry.go index 7b3ae3559..89c64fe46 100644 --- a/modules/runs/dry.go +++ b/modules/runs/dry.go @@ -73,6 +73,9 @@ type DryRequest struct { // StartDry queues a dry run of a document and returns its run's id. A document // that doesn't validate starts nothing. func (e *Engine) StartDry(ctx context.Context, req DryRequest) (uuid.UUID, error) { + if err := e.mayStart(ctx, req.Workflow); err != nil { + return uuid.Nil, err + } doc, err := e.validateDocument(ctx, req.Document) if err != nil { return uuid.Nil, err @@ -93,7 +96,7 @@ func (e *Engine) StartDry(ctx context.Context, req DryRequest) (uuid.UUID, error if err := doc.checkInputs(decodeInputs(input)); err != nil { return uuid.Nil, fmt.Errorf("workflow %q: %w", req.Workflow, err) } - return e.db.Start(ctx, store.NewRun{ + return e.queue.Start(ctx, store.NewRun{ OrgID: req.OrgID, Workflow: req.Workflow, Input: input, Dry: true, Overlay: overlay, }) } diff --git a/modules/runs/dry_test.go b/modules/runs/dry_test.go index ca3132312..fd4723bd5 100644 --- a/modules/runs/dry_test.go +++ b/modules/runs/dry_test.go @@ -10,6 +10,7 @@ import ( "github.com/google/uuid" "github.com/samcharles93/archie-core/modules/connectors" + "github.com/samcharles93/archie-core/modules/governance" "github.com/samcharles93/archie-core/modules/judgement" "github.com/samcharles93/archie-core/modules/runs/internal/store" ) @@ -48,6 +49,7 @@ func (anyBox) Tool(string) (connectors.Tool, bool) { return nil, true } // is queued. func TestStartDryRefusesWhatCannotStart(t *testing.T) { engine := newEngine(nil, &fakeWork{}, slog.New(slog.DiscardHandler), Options{}) + engine.Use(Deps{Access: &recordingAccess{decision: governance.Allowed()}}) tests := []struct { name string doc string @@ -71,7 +73,8 @@ func TestStartDryRefusesWhatCannotStart(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := engine.StartDry(context.Background(), DryRequest{ + ctx := governance.WithPrincipal(context.Background(), governance.SharedTokenOwner()) + _, err := engine.StartDry(ctx, DryRequest{ OrgID: "org", Workflow: "draft", Document: tt.doc, }) if err == nil || !strings.Contains(err.Error(), tt.want) { diff --git a/modules/runs/interpreter.go b/modules/runs/interpreter.go index cbe59adb1..91fe269f6 100644 --- a/modules/runs/interpreter.go +++ b/modules/runs/interpreter.go @@ -103,6 +103,10 @@ type Deps struct { // Capabilities is what saving a workflow checks connector references // against. Capabilities Capabilities + // Access decides whether a run may start and whether a claimed run may + // execute, for the principal acting or else the workload principal. With + // none, no run starts or executes. + Access governance.Authorizer } // Workflow is a document ready to run: the code that executes its steps for diff --git a/modules/runs/runs.go b/modules/runs/runs.go index 3b2a047f1..2e519833a 100644 --- a/modules/runs/runs.go +++ b/modules/runs/runs.go @@ -64,12 +64,14 @@ type runStarter interface { // Engine starts, executes and resumes runs. type Engine struct { - db *store.Store - work runWork - log *slog.Logger - opts Options - mu sync.RWMutex - deps *Deps + db *store.Store + // queue starts runs; it is db except in tests. + queue runQueue + work runWork + log *slog.Logger + opts Options + mu sync.RWMutex + deps *Deps // load turns a claimed run into the code that executes it. load func(ctx context.Context, run store.Run) (Workflow, error) } @@ -93,7 +95,7 @@ func newEngine(db *store.Store, work runWork, log *slog.Logger, opts Options) *E if opts.Owner == "" { opts.Owner = "archie" } - e := &Engine{db: db, work: work, log: log, opts: opts} + e := &Engine{db: db, queue: db, work: work, log: log, opts: opts} e.load = e.loadDocument return e } @@ -185,14 +187,42 @@ func (e *Engine) Has(ctx context.Context, orgID, name string) (bool, error) { return err == nil, err } +// runQueue reads the workflow a run starts under and queues the run. +type runQueue interface { + workflowFinder + Start(ctx context.Context, n store.NewRun) (uuid.UUID, error) +} + // Start queues a run under the latest stored version of its workflow and -// returns its id. +// returns its id. The principal in ctx must be allowed to run the workflow; +// work that starts runs on its own, such as a trigger, passes the workload +// principal. A start the policy chain can't decide queues nothing, and the +// error wraps governance.ErrChainUnavailable so the caller tries again. func (e *Engine) Start(ctx context.Context, req StartRequest) (uuid.UUID, error) { - n, err := newRun(ctx, e.db, req) + if err := e.mayStart(ctx, req.Workflow); err != nil { + return uuid.Nil, err + } + n, err := newRun(ctx, e.queue, req) if err != nil { return uuid.Nil, err } - return e.db.Start(ctx, n) + return e.queue.Start(ctx, n) +} + +// mayStart refuses a start by a caller who may not run workflow, or whose +// right can't be checked. +func (e *Engine) mayStart(ctx context.Context, workflow string) error { + p, ok := governance.PrincipalFromContext(ctx) + if !ok { + return fmt.Errorf("%w: no principal starts the run", governance.ErrDenied) + } + deps, err := e.steps() + if err != nil { + return fmt.Errorf("%w: %w", governance.ErrChainUnavailable, err) + } + return governance.Require(ctx, deps.Access, p, governance.ActionRun, governance.Resource{ + Kind: governance.KindWorkflow, ID: workflow, Org: p.Org, + }) } // workflowFinder reads stored workflow versions. @@ -320,10 +350,41 @@ func (e *Engine) claimAndExecute(ctx context.Context) (bool, error) { if err != nil || !ok { return false, err } + if err := e.mayExecute(ctx, run); err != nil { + return e.refuseClaim(ctx, run, token, err) + } e.execute(ctx, run, token) return true, nil } +// mayExecute rechecks, each time a worker claims a run, that the workload +// principal may still run its workflow. A new run, a retry and a resume after +// a crash or a wake-up all pass through here. +func (e *Engine) mayExecute(ctx context.Context, run store.Run) error { + deps, err := e.steps() + if err != nil { + return fmt.Errorf("%w: %w", governance.ErrChainUnavailable, err) + } + return governance.RequireWorkload(ctx, deps.Access, governance.ActionRun, governance.Resource{ + Kind: governance.KindWorkflow, ID: run.Workflow, + }) +} + +// refuseClaim handles a claimed run that may not execute. A policy chain that +// can't decide leaves the run unfinished, so its lease lapses and a worker +// claims it again. A refusal ends the run as failed, and its trail says why. +func (e *Engine) refuseClaim(ctx context.Context, run store.Run, token uuid.UUID, cause error) (bool, error) { + if !errors.Is(cause, governance.ErrDenied) { + return false, fmt.Errorf("run %s waits for access policies: %w", run.ID, cause) + } + ended := store.Ended{State: store.StateFailed, Error: cause.Error()} + if err := e.work.Finish(ctx, run.ID, token, ended); err != nil { + return false, fmt.Errorf("finish refused run %s: %w", run.ID, err) + } + e.log.WarnContext(ctx, "run refused", "run", run.ID, "workflow", run.Workflow, "error", cause) + return true, nil +} + // execute runs one claimed run to its next suspension or end. The run's // context ends if the lease can't be renewed, so a stale worker stops. func (e *Engine) execute(ctx context.Context, run store.Run, token uuid.UUID) {