feat(runs): authorize runs, deliveries and intake as the workload principal (#2078)
Trigger dispatch, run start, every run claim, every delivery attempt and webhook intake now ask the policy chain. Background work acts as the workload principal; a refusal is final and recorded on the run or the delivery, and an undecidable chain leaves the work queued for another attempt. Intake applies the per-source delivery policy after source verification.
This commit is contained in:
12 files changed
+639
-42
No files matched your search
@@ -55,11 +55,15 @@ func (o *operations) Start(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Every entry point asks the same chain, and each refusal is stored as a
|
||||
// denial.
|
||||
checked := governance.Recording(access, o.settings, o.log.With("module", "governance"))
|
||||
// A workflow takes its context from the run handle, as a handler takes it
|
||||
// from the request.
|
||||
o.engine.Use(runs.Deps{
|
||||
Settings: o.settings, Toolbox: o.conns, Judge: o.judge, Approver: o.allow,
|
||||
Notifier: o.tell, Messenger: o.tell, Memory: o.remember, Capabilities: o.conns,
|
||||
Access: checked,
|
||||
})
|
||||
if err := installStarter(ctx, o.engine); err != nil {
|
||||
return err
|
||||
@@ -76,9 +80,12 @@ func (o *operations) Start(ctx context.Context) error {
|
||||
|
||||
// Nothing below can fail, so a retry cannot launch a connector or a loop twice.
|
||||
o.checks.Register("connectors", o.conns.Ready)
|
||||
// Administrative writes are checked against the same live policy chain as
|
||||
// every other request.
|
||||
o.settings.UseAuthorizer(access)
|
||||
// Administrative writes, and every API request, are checked against the same
|
||||
// live policy chain as chat and background work. The webhook intake applies
|
||||
// each source's delivery policy from it.
|
||||
o.settings.UseAuthorizer(checked)
|
||||
o.tell.UseAccess(checked)
|
||||
o.hear.UseDeliveryPolicy(access)
|
||||
o.talk.Use(chat.Deps{
|
||||
People: o.settings, Access: access, Settings: o.settings, Toolbox: o.conns, Judge: o.judge,
|
||||
Approver: o.allow, Memory: o.remember, Runs: o.engine, Notifier: o.tell,
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package deliveries
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/samcharles93/archie-core/modules/connectors"
|
||||
"github.com/samcharles93/archie-core/modules/deliveries/internal/store"
|
||||
"github.com/samcharles93/archie-core/modules/governance"
|
||||
)
|
||||
|
||||
// sequencedAccess answers each Authorize with the next decision, the last one
|
||||
// repeating, and records what was asked.
|
||||
type sequencedAccess struct {
|
||||
decisions []governance.Decision
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (s *sequencedAccess) Authorize(_ context.Context, p governance.Principal, a governance.Action, r governance.Resource, _ governance.AccessContext) governance.Decision {
|
||||
s.asked = append(s.asked, string(p.IdentityID)+" "+string(a)+" "+string(r.Kind)+" "+r.ID)
|
||||
d := s.decisions[0]
|
||||
if len(s.decisions) > 1 {
|
||||
s.decisions = s.decisions[1:]
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// ledger records each attempt's outcome by kind.
|
||||
type ledger struct{ outcomes []string }
|
||||
|
||||
func (l *ledger) Sent(context.Context, int64, int, string, time.Duration) error {
|
||||
l.outcomes = append(l.outcomes, "sent")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *ledger) Failed(_ context.Context, _ int64, _ int, reason string, _ time.Duration) error {
|
||||
l.outcomes = append(l.outcomes, "failed: "+reason)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *ledger) GaveUp(_ context.Context, _ int64, _ int, reason string, _ time.Duration) error {
|
||||
l.outcomes = append(l.outcomes, "gave up: "+reason)
|
||||
return nil
|
||||
}
|
||||
|
||||
// channelLookups counts the sends a worker tried: it has no connector running,
|
||||
// so a send that was attempted fails and a send that was refused never looks.
|
||||
type channelLookups struct{ looked int }
|
||||
|
||||
func (c *channelLookups) Delivering(string) []*connectors.Connector { return nil }
|
||||
|
||||
func (c *channelLookups) Get(string) (*connectors.Connector, bool) {
|
||||
c.looked++
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// TestDeliveryIsAuthorizedOnEveryAttempt guards that a queued message is sent
|
||||
// only while the workload principal may still deliver on its channel. Each
|
||||
// attempt, including a retry after a backoff or a restart, asks again: a
|
||||
// revoked grant ends the message as failed, and a chain that can't decide
|
||||
// leaves it queued for its next attempt instead of dropping it.
|
||||
func TestDeliveryIsAuthorizedOnEveryAttempt(t *testing.T) {
|
||||
allowed, revoked, unavailable := governance.Allowed(), governance.DeniedAt(governance.LevelOrg, []string{"org-workload"}), governance.Unavailable()
|
||||
tests := []struct {
|
||||
name string
|
||||
decisions []governance.Decision
|
||||
unwired bool
|
||||
attempts int
|
||||
wantSends int
|
||||
wantOutcomes []string
|
||||
}{
|
||||
{name: "an allowed attempt is sent", decisions: []governance.Decision{allowed}, attempts: 1, wantSends: 1, wantOutcomes: []string{"failed: connector \"slack\" isn't running"}},
|
||||
{name: "a revoked grant ends the message", decisions: []governance.Decision{revoked}, attempts: 1, wantOutcomes: []string{"gave up: " + connectors.ErrRejected.Error() + ": " + governance.ErrDenied.Error() + ": deliver delivery slack"}},
|
||||
{name: "a retry after revocation is refused", decisions: []governance.Decision{allowed, revoked}, attempts: 2, wantSends: 1, wantOutcomes: []string{"failed: connector \"slack\" isn't running", "gave up: " + connectors.ErrRejected.Error() + ": " + governance.ErrDenied.Error() + ": deliver delivery slack"}},
|
||||
{name: "an undecidable chain leaves the message queued", decisions: []governance.Decision{unavailable}, attempts: 3},
|
||||
{name: "no policy wired leaves the message queued", unwired: true, attempts: 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
access, out, channels := &sequencedAccess{decisions: tt.decisions}, &ledger{}, &channelLookups{}
|
||||
m := &Module{outcomes: out, channels: channels, log: slog.New(slog.DiscardHandler)}
|
||||
if !tt.unwired {
|
||||
m.UseAccess(access)
|
||||
}
|
||||
for range tt.attempts {
|
||||
m.deliver(t.Context(), store.Message{Seq: 1, ID: uuid.New(), Channel: "slack", Attempts: 1})
|
||||
}
|
||||
if channels.looked != tt.wantSends {
|
||||
t.Fatalf("sends tried = %d, want %d", channels.looked, tt.wantSends)
|
||||
}
|
||||
if !slices.Equal(out.outcomes, tt.wantOutcomes) {
|
||||
t.Fatalf("outcomes = %q, want %q", out.outcomes, tt.wantOutcomes)
|
||||
}
|
||||
for _, call := range access.asked {
|
||||
if !strings.HasPrefix(call, "workload deliver delivery ") {
|
||||
t.Fatalf("asked %q, want the workload principal to deliver", call)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/platform/hub"
|
||||
"github.com/samcharles93/archie-core/modules/connectors"
|
||||
"github.com/samcharles93/archie-core/modules/deliveries/internal/store"
|
||||
"github.com/samcharles93/archie-core/modules/governance"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -111,16 +112,31 @@ type Channels interface {
|
||||
type Module struct {
|
||||
orgID string
|
||||
store *store.Store
|
||||
outcomes outcomes
|
||||
access governance.Authorizer
|
||||
hub *hub.Hub
|
||||
channels Channels
|
||||
log *slog.Logger
|
||||
poll time.Duration
|
||||
}
|
||||
|
||||
// outcomes records how each attempt to deliver a message ended.
|
||||
type outcomes interface {
|
||||
Sent(ctx context.Context, seq int64, attempt int, externalID string, took time.Duration) error
|
||||
Failed(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error
|
||||
GaveUp(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error
|
||||
}
|
||||
|
||||
// UseAccess gives the module the policy chain that decides, each time a
|
||||
// worker claims a message, whether it may go out. Until it is set nothing is
|
||||
// delivered.
|
||||
func (m *Module) UseAccess(a governance.Authorizer) { m.access = a }
|
||||
|
||||
// NewModule returns the deliveries module for orgID, announcing new messages
|
||||
// on h and sending them on every connector in channels that delivers them.
|
||||
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, h *hub.Hub, channels Channels) *Module {
|
||||
return &Module{orgID: orgID, store: store.New(db), hub: h, channels: channels, log: log, poll: time.Second}
|
||||
st := store.New(db)
|
||||
return &Module{orgID: orgID, store: st, outcomes: st, hub: h, channels: channels, log: log, poll: time.Second}
|
||||
}
|
||||
|
||||
// Notify delivers n on every channel. Delivering the same key again changes
|
||||
@@ -231,7 +247,23 @@ func (m *Module) Work(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// deliver sends a claimed message when the workload principal may still deliver
|
||||
// on its channel. Each attempt is decided afresh, so a retry after a backoff or
|
||||
// a restart honours a revocation made meanwhile. A refusal ends the message as
|
||||
// failed. When the policy chain can't decide, the message stays queued for its
|
||||
// next attempt.
|
||||
func (m *Module) deliver(ctx context.Context, msg store.Message) {
|
||||
err := governance.RequireWorkload(ctx, m.access, governance.ActionDeliver, governance.Resource{
|
||||
Kind: governance.KindDelivery, ID: msg.Channel,
|
||||
})
|
||||
switch {
|
||||
case errors.Is(err, governance.ErrDenied):
|
||||
m.recordFailure(ctx, msg, fmt.Errorf("%w: %w", connectors.ErrRejected, err), 0)
|
||||
return
|
||||
case err != nil:
|
||||
m.log.WarnContext(ctx, "delivery waits for access policies", "channel", msg.Channel, "message", msg.ID, "error", err)
|
||||
return
|
||||
}
|
||||
start := time.Now()
|
||||
external, err := m.sendTo(ctx, msg)
|
||||
took := time.Since(start)
|
||||
@@ -240,7 +272,7 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) {
|
||||
m.recordFailure(ctx, msg, err, took)
|
||||
return
|
||||
}
|
||||
if err := m.store.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil {
|
||||
if err := m.outcomes.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil {
|
||||
m.log.ErrorContext(ctx, "record delivery", "error", err)
|
||||
}
|
||||
}
|
||||
@@ -248,13 +280,13 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) {
|
||||
// recordFailure records why an attempt failed. It marks the delivery failed
|
||||
// when the channel rejected it permanently or the retry limit is reached.
|
||||
func (m *Module) recordFailure(ctx context.Context, msg store.Message, cause error, took time.Duration) {
|
||||
record := m.store.Failed
|
||||
record := m.outcomes.Failed
|
||||
reason := cause.Error()
|
||||
switch {
|
||||
case errors.Is(cause, connectors.ErrRejected):
|
||||
record = m.store.GaveUp
|
||||
record = m.outcomes.GaveUp
|
||||
case msg.Attempts >= maxAttempts:
|
||||
record = m.store.GaveUp
|
||||
record = m.outcomes.GaveUp
|
||||
reason = fmt.Sprintf("gave up after %d attempts: %s", msg.Attempts, reason)
|
||||
}
|
||||
if err := record(ctx, msg.Seq, msg.Attempts, reason, took); err != nil {
|
||||
@@ -292,11 +324,11 @@ func backoff(attempts int) time.Duration {
|
||||
// Register adds the composer API.
|
||||
func (m *Module) Register(api huma.API) {
|
||||
tags := []string{"Composer"}
|
||||
huma.Register(api, huma.Operation{
|
||||
httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindDelivery), huma.Operation{
|
||||
OperationID: "list-composer-messages", Method: http.MethodGet, Path: "/api/composer/messages",
|
||||
Summary: "List composer messages with their delivery attempts", Tags: tags,
|
||||
}, m.messages)
|
||||
huma.Register(api, huma.Operation{
|
||||
httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindDelivery), huma.Operation{
|
||||
OperationID: "stream-composer", Method: http.MethodGet, Path: "/api/composer/stream",
|
||||
Summary: "Stream the seq of each new composer message", Tags: tags,
|
||||
Responses: map[string]*huma.Response{"200": {
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/platform/httpjson"
|
||||
"github.com/samcharles93/archie-core/modules/connectors"
|
||||
"github.com/samcharles93/archie-core/modules/events/internal/store"
|
||||
"github.com/samcharles93/archie-core/modules/governance"
|
||||
"github.com/samcharles93/archie-core/modules/runs"
|
||||
)
|
||||
|
||||
// deliveryPolicy answers every delivery with one decision and records who
|
||||
// asked.
|
||||
type deliveryPolicy struct {
|
||||
decision governance.Decision
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (d *deliveryPolicy) AuthorizeDelivery(_ context.Context, org governance.OrgID, source, addr string) governance.Decision {
|
||||
d.asked = append(d.asked, string(org)+" "+source+" "+addr)
|
||||
return d.decision
|
||||
}
|
||||
|
||||
// onceInserter stores an event once per source event id, as the table's
|
||||
// uniqueness constraint does.
|
||||
type onceInserter struct{ stored map[string]bool }
|
||||
|
||||
func (o *onceInserter) Insert(_ context.Context, e store.Event) (bool, error) {
|
||||
if o.stored[e.SourceEventID] {
|
||||
return false, nil
|
||||
}
|
||||
o.stored[e.SourceEventID] = true
|
||||
return true, nil
|
||||
}
|
||||
|
||||
type noEcho struct{}
|
||||
|
||||
func (noEcho) SentBy(context.Context, string, ...string) (uuid.UUID, bool, error) {
|
||||
return uuid.Nil, false, nil
|
||||
}
|
||||
|
||||
// TestIntakeAppliesTheDeliveryPolicyBeforeStoring guards the webhook entry
|
||||
// point: a delivery that passes source verification is still refused when the
|
||||
// per-source, per-address delivery policy denies it or can't be read, and
|
||||
// nothing is parsed or stored then. A redelivery of a stored event id is
|
||||
// counted as a duplicate and stored once, so it can't dispatch again.
|
||||
func TestIntakeAppliesTheDeliveryPolicyBeforeStoring(t *testing.T) {
|
||||
const token = "not-a-real-token" //nolint:gosec // fixture bearer token the verifier must accept
|
||||
denied := governance.DeniedAt(governance.LevelObject, []string{"hook-allowlist"})
|
||||
tests := []struct {
|
||||
name string
|
||||
bearer string
|
||||
policy governance.Decision
|
||||
unwired bool
|
||||
deliveries int
|
||||
want int
|
||||
wantBody string
|
||||
wantStored int
|
||||
wantAsked []string
|
||||
}{
|
||||
{name: "an allowed delivery is stored", bearer: token, policy: governance.Allowed(), deliveries: 1, want: http.StatusAccepted, wantBody: `"stored":1`, wantStored: 1, wantAsked: []string{"org-sys hook 203.0.113.9"}},
|
||||
{name: "a redelivered event id is counted, not stored again", bearer: token, policy: governance.Allowed(), deliveries: 2, want: http.StatusAccepted, wantBody: `"duplicates":1`, wantStored: 1, wantAsked: []string{"org-sys hook 203.0.113.9", "org-sys hook 203.0.113.9"}},
|
||||
{name: "a valid token with a denying policy", bearer: token, policy: denied, deliveries: 1, want: http.StatusForbidden, wantAsked: []string{"org-sys hook 203.0.113.9"}},
|
||||
{name: "a valid token while the chain is unavailable", bearer: token, policy: governance.Unavailable(), deliveries: 1, want: http.StatusServiceUnavailable, wantAsked: []string{"org-sys hook 203.0.113.9"}},
|
||||
{name: "a valid token with no policy wired", bearer: token, unwired: true, deliveries: 1, want: http.StatusServiceUnavailable},
|
||||
{name: "a wrong token never reaches the policy", bearer: "other", policy: governance.Allowed(), deliveries: 1, want: http.StatusUnauthorized},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
policy := &deliveryPolicy{decision: tt.policy}
|
||||
inserter := &onceInserter{stored: map[string]bool{}}
|
||||
m := &Module{
|
||||
orgID: "default", log: slog.New(slog.DiscardHandler), inserter: inserter, deliveries: noEcho{},
|
||||
reader: fakeSources{store.Source{OrgID: "default", Name: "hook", Connector: "c", Verification: store.Token, TokenSHA256: httpjson.TokenHash(token)}},
|
||||
parse: func(context.Context, string, []byte, map[string]string) ([]connectors.Event, error) {
|
||||
return []connectors.Event{{Type: "alert", SourceEventID: "evt-1"}}, nil
|
||||
},
|
||||
}
|
||||
if !tt.unwired {
|
||||
m.UseDeliveryPolicy(policy)
|
||||
}
|
||||
var rec *httptest.ResponseRecorder
|
||||
for range tt.deliveries {
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/hooks/hook", strings.NewReader(`{}`))
|
||||
req.SetPathValue("source", "hook")
|
||||
req.RemoteAddr = "203.0.113.9:4711"
|
||||
req.Header.Set("Authorization", "Bearer "+tt.bearer)
|
||||
rec = httptest.NewRecorder()
|
||||
m.intake(rec, req)
|
||||
}
|
||||
body, _ := io.ReadAll(rec.Body)
|
||||
if rec.Code != tt.want || !strings.Contains(string(body), tt.wantBody) {
|
||||
t.Fatalf("status %d body %s, want %d containing %q", rec.Code, body, tt.want, tt.wantBody)
|
||||
}
|
||||
if len(inserter.stored) != tt.wantStored {
|
||||
t.Fatalf("events stored = %d, want %d", len(inserter.stored), tt.wantStored)
|
||||
}
|
||||
if !slices.Equal(policy.asked, tt.wantAsked) {
|
||||
t.Fatalf("delivery policy asked %v, want %v", policy.asked, tt.wantAsked)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// startRecorder is the run starter a trigger calls: it records the principal
|
||||
// each start carried and answers with a fixed error.
|
||||
type startRecorder struct {
|
||||
err error
|
||||
principals []governance.IdentityID
|
||||
}
|
||||
|
||||
func (s *startRecorder) Start(ctx context.Context, _ runs.StartRequest) (uuid.UUID, error) {
|
||||
p, _ := governance.PrincipalFromContext(ctx)
|
||||
s.principals = append(s.principals, p.IdentityID)
|
||||
return uuid.New(), s.err
|
||||
}
|
||||
|
||||
func (*startRecorder) CheckTrigger(context.Context, string, string) error { return nil }
|
||||
|
||||
// TestTriggersStartRunsAsTheWorkload guards that dispatching an event never
|
||||
// starts a run without a principal: the workload principal acts, a refusal
|
||||
// skips the trigger for good, and a chain that can't decide fails the dispatch
|
||||
// so the event stays queued instead of being dropped.
|
||||
func TestTriggersStartRunsAsTheWorkload(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
wantStarted bool
|
||||
wantErr error
|
||||
}{
|
||||
{name: "an allowed start", wantStarted: true},
|
||||
{name: "a refused start is final", err: governance.ErrDenied},
|
||||
{name: "an undecidable start is retried", err: governance.ErrChainUnavailable, wantErr: governance.ErrChainUnavailable},
|
||||
{name: "another failure is retried", err: errors.New("database down"), wantErr: errors.New("database down")},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
starts := &startRecorder{err: tt.err}
|
||||
m := &Module{runs: starts, log: slog.New(slog.DiscardHandler)}
|
||||
_, started, err := m.startRun(t.Context(), store.Event{ID: uuid.New(), OrgID: "default"}, store.Trigger{Name: "t", Workflow: "triage"})
|
||||
if started != tt.wantStarted {
|
||||
t.Fatalf("started = %v, want %v", started, tt.wantStarted)
|
||||
}
|
||||
if (err == nil) != (tt.wantErr == nil) || (tt.wantErr != nil && !strings.Contains(err.Error(), tt.wantErr.Error())) {
|
||||
t.Fatalf("error = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
if !slices.Equal(starts.principals, []governance.IdentityID{governance.WorkloadID}) {
|
||||
t.Fatalf("start acted as %v, want the workload principal", starts.principals)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+94
-18
@@ -16,6 +16,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"sync"
|
||||
@@ -28,6 +29,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/modules/connectors"
|
||||
"github.com/samcharles93/archie-core/modules/events/internal/match"
|
||||
"github.com/samcharles93/archie-core/modules/events/internal/store"
|
||||
"github.com/samcharles93/archie-core/modules/governance"
|
||||
"github.com/samcharles93/archie-core/modules/runs"
|
||||
)
|
||||
|
||||
@@ -71,6 +73,8 @@ type Module struct {
|
||||
orgID string
|
||||
store *store.Store
|
||||
reader sourceReader
|
||||
inserter eventInserter
|
||||
delivery governance.DeliveryAuthorizer
|
||||
sealer Sealer
|
||||
parse func(ctx context.Context, connector string, body []byte, headers map[string]string) ([]connectors.Event, error)
|
||||
connectors Connectors
|
||||
@@ -84,7 +88,7 @@ type Module struct {
|
||||
// NewModule returns the events module for orgID.
|
||||
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, sealer Sealer, c Connectors, r Runs, a Answers, d Deliveries) *Module {
|
||||
st := store.New(db)
|
||||
m := &Module{orgID: orgID, store: st, reader: st, sealer: sealer, connectors: c, runs: r, answers: a, deliveries: d, log: log, poll: time.Second}
|
||||
m := &Module{orgID: orgID, store: st, reader: st, inserter: st, sealer: sealer, connectors: c, runs: r, answers: a, deliveries: d, log: log, poll: time.Second}
|
||||
m.parse = m.parseWith
|
||||
return m
|
||||
}
|
||||
@@ -94,15 +98,26 @@ type sourceReader interface {
|
||||
Source(ctx context.Context, orgID, name string) (store.Source, error)
|
||||
}
|
||||
|
||||
// eventInserter stores an event, and reports false when its source event id
|
||||
// was stored before.
|
||||
type eventInserter interface {
|
||||
Insert(ctx context.Context, e store.Event) (isNew bool, err error)
|
||||
}
|
||||
|
||||
// UseDeliveryPolicy gives intake the policy that decides, per source and
|
||||
// sender address, who may deliver. Until it is set every delivery is refused.
|
||||
func (m *Module) UseDeliveryPolicy(d governance.DeliveryAuthorizer) { m.delivery = d }
|
||||
|
||||
// Routes adds the webhook intake. It stays outside the API because each
|
||||
// source verifies its own deliveries.
|
||||
func (m *Module) Routes(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /hooks/{source}", m.intake)
|
||||
}
|
||||
|
||||
// intake verifies a delivery, parses it with the source's connector and stores
|
||||
// each event before responding. A redelivered event violates a uniqueness
|
||||
// constraint and is counted as a duplicate.
|
||||
// intake verifies a delivery, checks the source's delivery policy, parses it
|
||||
// with the source's connector and stores each event before responding. A
|
||||
// redelivered event violates a uniqueness constraint, is counted as a
|
||||
// duplicate and is not dispatched again.
|
||||
func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
|
||||
src, parsed, body, status, err := m.accept(r)
|
||||
if err != nil {
|
||||
@@ -151,6 +166,9 @@ func (m *Module) accept(r *http.Request) (src store.Source, parsed []connectors.
|
||||
return m.refuse(ctx, src, err)
|
||||
}
|
||||
}
|
||||
if status, err := m.deliveryAllowed(ctx, src, r); err != nil {
|
||||
return src, nil, nil, status, err
|
||||
}
|
||||
parsed, err = m.parse(ctx, src.Connector, body, map[string]string{"Content-Type": r.Header.Get("Content-Type")})
|
||||
if errors.Is(err, errConnectorDown) {
|
||||
return src, nil, nil, http.StatusServiceUnavailable, err
|
||||
@@ -176,6 +194,36 @@ func (m *Module) refuse(ctx context.Context, src store.Source, err error) (store
|
||||
return src, nil, nil, http.StatusServiceUnavailable, errors.New("couldn't verify the delivery")
|
||||
}
|
||||
|
||||
var (
|
||||
errDeliveryDenied = errors.New("delivery not permitted")
|
||||
errDeliveryUnavailable = errors.New("delivery policy unavailable")
|
||||
)
|
||||
|
||||
// deliveryAllowed applies the source's delivery policy to the sender's address.
|
||||
// A verified delivery still needs it. When the policy chain can't decide the
|
||||
// delivery is refused with 503, so the sender retries and nothing is stored.
|
||||
func (m *Module) deliveryAllowed(ctx context.Context, src store.Source, r *http.Request) (int, error) {
|
||||
if m.delivery == nil {
|
||||
return http.StatusServiceUnavailable, errDeliveryUnavailable
|
||||
}
|
||||
addr, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||
if err != nil {
|
||||
addr = r.RemoteAddr
|
||||
}
|
||||
// Delivery policies belong to the operating organisation, whatever the
|
||||
// source's own org id says.
|
||||
d := m.delivery.AuthorizeDelivery(ctx, governance.DefaultOrgID, src.Name, addr)
|
||||
switch {
|
||||
case d.Allowed:
|
||||
return 0, nil
|
||||
case errors.Is(d.Err, governance.ErrChainUnavailable):
|
||||
m.log.WarnContext(ctx, "delivery policy unavailable", "source", src.Name)
|
||||
return http.StatusServiceUnavailable, errDeliveryUnavailable
|
||||
}
|
||||
m.log.WarnContext(ctx, "refused webhook delivery", "source", src.Name, "reason", "delivery policy", "level", d.Level, "policies", d.Policies)
|
||||
return http.StatusForbidden, errDeliveryDenied
|
||||
}
|
||||
|
||||
// parseWith has a running connector parse a delivery.
|
||||
func (m *Module) parseWith(ctx context.Context, connector string, body []byte, headers map[string]string) ([]connectors.Event, error) {
|
||||
conn, ok := m.connectors.Get(connector)
|
||||
@@ -193,7 +241,7 @@ func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors
|
||||
if err != nil {
|
||||
return stored, duplicates, fmt.Errorf("check delivery provenance: %w", err)
|
||||
}
|
||||
isNew, err := m.store.Insert(ctx, newRow(m.orgID, src, e, body, own, run))
|
||||
isNew, err := m.inserter.Insert(ctx, newRow(m.orgID, src, e, body, own, run))
|
||||
if err != nil {
|
||||
return stored, duplicates, err
|
||||
}
|
||||
@@ -332,7 +380,7 @@ func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) {
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
matched := false
|
||||
matched, refused := false, false
|
||||
for _, t := range triggers {
|
||||
var filter map[string]string
|
||||
if err := json.Unmarshal(t.Filter, &filter); err != nil {
|
||||
@@ -341,11 +389,13 @@ func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) {
|
||||
if !match.Fields(e.Fields, filter) {
|
||||
continue
|
||||
}
|
||||
runID, err := m.runs.Start(ctx, runs.StartRequest{
|
||||
OrgID: e.OrgID, Workflow: t.Workflow, Key: e.ID.String() + ":" + t.Name, Input: runInput(e),
|
||||
})
|
||||
runID, started, err := m.startRun(ctx, e, t)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("start run for trigger %q: %w", t.Name, err)
|
||||
return false, err
|
||||
}
|
||||
if !started {
|
||||
refused = true
|
||||
continue
|
||||
}
|
||||
if err := store.RecordDispatch(ctx, tx, e.ID, t.Name, runID); err != nil {
|
||||
return false, err
|
||||
@@ -353,18 +403,44 @@ func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) {
|
||||
m.log.InfoContext(ctx, "trigger started run", "event", e.ID, "trigger", t.Name, "run", runID)
|
||||
matched = true
|
||||
}
|
||||
if !matched {
|
||||
created, err := store.NoteUnmatched(ctx, tx, e)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if created {
|
||||
m.log.InfoContext(ctx, "unmatched events started a group", "source", e.Source, "type", e.Type)
|
||||
}
|
||||
if !matched && !refused {
|
||||
return false, m.noteUnmatched(ctx, tx, e)
|
||||
}
|
||||
return matched, nil
|
||||
}
|
||||
|
||||
// noteUnmatched files an event no trigger took into a group for review.
|
||||
func (m *Module) noteUnmatched(ctx context.Context, tx *sqlx.Tx, e store.Event) error {
|
||||
created, err := store.NoteUnmatched(ctx, tx, e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if created {
|
||||
m.log.InfoContext(ctx, "unmatched events started a group", "source", e.Source, "type", e.Type)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// startRun starts the run trigger t asks for, as the workload principal, so
|
||||
// the start is decided by the same policy chain as any other caller. A refusal
|
||||
// is final: it reports no run and no error, and the trigger is skipped. A policy
|
||||
// chain that can't decide returns an error instead, which rolls the dispatch
|
||||
// back so the event stays queued and is dispatched again.
|
||||
func (m *Module) startRun(ctx context.Context, e store.Event, t store.Trigger) (runID uuid.UUID, started bool, err error) {
|
||||
ctx = governance.WithPrincipal(ctx, governance.WorkloadPrincipal(governance.DefaultOrgID))
|
||||
runID, err = m.runs.Start(ctx, runs.StartRequest{
|
||||
OrgID: e.OrgID, Workflow: t.Workflow, Key: e.ID.String() + ":" + t.Name, Input: runInput(e),
|
||||
})
|
||||
if errors.Is(err, governance.ErrDenied) {
|
||||
m.log.WarnContext(ctx, "trigger not permitted to start its workflow", "event", e.ID, "trigger", t.Name, "error", err)
|
||||
return uuid.Nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return uuid.Nil, false, fmt.Errorf("start run for trigger %q: %w", t.Name, err)
|
||||
}
|
||||
return runID, true, nil
|
||||
}
|
||||
|
||||
// Origin returns the event and trigger that started a run.
|
||||
func (m *Module) Origin(ctx context.Context, runID uuid.UUID) (runs.Origin, bool, error) {
|
||||
o, ok, err := m.store.OriginOf(ctx, m.orgID, runID)
|
||||
|
||||
@@ -50,7 +50,7 @@ func (m *Module) tick(ctx context.Context, now time.Time) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode tick: %w", err)
|
||||
}
|
||||
isNew, err := m.store.Insert(ctx, tickEvent(m.orgID, t.Name, slot, fields))
|
||||
isNew, err := m.inserter.Insert(ctx, tickEvent(m.orgID, t.Name, slot, fields))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/platform/seal"
|
||||
"github.com/samcharles93/archie-core/modules/connectors"
|
||||
"github.com/samcharles93/archie-core/modules/events/internal/store"
|
||||
"github.com/samcharles93/archie-core/modules/governance"
|
||||
)
|
||||
|
||||
type fakeSources struct{ src store.Source }
|
||||
@@ -71,7 +72,7 @@ func TestHMACSourceVerifiesBeforeParsing(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
parsed := false
|
||||
m := &Module{
|
||||
orgID: "org", reader: fakeSources{src}, sealer: sealer,
|
||||
orgID: "org", reader: fakeSources{src}, sealer: sealer, delivery: permitAll{},
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
parse: func(context.Context, string, []byte, map[string]string) ([]connectors.Event, error) {
|
||||
parsed = true
|
||||
@@ -93,3 +94,10 @@ func TestHMACSourceVerifiesBeforeParsing(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// permitAll accepts a delivery from any address.
|
||||
type permitAll struct{}
|
||||
|
||||
func (permitAll) AuthorizeDelivery(context.Context, governance.OrgID, string, string) governance.Decision {
|
||||
return governance.Allowed()
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package runs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json/jsontext"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/samcharles93/archie-core/modules/governance"
|
||||
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
||||
)
|
||||
|
||||
// asked is one decision the engine requested.
|
||||
type asked struct {
|
||||
principal governance.IdentityID
|
||||
action governance.Action
|
||||
kind governance.ResourceKind
|
||||
id string
|
||||
}
|
||||
|
||||
// recordingAccess answers every Authorize with one fixed decision and records
|
||||
// what was asked.
|
||||
type recordingAccess struct {
|
||||
decision governance.Decision
|
||||
calls []asked
|
||||
}
|
||||
|
||||
func (r *recordingAccess) Authorize(_ context.Context, p governance.Principal, a governance.Action, res governance.Resource, _ governance.AccessContext) governance.Decision {
|
||||
r.calls = append(r.calls, asked{p.IdentityID, a, res.Kind, res.ID})
|
||||
return r.decision
|
||||
}
|
||||
|
||||
// queueingStore holds the runs the engine queues.
|
||||
type queueingStore struct {
|
||||
fakeWork
|
||||
queued []store.NewRun
|
||||
}
|
||||
|
||||
func (q *queueingStore) Start(_ context.Context, n store.NewRun) (uuid.UUID, error) {
|
||||
q.queued = append(q.queued, n)
|
||||
return uuid.New(), nil
|
||||
}
|
||||
|
||||
// claimsOne hands out one queued run to the worker, as the store does after a
|
||||
// restart or a retry.
|
||||
type claimsOne struct {
|
||||
*queueingStore
|
||||
run store.Run
|
||||
}
|
||||
|
||||
func (c claimsOne) Claim(context.Context, string, uuid.UUID, time.Duration) (store.Run, bool, error) {
|
||||
return c.run, true, nil
|
||||
}
|
||||
|
||||
// TestRunsAreAuthorizedAtStartAndAtEveryClaim guards that a run started by a
|
||||
// trigger and the same run claimed again after a restart or a retry are decided
|
||||
// by the same live check, as the workload principal, and that a refusal stops
|
||||
// the work. A policy chain that can't decide starts nothing, finishes nothing
|
||||
// and leaves the run for another attempt. Without it a revoked grant would
|
||||
// keep running queued work.
|
||||
func TestRunsAreAuthorizedAtStartAndAtEveryClaim(t *testing.T) {
|
||||
workload := asked{governance.WorkloadID, governance.ActionRun, governance.KindWorkflow, "triage"}
|
||||
tests := []struct {
|
||||
name string
|
||||
decision governance.Decision
|
||||
noPrincipal bool
|
||||
claim bool
|
||||
wantCalls []asked
|
||||
wantQueued int
|
||||
wantRan bool
|
||||
wantFinished []string
|
||||
wantErr error
|
||||
}{
|
||||
{name: "a trigger starts its run", decision: governance.Allowed(), wantCalls: []asked{workload}, wantQueued: 1},
|
||||
{name: "a start with no principal", decision: governance.Allowed(), noPrincipal: true, wantErr: governance.ErrDenied},
|
||||
{name: "a revoked start", decision: governance.DeniedAt(governance.LevelOrg, []string{"org-workload"}), wantCalls: []asked{workload}, wantErr: governance.ErrDenied},
|
||||
{name: "a start the chain can't decide", decision: governance.Unavailable(), wantCalls: []asked{workload}, wantErr: governance.ErrChainUnavailable},
|
||||
{name: "a claim after a restart", decision: governance.Allowed(), claim: true, wantCalls: []asked{workload}, wantRan: true, wantFinished: []string{store.StateSucceeded}},
|
||||
{name: "a claim after revocation", decision: governance.DeniedAt(governance.LevelOrg, []string{"org-workload"}), claim: true, wantCalls: []asked{workload}, wantFinished: []string{store.StateFailed}},
|
||||
{name: "a claim the chain can't decide", decision: governance.Unavailable(), claim: true, wantCalls: []asked{workload}, wantErr: governance.ErrChainUnavailable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
work := &queueingStore{}
|
||||
access := &recordingAccess{decision: tt.decision}
|
||||
engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{})
|
||||
engine.queue = work
|
||||
engine.Use(Deps{Access: access})
|
||||
ran := false
|
||||
useWorkflow(engine, func(*Context, jsontext.Value) (jsontext.Value, error) {
|
||||
ran = true
|
||||
return jsontext.Value(`"done"`), nil
|
||||
})
|
||||
|
||||
var err error
|
||||
if tt.claim {
|
||||
engine.work = claimsOne{work, store.Run{ID: uuid.New(), OrgID: "org", Workflow: "triage", Version: 1, Input: jsontext.Value(`null`)}}
|
||||
_, err = engine.claimAndExecute(t.Context())
|
||||
} else {
|
||||
ctx := t.Context()
|
||||
if !tt.noPrincipal {
|
||||
ctx = governance.WithPrincipal(ctx, governance.WorkloadPrincipal(governance.DefaultOrgID))
|
||||
}
|
||||
_, err = engine.Start(ctx, StartRequest{OrgID: "org", Workflow: "triage", Key: "event:trigger"})
|
||||
}
|
||||
|
||||
if !errors.Is(err, tt.wantErr) {
|
||||
t.Fatalf("error = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
if !slices.Equal(access.calls, tt.wantCalls) {
|
||||
t.Fatalf("decisions asked = %v, want %v", access.calls, tt.wantCalls)
|
||||
}
|
||||
if len(work.queued) != tt.wantQueued {
|
||||
t.Fatalf("runs queued = %d, want %d", len(work.queued), tt.wantQueued)
|
||||
}
|
||||
if ran != tt.wantRan {
|
||||
t.Fatalf("workflow ran = %v, want %v", ran, tt.wantRan)
|
||||
}
|
||||
if !slices.Equal(work.finished, tt.wantFinished) {
|
||||
t.Fatalf("finished = %v, want %v", work.finished, tt.wantFinished)
|
||||
}
|
||||
if slices.Contains(tt.wantFinished, store.StateFailed) && !strings.Contains(work.errors[0], "not permitted") {
|
||||
t.Fatalf("the run's trail says %q, want the refusal", work.errors[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+4
-1
@@ -73,6 +73,9 @@ type DryRequest struct {
|
||||
// StartDry queues a dry run of a document and returns its run's id. A document
|
||||
// that doesn't validate starts nothing.
|
||||
func (e *Engine) StartDry(ctx context.Context, req DryRequest) (uuid.UUID, error) {
|
||||
if err := e.mayStart(ctx, req.Workflow); err != nil {
|
||||
return uuid.Nil, err
|
||||
}
|
||||
doc, err := e.validateDocument(ctx, req.Document)
|
||||
if err != nil {
|
||||
return uuid.Nil, err
|
||||
@@ -93,7 +96,7 @@ func (e *Engine) StartDry(ctx context.Context, req DryRequest) (uuid.UUID, error
|
||||
if err := doc.checkInputs(decodeInputs(input)); err != nil {
|
||||
return uuid.Nil, fmt.Errorf("workflow %q: %w", req.Workflow, err)
|
||||
}
|
||||
return e.db.Start(ctx, store.NewRun{
|
||||
return e.queue.Start(ctx, store.NewRun{
|
||||
OrgID: req.OrgID, Workflow: req.Workflow, Input: input, Dry: true, Overlay: overlay,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/samcharles93/archie-core/modules/connectors"
|
||||
"github.com/samcharles93/archie-core/modules/governance"
|
||||
"github.com/samcharles93/archie-core/modules/judgement"
|
||||
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
||||
)
|
||||
@@ -48,6 +49,7 @@ func (anyBox) Tool(string) (connectors.Tool, bool) { return nil, true }
|
||||
// is queued.
|
||||
func TestStartDryRefusesWhatCannotStart(t *testing.T) {
|
||||
engine := newEngine(nil, &fakeWork{}, slog.New(slog.DiscardHandler), Options{})
|
||||
engine.Use(Deps{Access: &recordingAccess{decision: governance.Allowed()}})
|
||||
tests := []struct {
|
||||
name string
|
||||
doc string
|
||||
@@ -71,7 +73,8 @@ func TestStartDryRefusesWhatCannotStart(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := engine.StartDry(context.Background(), DryRequest{
|
||||
ctx := governance.WithPrincipal(context.Background(), governance.SharedTokenOwner())
|
||||
_, err := engine.StartDry(ctx, DryRequest{
|
||||
OrgID: "org", Workflow: "draft", Document: tt.doc,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
|
||||
@@ -103,6 +103,10 @@ type Deps struct {
|
||||
// Capabilities is what saving a workflow checks connector references
|
||||
// against.
|
||||
Capabilities Capabilities
|
||||
// Access decides whether a run may start and whether a claimed run may
|
||||
// execute, for the principal acting or else the workload principal. With
|
||||
// none, no run starts or executes.
|
||||
Access governance.Authorizer
|
||||
}
|
||||
|
||||
// Workflow is a document ready to run: the code that executes its steps for
|
||||
|
||||
+71
-10
@@ -64,12 +64,14 @@ type runStarter interface {
|
||||
|
||||
// Engine starts, executes and resumes runs.
|
||||
type Engine struct {
|
||||
db *store.Store
|
||||
work runWork
|
||||
log *slog.Logger
|
||||
opts Options
|
||||
mu sync.RWMutex
|
||||
deps *Deps
|
||||
db *store.Store
|
||||
// queue starts runs; it is db except in tests.
|
||||
queue runQueue
|
||||
work runWork
|
||||
log *slog.Logger
|
||||
opts Options
|
||||
mu sync.RWMutex
|
||||
deps *Deps
|
||||
// load turns a claimed run into the code that executes it.
|
||||
load func(ctx context.Context, run store.Run) (Workflow, error)
|
||||
}
|
||||
@@ -93,7 +95,7 @@ func newEngine(db *store.Store, work runWork, log *slog.Logger, opts Options) *E
|
||||
if opts.Owner == "" {
|
||||
opts.Owner = "archie"
|
||||
}
|
||||
e := &Engine{db: db, work: work, log: log, opts: opts}
|
||||
e := &Engine{db: db, queue: db, work: work, log: log, opts: opts}
|
||||
e.load = e.loadDocument
|
||||
return e
|
||||
}
|
||||
@@ -185,14 +187,42 @@ func (e *Engine) Has(ctx context.Context, orgID, name string) (bool, error) {
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
// runQueue reads the workflow a run starts under and queues the run.
|
||||
type runQueue interface {
|
||||
workflowFinder
|
||||
Start(ctx context.Context, n store.NewRun) (uuid.UUID, error)
|
||||
}
|
||||
|
||||
// Start queues a run under the latest stored version of its workflow and
|
||||
// returns its id.
|
||||
// returns its id. The principal in ctx must be allowed to run the workflow;
|
||||
// work that starts runs on its own, such as a trigger, passes the workload
|
||||
// principal. A start the policy chain can't decide queues nothing, and the
|
||||
// error wraps governance.ErrChainUnavailable so the caller tries again.
|
||||
func (e *Engine) Start(ctx context.Context, req StartRequest) (uuid.UUID, error) {
|
||||
n, err := newRun(ctx, e.db, req)
|
||||
if err := e.mayStart(ctx, req.Workflow); err != nil {
|
||||
return uuid.Nil, err
|
||||
}
|
||||
n, err := newRun(ctx, e.queue, req)
|
||||
if err != nil {
|
||||
return uuid.Nil, err
|
||||
}
|
||||
return e.db.Start(ctx, n)
|
||||
return e.queue.Start(ctx, n)
|
||||
}
|
||||
|
||||
// mayStart refuses a start by a caller who may not run workflow, or whose
|
||||
// right can't be checked.
|
||||
func (e *Engine) mayStart(ctx context.Context, workflow string) error {
|
||||
p, ok := governance.PrincipalFromContext(ctx)
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: no principal starts the run", governance.ErrDenied)
|
||||
}
|
||||
deps, err := e.steps()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", governance.ErrChainUnavailable, err)
|
||||
}
|
||||
return governance.Require(ctx, deps.Access, p, governance.ActionRun, governance.Resource{
|
||||
Kind: governance.KindWorkflow, ID: workflow, Org: p.Org,
|
||||
})
|
||||
}
|
||||
|
||||
// workflowFinder reads stored workflow versions.
|
||||
@@ -320,10 +350,41 @@ func (e *Engine) claimAndExecute(ctx context.Context) (bool, error) {
|
||||
if err != nil || !ok {
|
||||
return false, err
|
||||
}
|
||||
if err := e.mayExecute(ctx, run); err != nil {
|
||||
return e.refuseClaim(ctx, run, token, err)
|
||||
}
|
||||
e.execute(ctx, run, token)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// mayExecute rechecks, each time a worker claims a run, that the workload
|
||||
// principal may still run its workflow. A new run, a retry and a resume after
|
||||
// a crash or a wake-up all pass through here.
|
||||
func (e *Engine) mayExecute(ctx context.Context, run store.Run) error {
|
||||
deps, err := e.steps()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", governance.ErrChainUnavailable, err)
|
||||
}
|
||||
return governance.RequireWorkload(ctx, deps.Access, governance.ActionRun, governance.Resource{
|
||||
Kind: governance.KindWorkflow, ID: run.Workflow,
|
||||
})
|
||||
}
|
||||
|
||||
// refuseClaim handles a claimed run that may not execute. A policy chain that
|
||||
// can't decide leaves the run unfinished, so its lease lapses and a worker
|
||||
// claims it again. A refusal ends the run as failed, and its trail says why.
|
||||
func (e *Engine) refuseClaim(ctx context.Context, run store.Run, token uuid.UUID, cause error) (bool, error) {
|
||||
if !errors.Is(cause, governance.ErrDenied) {
|
||||
return false, fmt.Errorf("run %s waits for access policies: %w", run.ID, cause)
|
||||
}
|
||||
ended := store.Ended{State: store.StateFailed, Error: cause.Error()}
|
||||
if err := e.work.Finish(ctx, run.ID, token, ended); err != nil {
|
||||
return false, fmt.Errorf("finish refused run %s: %w", run.ID, err)
|
||||
}
|
||||
e.log.WarnContext(ctx, "run refused", "run", run.ID, "workflow", run.Workflow, "error", cause)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// execute runs one claimed run to its next suspension or end. The run's
|
||||
// context ends if the lease can't be renewed, so a stale worker stops.
|
||||
func (e *Engine) execute(ctx context.Context, run store.Run, token uuid.UUID) {
|
||||
|
||||
Reference in new issue
Block a user