From 43769e902b40524f5223b364e9e213e2d1dbe14f Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 10 Oct 2026 22:19:02 +1100 Subject: [PATCH] fix(access): an unloaded chain is a refusing engine, not nil Webhook intake answers 503 and counts no refusal while the chain loads. The readiness probe reads the chain on every check. The UI never exits on a first-load failure. An invalid instance policy closes the instance but lets an instance admin repair policies. Outage pages are 503, and the health detail names the unloaded chain. --- internal/app/archieui/compose.go | 17 ++---- internal/app/archieui/readiness.go | 12 +---- internal/app/archieui/run.go | 26 +++------ .../app/controlplane/workflow_org_test.go | 5 +- internal/app/statestore/policy_guard.go | 11 +--- internal/app/statestore/server.go | 5 +- internal/domain/access/request.go | 6 +++ .../infrastructure/access/delivery_test.go | 5 +- internal/infrastructure/access/engine.go | 40 +++++++++----- internal/infrastructure/access/engine_test.go | 47 ++++++++++++++++ internal/infrastructure/access/live.go | 53 ++++++------------- .../infrastructure/captureintake/org_test.go | 53 +++++++++++++++---- .../infrastructure/captureintake/receiver.go | 26 +++++---- internal/infrastructure/readiness/probes.go | 6 ++- .../infrastructure/readiness/probes_test.go | 37 +++++++++++++ internal/webui/api_orgs_test.go | 10 +--- internal/webui/auth.go | 13 ++--- internal/webui/authorize.go | 23 ++++++-- internal/webui/authorize_test.go | 31 ++++++++--- internal/webui/server.go | 18 +++++-- 20 files changed, 277 insertions(+), 167 deletions(-) create mode 100644 internal/infrastructure/access/engine_test.go create mode 100644 internal/infrastructure/readiness/probes_test.go diff --git a/internal/app/archieui/compose.go b/internal/app/archieui/compose.go index f6884040a..0da572098 100644 --- a/internal/app/archieui/compose.go +++ b/internal/app/archieui/compose.go @@ -116,10 +116,8 @@ func compose(ctx context.Context, d deps) *webui.Server { wireCatalog(d, srv) // The policy chain: wired together or not at all. srv.Refusals = refusalStore(d.Store) - if d.Access != nil { - srv.Access = d.Access - srv.ReloadAccess = d.Access.Reload - } + srv.Access = d.Access + srv.ReloadAccess = d.Access.Reload srv.Principals = d.Principals srv.Denials = d.Denials wireTaskLogs(d, srv) @@ -195,7 +193,7 @@ func wireCaptureSurfaces(d deps, srv *webui.Server) { MaxEvents: capture.MaxEvents, MaxBodyBytes: int64(capture.MaxBodyBytes), Publish: captureArrivalPublisher(d), - Delivery: deliveryAuthorizer(d.Access), + Delivery: d.Access, Refusals: refusalStore(d.Store), Log: d.Log, } @@ -221,15 +219,6 @@ func captureArrivalPublisher(d deps) func(context.Context, events.Event) { // a hung State Store must not hold the sender's webhook past this. const captureEventTimeout = 5 * time.Second -// deliveryAuthorizer keeps a missing chain a nil authorizer, which admits -// every sender. -func deliveryAuthorizer(chain *infraaccess.Live) access.DeliveryAuthorizer { - if chain == nil { - return nil - } - return chain -} - func refusalStore(store storecontract.TaskStore) storecontract.CaptureRefusalStore { refusals, _ := store.(storecontract.CaptureRefusalStore) return refusals diff --git a/internal/app/archieui/readiness.go b/internal/app/archieui/readiness.go index 0536fa350..dfd49fa4d 100644 --- a/internal/app/archieui/readiness.go +++ b/internal/app/archieui/readiness.go @@ -18,7 +18,7 @@ import ( // Store and, when one is configured, the sign-in provider. func newReadinessRegistry(o Options, tasks storecontract.TaskStore, chat messaging.ChatContract, chain *infraaccess.Live, provider *oidc.Provider) *health.Registry { probes := []health.Probe{ - readiness.NewProblemProbe("access_policies", problemSource(chain)), + readiness.NewProblemProbe("access_policies", chain), captureRefusalProbe{store: refusalStore(tasks)}, readiness.NewContractProbe("state_db", o.DependencyTimeout, func(ctx context.Context) error { _, err := tasks.StatusCounts(ctx) @@ -35,16 +35,6 @@ func newReadinessRegistry(o Options, tasks storecontract.TaskStore, chat messagi return health.NewRegistry(probes...) } -// problemSource reports the chain's invalid policies; a chain that has not -// loaded yet is reported as not wired, so readiness is degraded while requests -// are refused. -func problemSource(chain *infraaccess.Live) readiness.PolicyProblems { - if chain == nil || !chain.Ready() { - return nil - } - return chain -} - // captureRefusalProbe degrades when any source refused more events than it // accepted over the last hour: a sender being blocked, or someone probing. type captureRefusalProbe struct { diff --git a/internal/app/archieui/run.go b/internal/app/archieui/run.go index 9d6b7cd7d..07fc0af99 100644 --- a/internal/app/archieui/run.go +++ b/internal/app/archieui/run.go @@ -12,9 +12,6 @@ import ( "slices" "time" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "github.com/samcharles93/archie-core/internal/app/servicekit" "github.com/samcharles93/archie-core/internal/domain/applystatus" "github.com/samcharles93/archie-core/internal/domain/presence" @@ -27,20 +24,14 @@ import ( "github.com/samcharles93/archie-core/internal/webui" ) -// buildAccessChain builds the live access engine over the stored policies. A -// State Store that is not up yet yields a chain with no engine: it refuses -// every request until the first load succeeds and keeps retrying, so a slow -// database or a bad start order closes the dashboard rather than opening it. -func buildAccessChain(ctx context.Context, store *staterpc.Client, log *slog.Logger) (*infraaccess.Live, error) { +// buildAccessChain starts the live access engine over the stored policies. +// It refuses every request until Run's first load succeeds and keeps +// retrying, so a State Store or database that is down or slow at start +// closes the dashboard rather than opening it or stopping it. +func buildAccessChain(ctx context.Context, store *staterpc.Client, log *slog.Logger) *infraaccess.Live { live := infraaccess.NewPending(store.ListPolicies, log) - if err := live.Reload(ctx); err != nil { - if status.Code(err) != codes.Unavailable { - return nil, err - } - log.Warn("access policies unavailable; the dashboard refuses API requests until they load", "err", err) - } go live.Run(ctx, applystatus.RestampInterval) - return live, nil + return live } // Run serves the dashboard until ctx is cancelled. It resolves its own @@ -71,10 +62,7 @@ func Run(ctx context.Context, options Options) error { return err } cleanups = append(cleanups, closeGateway) - chain, err := buildAccessChain(ctx, tasks, log) - if err != nil { - return err - } + chain := buildAccessChain(ctx, tasks, log) provider, err := signInProvider(opts) if err != nil { return err diff --git a/internal/app/controlplane/workflow_org_test.go b/internal/app/controlplane/workflow_org_test.go index df13237d5..eb12884be 100644 --- a/internal/app/controlplane/workflow_org_test.go +++ b/internal/app/controlplane/workflow_org_test.go @@ -169,10 +169,7 @@ func checkWorkflowDashboard(t *testing.T, db *pgstore.TaskDB, client *staterpc.C t.Helper() ctx := t.Context() policies := append(access.ShippedOrgPolicies("acme"), access.ShippedOrgPolicies(org.DefaultOrgID)...) - chain, err := infraaccess.New(policies) - if err != nil { - t.Fatal(err) - } + chain := infraaccess.New(policies) srv := &webui.Server{Store: client, ControlPlane: client.ControlPlane(), Access: chain, Principals: client, Authenticate: func(ctx context.Context, token string) (identity.Identity, error) { return db.Get(ctx, identity.IdentityID(token)) }} diff --git a/internal/app/statestore/policy_guard.go b/internal/app/statestore/policy_guard.go index cb9258d50..f16897527 100644 --- a/internal/app/statestore/policy_guard.go +++ b/internal/app/statestore/policy_guard.go @@ -26,11 +26,7 @@ type guardedPolicies struct { } func (g guardedPolicies) PutPolicy(ctx context.Context, p access.Policy) (int64, error) { - validator, err := accessengine.New(nil) - if err != nil { - return 0, err - } - if err := validator.Validate(p); err != nil { + if err := accessengine.New(nil).Validate(p); err != nil { return 0, err } if err := g.checkLockout(ctx, func(current []access.Policy) []access.Policy { @@ -55,10 +51,7 @@ func (g guardedPolicies) checkLockout(ctx context.Context, change func([]access. if err != nil { return err } - engine, err := accessengine.New(change(current)) - if err != nil { - return err - } + engine := accessengine.New(change(current)) orgs, err := g.owners.ListOrgs(ctx) if err != nil { return err diff --git a/internal/app/statestore/server.go b/internal/app/statestore/server.go index 5f581ebc7..815dd5f73 100644 --- a/internal/app/statestore/server.go +++ b/internal/app/statestore/server.go @@ -181,10 +181,7 @@ func (b *server) seedAndValidatePolicies(ctx context.Context) error { if err != nil { return fmt.Errorf("load stored policies: %w", err) } - engine, err := infraaccess.New(stored) - if err != nil { - return fmt.Errorf("validate stored policies: %w", err) - } + engine := infraaccess.New(stored) for _, problem := range engine.Problems() { b.log.Error("stored access policy is invalid and denies its level", "policy", problem.Policy.ID, "level", problem.Policy.Level, "err", problem.Err) diff --git a/internal/domain/access/request.go b/internal/domain/access/request.go index 4cefaeb82..bbf85a697 100644 --- a/internal/domain/access/request.go +++ b/internal/domain/access/request.go @@ -131,6 +131,12 @@ type Decision struct { // Allowed is the shared permit decision: no level denied. func Allowed() Decision { return Decision{Allowed: true} } +// Unavailable is the refusal of a chain that has not loaded: nothing was +// evaluated, so it is an availability failure rather than a denial. +func Unavailable() Decision { + return Decision{Level: LevelInstance, Err: ErrChainUnavailable} +} + // DeniedAt records a denial decided at a level by the given policies. func DeniedAt(level Level, policies []string) Decision { return Decision{Level: level, Policies: policies} diff --git a/internal/infrastructure/access/delivery_test.go b/internal/infrastructure/access/delivery_test.go index 0a0f30358..2c2cec7c8 100644 --- a/internal/infrastructure/access/delivery_test.go +++ b/internal/infrastructure/access/delivery_test.go @@ -40,10 +40,7 @@ func TestAuthorizeDelivery(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - engine, err := New(tt.policies) - if err != nil { - t.Fatal(err) - } + engine := New(tt.policies) acting := tt.org if acting == "" { acting = org.DefaultOrgID diff --git a/internal/infrastructure/access/engine.go b/internal/infrastructure/access/engine.go index f35e72c35..0c7f63c6f 100644 --- a/internal/infrastructure/access/engine.go +++ b/internal/infrastructure/access/engine.go @@ -65,8 +65,14 @@ type Engine struct { spaces map[spaceKey]*levelSet objects map[objectKey]*levelSet problems []Problem + // pending marks the engine of a chain whose policies have not loaded: + // it refuses everything without evaluating. + pending bool } +// pendingEngine refuses every request until the stored policies load. +var pendingEngine = &Engine{pending: true} + type ( spaceKey struct { org org.OrgID @@ -85,10 +91,9 @@ var ( _ access.Validator = (*Engine)(nil) ) -// New builds an engine from the stored policies. An invalid instance policy -// is an error; an invalid policy at another level makes that level deny -// everything. -func New(policies []access.Policy) (*Engine, error) { +// New builds an engine from the stored policies. An invalid policy makes its +// level deny everything and is reported as a Problem. +func New(policies []access.Policy) *Engine { e := &Engine{ orgs: map[org.OrgID]*levelSet{}, spaces: map[spaceKey]*levelSet{}, @@ -98,9 +103,6 @@ func New(policies []access.Policy) (*Engine, error) { lvl := e.level(p) policy, err := compile(p) if err != nil { - if p.Level == access.LevelInstance { - return nil, fmt.Errorf("access: instance policy %q: %w", p.ID, err) - } // The refusal must name the policy that poisoned the level. lvl.ids = append(lvl.ids, p.ID) lvl.invalid = err @@ -110,9 +112,12 @@ func New(policies []access.Policy) (*Engine, error) { lvl.ids = append(lvl.ids, p.ID) lvl.set.Add(types.PolicyID(p.ID), policy) } - return e, nil + return e } +// Ready reports whether the engine was built from stored policies. +func (e *Engine) Ready() bool { return !e.pending } + // Problems reports every stored policy the engine could not compile. Empty // when every stored policy is valid. func (e *Engine) Problems() []Problem { return e.problems } @@ -186,6 +191,9 @@ func compile(p access.Policy) (*cedar.Policy, error) { // Authorize evaluates the chain for one request. func (e *Engine) Authorize(p access.Principal, a access.Action, r access.Resource, c access.Context) access.Decision { + if e.pending { + return access.Unavailable() + } // The cross-org forbid, enforced structurally: a principal's requests // only ever reach its own org. An unset resource org means the // principal's org (a create request names a record that does not exist @@ -198,7 +206,7 @@ func (e *Engine) Authorize(p access.Principal, a access.Action, r access.Resourc return access.DeniedAt(access.LevelInstance, []string{access.CrossOrgForbidID}) } - return evaluate(e.chain(r), buildEntities(p, r), buildRequest(p, a, r, c)) + return evaluate(e.chain(p, r), buildEntities(p, r), buildRequest(p, a, r, c)) } // AuthorizeDelivery decides whether addr may deliver an event to a source: @@ -206,6 +214,9 @@ func (e *Engine) Authorize(p access.Principal, a access.Action, r access.Resourc // sender has no identity or role, so the org and workspace role policies // never apply to it. func (e *Engine) AuthorizeDelivery(orgID org.OrgID, sourcePath, addr string) access.Decision { + if e.pending { + return access.Unavailable() + } sender := access.Principal{IdentityID: access.SenderID, Org: orgID} resource := access.Resource{Kind: access.KindSource, ID: sourcePath, Org: orgID} levels := []*levelSet{e.instance} @@ -247,9 +258,14 @@ func evaluate(levels []*levelSet, entities types.EntityMap, request types.Reques return access.Allowed() } -// chain returns the level sets the request passes through, in order. -func (e *Engine) chain(r access.Resource) []*levelSet { - out := []*levelSet{e.instance} +// chain returns the level sets the request passes through, in order. An +// invalid instance level refuses everything except an instance admin working +// on the policies, so the operator can repair it from the dashboard. +func (e *Engine) chain(p access.Principal, r access.Resource) []*levelSet { + var out []*levelSet + if e.instance == nil || e.instance.invalid == nil || r.Kind != access.KindPolicy || !p.InstanceAdmin() { + out = append(out, e.instance) + } if l, ok := e.orgs[r.Org]; ok { out = append(out, l) } diff --git a/internal/infrastructure/access/engine_test.go b/internal/infrastructure/access/engine_test.go new file mode 100644 index 000000000..0cbb4689f --- /dev/null +++ b/internal/infrastructure/access/engine_test.go @@ -0,0 +1,47 @@ +package access + +import ( + "testing" + + "github.com/samcharles93/archie-core/internal/domain/access" + "github.com/samcharles93/archie-core/internal/domain/identity" + "github.com/samcharles93/archie-core/internal/domain/org" +) + +// TestInvalidInstancePolicy pins that an instance policy that no longer +// compiles closes the instance without locking the operator out: only an +// instance admin working on the policies passes, so the policy can be +// repaired from the dashboard. +func TestInvalidInstancePolicy(t *testing.T) { + broken := access.Policy{ID: "broken", Level: access.LevelInstance, Text: `permit(principal, action == Archie::Action::"nope", resource);`} + policies := append(access.ShippedOrgPolicies(org.DefaultOrgID), access.ShippedOrgPolicies("acme")...) + engine := New(append(policies, broken)) + if len(engine.Problems()) != 1 { + t.Fatalf("problems = %v, want the broken instance policy", engine.Problems()) + } + + owner := func(id identity.IdentityID, o org.OrgID) access.Principal { + return access.Principal{IdentityID: id, Org: o, Memberships: []org.Membership{{IdentityID: id, OrgID: o, Role: org.RoleOwner}}} + } + admin := owner("admin", org.DefaultOrgID) + tests := []struct { + name string + principal access.Principal + action access.Action + kind access.ResourceKind + allowed bool + }{ + {"instance admin edits policies", admin, access.ActionManagePolicies, access.KindPolicy, true}, + {"instance admin reads policies", admin, access.ActionRead, access.KindPolicy, true}, + {"instance admin reads anything else", admin, access.ActionRead, access.KindDashboard, false}, + {"another org's owner edits policies", owner("acme-owner", "acme"), access.ActionManagePolicies, access.KindPolicy, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := engine.Authorize(tt.principal, tt.action, access.Resource{Kind: tt.kind}, access.Context{}) + if got.Allowed != tt.allowed { + t.Fatalf("allowed = %v, want %v (%+v)", got.Allowed, tt.allowed, got) + } + }) + } +} diff --git a/internal/infrastructure/access/live.go b/internal/infrastructure/access/live.go index 36a824711..6cb912568 100644 --- a/internal/infrastructure/access/live.go +++ b/internal/infrastructure/access/live.go @@ -21,22 +21,24 @@ type Live struct { // NewLive builds the first engine from load. func NewLive(ctx context.Context, load func(context.Context) ([]access.Policy, error), log *slog.Logger) (*Live, error) { - l := &Live{load: load, log: log} + l := NewPending(load, log) if err := l.Reload(ctx); err != nil { return nil, err } return l, nil } -// NewPending returns a chain with no engine: Run loads it, retrying until the -// store answers. A chain that is not ready refuses every request, so a process -// that starts before its State Store is closed rather than opened. +// NewPending returns a chain whose engine refuses every request until Run or +// Reload loads the stored policies, so a process that starts before its State +// Store is closed rather than opened. func NewPending(load func(context.Context) ([]access.Policy, error), log *slog.Logger) *Live { - return &Live{load: load, log: log} + l := &Live{load: load, log: log} + l.engine.Store(pendingEngine) + return l } -// Ready reports whether an engine has loaded. -func (l *Live) Ready() bool { return l.engine.Load() != nil } +// Ready reports whether the stored policies have loaded. +func (l *Live) Ready() bool { return l.engine.Load().Ready() } // Reload rebuilds the engine from the store now. func (l *Live) Reload(ctx context.Context) error { @@ -44,10 +46,7 @@ func (l *Live) Reload(ctx context.Context) error { if err != nil { return err } - engine, err := New(stored) - if err != nil { - return err - } + engine := New(stored) for _, problem := range engine.Problems() { l.log.Error("stored access policy is invalid and denies its level", "policy", problem.Policy.ID, "level", problem.Policy.Level, "err", problem.Err) @@ -56,9 +55,8 @@ func (l *Live) Reload(ctx context.Context) error { return nil } -// Run reloads on every interval until ctx ends. A chain with no engine yet -// tries at once and keeps trying, so the first load does not wait an interval -// after the State Store returns. +// Run loads at once, then reloads on every interval until ctx ends, so the +// first load does not wait an interval after the State Store returns. func (l *Live) Run(ctx context.Context, interval time.Duration) { ticker := time.NewTicker(interval) defer ticker.Stop() @@ -85,33 +83,12 @@ func (l *Live) reload(ctx context.Context) { } func (l *Live) Authorize(p access.Principal, a access.Action, r access.Resource, c access.Context) access.Decision { - engine := l.engine.Load() - if engine == nil { - return notLoaded() - } - return engine.Authorize(p, a, r, c) + return l.engine.Load().Authorize(p, a, r, c) } func (l *Live) AuthorizeDelivery(orgID org.OrgID, sourcePath, addr string) access.Decision { - engine := l.engine.Load() - if engine == nil { - return notLoaded() - } - return engine.AuthorizeDelivery(orgID, sourcePath, addr) -} - -// notLoaded is the refusal of a chain with no engine: nothing was evaluated. -func notLoaded() access.Decision { - decision := access.DeniedAt(access.LevelInstance, nil) - decision.Err = access.ErrChainUnavailable - return decision + return l.engine.Load().AuthorizeDelivery(orgID, sourcePath, addr) } // Problems reports the current engine's invalid policies. -func (l *Live) Problems() []Problem { - engine := l.engine.Load() - if engine == nil { - return nil - } - return engine.Problems() -} +func (l *Live) Problems() []Problem { return l.engine.Load().Problems() } diff --git a/internal/infrastructure/captureintake/org_test.go b/internal/infrastructure/captureintake/org_test.go index 0859c8d1f..ca75f4ec8 100644 --- a/internal/infrastructure/captureintake/org_test.go +++ b/internal/infrastructure/captureintake/org_test.go @@ -36,21 +36,34 @@ func (s *stubCaptures) ListCaptures(_ context.Context, _ int) ([]storecontract.C } type stubDelivery struct { - orgID org.OrgID - allow bool + orgID org.OrgID + decision access.Decision } func (s *stubDelivery) AuthorizeDelivery(orgID org.OrgID, _, _ string) access.Decision { s.orgID = orgID - return access.Decision{Allowed: s.allow} + return s.decision +} + +type stubRefusals struct{ recorded int } + +func (s *stubRefusals) RecordCaptureRefusal(context.Context, string, string, time.Time) error { + s.recorded++ + return nil +} + +func (*stubRefusals) CaptureRefusals(context.Context, time.Time) ([]storecontract.CaptureRefusals, error) { + return nil, nil } // Delivery runs against the source's owning org, falling back to the // default org when the source is unknown. func TestDeliveryUsesSourceOrg(t *testing.T) { + refusals := &stubRefusals{} newReceiver := func(delivery *stubDelivery) (*Receiver, *stubCaptures) { captures := &stubCaptures{} return &Receiver{ + Refusals: refusals, Captures: captures, Sources: stubSources{byPath: map[string]*source.Source{ "acme-hook": {Path: "acme-hook", Signing: source.SigningUnsigned, OrgID: "acme"}, @@ -69,7 +82,7 @@ func TestDeliveryUsesSourceOrg(t *testing.T) { return response } - delivery := &stubDelivery{allow: true} + delivery := &stubDelivery{decision: access.Allowed()} rc, _ := newReceiver(delivery) if response := post(rc, "acme-hook"); response.Code != http.StatusAccepted { t.Fatalf("capture: %d %s", response.Code, response.Body.String()) @@ -78,7 +91,7 @@ func TestDeliveryUsesSourceOrg(t *testing.T) { t.Fatalf("authorized against %q, want acme", delivery.orgID) } - delivery = &stubDelivery{allow: true} + delivery = &stubDelivery{decision: access.Allowed()} rc, _ = newReceiver(delivery) if response := post(rc, "unknown"); response.Code != http.StatusAccepted { t.Fatalf("capture: %d %s", response.Code, response.Body.String()) @@ -87,12 +100,30 @@ func TestDeliveryUsesSourceOrg(t *testing.T) { t.Fatalf("authorized against %q, want the default org", delivery.orgID) } - delivery = &stubDelivery{} - rc, captures := newReceiver(delivery) - if response := post(rc, "acme-hook"); response.Code != http.StatusForbidden { - t.Fatalf("refused capture: %d %s", response.Code, response.Body.String()) + // A denial is refused and counted; a chain that has not loaded evaluated + // nothing, so the sender may retry and no refusal is counted. + refusalTests := []struct { + name string + decision access.Decision + code int + counted int + }{ + {"denied", access.DeniedAt(access.LevelObject, []string{"lan-only"}), http.StatusForbidden, 1}, + {"chain not loaded", access.Unavailable(), http.StatusServiceUnavailable, 0}, } - if captures.stored != 0 { - t.Fatalf("refused delivery stored %d captures", captures.stored) + for _, tt := range refusalTests { + t.Run(tt.name, func(t *testing.T) { + refusals.recorded = 0 + rc, captures := newReceiver(&stubDelivery{decision: tt.decision}) + if response := post(rc, "acme-hook"); response.Code != tt.code { + t.Fatalf("refused capture: %d %s, want %d", response.Code, response.Body.String(), tt.code) + } + if captures.stored != 0 { + t.Fatalf("refused delivery stored %d captures", captures.stored) + } + if refusals.recorded != tt.counted { + t.Fatalf("refusals counted = %d, want %d", refusals.recorded, tt.counted) + } + }) } } diff --git a/internal/infrastructure/captureintake/receiver.go b/internal/infrastructure/captureintake/receiver.go index e1773ec03..824a861e0 100644 --- a/internal/infrastructure/captureintake/receiver.go +++ b/internal/infrastructure/captureintake/receiver.go @@ -5,6 +5,7 @@ package captureintake import ( "context" "encoding/json" + "errors" "io" "log/slog" "mime" @@ -91,8 +92,8 @@ func (rc *Receiver) ServeHTTP(w http.ResponseWriter, r *http.Request) { // authorizes as the default org. src := rc.resolveSource(r, path) - if rc.refused(r, path, orgOf(src)) { - http.Error(w, "forbidden", http.StatusForbidden) + if code := rc.refusal(r, path, orgOf(src)); code != 0 { + http.Error(w, http.StatusText(code), code) return } @@ -158,22 +159,29 @@ func (rc *Receiver) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) } -// refused applies the network rules before the body is read. A refused event -// is not stored, only counted on its source with the sender's address. -func (rc *Receiver) refused(r *http.Request, path string, owner org.OrgID) bool { +// refusal applies the network rules before the body is read and returns the +// status to refuse with, or 0 to admit. A refused event is not stored, only +// counted on its source with the sender's address. A chain that has not +// loaded evaluated nothing: the sender gets a retryable 503 and no refusal is +// counted. +func (rc *Receiver) refusal(r *http.Request, path string, owner org.OrgID) int { if rc.Delivery == nil { - return false + return 0 } addr := remoteAddrHost(r) - if rc.Delivery.AuthorizeDelivery(owner, path, addr).Allowed { - return false + decision := rc.Delivery.AuthorizeDelivery(owner, path, addr) + if decision.Allowed { + return 0 + } + if errors.Is(decision.Err, access.ErrChainUnavailable) { + return http.StatusServiceUnavailable } if rc.Refusals != nil { if err := rc.Refusals.RecordCaptureRefusal(r.Context(), path, addr, time.Now()); err != nil { rc.logger().Warn("capture refusal not counted", "source", path, "err", err) } } - return true + return http.StatusForbidden } // orgOf reports the org that owns src, or the default org when the source diff --git a/internal/infrastructure/readiness/probes.go b/internal/infrastructure/readiness/probes.go index d8dbaa865..9a09eed54 100644 --- a/internal/infrastructure/readiness/probes.go +++ b/internal/infrastructure/readiness/probes.go @@ -86,8 +86,9 @@ func (p *ContractProbe) Check(ctx context.Context) health.Result { // PolicyProblems reports the stored access policies this process's engine // could not validate. The policy chain is the source; a policy that failed // startup re-validation denies its level and must be named here with its -// error. +// error. Ready is false until the stored policies have loaded. type PolicyProblems interface { + Ready() bool Problems() []access.Problem } @@ -109,6 +110,9 @@ func (p *ProblemProbe) Check(ctx context.Context) health.Result { if p.Source == nil { return health.Result{Status: health.StatusDegraded, Detail: "access engine not wired"} } + if !p.Source.Ready() { + return health.Result{Status: health.StatusDegraded, Detail: "access policies have not loaded; requests are refused"} + } problems := p.Source.Problems() if len(problems) == 0 { return health.Result{Status: health.StatusOK} diff --git a/internal/infrastructure/readiness/probes_test.go b/internal/infrastructure/readiness/probes_test.go new file mode 100644 index 000000000..b828b6f96 --- /dev/null +++ b/internal/infrastructure/readiness/probes_test.go @@ -0,0 +1,37 @@ +package readiness + +import ( + "context" + "errors" + "log/slog" + "testing" + + "github.com/samcharles93/archie-core/internal/domain/access" + "github.com/samcharles93/archie-core/internal/domain/health" + infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access" +) + +// TestProblemProbeFollowsTheChain pins that the access_policies probe reads +// the chain on every check: degraded while the policies have not loaded, ok +// once they do, without rebuilding the probe. +func TestProblemProbeFollowsTheChain(t *testing.T) { + up := false + chain := infraaccess.NewPending(func(context.Context) ([]access.Policy, error) { + if !up { + return nil, errors.New("store down") + } + return nil, nil + }, slog.New(slog.DiscardHandler)) + probe := NewProblemProbe("access_policies", chain) + + if got := probe.Check(t.Context()); got.Status != health.StatusDegraded { + t.Fatalf("before load = %+v, want degraded", got) + } + up = true + if err := chain.Reload(t.Context()); err != nil { + t.Fatal(err) + } + if got := probe.Check(t.Context()); got.Status != health.StatusOK { + t.Fatalf("after load = %+v, want ok", got) + } +} diff --git a/internal/webui/api_orgs_test.go b/internal/webui/api_orgs_test.go index ee6416fa3..3ee079e81 100644 --- a/internal/webui/api_orgs_test.go +++ b/internal/webui/api_orgs_test.go @@ -179,10 +179,7 @@ func (rolePrincipals) PrincipalFor(_ context.Context, id identity.IdentityID) (a // policies: a developer manages no members, an admin manages members but can // neither grant the owner role nor change an owner's, and an owner can. func TestOrgMembershipNeedsItsRole(t *testing.T) { - engine, err := infraaccess.New(access.ShippedOrgPolicies(org.DefaultOrgID)) - if err != nil { - t.Fatal(err) - } + engine := infraaccess.New(access.ShippedOrgPolicies(org.DefaultOrgID)) members := "/api/orgs/" + string(org.DefaultOrgID) + "/members/" tests := []struct { name string @@ -233,10 +230,7 @@ func TestOrgMembershipNeedsItsRole(t *testing.T) { // lists and opens every org and creates one with its owner; an admin of // org-sys sees only its own org and cannot create one. func TestInstanceAdminOrgs(t *testing.T) { - engine, err := infraaccess.New(access.ShippedOrgPolicies(org.DefaultOrgID)) - if err != nil { - t.Fatal(err) - } + engine := infraaccess.New(access.ShippedOrgPolicies(org.DefaultOrgID)) tests := []struct { name string caller string diff --git a/internal/webui/auth.go b/internal/webui/auth.go index c87a45023..8f87b1382 100644 --- a/internal/webui/auth.go +++ b/internal/webui/auth.go @@ -116,7 +116,7 @@ func (s *Server) requireToken(h http.Handler) http.Handler { if tok := r.URL.Query().Get("t"); tok != "" { if !tokenEqual(tok, s.Token) { if wantsDocument(r) { - s.authPage(w, "That token was not accepted.") + s.authPage(w, http.StatusUnauthorized, "That token was not accepted.") return } w.Header().Set("Cache-Control", "no-store") @@ -150,7 +150,7 @@ func (s *Server) requireToken(h http.Handler) http.Handler { } if wantsDocument(r) { - s.authPage(w, "") + s.authPage(w, http.StatusUnauthorized, "") return } w.Header().Set("Content-Type", "text/plain; charset=utf-8") @@ -306,7 +306,7 @@ func (s *Server) refuseUnidentified(w http.ResponseWriter, r *http.Request, err http.Redirect(w, r, loginPath, http.StatusSeeOther) return } - s.authPage(w, reason) + s.authPage(w, http.StatusUnauthorized, reason) return } http.Error(w, reason, status) @@ -335,12 +335,13 @@ func wantsDocument(r *http.Request) bool { return strings.Contains(r.Header.Get("Accept"), "text/html") } -// authPage serves a self-contained 401 page for pasting the access token. -func (s *Server) authPage(w http.ResponseWriter, reason string) { +// authPage serves the self-contained page for pasting the access token, +// with code: 401 for a credential failure, 503 for an outage. +func (s *Server) authPage(w http.ResponseWriter, code int, reason string) { w.Header().Set("Content-Type", "text/html; charset=utf-8") w.Header().Set("Cache-Control", "no-store") w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; form-action 'self'") - w.WriteHeader(http.StatusUnauthorized) + w.WriteHeader(code) notice := "" if reason != "" { notice = `

` + reason + `

` diff --git a/internal/webui/authorize.go b/internal/webui/authorize.go index f3de3ea35..1ddcf405d 100644 --- a/internal/webui/authorize.go +++ b/internal/webui/authorize.go @@ -2,11 +2,13 @@ package webui import ( "context" + "encoding/json" "errors" "net/http" "strings" "github.com/samcharles93/archie-core/internal/domain/access" + "github.com/samcharles93/archie-core/internal/domain/health" "github.com/samcharles93/archie-core/internal/domain/identity" "github.com/samcharles93/archie-core/internal/domain/org" ) @@ -17,14 +19,16 @@ import ( // letting every request through as the system org. func (s *Server) authorize(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if s.Access == nil { + // Checked before the principal: resolving it is a State Store round + // trip, which buys nothing when the answer is a refusal anyway. + if s.Access == nil || !s.Access.Ready() { s.refuseUngated(w, r) return } principal, err := s.requestPrincipal(r.Context()) if err != nil { if wantsDocument(r) { - s.authPage(w, "Sign-in could not resolve your org.") + s.authPage(w, http.StatusUnauthorized, "Sign-in could not resolve your org.") return } http.Error(w, "principal unavailable", http.StatusServiceUnavailable) @@ -46,7 +50,7 @@ func (s *Server) authorize(h http.Handler) http.Handler { } s.recordDenial(r.Context(), principal, action, resource, decision) if wantsDocument(r) { - s.authPage(w, "You do not have permission to do that.") + s.authPage(w, http.StatusUnauthorized, "You do not have permission to do that.") return } // A denial inside the caller's org is forbidden, without the @@ -63,13 +67,22 @@ func (s *Server) authorize(h http.Handler) http.Handler { // refuseUngated answers a request the dashboard cannot authorize because no // policy chain is available. It serves nothing ungated: without a chain the // credential check would be the whole gate and every call would act as the -// system org. +// system org. The detailed health report names only the unloaded chain, so +// the operator can see why without any other probe being served ungated. func (s *Server) refuseUngated(w http.ResponseWriter, r *http.Request) { if wantsDocument(r) { - s.authPage(w, "The access policy chain is not available yet.") + s.authPage(w, http.StatusServiceUnavailable, "The access policy chain is not available yet.") return } w.Header().Set("Cache-Control", "no-store") + if r.URL.Path == healthDetailedRoute { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + _ = json.NewEncoder(w).Encode(health.Report{Status: health.StatusDegraded, Components: []health.Component{{ + Name: "access_policies", Status: health.StatusDegraded, Detail: "access policies have not loaded; requests are refused", + }}}) + return + } http.Error(w, "access policies are not available", http.StatusServiceUnavailable) } diff --git a/internal/webui/authorize_test.go b/internal/webui/authorize_test.go index 80091d889..02e84af7c 100644 --- a/internal/webui/authorize_test.go +++ b/internal/webui/authorize_test.go @@ -45,16 +45,18 @@ func (allowAll) Authorize(access.Principal, access.Action, access.Resource, acce return access.Allowed() } +func (allowAll) Ready() bool { return true } + // notReadyChain is an authorizer whose engine has not loaded: every decision // is the not-loaded refusal, which no policy evaluated. type notReadyChain struct{} func (notReadyChain) Authorize(access.Principal, access.Action, access.Resource, access.Context) access.Decision { - decision := access.DeniedAt(access.LevelInstance, nil) - decision.Err = access.ErrChainUnavailable - return decision + return access.Unavailable() } +func (notReadyChain) Ready() bool { return false } + type recordingTokens struct{ owner identity.IdentityID } // AddPersonalToken mirrors the State Store's rule: only a signed-in person's @@ -138,11 +140,18 @@ func TestSharedTokenOperatorPostsPersonalToken(t *testing.T) { // served as the system org, and nothing is written. func TestAuthorizeRefusesWithoutAChain(t *testing.T) { tests := []struct { - name string - srv *Server + name string + srv *Server + method, path, accept string + body string }{ - {"no chain", &Server{}}, - {"chain not loaded", &Server{Access: notReadyChain{}}}, + {name: "no chain", srv: &Server{}, method: http.MethodPost, path: "/api/tokens"}, + {name: "chain not loaded", srv: &Server{Access: notReadyChain{}}, method: http.MethodPost, path: "/api/tokens"}, + // An outage is not a credential failure, so the browser notice is 503 + // too. + {name: "chain not loaded, browser", srv: &Server{Access: notReadyChain{}}, method: http.MethodGet, path: "/tasks", accept: "text/html"}, + // The health report still says why, naming only the chain. + {name: "chain not loaded, health detail", srv: &Server{Access: notReadyChain{}}, method: http.MethodGet, path: healthDetailedRoute, body: `"name":"access_policies"`}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -154,12 +163,18 @@ func TestAuthorizeRefusesWithoutAChain(t *testing.T) { t.Fatal("the handler must not be reached") }))) rec := httptest.NewRecorder() - req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/api/tokens", nil) + req := httptest.NewRequestWithContext(t.Context(), tt.method, tt.path, nil) req.Header.Set("Authorization", "Bearer shared") + if tt.accept != "" { + req.Header.Set("Accept", tt.accept) + } handler.ServeHTTP(rec, req) if rec.Code != http.StatusServiceUnavailable { t.Fatalf("status = %d, want 503", rec.Code) } + if !strings.Contains(rec.Body.String(), tt.body) { + t.Fatalf("body = %q, want it to contain %q", rec.Body.String(), tt.body) + } if tokens.owner != "" { t.Fatalf("token owner = %q, want no write", tokens.owner) } diff --git a/internal/webui/server.go b/internal/webui/server.go index 4151fadad..7eac7fedf 100644 --- a/internal/webui/server.go +++ b/internal/webui/server.go @@ -127,9 +127,9 @@ type Server struct { Login identity.LoginFlow // Access evaluates the policy chain, Principals builds the request principal - // and Denials records refusals. All three or none; nil leaves the credential - // check as the only gate. - Access access.Authorizer + // and Denials records refusals. All three or none; a nil or unready chain + // refuses every gated request. + Access Chain Principals access.PrincipalSource Denials access.DenialRecorder @@ -366,7 +366,7 @@ func (s *Server) Handler() http.Handler { s.registerConfigAndLogRoutes(mux) s.registerChatRoutes(mux) - mux.HandleFunc("GET /health/detailed", instanceOnly(s.handleHealthDetailed)) + mux.HandleFunc("GET "+healthDetailedRoute, instanceOnly(s.handleHealthDetailed)) mux.HandleFunc("GET /api/stream", s.handleSSE) mux.Handle("GET /", s.assets()) @@ -428,6 +428,16 @@ func trimLeadingSlash(p string) string { return p } +// healthDetailedRoute is the instance health report. +const healthDetailedRoute = "/health/detailed" + +// Chain is the policy chain the dashboard gates on. Ready is false until the +// stored policies have loaded. +type Chain interface { + access.Authorizer + Ready() bool +} + func writeJSON(w http.ResponseWriter, v any) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(v)