fix(access): an unloaded chain is a refusing engine, not nil

Webhook intake answers 503 and counts no refusal while the chain loads.
The readiness probe reads the chain on every check. The UI never exits on
a first-load failure. An invalid instance policy closes the instance but
lets an instance admin repair policies. Outage pages are 503, and the
health detail names the unloaded chain.
This commit is contained in:
sam committed 2026-10-10 22:19:02 +11:00
1 parent e01008ee92
commit 43769e902b
20 files changed
+277 -167

No files matched your search

+3 -14
View File
@@ -116,10 +116,8 @@ func compose(ctx context.Context, d deps) *webui.Server {
wireCatalog(d, srv) wireCatalog(d, srv)
// The policy chain: wired together or not at all. // The policy chain: wired together or not at all.
srv.Refusals = refusalStore(d.Store) srv.Refusals = refusalStore(d.Store)
if d.Access != nil { srv.Access = d.Access
srv.Access = d.Access srv.ReloadAccess = d.Access.Reload
srv.ReloadAccess = d.Access.Reload
}
srv.Principals = d.Principals srv.Principals = d.Principals
srv.Denials = d.Denials srv.Denials = d.Denials
wireTaskLogs(d, srv) wireTaskLogs(d, srv)
@@ -195,7 +193,7 @@ func wireCaptureSurfaces(d deps, srv *webui.Server) {
MaxEvents: capture.MaxEvents, MaxEvents: capture.MaxEvents,
MaxBodyBytes: int64(capture.MaxBodyBytes), MaxBodyBytes: int64(capture.MaxBodyBytes),
Publish: captureArrivalPublisher(d), Publish: captureArrivalPublisher(d),
Delivery: deliveryAuthorizer(d.Access), Delivery: d.Access,
Refusals: refusalStore(d.Store), Refusals: refusalStore(d.Store),
Log: d.Log, Log: d.Log,
} }
@@ -221,15 +219,6 @@ func captureArrivalPublisher(d deps) func(context.Context, events.Event) {
// a hung State Store must not hold the sender's webhook past this. // a hung State Store must not hold the sender's webhook past this.
const captureEventTimeout = 5 * time.Second const captureEventTimeout = 5 * time.Second
// deliveryAuthorizer keeps a missing chain a nil authorizer, which admits
// every sender.
func deliveryAuthorizer(chain *infraaccess.Live) access.DeliveryAuthorizer {
if chain == nil {
return nil
}
return chain
}
func refusalStore(store storecontract.TaskStore) storecontract.CaptureRefusalStore { func refusalStore(store storecontract.TaskStore) storecontract.CaptureRefusalStore {
refusals, _ := store.(storecontract.CaptureRefusalStore) refusals, _ := store.(storecontract.CaptureRefusalStore)
return refusals return refusals
+1 -11
View File
@@ -18,7 +18,7 @@ import (
// Store and, when one is configured, the sign-in provider. // Store and, when one is configured, the sign-in provider.
func newReadinessRegistry(o Options, tasks storecontract.TaskStore, chat messaging.ChatContract, chain *infraaccess.Live, provider *oidc.Provider) *health.Registry { func newReadinessRegistry(o Options, tasks storecontract.TaskStore, chat messaging.ChatContract, chain *infraaccess.Live, provider *oidc.Provider) *health.Registry {
probes := []health.Probe{ probes := []health.Probe{
readiness.NewProblemProbe("access_policies", problemSource(chain)), readiness.NewProblemProbe("access_policies", chain),
captureRefusalProbe{store: refusalStore(tasks)}, captureRefusalProbe{store: refusalStore(tasks)},
readiness.NewContractProbe("state_db", o.DependencyTimeout, func(ctx context.Context) error { readiness.NewContractProbe("state_db", o.DependencyTimeout, func(ctx context.Context) error {
_, err := tasks.StatusCounts(ctx) _, err := tasks.StatusCounts(ctx)
@@ -35,16 +35,6 @@ func newReadinessRegistry(o Options, tasks storecontract.TaskStore, chat messagi
return health.NewRegistry(probes...) return health.NewRegistry(probes...)
} }
// problemSource reports the chain's invalid policies; a chain that has not
// loaded yet is reported as not wired, so readiness is degraded while requests
// are refused.
func problemSource(chain *infraaccess.Live) readiness.PolicyProblems {
if chain == nil || !chain.Ready() {
return nil
}
return chain
}
// captureRefusalProbe degrades when any source refused more events than it // captureRefusalProbe degrades when any source refused more events than it
// accepted over the last hour: a sender being blocked, or someone probing. // accepted over the last hour: a sender being blocked, or someone probing.
type captureRefusalProbe struct { type captureRefusalProbe struct {
+7 -19
View File
@@ -12,9 +12,6 @@ import (
"slices" "slices"
"time" "time"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/samcharles93/archie-core/internal/app/servicekit" "github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/domain/applystatus" "github.com/samcharles93/archie-core/internal/domain/applystatus"
"github.com/samcharles93/archie-core/internal/domain/presence" "github.com/samcharles93/archie-core/internal/domain/presence"
@@ -27,20 +24,14 @@ import (
"github.com/samcharles93/archie-core/internal/webui" "github.com/samcharles93/archie-core/internal/webui"
) )
// buildAccessChain builds the live access engine over the stored policies. A // buildAccessChain starts the live access engine over the stored policies.
// State Store that is not up yet yields a chain with no engine: it refuses // It refuses every request until Run's first load succeeds and keeps
// every request until the first load succeeds and keeps retrying, so a slow // retrying, so a State Store or database that is down or slow at start
// database or a bad start order closes the dashboard rather than opening it. // closes the dashboard rather than opening it or stopping it.
func buildAccessChain(ctx context.Context, store *staterpc.Client, log *slog.Logger) (*infraaccess.Live, error) { func buildAccessChain(ctx context.Context, store *staterpc.Client, log *slog.Logger) *infraaccess.Live {
live := infraaccess.NewPending(store.ListPolicies, log) live := infraaccess.NewPending(store.ListPolicies, log)
if err := live.Reload(ctx); err != nil {
if status.Code(err) != codes.Unavailable {
return nil, err
}
log.Warn("access policies unavailable; the dashboard refuses API requests until they load", "err", err)
}
go live.Run(ctx, applystatus.RestampInterval) go live.Run(ctx, applystatus.RestampInterval)
return live, nil return live
} }
// Run serves the dashboard until ctx is cancelled. It resolves its own // Run serves the dashboard until ctx is cancelled. It resolves its own
@@ -71,10 +62,7 @@ func Run(ctx context.Context, options Options) error {
return err return err
} }
cleanups = append(cleanups, closeGateway) cleanups = append(cleanups, closeGateway)
chain, err := buildAccessChain(ctx, tasks, log) chain := buildAccessChain(ctx, tasks, log)
if err != nil {
return err
}
provider, err := signInProvider(opts) provider, err := signInProvider(opts)
if err != nil { if err != nil {
return err return err
@@ -169,10 +169,7 @@ func checkWorkflowDashboard(t *testing.T, db *pgstore.TaskDB, client *staterpc.C
t.Helper() t.Helper()
ctx := t.Context() ctx := t.Context()
policies := append(access.ShippedOrgPolicies("acme"), access.ShippedOrgPolicies(org.DefaultOrgID)...) policies := append(access.ShippedOrgPolicies("acme"), access.ShippedOrgPolicies(org.DefaultOrgID)...)
chain, err := infraaccess.New(policies) chain := infraaccess.New(policies)
if err != nil {
t.Fatal(err)
}
srv := &webui.Server{Store: client, ControlPlane: client.ControlPlane(), Access: chain, Principals: client, Authenticate: func(ctx context.Context, token string) (identity.Identity, error) { srv := &webui.Server{Store: client, ControlPlane: client.ControlPlane(), Access: chain, Principals: client, Authenticate: func(ctx context.Context, token string) (identity.Identity, error) {
return db.Get(ctx, identity.IdentityID(token)) return db.Get(ctx, identity.IdentityID(token))
}} }}
+2 -9
View File
@@ -26,11 +26,7 @@ type guardedPolicies struct {
} }
func (g guardedPolicies) PutPolicy(ctx context.Context, p access.Policy) (int64, error) { func (g guardedPolicies) PutPolicy(ctx context.Context, p access.Policy) (int64, error) {
validator, err := accessengine.New(nil) if err := accessengine.New(nil).Validate(p); err != nil {
if err != nil {
return 0, err
}
if err := validator.Validate(p); err != nil {
return 0, err return 0, err
} }
if err := g.checkLockout(ctx, func(current []access.Policy) []access.Policy { if err := g.checkLockout(ctx, func(current []access.Policy) []access.Policy {
@@ -55,10 +51,7 @@ func (g guardedPolicies) checkLockout(ctx context.Context, change func([]access.
if err != nil { if err != nil {
return err return err
} }
engine, err := accessengine.New(change(current)) engine := accessengine.New(change(current))
if err != nil {
return err
}
orgs, err := g.owners.ListOrgs(ctx) orgs, err := g.owners.ListOrgs(ctx)
if err != nil { if err != nil {
return err return err
+1 -4
View File
@@ -181,10 +181,7 @@ func (b *server) seedAndValidatePolicies(ctx context.Context) error {
if err != nil { if err != nil {
return fmt.Errorf("load stored policies: %w", err) return fmt.Errorf("load stored policies: %w", err)
} }
engine, err := infraaccess.New(stored) engine := infraaccess.New(stored)
if err != nil {
return fmt.Errorf("validate stored policies: %w", err)
}
for _, problem := range engine.Problems() { for _, problem := range engine.Problems() {
b.log.Error("stored access policy is invalid and denies its level", b.log.Error("stored access policy is invalid and denies its level",
"policy", problem.Policy.ID, "level", problem.Policy.Level, "err", problem.Err) "policy", problem.Policy.ID, "level", problem.Policy.Level, "err", problem.Err)
+6
View File
@@ -131,6 +131,12 @@ type Decision struct {
// Allowed is the shared permit decision: no level denied. // Allowed is the shared permit decision: no level denied.
func Allowed() Decision { return Decision{Allowed: true} } func Allowed() Decision { return Decision{Allowed: true} }
// Unavailable is the refusal of a chain that has not loaded: nothing was
// evaluated, so it is an availability failure rather than a denial.
func Unavailable() Decision {
return Decision{Level: LevelInstance, Err: ErrChainUnavailable}
}
// DeniedAt records a denial decided at a level by the given policies. // DeniedAt records a denial decided at a level by the given policies.
func DeniedAt(level Level, policies []string) Decision { func DeniedAt(level Level, policies []string) Decision {
return Decision{Level: level, Policies: policies} return Decision{Level: level, Policies: policies}
@@ -40,10 +40,7 @@ func TestAuthorizeDelivery(t *testing.T) {
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
engine, err := New(tt.policies) engine := New(tt.policies)
if err != nil {
t.Fatal(err)
}
acting := tt.org acting := tt.org
if acting == "" { if acting == "" {
acting = org.DefaultOrgID acting = org.DefaultOrgID
+28 -12
View File
@@ -65,8 +65,14 @@ type Engine struct {
spaces map[spaceKey]*levelSet spaces map[spaceKey]*levelSet
objects map[objectKey]*levelSet objects map[objectKey]*levelSet
problems []Problem problems []Problem
// pending marks the engine of a chain whose policies have not loaded:
// it refuses everything without evaluating.
pending bool
} }
// pendingEngine refuses every request until the stored policies load.
var pendingEngine = &Engine{pending: true}
type ( type (
spaceKey struct { spaceKey struct {
org org.OrgID org org.OrgID
@@ -85,10 +91,9 @@ var (
_ access.Validator = (*Engine)(nil) _ access.Validator = (*Engine)(nil)
) )
// New builds an engine from the stored policies. An invalid instance policy // New builds an engine from the stored policies. An invalid policy makes its
// is an error; an invalid policy at another level makes that level deny // level deny everything and is reported as a Problem.
// everything. func New(policies []access.Policy) *Engine {
func New(policies []access.Policy) (*Engine, error) {
e := &Engine{ e := &Engine{
orgs: map[org.OrgID]*levelSet{}, orgs: map[org.OrgID]*levelSet{},
spaces: map[spaceKey]*levelSet{}, spaces: map[spaceKey]*levelSet{},
@@ -98,9 +103,6 @@ func New(policies []access.Policy) (*Engine, error) {
lvl := e.level(p) lvl := e.level(p)
policy, err := compile(p) policy, err := compile(p)
if err != nil { if err != nil {
if p.Level == access.LevelInstance {
return nil, fmt.Errorf("access: instance policy %q: %w", p.ID, err)
}
// The refusal must name the policy that poisoned the level. // The refusal must name the policy that poisoned the level.
lvl.ids = append(lvl.ids, p.ID) lvl.ids = append(lvl.ids, p.ID)
lvl.invalid = err lvl.invalid = err
@@ -110,9 +112,12 @@ func New(policies []access.Policy) (*Engine, error) {
lvl.ids = append(lvl.ids, p.ID) lvl.ids = append(lvl.ids, p.ID)
lvl.set.Add(types.PolicyID(p.ID), policy) lvl.set.Add(types.PolicyID(p.ID), policy)
} }
return e, nil return e
} }
// Ready reports whether the engine was built from stored policies.
func (e *Engine) Ready() bool { return !e.pending }
// Problems reports every stored policy the engine could not compile. Empty // Problems reports every stored policy the engine could not compile. Empty
// when every stored policy is valid. // when every stored policy is valid.
func (e *Engine) Problems() []Problem { return e.problems } func (e *Engine) Problems() []Problem { return e.problems }
@@ -186,6 +191,9 @@ func compile(p access.Policy) (*cedar.Policy, error) {
// Authorize evaluates the chain for one request. // Authorize evaluates the chain for one request.
func (e *Engine) Authorize(p access.Principal, a access.Action, r access.Resource, c access.Context) access.Decision { func (e *Engine) Authorize(p access.Principal, a access.Action, r access.Resource, c access.Context) access.Decision {
if e.pending {
return access.Unavailable()
}
// The cross-org forbid, enforced structurally: a principal's requests // The cross-org forbid, enforced structurally: a principal's requests
// only ever reach its own org. An unset resource org means the // only ever reach its own org. An unset resource org means the
// principal's org (a create request names a record that does not exist // principal's org (a create request names a record that does not exist
@@ -198,7 +206,7 @@ func (e *Engine) Authorize(p access.Principal, a access.Action, r access.Resourc
return access.DeniedAt(access.LevelInstance, []string{access.CrossOrgForbidID}) return access.DeniedAt(access.LevelInstance, []string{access.CrossOrgForbidID})
} }
return evaluate(e.chain(r), buildEntities(p, r), buildRequest(p, a, r, c)) return evaluate(e.chain(p, r), buildEntities(p, r), buildRequest(p, a, r, c))
} }
// AuthorizeDelivery decides whether addr may deliver an event to a source: // AuthorizeDelivery decides whether addr may deliver an event to a source:
@@ -206,6 +214,9 @@ func (e *Engine) Authorize(p access.Principal, a access.Action, r access.Resourc
// sender has no identity or role, so the org and workspace role policies // sender has no identity or role, so the org and workspace role policies
// never apply to it. // never apply to it.
func (e *Engine) AuthorizeDelivery(orgID org.OrgID, sourcePath, addr string) access.Decision { func (e *Engine) AuthorizeDelivery(orgID org.OrgID, sourcePath, addr string) access.Decision {
if e.pending {
return access.Unavailable()
}
sender := access.Principal{IdentityID: access.SenderID, Org: orgID} sender := access.Principal{IdentityID: access.SenderID, Org: orgID}
resource := access.Resource{Kind: access.KindSource, ID: sourcePath, Org: orgID} resource := access.Resource{Kind: access.KindSource, ID: sourcePath, Org: orgID}
levels := []*levelSet{e.instance} levels := []*levelSet{e.instance}
@@ -247,9 +258,14 @@ func evaluate(levels []*levelSet, entities types.EntityMap, request types.Reques
return access.Allowed() return access.Allowed()
} }
// chain returns the level sets the request passes through, in order. // chain returns the level sets the request passes through, in order. An
func (e *Engine) chain(r access.Resource) []*levelSet { // invalid instance level refuses everything except an instance admin working
out := []*levelSet{e.instance} // on the policies, so the operator can repair it from the dashboard.
func (e *Engine) chain(p access.Principal, r access.Resource) []*levelSet {
var out []*levelSet
if e.instance == nil || e.instance.invalid == nil || r.Kind != access.KindPolicy || !p.InstanceAdmin() {
out = append(out, e.instance)
}
if l, ok := e.orgs[r.Org]; ok { if l, ok := e.orgs[r.Org]; ok {
out = append(out, l) out = append(out, l)
} }
@@ -0,0 +1,47 @@
package access
import (
"testing"
"github.com/samcharles93/archie-core/internal/domain/access"
"github.com/samcharles93/archie-core/internal/domain/identity"
"github.com/samcharles93/archie-core/internal/domain/org"
)
// TestInvalidInstancePolicy pins that an instance policy that no longer
// compiles closes the instance without locking the operator out: only an
// instance admin working on the policies passes, so the policy can be
// repaired from the dashboard.
func TestInvalidInstancePolicy(t *testing.T) {
broken := access.Policy{ID: "broken", Level: access.LevelInstance, Text: `permit(principal, action == Archie::Action::"nope", resource);`}
policies := append(access.ShippedOrgPolicies(org.DefaultOrgID), access.ShippedOrgPolicies("acme")...)
engine := New(append(policies, broken))
if len(engine.Problems()) != 1 {
t.Fatalf("problems = %v, want the broken instance policy", engine.Problems())
}
owner := func(id identity.IdentityID, o org.OrgID) access.Principal {
return access.Principal{IdentityID: id, Org: o, Memberships: []org.Membership{{IdentityID: id, OrgID: o, Role: org.RoleOwner}}}
}
admin := owner("admin", org.DefaultOrgID)
tests := []struct {
name string
principal access.Principal
action access.Action
kind access.ResourceKind
allowed bool
}{
{"instance admin edits policies", admin, access.ActionManagePolicies, access.KindPolicy, true},
{"instance admin reads policies", admin, access.ActionRead, access.KindPolicy, true},
{"instance admin reads anything else", admin, access.ActionRead, access.KindDashboard, false},
{"another org's owner edits policies", owner("acme-owner", "acme"), access.ActionManagePolicies, access.KindPolicy, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := engine.Authorize(tt.principal, tt.action, access.Resource{Kind: tt.kind}, access.Context{})
if got.Allowed != tt.allowed {
t.Fatalf("allowed = %v, want %v (%+v)", got.Allowed, tt.allowed, got)
}
})
}
}
+15 -38
View File
@@ -21,22 +21,24 @@ type Live struct {
// NewLive builds the first engine from load. // NewLive builds the first engine from load.
func NewLive(ctx context.Context, load func(context.Context) ([]access.Policy, error), log *slog.Logger) (*Live, error) { func NewLive(ctx context.Context, load func(context.Context) ([]access.Policy, error), log *slog.Logger) (*Live, error) {
l := &Live{load: load, log: log} l := NewPending(load, log)
if err := l.Reload(ctx); err != nil { if err := l.Reload(ctx); err != nil {
return nil, err return nil, err
} }
return l, nil return l, nil
} }
// NewPending returns a chain with no engine: Run loads it, retrying until the // NewPending returns a chain whose engine refuses every request until Run or
// store answers. A chain that is not ready refuses every request, so a process // Reload loads the stored policies, so a process that starts before its State
// that starts before its State Store is closed rather than opened. // Store is closed rather than opened.
func NewPending(load func(context.Context) ([]access.Policy, error), log *slog.Logger) *Live { func NewPending(load func(context.Context) ([]access.Policy, error), log *slog.Logger) *Live {
return &Live{load: load, log: log} l := &Live{load: load, log: log}
l.engine.Store(pendingEngine)
return l
} }
// Ready reports whether an engine has loaded. // Ready reports whether the stored policies have loaded.
func (l *Live) Ready() bool { return l.engine.Load() != nil } func (l *Live) Ready() bool { return l.engine.Load().Ready() }
// Reload rebuilds the engine from the store now. // Reload rebuilds the engine from the store now.
func (l *Live) Reload(ctx context.Context) error { func (l *Live) Reload(ctx context.Context) error {
@@ -44,10 +46,7 @@ func (l *Live) Reload(ctx context.Context) error {
if err != nil { if err != nil {
return err return err
} }
engine, err := New(stored) engine := New(stored)
if err != nil {
return err
}
for _, problem := range engine.Problems() { for _, problem := range engine.Problems() {
l.log.Error("stored access policy is invalid and denies its level", l.log.Error("stored access policy is invalid and denies its level",
"policy", problem.Policy.ID, "level", problem.Policy.Level, "err", problem.Err) "policy", problem.Policy.ID, "level", problem.Policy.Level, "err", problem.Err)
@@ -56,9 +55,8 @@ func (l *Live) Reload(ctx context.Context) error {
return nil return nil
} }
// Run reloads on every interval until ctx ends. A chain with no engine yet // Run loads at once, then reloads on every interval until ctx ends, so the
// tries at once and keeps trying, so the first load does not wait an interval // first load does not wait an interval after the State Store returns.
// after the State Store returns.
func (l *Live) Run(ctx context.Context, interval time.Duration) { func (l *Live) Run(ctx context.Context, interval time.Duration) {
ticker := time.NewTicker(interval) ticker := time.NewTicker(interval)
defer ticker.Stop() defer ticker.Stop()
@@ -85,33 +83,12 @@ func (l *Live) reload(ctx context.Context) {
} }
func (l *Live) Authorize(p access.Principal, a access.Action, r access.Resource, c access.Context) access.Decision { func (l *Live) Authorize(p access.Principal, a access.Action, r access.Resource, c access.Context) access.Decision {
engine := l.engine.Load() return l.engine.Load().Authorize(p, a, r, c)
if engine == nil {
return notLoaded()
}
return engine.Authorize(p, a, r, c)
} }
func (l *Live) AuthorizeDelivery(orgID org.OrgID, sourcePath, addr string) access.Decision { func (l *Live) AuthorizeDelivery(orgID org.OrgID, sourcePath, addr string) access.Decision {
engine := l.engine.Load() return l.engine.Load().AuthorizeDelivery(orgID, sourcePath, addr)
if engine == nil {
return notLoaded()
}
return engine.AuthorizeDelivery(orgID, sourcePath, addr)
}
// notLoaded is the refusal of a chain with no engine: nothing was evaluated.
func notLoaded() access.Decision {
decision := access.DeniedAt(access.LevelInstance, nil)
decision.Err = access.ErrChainUnavailable
return decision
} }
// Problems reports the current engine's invalid policies. // Problems reports the current engine's invalid policies.
func (l *Live) Problems() []Problem { func (l *Live) Problems() []Problem { return l.engine.Load().Problems() }
engine := l.engine.Load()
if engine == nil {
return nil
}
return engine.Problems()
}
@@ -36,21 +36,34 @@ func (s *stubCaptures) ListCaptures(_ context.Context, _ int) ([]storecontract.C
} }
type stubDelivery struct { type stubDelivery struct {
orgID org.OrgID orgID org.OrgID
allow bool decision access.Decision
} }
func (s *stubDelivery) AuthorizeDelivery(orgID org.OrgID, _, _ string) access.Decision { func (s *stubDelivery) AuthorizeDelivery(orgID org.OrgID, _, _ string) access.Decision {
s.orgID = orgID s.orgID = orgID
return access.Decision{Allowed: s.allow} return s.decision
}
type stubRefusals struct{ recorded int }
func (s *stubRefusals) RecordCaptureRefusal(context.Context, string, string, time.Time) error {
s.recorded++
return nil
}
func (*stubRefusals) CaptureRefusals(context.Context, time.Time) ([]storecontract.CaptureRefusals, error) {
return nil, nil
} }
// Delivery runs against the source's owning org, falling back to the // Delivery runs against the source's owning org, falling back to the
// default org when the source is unknown. // default org when the source is unknown.
func TestDeliveryUsesSourceOrg(t *testing.T) { func TestDeliveryUsesSourceOrg(t *testing.T) {
refusals := &stubRefusals{}
newReceiver := func(delivery *stubDelivery) (*Receiver, *stubCaptures) { newReceiver := func(delivery *stubDelivery) (*Receiver, *stubCaptures) {
captures := &stubCaptures{} captures := &stubCaptures{}
return &Receiver{ return &Receiver{
Refusals: refusals,
Captures: captures, Captures: captures,
Sources: stubSources{byPath: map[string]*source.Source{ Sources: stubSources{byPath: map[string]*source.Source{
"acme-hook": {Path: "acme-hook", Signing: source.SigningUnsigned, OrgID: "acme"}, "acme-hook": {Path: "acme-hook", Signing: source.SigningUnsigned, OrgID: "acme"},
@@ -69,7 +82,7 @@ func TestDeliveryUsesSourceOrg(t *testing.T) {
return response return response
} }
delivery := &stubDelivery{allow: true} delivery := &stubDelivery{decision: access.Allowed()}
rc, _ := newReceiver(delivery) rc, _ := newReceiver(delivery)
if response := post(rc, "acme-hook"); response.Code != http.StatusAccepted { if response := post(rc, "acme-hook"); response.Code != http.StatusAccepted {
t.Fatalf("capture: %d %s", response.Code, response.Body.String()) t.Fatalf("capture: %d %s", response.Code, response.Body.String())
@@ -78,7 +91,7 @@ func TestDeliveryUsesSourceOrg(t *testing.T) {
t.Fatalf("authorized against %q, want acme", delivery.orgID) t.Fatalf("authorized against %q, want acme", delivery.orgID)
} }
delivery = &stubDelivery{allow: true} delivery = &stubDelivery{decision: access.Allowed()}
rc, _ = newReceiver(delivery) rc, _ = newReceiver(delivery)
if response := post(rc, "unknown"); response.Code != http.StatusAccepted { if response := post(rc, "unknown"); response.Code != http.StatusAccepted {
t.Fatalf("capture: %d %s", response.Code, response.Body.String()) t.Fatalf("capture: %d %s", response.Code, response.Body.String())
@@ -87,12 +100,30 @@ func TestDeliveryUsesSourceOrg(t *testing.T) {
t.Fatalf("authorized against %q, want the default org", delivery.orgID) t.Fatalf("authorized against %q, want the default org", delivery.orgID)
} }
delivery = &stubDelivery{} // A denial is refused and counted; a chain that has not loaded evaluated
rc, captures := newReceiver(delivery) // nothing, so the sender may retry and no refusal is counted.
if response := post(rc, "acme-hook"); response.Code != http.StatusForbidden { refusalTests := []struct {
t.Fatalf("refused capture: %d %s", response.Code, response.Body.String()) name string
decision access.Decision
code int
counted int
}{
{"denied", access.DeniedAt(access.LevelObject, []string{"lan-only"}), http.StatusForbidden, 1},
{"chain not loaded", access.Unavailable(), http.StatusServiceUnavailable, 0},
} }
if captures.stored != 0 { for _, tt := range refusalTests {
t.Fatalf("refused delivery stored %d captures", captures.stored) t.Run(tt.name, func(t *testing.T) {
refusals.recorded = 0
rc, captures := newReceiver(&stubDelivery{decision: tt.decision})
if response := post(rc, "acme-hook"); response.Code != tt.code {
t.Fatalf("refused capture: %d %s, want %d", response.Code, response.Body.String(), tt.code)
}
if captures.stored != 0 {
t.Fatalf("refused delivery stored %d captures", captures.stored)
}
if refusals.recorded != tt.counted {
t.Fatalf("refusals counted = %d, want %d", refusals.recorded, tt.counted)
}
})
} }
} }
@@ -5,6 +5,7 @@ package captureintake
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"io" "io"
"log/slog" "log/slog"
"mime" "mime"
@@ -91,8 +92,8 @@ func (rc *Receiver) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// authorizes as the default org. // authorizes as the default org.
src := rc.resolveSource(r, path) src := rc.resolveSource(r, path)
if rc.refused(r, path, orgOf(src)) { if code := rc.refusal(r, path, orgOf(src)); code != 0 {
http.Error(w, "forbidden", http.StatusForbidden) http.Error(w, http.StatusText(code), code)
return return
} }
@@ -158,22 +159,29 @@ func (rc *Receiver) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusAccepted) w.WriteHeader(http.StatusAccepted)
} }
// refused applies the network rules before the body is read. A refused event // refusal applies the network rules before the body is read and returns the
// is not stored, only counted on its source with the sender's address. // status to refuse with, or 0 to admit. A refused event is not stored, only
func (rc *Receiver) refused(r *http.Request, path string, owner org.OrgID) bool { // counted on its source with the sender's address. A chain that has not
// loaded evaluated nothing: the sender gets a retryable 503 and no refusal is
// counted.
func (rc *Receiver) refusal(r *http.Request, path string, owner org.OrgID) int {
if rc.Delivery == nil { if rc.Delivery == nil {
return false return 0
} }
addr := remoteAddrHost(r) addr := remoteAddrHost(r)
if rc.Delivery.AuthorizeDelivery(owner, path, addr).Allowed { decision := rc.Delivery.AuthorizeDelivery(owner, path, addr)
return false if decision.Allowed {
return 0
}
if errors.Is(decision.Err, access.ErrChainUnavailable) {
return http.StatusServiceUnavailable
} }
if rc.Refusals != nil { if rc.Refusals != nil {
if err := rc.Refusals.RecordCaptureRefusal(r.Context(), path, addr, time.Now()); err != nil { if err := rc.Refusals.RecordCaptureRefusal(r.Context(), path, addr, time.Now()); err != nil {
rc.logger().Warn("capture refusal not counted", "source", path, "err", err) rc.logger().Warn("capture refusal not counted", "source", path, "err", err)
} }
} }
return true return http.StatusForbidden
} }
// orgOf reports the org that owns src, or the default org when the source // orgOf reports the org that owns src, or the default org when the source
+5 -1
View File
@@ -86,8 +86,9 @@ func (p *ContractProbe) Check(ctx context.Context) health.Result {
// PolicyProblems reports the stored access policies this process's engine // PolicyProblems reports the stored access policies this process's engine
// could not validate. The policy chain is the source; a policy that failed // could not validate. The policy chain is the source; a policy that failed
// startup re-validation denies its level and must be named here with its // startup re-validation denies its level and must be named here with its
// error. // error. Ready is false until the stored policies have loaded.
type PolicyProblems interface { type PolicyProblems interface {
Ready() bool
Problems() []access.Problem Problems() []access.Problem
} }
@@ -109,6 +110,9 @@ func (p *ProblemProbe) Check(ctx context.Context) health.Result {
if p.Source == nil { if p.Source == nil {
return health.Result{Status: health.StatusDegraded, Detail: "access engine not wired"} return health.Result{Status: health.StatusDegraded, Detail: "access engine not wired"}
} }
if !p.Source.Ready() {
return health.Result{Status: health.StatusDegraded, Detail: "access policies have not loaded; requests are refused"}
}
problems := p.Source.Problems() problems := p.Source.Problems()
if len(problems) == 0 { if len(problems) == 0 {
return health.Result{Status: health.StatusOK} return health.Result{Status: health.StatusOK}
@@ -0,0 +1,37 @@
package readiness
import (
"context"
"errors"
"log/slog"
"testing"
"github.com/samcharles93/archie-core/internal/domain/access"
"github.com/samcharles93/archie-core/internal/domain/health"
infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access"
)
// TestProblemProbeFollowsTheChain pins that the access_policies probe reads
// the chain on every check: degraded while the policies have not loaded, ok
// once they do, without rebuilding the probe.
func TestProblemProbeFollowsTheChain(t *testing.T) {
up := false
chain := infraaccess.NewPending(func(context.Context) ([]access.Policy, error) {
if !up {
return nil, errors.New("store down")
}
return nil, nil
}, slog.New(slog.DiscardHandler))
probe := NewProblemProbe("access_policies", chain)
if got := probe.Check(t.Context()); got.Status != health.StatusDegraded {
t.Fatalf("before load = %+v, want degraded", got)
}
up = true
if err := chain.Reload(t.Context()); err != nil {
t.Fatal(err)
}
if got := probe.Check(t.Context()); got.Status != health.StatusOK {
t.Fatalf("after load = %+v, want ok", got)
}
}
+2 -8
View File
@@ -179,10 +179,7 @@ func (rolePrincipals) PrincipalFor(_ context.Context, id identity.IdentityID) (a
// policies: a developer manages no members, an admin manages members but can // policies: a developer manages no members, an admin manages members but can
// neither grant the owner role nor change an owner's, and an owner can. // neither grant the owner role nor change an owner's, and an owner can.
func TestOrgMembershipNeedsItsRole(t *testing.T) { func TestOrgMembershipNeedsItsRole(t *testing.T) {
engine, err := infraaccess.New(access.ShippedOrgPolicies(org.DefaultOrgID)) engine := infraaccess.New(access.ShippedOrgPolicies(org.DefaultOrgID))
if err != nil {
t.Fatal(err)
}
members := "/api/orgs/" + string(org.DefaultOrgID) + "/members/" members := "/api/orgs/" + string(org.DefaultOrgID) + "/members/"
tests := []struct { tests := []struct {
name string name string
@@ -233,10 +230,7 @@ func TestOrgMembershipNeedsItsRole(t *testing.T) {
// lists and opens every org and creates one with its owner; an admin of // lists and opens every org and creates one with its owner; an admin of
// org-sys sees only its own org and cannot create one. // org-sys sees only its own org and cannot create one.
func TestInstanceAdminOrgs(t *testing.T) { func TestInstanceAdminOrgs(t *testing.T) {
engine, err := infraaccess.New(access.ShippedOrgPolicies(org.DefaultOrgID)) engine := infraaccess.New(access.ShippedOrgPolicies(org.DefaultOrgID))
if err != nil {
t.Fatal(err)
}
tests := []struct { tests := []struct {
name string name string
caller string caller string
+7 -6
View File
@@ -116,7 +116,7 @@ func (s *Server) requireToken(h http.Handler) http.Handler {
if tok := r.URL.Query().Get("t"); tok != "" { if tok := r.URL.Query().Get("t"); tok != "" {
if !tokenEqual(tok, s.Token) { if !tokenEqual(tok, s.Token) {
if wantsDocument(r) { if wantsDocument(r) {
s.authPage(w, "That token was not accepted.") s.authPage(w, http.StatusUnauthorized, "That token was not accepted.")
return return
} }
w.Header().Set("Cache-Control", "no-store") w.Header().Set("Cache-Control", "no-store")
@@ -150,7 +150,7 @@ func (s *Server) requireToken(h http.Handler) http.Handler {
} }
if wantsDocument(r) { if wantsDocument(r) {
s.authPage(w, "") s.authPage(w, http.StatusUnauthorized, "")
return return
} }
w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("Content-Type", "text/plain; charset=utf-8")
@@ -306,7 +306,7 @@ func (s *Server) refuseUnidentified(w http.ResponseWriter, r *http.Request, err
http.Redirect(w, r, loginPath, http.StatusSeeOther) http.Redirect(w, r, loginPath, http.StatusSeeOther)
return return
} }
s.authPage(w, reason) s.authPage(w, http.StatusUnauthorized, reason)
return return
} }
http.Error(w, reason, status) http.Error(w, reason, status)
@@ -335,12 +335,13 @@ func wantsDocument(r *http.Request) bool {
return strings.Contains(r.Header.Get("Accept"), "text/html") return strings.Contains(r.Header.Get("Accept"), "text/html")
} }
// authPage serves a self-contained 401 page for pasting the access token. // authPage serves the self-contained page for pasting the access token,
func (s *Server) authPage(w http.ResponseWriter, reason string) { // with code: 401 for a credential failure, 503 for an outage.
func (s *Server) authPage(w http.ResponseWriter, code int, reason string) {
w.Header().Set("Content-Type", "text/html; charset=utf-8") w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.Header().Set("Cache-Control", "no-store") w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; form-action 'self'") w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; form-action 'self'")
w.WriteHeader(http.StatusUnauthorized) w.WriteHeader(code)
notice := "" notice := ""
if reason != "" { if reason != "" {
notice = `<p class="error">` + reason + `</p>` notice = `<p class="error">` + reason + `</p>`
+18 -5
View File
@@ -2,11 +2,13 @@ package webui
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"net/http" "net/http"
"strings" "strings"
"github.com/samcharles93/archie-core/internal/domain/access" "github.com/samcharles93/archie-core/internal/domain/access"
"github.com/samcharles93/archie-core/internal/domain/health"
"github.com/samcharles93/archie-core/internal/domain/identity" "github.com/samcharles93/archie-core/internal/domain/identity"
"github.com/samcharles93/archie-core/internal/domain/org" "github.com/samcharles93/archie-core/internal/domain/org"
) )
@@ -17,14 +19,16 @@ import (
// letting every request through as the system org. // letting every request through as the system org.
func (s *Server) authorize(h http.Handler) http.Handler { func (s *Server) authorize(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if s.Access == nil { // Checked before the principal: resolving it is a State Store round
// trip, which buys nothing when the answer is a refusal anyway.
if s.Access == nil || !s.Access.Ready() {
s.refuseUngated(w, r) s.refuseUngated(w, r)
return return
} }
principal, err := s.requestPrincipal(r.Context()) principal, err := s.requestPrincipal(r.Context())
if err != nil { if err != nil {
if wantsDocument(r) { if wantsDocument(r) {
s.authPage(w, "Sign-in could not resolve your org.") s.authPage(w, http.StatusUnauthorized, "Sign-in could not resolve your org.")
return return
} }
http.Error(w, "principal unavailable", http.StatusServiceUnavailable) http.Error(w, "principal unavailable", http.StatusServiceUnavailable)
@@ -46,7 +50,7 @@ func (s *Server) authorize(h http.Handler) http.Handler {
} }
s.recordDenial(r.Context(), principal, action, resource, decision) s.recordDenial(r.Context(), principal, action, resource, decision)
if wantsDocument(r) { if wantsDocument(r) {
s.authPage(w, "You do not have permission to do that.") s.authPage(w, http.StatusUnauthorized, "You do not have permission to do that.")
return return
} }
// A denial inside the caller's org is forbidden, without the // A denial inside the caller's org is forbidden, without the
@@ -63,13 +67,22 @@ func (s *Server) authorize(h http.Handler) http.Handler {
// refuseUngated answers a request the dashboard cannot authorize because no // refuseUngated answers a request the dashboard cannot authorize because no
// policy chain is available. It serves nothing ungated: without a chain the // policy chain is available. It serves nothing ungated: without a chain the
// credential check would be the whole gate and every call would act as the // credential check would be the whole gate and every call would act as the
// system org. // system org. The detailed health report names only the unloaded chain, so
// the operator can see why without any other probe being served ungated.
func (s *Server) refuseUngated(w http.ResponseWriter, r *http.Request) { func (s *Server) refuseUngated(w http.ResponseWriter, r *http.Request) {
if wantsDocument(r) { if wantsDocument(r) {
s.authPage(w, "The access policy chain is not available yet.") s.authPage(w, http.StatusServiceUnavailable, "The access policy chain is not available yet.")
return return
} }
w.Header().Set("Cache-Control", "no-store") w.Header().Set("Cache-Control", "no-store")
if r.URL.Path == healthDetailedRoute {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusServiceUnavailable)
_ = json.NewEncoder(w).Encode(health.Report{Status: health.StatusDegraded, Components: []health.Component{{
Name: "access_policies", Status: health.StatusDegraded, Detail: "access policies have not loaded; requests are refused",
}}})
return
}
http.Error(w, "access policies are not available", http.StatusServiceUnavailable) http.Error(w, "access policies are not available", http.StatusServiceUnavailable)
} }
+23 -8
View File
@@ -45,16 +45,18 @@ func (allowAll) Authorize(access.Principal, access.Action, access.Resource, acce
return access.Allowed() return access.Allowed()
} }
func (allowAll) Ready() bool { return true }
// notReadyChain is an authorizer whose engine has not loaded: every decision // notReadyChain is an authorizer whose engine has not loaded: every decision
// is the not-loaded refusal, which no policy evaluated. // is the not-loaded refusal, which no policy evaluated.
type notReadyChain struct{} type notReadyChain struct{}
func (notReadyChain) Authorize(access.Principal, access.Action, access.Resource, access.Context) access.Decision { func (notReadyChain) Authorize(access.Principal, access.Action, access.Resource, access.Context) access.Decision {
decision := access.DeniedAt(access.LevelInstance, nil) return access.Unavailable()
decision.Err = access.ErrChainUnavailable
return decision
} }
func (notReadyChain) Ready() bool { return false }
type recordingTokens struct{ owner identity.IdentityID } type recordingTokens struct{ owner identity.IdentityID }
// AddPersonalToken mirrors the State Store's rule: only a signed-in person's // AddPersonalToken mirrors the State Store's rule: only a signed-in person's
@@ -138,11 +140,18 @@ func TestSharedTokenOperatorPostsPersonalToken(t *testing.T) {
// served as the system org, and nothing is written. // served as the system org, and nothing is written.
func TestAuthorizeRefusesWithoutAChain(t *testing.T) { func TestAuthorizeRefusesWithoutAChain(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
srv *Server srv *Server
method, path, accept string
body string
}{ }{
{"no chain", &Server{}}, {name: "no chain", srv: &Server{}, method: http.MethodPost, path: "/api/tokens"},
{"chain not loaded", &Server{Access: notReadyChain{}}}, {name: "chain not loaded", srv: &Server{Access: notReadyChain{}}, method: http.MethodPost, path: "/api/tokens"},
// An outage is not a credential failure, so the browser notice is 503
// too.
{name: "chain not loaded, browser", srv: &Server{Access: notReadyChain{}}, method: http.MethodGet, path: "/tasks", accept: "text/html"},
// The health report still says why, naming only the chain.
{name: "chain not loaded, health detail", srv: &Server{Access: notReadyChain{}}, method: http.MethodGet, path: healthDetailedRoute, body: `"name":"access_policies"`},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
@@ -154,12 +163,18 @@ func TestAuthorizeRefusesWithoutAChain(t *testing.T) {
t.Fatal("the handler must not be reached") t.Fatal("the handler must not be reached")
}))) })))
rec := httptest.NewRecorder() rec := httptest.NewRecorder()
req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/api/tokens", nil) req := httptest.NewRequestWithContext(t.Context(), tt.method, tt.path, nil)
req.Header.Set("Authorization", "Bearer shared") req.Header.Set("Authorization", "Bearer shared")
if tt.accept != "" {
req.Header.Set("Accept", tt.accept)
}
handler.ServeHTTP(rec, req) handler.ServeHTTP(rec, req)
if rec.Code != http.StatusServiceUnavailable { if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("status = %d, want 503", rec.Code) t.Fatalf("status = %d, want 503", rec.Code)
} }
if !strings.Contains(rec.Body.String(), tt.body) {
t.Fatalf("body = %q, want it to contain %q", rec.Body.String(), tt.body)
}
if tokens.owner != "" { if tokens.owner != "" {
t.Fatalf("token owner = %q, want no write", tokens.owner) t.Fatalf("token owner = %q, want no write", tokens.owner)
} }
+14 -4
View File
@@ -127,9 +127,9 @@ type Server struct {
Login identity.LoginFlow Login identity.LoginFlow
// Access evaluates the policy chain, Principals builds the request principal // Access evaluates the policy chain, Principals builds the request principal
// and Denials records refusals. All three or none; nil leaves the credential // and Denials records refusals. All three or none; a nil or unready chain
// check as the only gate. // refuses every gated request.
Access access.Authorizer Access Chain
Principals access.PrincipalSource Principals access.PrincipalSource
Denials access.DenialRecorder Denials access.DenialRecorder
@@ -366,7 +366,7 @@ func (s *Server) Handler() http.Handler {
s.registerConfigAndLogRoutes(mux) s.registerConfigAndLogRoutes(mux)
s.registerChatRoutes(mux) s.registerChatRoutes(mux)
mux.HandleFunc("GET /health/detailed", instanceOnly(s.handleHealthDetailed)) mux.HandleFunc("GET "+healthDetailedRoute, instanceOnly(s.handleHealthDetailed))
mux.HandleFunc("GET /api/stream", s.handleSSE) mux.HandleFunc("GET /api/stream", s.handleSSE)
mux.Handle("GET /", s.assets()) mux.Handle("GET /", s.assets())
@@ -428,6 +428,16 @@ func trimLeadingSlash(p string) string {
return p return p
} }
// healthDetailedRoute is the instance health report.
const healthDetailedRoute = "/health/detailed"
// Chain is the policy chain the dashboard gates on. Ready is false until the
// stored policies have loaded.
type Chain interface {
access.Authorizer
Ready() bool
}
func writeJSON(w http.ResponseWriter, v any) { func writeJSON(w http.ResponseWriter, v any) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(v) _ = json.NewEncoder(w).Encode(v)