fix(app): retry startup and atomically seed policies (#2081)

This commit is contained in:
sam committed 2026-11-21 11:45:18 +11:00
1 parent 32fbf8b69c
commit 81bca2251e
9 files changed
+458 -122

No files matched your search

+110 -24
View File
@@ -8,29 +8,44 @@ import (
"net/http"
"strings"
"sync"
"time"
"github.com/samcharles93/archie-core/internal/platform/health"
"github.com/samcharles93/archie-core/internal/platform/httpjson"
)
// The two account operations the setup page calls. Everything else under
// /api/ needs the instance it would be operating.
// The two account operations the setup page calls. Every other /api/ route
// needs an instance that already has an owner.
const (
authStatusPath = "/api/auth/status"
setUpAccountPath = "/api/auth/setup"
)
// What stopped operations from starting. Readiness reports the stage; the error
// text stays in the log, since it can name hosts, roles and tables.
const (
stageSetupRead = "cannot read whether owner setup is required"
stageStartup = "starting operations failed"
)
// How long a failed boot step waits before it runs again. The delay doubles up
// to a ceiling.
const (
startupRetryMin = 100 * time.Millisecond
startupRetryMax = 5 * time.Second
)
// gate is the surface the binary serves. Until the owner setup transaction has
// committed it answers only what the setup page needs, and refuses every
// operational route before its handler runs, whatever credential the request
// carries. Once operations are started and wired it hands every request to
// them.
// committed it answers only what the setup page needs, and it refuses every
// operational route before the handler runs, whatever credential the request
// carries. Once the operations are running it passes every request to them.
type gate struct {
ops http.Handler
setup http.Handler
mu sync.RWMutex
open bool
mu sync.RWMutex
open bool
problem string
}
// newGate returns a closed gate over the operational handler and the dashboard
@@ -53,6 +68,7 @@ func (g *gate) publish() {
g.mu.Lock()
defer g.mu.Unlock()
g.open = true
g.problem = ""
}
func (g *gate) opened() bool {
@@ -61,11 +77,24 @@ func (g *gate) opened() bool {
return g.open
}
// stall records the stage that kept the gate closed, so readiness names it rather
// than only the missing owner. An empty stage clears it.
func (g *gate) stall(stage string) {
g.mu.Lock()
defer g.mu.Unlock()
g.problem = stage
}
// check is the readiness check for the gate. An instance still waiting for its
// owner is alive but has nothing ready to serve, and that is not a crash.
func (g *gate) check(context.Context) error {
if g.opened() {
g.mu.RLock()
defer g.mu.RUnlock()
switch {
case g.open:
return nil
case g.problem != "":
return errors.New(g.problem)
}
return errors.New("owner setup is incomplete")
}
@@ -91,10 +120,10 @@ func setupSurface(ops http.Handler, dist fs.FS) http.Handler {
})
}
// setupRequest is the whole routing decision while the gate is closed: the
// setup page and the files it loads, and the two account operations it calls.
// Nothing else passes, so a route inside the SPA is not a way in and neither is
// the wrong method on a route that is.
// setupRequest is the whole routing decision while the gate is closed: the setup
// page, the files it loads, and the two account operations it calls. Nothing else
// passes, so neither a deep link into the SPA nor a setup route called with the
// wrong method reaches a handler.
func setupRequest(r *http.Request, dist fs.FS) bool {
switch r.URL.Path {
case setUpAccountPath:
@@ -108,9 +137,8 @@ func setupRequest(r *http.Request, dist fs.FS) bool {
return r.URL.Path == "/" || builtFile(r.URL.Path, dist)
}
// builtFile reports whether path names a file in the dashboard build. The
// page's assets and icon are files; a route inside the SPA is not, so a deep
// link cannot borrow the setup entry's page.
// builtFile reports whether path names a file in the dashboard build. A route
// inside the SPA does not, so a deep link cannot be served the setup page.
func builtFile(path string, dist fs.FS) bool {
if dist == nil {
return false
@@ -119,27 +147,85 @@ func builtFile(path string, dist fs.FS) bool {
return err == nil && !info.IsDir()
}
// starter is what activation brings up: the operations Run wired.
// setupLookup reads whether owner setup is still required. Durable state is the
// only source: nothing a request carries can answer it.
type setupLookup func(context.Context) (bool, error)
// starter is what activation brings up: the operations Run wired. Start is safe
// to call again after it returns an error.
type starter interface {
Start(context.Context) error
}
// activate waits for the owner setup transaction to commit, starts the
// operations, and then opens the gate. It runs on the process context, never
// the setup request's: the request that triggers it is answered long before the
// work it starts is done.
func activate(ctx context.Context, log *slog.Logger, setup <-chan struct{}, setUp bool, ops starter, g *gate) error {
if !setUp {
// operations, and then opens the gate. It runs on the process context: the setup
// request that triggered it has been answered by then.
func activate(ctx context.Context, log *slog.Logger, lookup setupLookup, done <-chan struct{}, ops starter, g *gate) error {
required, err := waitForSetup(ctx, log, lookup, done, g)
if err != nil {
return err
}
if required {
select {
case <-setup:
case <-done:
case <-ctx.Done():
return ctx.Err()
}
}
if err := ops.Start(ctx); err != nil {
if err := retryStartup(ctx, log, g, stageStartup, ops.Start); err != nil {
return err
}
g.publish()
log.InfoContext(ctx, "owner setup is complete; serving operations")
return nil
}
// waitForSetup answers whether owner setup is still required. A failed read
// cannot rule setup out, so it is retried rather than trusted.
func waitForSetup(ctx context.Context, log *slog.Logger, lookup setupLookup, done <-chan struct{}, g *gate) (bool, error) {
for delay := startupRetryMin; ; delay = min(delay*2, startupRetryMax) {
required, err := lookup(ctx)
if err == nil {
g.stall("")
return required, nil
}
if ctx.Err() != nil {
return false, ctx.Err()
}
g.stall(stageSetupRead)
log.WarnContext(ctx, "startup step failed; retrying", "stage", stageSetupRead, "err", err, "in", delay)
select {
case <-time.After(delay):
case <-done:
// A setup has committed in this process while the reads failed, so an
// owner exists whatever the database can answer.
g.stall("")
return false, nil
case <-ctx.Done():
return false, ctx.Err()
}
}
}
// retryStartup runs a boot step until it succeeds, recording the stage that
// failed for readiness. A boot step that reads the database must not turn a brief
// outage into a terminated process.
func retryStartup(ctx context.Context, log *slog.Logger, g *gate, stage string, attempt func(context.Context) error) error {
for delay := startupRetryMin; ; delay = min(delay*2, startupRetryMax) {
err := attempt(ctx)
if err == nil {
g.stall("")
return nil
}
if ctx.Err() != nil {
return ctx.Err()
}
g.stall(stage)
log.WarnContext(ctx, "startup step failed; retrying", "stage", stage, "err", err, "in", delay)
select {
case <-time.After(delay):
case <-ctx.Done():
return ctx.Err()
}
}
}
+197 -34
View File
@@ -12,6 +12,7 @@ import (
"sync/atomic"
"testing"
"testing/fstest"
"time"
"github.com/samcharles93/archie-core/internal/platform/health"
"github.com/samcharles93/archie-core/modules/dashboard"
@@ -24,6 +25,12 @@ const testAdminToken = "Bearer a-bootstrap-admin-token"
// testLog discards what the gate and activation log.
func testLog() *slog.Logger { return slog.New(slog.DiscardHandler) }
// setUpRequired is durable state on an instance with no owner.
func setUpRequired(context.Context) (bool, error) { return true, nil }
// setUpDone is durable state on an instance whose owner setup has committed.
func setUpDone(context.Context) (bool, error) { return false, nil }
// reached records the requests that get past the gate, so a test can tell a
// refusal from a handler that ran.
type reached struct {
@@ -105,13 +112,16 @@ func TestClosedGateServesOnlySetup(t *testing.T) {
admin bool
status int
body string
// servedAsPage marks a response the dashboard handler writes, which must
// keep the CSP that stops chat-rendered Markdown loading a remote image.
servedAsPage bool
}{
{name: "auth status", method: http.MethodGet, path: authStatusPath, status: http.StatusOK, body: `"setup_required":true`},
{name: "auth status by head", method: http.MethodHead, path: authStatusPath, status: http.StatusOK},
{name: "set up the owner", method: http.MethodPost, path: setUpAccountPath, status: http.StatusCreated},
{name: "the setup page", method: http.MethodGet, path: "/", status: http.StatusOK, body: "archie setup"},
{name: "a script the page loads", method: http.MethodGet, path: "/assets/app.js", status: http.StatusOK, body: "console.log(1)"},
{name: "the page icon", method: http.MethodGet, path: "/favicon.svg", status: http.StatusOK, body: "<svg>"},
{name: "the setup page", method: http.MethodGet, path: "/", status: http.StatusOK, body: "archie setup", servedAsPage: true},
{name: "a script the page loads", method: http.MethodGet, path: "/assets/app.js", status: http.StatusOK, body: "console.log(1)", servedAsPage: true},
{name: "the page icon", method: http.MethodGet, path: "/favicon.svg", status: http.StatusOK, body: "<svg>", servedAsPage: true},
{name: "liveness", method: http.MethodGet, path: "/healthz", status: http.StatusOK, body: "ok\n"},
// From here down nothing reaches a handler, credential or not.
@@ -147,6 +157,12 @@ func TestClosedGateServesOnlySetup(t *testing.T) {
if got := hit.count(); got != 0 {
t.Fatalf("%s %s: %d operational handlers ran, want none (reached %v)", tc.method, tc.path, got, hit.paths())
}
if tc.servedAsPage {
// Mirrors the dashboard's own header: the setup page must not lose it.
if got := rec.Header().Get("Content-Security-Policy"); got != "img-src 'self' data:" {
t.Fatalf("%s %s: Content-Security-Policy = %q, want the dashboard's", tc.method, tc.path, got)
}
}
})
}
}
@@ -247,16 +263,20 @@ func TestReadinessReportsSetupUntilOperationsStart(t *testing.T) {
}
}
// fakeOperations stands in for the connectors, the access chain, the shipped
// workflow and the worker loops, and counts how often they would start.
// fakeOperations stands in for the startup work and counts how often it runs.
type fakeOperations struct {
starts atomic.Int32
err error
starts atomic.Int32
startFails atomic.Int32
startErr error
}
func (f *fakeOperations) Start(context.Context) error {
f.starts.Add(1)
return f.err
if f.startFails.Load() > 0 {
f.startFails.Add(-1)
return f.startErr
}
return nil
}
func TestActivateStartsOperationsWhenSetupCommits(t *testing.T) {
@@ -267,7 +287,7 @@ func TestActivateStartsOperationsWhenSetupCommits(t *testing.T) {
served, hit, g := testServer(t)
done := make(chan error, 1)
go func() { done <- activate(context.Background(), testLog(), signal.Done(), false, ops, g) }()
go func() { done <- activate(context.Background(), testLog(), setUpRequired, signal.Done(), ops, g) }()
if got := ops.starts.Load(); got != 0 {
t.Fatalf("starts before the commit = %d, want 0", got)
@@ -305,7 +325,7 @@ func TestConcurrentSetupCommitsStartOperationsOnce(t *testing.T) {
_, _, g := testServer(t)
done := make(chan error, 1)
go func() { done <- activate(context.Background(), testLog(), signal.Done(), false, ops, g) }()
go func() { done <- activate(context.Background(), testLog(), setUpRequired, signal.Done(), ops, g) }()
var completing sync.WaitGroup
for range 8 {
@@ -322,28 +342,6 @@ func TestConcurrentSetupCommitsStartOperationsOnce(t *testing.T) {
}
}
func TestFailedStartLeavesTheGateClosed(t *testing.T) {
// A connector or policy failure must not leave an instance answering
// operational requests with half its wiring up.
signal := governance.NewSetupSignal()
ops := &fakeOperations{err: errors.New("connectors refused")}
served, hit, g := testServer(t)
signal.Complete()
if err := activate(context.Background(), testLog(), signal.Done(), false, ops, g); err == nil {
t.Fatalf("activate returned no error, want the start failure")
}
if g.opened() {
t.Fatalf("the gate opened after a failed start")
}
if rec := do(t, served, http.MethodGet, "/api/system/bootstrap", true); rec.Code != http.StatusNotFound {
t.Fatalf("operational API after a failed start: status = %d, want %d", rec.Code, http.StatusNotFound)
}
if got := hit.count(); got != 0 {
t.Fatalf("operational handlers after a failed start = %d, want none", got)
}
}
func TestCancelledBeforeSetupStartsNothing(t *testing.T) {
signal := governance.NewSetupSignal()
ops := &fakeOperations{}
@@ -351,7 +349,7 @@ func TestCancelledBeforeSetupStartsNothing(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- activate(ctx, testLog(), signal.Done(), false, ops, g) }()
go func() { done <- activate(ctx, testLog(), setUpRequired, signal.Done(), ops, g) }()
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
@@ -373,7 +371,7 @@ func TestCompletedSetupStartsWithoutWaiting(t *testing.T) {
ops := &fakeOperations{}
served, hit, g := testServer(t)
if err := activate(context.Background(), testLog(), signal.Done(), true, ops, g); err != nil {
if err := activate(context.Background(), testLog(), setUpDone, signal.Done(), ops, g); err != nil {
t.Fatalf("activate: %v", err)
}
if got := ops.starts.Load(); got != 1 {
@@ -389,3 +387,168 @@ func TestCompletedSetupStartsWithoutWaiting(t *testing.T) {
t.Fatalf("operational handlers = %d, want 1", got)
}
}
// flakyLookup fails a set number of reads before answering, as a database that
// is briefly unreachable does.
type flakyLookup struct {
failures atomic.Int32
required bool
reads atomic.Int32
}
// newFlakyLookup fails failures reads before answering required.
func newFlakyLookup(failures int32, required bool) *flakyLookup {
l := &flakyLookup{required: required}
l.failures.Store(failures)
return l
}
func (f *flakyLookup) lookup(context.Context) (bool, error) {
f.reads.Add(1)
if f.failures.Load() > 0 {
f.failures.Add(-1)
return false, errors.New("connection refused")
}
return f.required, nil
}
func TestUnreadableSetupStateIsRetried(t *testing.T) {
// A database that is briefly unavailable must not hold an instance shut
// until someone restarts it: setup state that can't be read is retried, and
// readiness reports the failure meanwhile.
lookup := newFlakyLookup(2, false)
ops := &fakeOperations{}
served, hit, g := testServer(t)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
done := make(chan error, 1)
go func() { done <- activate(ctx, testLog(), lookup.lookup, nil, ops, g) }()
// While the read fails, nothing starts and readiness names the failure.
if got := ops.starts.Load(); got != 0 {
t.Fatalf("starts while setup state is unreadable = %d, want 0", got)
}
for range 100 {
if strings.Contains(readyzBody(t, served), stageSetupRead) {
break
}
time.Sleep(10 * time.Millisecond)
}
body := readyzBody(t, served)
if !strings.Contains(body, stageSetupRead) {
t.Fatalf("readiness while setup state is unreadable = %q, want %q", body, stageSetupRead)
}
if strings.Contains(body, "connection refused") {
t.Fatalf("readiness = %q, leaked the database's own error", body)
}
if err := <-done; err != nil {
t.Fatalf("activate: %v", err)
}
if got := ops.starts.Load(); got != 1 {
t.Fatalf("starts after the read recovered = %d, want 1", got)
}
if got := lookup.reads.Load(); got < 3 {
t.Fatalf("reads = %d, want the failed read retried", got)
}
if rec := do(t, served, http.MethodGet, "/api/system/bootstrap", true); rec.Code != http.StatusOK {
t.Fatalf("operational API after the read recovered: status = %d, want %d", rec.Code, http.StatusOK)
}
if got := hit.count(); got != 1 {
t.Fatalf("operational handlers after the read recovered = %d, want 1", got)
}
}
func TestUnreadableSetupStateWaitsForTheCommit(t *testing.T) {
// When the read keeps failing, a setup that commits in this process is
// still enough: the owner exists, so operations may start.
lookup := newFlakyLookup(1000, true)
ops := &fakeOperations{}
_, _, g := testServer(t)
signal := governance.NewSetupSignal()
done := make(chan error, 1)
go func() { done <- activate(context.Background(), testLog(), lookup.lookup, signal.Done(), ops, g) }()
time.Sleep(50 * time.Millisecond)
if got := ops.starts.Load(); got != 0 {
t.Fatalf("starts while setup state is unreadable = %d, want 0", got)
}
signal.Complete()
if err := <-done; err != nil {
t.Fatalf("activate: %v", err)
}
if got := ops.starts.Load(); got != 1 {
t.Fatalf("starts after the commit = %d, want 1", got)
}
}
func readyzBody(t *testing.T, served http.Handler) string {
t.Helper()
return do(t, served, http.MethodGet, "/readyz", false).Body.String()
}
func TestOperationsStartIsRetried(t *testing.T) {
// connectors.Module.Start returns an error only for the read of accepted
// connectors, before anything launches, so a failed Start leaves nothing
// running and the next attempt is safe. Without that retry a transient read
// error took the process down.
ops := &fakeOperations{startErr: errors.New("connection refused")}
ops.startFails.Store(2)
served, hit, g := testServer(t)
signal := governance.NewSetupSignal()
signal.Complete()
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
done := make(chan error, 1)
go func() { done <- activate(ctx, testLog(), setUpDone, signal.Done(), ops, g) }()
if err := <-done; err != nil {
t.Fatalf("activate: %v", err)
}
if got := ops.starts.Load(); got != 3 {
t.Fatalf("start calls = %d, want the two failures retried", got)
}
if !g.opened() {
t.Fatalf("the gate is shut after a retried start")
}
if rec := do(t, served, http.MethodGet, "/api/system/bootstrap", true); rec.Code != http.StatusOK {
t.Fatalf("operational API after a retried start: status = %d, want %d", rec.Code, http.StatusOK)
}
if got := hit.count(); got != 1 {
t.Fatalf("operational handlers = %d, want 1: one surface, not one per attempt", got)
}
}
func TestReadinessNamesTheStageWithoutTheRawError(t *testing.T) {
// Readiness answers whoever can reach the port, so it names the stage that
// failed and keeps the database's own error text in the log.
ops := &fakeOperations{startErr: errors.New("pq: password authentication failed for user archie")}
ops.startFails.Store(1000)
served, _, g := testServer(t)
signal := governance.NewSetupSignal()
signal.Complete()
done := make(chan error, 1)
go func() { done <- activate(context.Background(), testLog(), setUpDone, signal.Done(), ops, g) }()
for range 200 {
if strings.Contains(readyzBody(t, served), "starting operations failed") {
break
}
time.Sleep(10 * time.Millisecond)
}
body := readyzBody(t, served)
if !strings.Contains(body, "starting operations failed") {
t.Fatalf("readiness = %q, want it to name the stage", body)
}
for _, leak := range []string{"pq:", "password", "user archie", "connection refused"} {
if strings.Contains(body, leak) {
t.Fatalf("readiness = %q, leaked %q", body, leak)
}
}
if g.opened() {
t.Fatalf("the gate opened while a startup stage fails")
}
}
+19 -16
View File
@@ -39,27 +39,18 @@ type operations struct {
}
// Start brings up every part of the binary that does work, in the order the
// rest of it expects: connectors first, then the access chain they are checked
// against, then the shipped workflow, then the loops. A failure returns with
// what started still running for the caller to shut down; nothing is left to
// report ready.
// other parts depend on: the role policies and access chain, the workflow that
// ships with the binary, the connectors, then the loops.
//
// Calling it again after it returns an error is safe. Every step that can fail
// runs before a connector launches, and connectors.Module.Start fails only for
// the read of accepted connectors, before launching any. An error therefore
// means nothing is running.
func (o *operations) Start(ctx context.Context) error {
if err := o.conns.Start(ctx); err != nil {
return fmt.Errorf("start connectors: %w", err)
}
o.checks.Register("connectors", o.conns.Ready)
access, err := startAccess(ctx, o.log, o.settings)
if err != nil {
return err
}
// Administrative writes are checked against the same live policy chain as
// every other request.
o.settings.UseAuthorizer(access)
o.talk.Use(chat.Deps{
People: o.settings, Access: access, Settings: o.settings, Toolbox: o.conns, Judge: o.judge,
Approver: o.allow, Memory: o.remember, Runs: o.engine, Notifier: o.tell,
})
// A workflow takes its context from the run handle, as a handler takes it
// from the request.
o.engine.Use(runs.Deps{
@@ -69,7 +60,19 @@ func (o *operations) Start(ctx context.Context) error {
if err := installStarter(ctx, o.engine); err != nil {
return err
}
if err := o.conns.Start(ctx); err != nil {
return fmt.Errorf("read accepted connectors: %w", err)
}
// Nothing below can fail, so a retry cannot launch a connector or a loop twice.
o.checks.Register("connectors", o.conns.Ready)
// Administrative writes are checked against the same live policy chain as
// every other request.
o.settings.UseAuthorizer(access)
o.talk.Use(chat.Deps{
People: o.settings, Access: access, Settings: o.settings, Toolbox: o.conns, Judge: o.judge,
Approver: o.allow, Memory: o.remember, Runs: o.engine, Notifier: o.tell,
})
for _, loop := range []func(context.Context){o.engine.Work, o.hear.Work, o.tell.Work, o.talk.Work} {
o.workers.Go(func() { loop(ctx) })
}
+8 -24
View File
@@ -46,8 +46,9 @@ const shutdownGrace = 10 * time.Second
//
// An instance whose owner setup has not committed serves the setup page and
// nothing else: connectors, module routes, the shipped workflow and the worker
// loops all wait for that transaction. Durable state says whether it already
// happened, so a restart needs no setup form and serves operations at once.
// loops all wait for that transaction. Durable state says whether it has already
// happened, so a restart reads that state and starts serving operations
// immediately.
func Run(ctx context.Context, configPath string) error {
log := slog.New(slog.NewJSONHandler(os.Stdout, nil)).With("service", "archie", "version", buildinfo.Version)
@@ -69,39 +70,22 @@ func Run(ctx context.Context, configPath string) error {
}
defer b.close()
// Durable state decides the first gate. A read that fails cannot rule setup
// out, so the gate stays closed: nothing operational runs on an instance
// that may have no owner. The failure is logged and readiness reports it,
// and a restart reads the state again.
required, setupErr := b.settings.AccountSetupRequired(ctx)
if setupErr != nil {
log.ErrorContext(ctx, "read whether owner setup is required", "err", setupErr)
}
setUp := setupErr == nil && !required
work, stop := context.WithCancel(ctx)
defer stop()
started := make(chan error, 1)
started := make(chan struct{})
go func() {
err := activate(work, log, b.settings.SetupDone(), setUp, b.ops, b.gate)
if err != nil {
// Whatever half of the operations came up is no use without the rest,
// so the binary stops rather than serving a partial instance.
stop()
}
started <- err
// Startup failures are retried, so only shutdown ends activation.
_ = activate(work, log, b.settings.AccountSetupRequired, b.settings.SetupDone(), b.ops, b.gate)
close(started)
}()
serveErr := serve(work, log, cfg.Listen, b.served)
stop()
startErr := <-started
<-started
// Every loop activation began has returned before the caller closes the
// connectors and the database.
b.ops.Wait()
if startErr != nil && !errors.Is(startErr, context.Canceled) {
return startErr
}
return serveErr
}
+4
View File
@@ -81,6 +81,10 @@ func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, sealing Sealing) (*M
// Start launches every accepted secret engine, then every accepted connector,
// since a connector's credentials may be resolved through an engine. One that
// fails to start is reported by Ready and logged; the others still run.
//
// Its only error is the read of the accepted connectors, which happens before
// anything launches, so a caller may call Start again after a failure without
// launching anything twice.
func (m *Module) Start(ctx context.Context) error {
accepted, err := m.store.List(ctx, m.orgID)
if err != nil {
+43
View File
@@ -6,6 +6,8 @@ import (
"log/slog"
"slices"
"testing"
"github.com/samcharles93/archie-core/modules/governance/internal/store"
)
// TestInvalidInstancePolicy pins that an instance policy that no longer
@@ -440,3 +442,44 @@ func TestCustomRoleAndMemberStatePolicy(t *testing.T) {
})
}
}
// fakePolicySeed records the one atomic seed the module asks for.
type fakePolicySeed struct {
calls int
ids []string
rows []store.PolicyRow
err error
}
func (f *fakePolicySeed) SeedOrgPolicies(_ context.Context, _ string, shippedIDs []string, rows []store.PolicyRow) error {
f.calls++
f.ids, f.rows = shippedIDs, rows
return f.err
}
// TestSeedShippedPolicies guards that the shipped set is written in one call
// rather than one policy at a time, which is what left an org holding only some
// of its role policies. The transaction, the zero-count check under the lock and
// how deletions and edits survive are proven against Postgres in the binary's
// smoke test.
func TestSeedShippedPolicies(t *testing.T) {
seeder := &fakePolicySeed{}
if err := seedShippedPolicies(t.Context(), seeder, DefaultOrgID); err != nil {
t.Fatalf("seed: %v", err)
}
shipped := ShippedOrgPolicies(DefaultOrgID)
if seeder.calls != 1 {
t.Fatalf("seed calls = %d, want 1", seeder.calls)
}
if len(seeder.rows) != len(shipped) || len(seeder.ids) != len(shipped) {
t.Fatalf("seeded %d rows and %d ids, want %d of each", len(seeder.rows), len(seeder.ids), len(shipped))
}
for i, p := range shipped {
if seeder.rows[i].PolicyID != p.ID || seeder.ids[i] != p.ID {
t.Fatalf("row %d = %s, want %s", i, seeder.rows[i].PolicyID, p.ID)
}
}
if err := seedShippedPolicies(t.Context(), &fakePolicySeed{err: errors.New("connection reset by peer")}, DefaultOrgID); err == nil {
t.Fatal("a failed seed returned no error")
}
}
+54 -8
View File
@@ -192,17 +192,63 @@ func (s *Store) DeleteAccessPolicy(ctx context.Context, p PolicyRow) (found bool
return found, nil
}
// CountShippedOrgPolicies returns how many of policyIDs are present at org level for orgID.
func (s *Store) CountShippedOrgPolicies(ctx context.Context, orgID string, policyIDs []string) (int64, error) {
var count int64
if err := s.db.GetContext(ctx, &count, `
SELECT count(*) FROM governance.access_policies
WHERE level = 'org' AND org_id = $1 AND policy_id = ANY($2::text[])`, orgID, policyIDs); err != nil {
return 0, fmt.Errorf("count shipped org policies: %w", err)
// SeedOrgPolicies stores an org's initial policies, unless the org already has
// any of them.
//
// The check and the writes share one transaction, and the lock is taken before
// the check, so a second seed waits for the first to commit and then finds its
// rows: it writes nothing, and an edit or a deletion made in between stands.
func (s *Store) SeedOrgPolicies(ctx context.Context, orgID string, shippedIDs []string, rows []PolicyRow) error {
if len(rows) == 0 {
return nil
}
return count, nil
err := s.withTx(ctx, func(tx *sqlx.Tx) error {
if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, seedLockKey(orgID)); err != nil {
return err
}
var present int64
if err := tx.GetContext(ctx, &present, `
SELECT count(*) FROM governance.access_policies
WHERE level = 'org' AND org_id = $1 AND policy_id = ANY($2::text[])`,
orgID, shippedIDs); err != nil {
return err
}
if present > 0 {
return nil
}
for _, p := range rows {
if _, err := tx.ExecContext(ctx, `
INSERT INTO governance.access_policies (level, org_id, workspace_id, object_kind, object_id, policy_id, text)
VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (level, org_id, workspace_id, object_kind, object_id, policy_id)
DO UPDATE SET text = EXCLUDED.text, updated_at = now()`,
p.Level, p.OrgID, p.WorkspaceID, p.ObjectKind, p.ObjectID, p.PolicyID, p.Text); err != nil {
return err
}
}
// One version for the whole seed: a reader sees none of the policies or
// all of them.
first := rows[0]
if _, err := tx.ExecContext(ctx, `
INSERT INTO governance.access_policy_versions (level, org_id, workspace_id, version)
VALUES ($1, $2, $3, 2)
ON CONFLICT (level, org_id, workspace_id)
DO UPDATE SET version = governance.access_policy_versions.version + 1, updated_at = now()`,
first.Level, first.OrgID, first.WorkspaceID); err != nil {
return err
}
return bumpPolicyRevision(ctx, tx)
})
if err != nil {
return fmt.Errorf("seed org policies: %w", err)
}
return nil
}
// seedLockKey is the advisory lock key a seed takes before it reads whether the
// org has policies, distinct from the per-policy keys.
func seedLockKey(orgID string) string { return "seed/" + orgID }
// DeleteOrgPoliciesNotShipped removes non-shipped org policies and bumps revision.
func (s *Store) DeleteOrgPoliciesNotShipped(ctx context.Context, orgID string, shippedIDs []string) error {
return s.withTx(ctx, func(tx *sqlx.Tx) error {
+20 -13
View File
@@ -113,21 +113,28 @@ func (m *Module) DeletePolicy(ctx context.Context, p Policy) error {
return nil
}
// EnsureShippedOrgPolicies seeds an org's shipped role policies when none exist yet.
// EnsureShippedOrgPolicies seeds the shipped role policies when the org has none
// of them.
func (m *Module) EnsureShippedOrgPolicies(ctx context.Context, orgID OrgID) error {
count, err := m.store.CountShippedOrgPolicies(ctx, string(orgID), shippedOrgPolicyIDs())
if err != nil {
return err
return seedShippedPolicies(ctx, m.store, orgID)
}
// policySeedStore is the part of the store that seeds the shipped policies.
type policySeedStore interface {
SeedOrgPolicies(context.Context, string, []string, []store.PolicyRow) error
}
// seedShippedPolicies writes the whole shipped set in one call, so a failure
// leaves none of it behind and the next attempt writes all of it. The store
// decides whether the seed already ran; an org that has any shipped policy is
// left alone, so a deletion or an edit an admin made stands across restarts.
func seedShippedPolicies(ctx context.Context, s policySeedStore, orgID OrgID) error {
shipped := ShippedOrgPolicies(orgID)
rows := make([]store.PolicyRow, 0, len(shipped))
for _, p := range shipped {
rows = append(rows, policyToRow(p))
}
if count > 0 {
return nil
}
for _, p := range ShippedOrgPolicies(orgID) {
if _, err := m.store.PutAccessPolicy(ctx, policyToRow(p)); err != nil {
return err
}
}
return nil
return s.SeedOrgPolicies(ctx, string(orgID), shippedOrgPolicyIDs(), rows)
}
// ResetOrgPolicies removes non-shipped org policies and restores the shipped role policies.
+3 -3
View File
@@ -381,9 +381,9 @@ func (m *Module) RegisterAccount(ctx context.Context, r AccountRegistration, boo
return Identity{}, mapIdentityErr(err)
}
if bootstrap {
// The owner is in durable state now. Whatever waits on this signal starts
// operations, so it is sent only after the transaction committed: a failed
// attempt returns above and leaves it closed.
// The owner is in durable state now. The composition root starts operations
// when this signal closes, so it is sent only after the transaction
// committed: a failed attempt returns above and leaves it closed.
m.setup.Complete()
}
return identityFromRow(row), nil