From 81bca2251ec12057c4fd36560dd95d85f529c359 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 21 Nov 2026 11:45:18 +1100 Subject: [PATCH] fix(app): retry startup and atomically seed policies (#2081) --- internal/app/archie/gate.go | 134 ++++++++++-- internal/app/archie/gate_test.go | 231 +++++++++++++++++--- internal/app/archie/operations.go | 35 +-- internal/app/archie/run.go | 32 +-- modules/connectors/module.go | 4 + modules/governance/access_test.go | 43 ++++ modules/governance/internal/store/access.go | 62 +++++- modules/governance/store_access.go | 33 +-- modules/governance/store_identities.go | 6 +- 9 files changed, 458 insertions(+), 122 deletions(-) diff --git a/internal/app/archie/gate.go b/internal/app/archie/gate.go index e6e3b6ac6..1bab3f746 100644 --- a/internal/app/archie/gate.go +++ b/internal/app/archie/gate.go @@ -8,29 +8,44 @@ import ( "net/http" "strings" "sync" + "time" "github.com/samcharles93/archie-core/internal/platform/health" "github.com/samcharles93/archie-core/internal/platform/httpjson" ) -// The two account operations the setup page calls. Everything else under -// /api/ needs the instance it would be operating. +// The two account operations the setup page calls. Every other /api/ route +// needs an instance that already has an owner. const ( authStatusPath = "/api/auth/status" setUpAccountPath = "/api/auth/setup" ) +// What stopped operations from starting. Readiness reports the stage; the error +// text stays in the log, since it can name hosts, roles and tables. +const ( + stageSetupRead = "cannot read whether owner setup is required" + stageStartup = "starting operations failed" +) + +// How long a failed boot step waits before it runs again. The delay doubles up +// to a ceiling. +const ( + startupRetryMin = 100 * time.Millisecond + startupRetryMax = 5 * time.Second +) + // gate is the surface the binary serves. Until the owner setup transaction has -// committed it answers only what the setup page needs, and refuses every -// operational route before its handler runs, whatever credential the request -// carries. Once operations are started and wired it hands every request to -// them. +// committed it answers only what the setup page needs, and it refuses every +// operational route before the handler runs, whatever credential the request +// carries. Once the operations are running it passes every request to them. type gate struct { ops http.Handler setup http.Handler - mu sync.RWMutex - open bool + mu sync.RWMutex + open bool + problem string } // newGate returns a closed gate over the operational handler and the dashboard @@ -53,6 +68,7 @@ func (g *gate) publish() { g.mu.Lock() defer g.mu.Unlock() g.open = true + g.problem = "" } func (g *gate) opened() bool { @@ -61,11 +77,24 @@ func (g *gate) opened() bool { return g.open } +// stall records the stage that kept the gate closed, so readiness names it rather +// than only the missing owner. An empty stage clears it. +func (g *gate) stall(stage string) { + g.mu.Lock() + defer g.mu.Unlock() + g.problem = stage +} + // check is the readiness check for the gate. An instance still waiting for its // owner is alive but has nothing ready to serve, and that is not a crash. func (g *gate) check(context.Context) error { - if g.opened() { + g.mu.RLock() + defer g.mu.RUnlock() + switch { + case g.open: return nil + case g.problem != "": + return errors.New(g.problem) } return errors.New("owner setup is incomplete") } @@ -91,10 +120,10 @@ func setupSurface(ops http.Handler, dist fs.FS) http.Handler { }) } -// setupRequest is the whole routing decision while the gate is closed: the -// setup page and the files it loads, and the two account operations it calls. -// Nothing else passes, so a route inside the SPA is not a way in and neither is -// the wrong method on a route that is. +// setupRequest is the whole routing decision while the gate is closed: the setup +// page, the files it loads, and the two account operations it calls. Nothing else +// passes, so neither a deep link into the SPA nor a setup route called with the +// wrong method reaches a handler. func setupRequest(r *http.Request, dist fs.FS) bool { switch r.URL.Path { case setUpAccountPath: @@ -108,9 +137,8 @@ func setupRequest(r *http.Request, dist fs.FS) bool { return r.URL.Path == "/" || builtFile(r.URL.Path, dist) } -// builtFile reports whether path names a file in the dashboard build. The -// page's assets and icon are files; a route inside the SPA is not, so a deep -// link cannot borrow the setup entry's page. +// builtFile reports whether path names a file in the dashboard build. A route +// inside the SPA does not, so a deep link cannot be served the setup page. func builtFile(path string, dist fs.FS) bool { if dist == nil { return false @@ -119,27 +147,85 @@ func builtFile(path string, dist fs.FS) bool { return err == nil && !info.IsDir() } -// starter is what activation brings up: the operations Run wired. +// setupLookup reads whether owner setup is still required. Durable state is the +// only source: nothing a request carries can answer it. +type setupLookup func(context.Context) (bool, error) + +// starter is what activation brings up: the operations Run wired. Start is safe +// to call again after it returns an error. type starter interface { Start(context.Context) error } // activate waits for the owner setup transaction to commit, starts the -// operations, and then opens the gate. It runs on the process context, never -// the setup request's: the request that triggers it is answered long before the -// work it starts is done. -func activate(ctx context.Context, log *slog.Logger, setup <-chan struct{}, setUp bool, ops starter, g *gate) error { - if !setUp { +// operations, and then opens the gate. It runs on the process context: the setup +// request that triggered it has been answered by then. +func activate(ctx context.Context, log *slog.Logger, lookup setupLookup, done <-chan struct{}, ops starter, g *gate) error { + required, err := waitForSetup(ctx, log, lookup, done, g) + if err != nil { + return err + } + if required { select { - case <-setup: + case <-done: case <-ctx.Done(): return ctx.Err() } } - if err := ops.Start(ctx); err != nil { + if err := retryStartup(ctx, log, g, stageStartup, ops.Start); err != nil { return err } g.publish() log.InfoContext(ctx, "owner setup is complete; serving operations") return nil } + +// waitForSetup answers whether owner setup is still required. A failed read +// cannot rule setup out, so it is retried rather than trusted. +func waitForSetup(ctx context.Context, log *slog.Logger, lookup setupLookup, done <-chan struct{}, g *gate) (bool, error) { + for delay := startupRetryMin; ; delay = min(delay*2, startupRetryMax) { + required, err := lookup(ctx) + if err == nil { + g.stall("") + return required, nil + } + if ctx.Err() != nil { + return false, ctx.Err() + } + g.stall(stageSetupRead) + log.WarnContext(ctx, "startup step failed; retrying", "stage", stageSetupRead, "err", err, "in", delay) + select { + case <-time.After(delay): + case <-done: + // A setup has committed in this process while the reads failed, so an + // owner exists whatever the database can answer. + g.stall("") + return false, nil + case <-ctx.Done(): + return false, ctx.Err() + } + } +} + +// retryStartup runs a boot step until it succeeds, recording the stage that +// failed for readiness. A boot step that reads the database must not turn a brief +// outage into a terminated process. +func retryStartup(ctx context.Context, log *slog.Logger, g *gate, stage string, attempt func(context.Context) error) error { + for delay := startupRetryMin; ; delay = min(delay*2, startupRetryMax) { + err := attempt(ctx) + if err == nil { + g.stall("") + return nil + } + if ctx.Err() != nil { + return ctx.Err() + } + g.stall(stage) + log.WarnContext(ctx, "startup step failed; retrying", "stage", stage, "err", err, "in", delay) + select { + case <-time.After(delay): + case <-ctx.Done(): + return ctx.Err() + } + } +} diff --git a/internal/app/archie/gate_test.go b/internal/app/archie/gate_test.go index 471972f6e..2b70ce920 100644 --- a/internal/app/archie/gate_test.go +++ b/internal/app/archie/gate_test.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "testing" "testing/fstest" + "time" "github.com/samcharles93/archie-core/internal/platform/health" "github.com/samcharles93/archie-core/modules/dashboard" @@ -24,6 +25,12 @@ const testAdminToken = "Bearer a-bootstrap-admin-token" // testLog discards what the gate and activation log. func testLog() *slog.Logger { return slog.New(slog.DiscardHandler) } +// setUpRequired is durable state on an instance with no owner. +func setUpRequired(context.Context) (bool, error) { return true, nil } + +// setUpDone is durable state on an instance whose owner setup has committed. +func setUpDone(context.Context) (bool, error) { return false, nil } + // reached records the requests that get past the gate, so a test can tell a // refusal from a handler that ran. type reached struct { @@ -105,13 +112,16 @@ func TestClosedGateServesOnlySetup(t *testing.T) { admin bool status int body string + // servedAsPage marks a response the dashboard handler writes, which must + // keep the CSP that stops chat-rendered Markdown loading a remote image. + servedAsPage bool }{ {name: "auth status", method: http.MethodGet, path: authStatusPath, status: http.StatusOK, body: `"setup_required":true`}, {name: "auth status by head", method: http.MethodHead, path: authStatusPath, status: http.StatusOK}, {name: "set up the owner", method: http.MethodPost, path: setUpAccountPath, status: http.StatusCreated}, - {name: "the setup page", method: http.MethodGet, path: "/", status: http.StatusOK, body: "archie setup"}, - {name: "a script the page loads", method: http.MethodGet, path: "/assets/app.js", status: http.StatusOK, body: "console.log(1)"}, - {name: "the page icon", method: http.MethodGet, path: "/favicon.svg", status: http.StatusOK, body: ""}, + {name: "the setup page", method: http.MethodGet, path: "/", status: http.StatusOK, body: "archie setup", servedAsPage: true}, + {name: "a script the page loads", method: http.MethodGet, path: "/assets/app.js", status: http.StatusOK, body: "console.log(1)", servedAsPage: true}, + {name: "the page icon", method: http.MethodGet, path: "/favicon.svg", status: http.StatusOK, body: "", servedAsPage: true}, {name: "liveness", method: http.MethodGet, path: "/healthz", status: http.StatusOK, body: "ok\n"}, // From here down nothing reaches a handler, credential or not. @@ -147,6 +157,12 @@ func TestClosedGateServesOnlySetup(t *testing.T) { if got := hit.count(); got != 0 { t.Fatalf("%s %s: %d operational handlers ran, want none (reached %v)", tc.method, tc.path, got, hit.paths()) } + if tc.servedAsPage { + // Mirrors the dashboard's own header: the setup page must not lose it. + if got := rec.Header().Get("Content-Security-Policy"); got != "img-src 'self' data:" { + t.Fatalf("%s %s: Content-Security-Policy = %q, want the dashboard's", tc.method, tc.path, got) + } + } }) } } @@ -247,16 +263,20 @@ func TestReadinessReportsSetupUntilOperationsStart(t *testing.T) { } } -// fakeOperations stands in for the connectors, the access chain, the shipped -// workflow and the worker loops, and counts how often they would start. +// fakeOperations stands in for the startup work and counts how often it runs. type fakeOperations struct { - starts atomic.Int32 - err error + starts atomic.Int32 + startFails atomic.Int32 + startErr error } func (f *fakeOperations) Start(context.Context) error { f.starts.Add(1) - return f.err + if f.startFails.Load() > 0 { + f.startFails.Add(-1) + return f.startErr + } + return nil } func TestActivateStartsOperationsWhenSetupCommits(t *testing.T) { @@ -267,7 +287,7 @@ func TestActivateStartsOperationsWhenSetupCommits(t *testing.T) { served, hit, g := testServer(t) done := make(chan error, 1) - go func() { done <- activate(context.Background(), testLog(), signal.Done(), false, ops, g) }() + go func() { done <- activate(context.Background(), testLog(), setUpRequired, signal.Done(), ops, g) }() if got := ops.starts.Load(); got != 0 { t.Fatalf("starts before the commit = %d, want 0", got) @@ -305,7 +325,7 @@ func TestConcurrentSetupCommitsStartOperationsOnce(t *testing.T) { _, _, g := testServer(t) done := make(chan error, 1) - go func() { done <- activate(context.Background(), testLog(), signal.Done(), false, ops, g) }() + go func() { done <- activate(context.Background(), testLog(), setUpRequired, signal.Done(), ops, g) }() var completing sync.WaitGroup for range 8 { @@ -322,28 +342,6 @@ func TestConcurrentSetupCommitsStartOperationsOnce(t *testing.T) { } } -func TestFailedStartLeavesTheGateClosed(t *testing.T) { - // A connector or policy failure must not leave an instance answering - // operational requests with half its wiring up. - signal := governance.NewSetupSignal() - ops := &fakeOperations{err: errors.New("connectors refused")} - served, hit, g := testServer(t) - signal.Complete() - - if err := activate(context.Background(), testLog(), signal.Done(), false, ops, g); err == nil { - t.Fatalf("activate returned no error, want the start failure") - } - if g.opened() { - t.Fatalf("the gate opened after a failed start") - } - if rec := do(t, served, http.MethodGet, "/api/system/bootstrap", true); rec.Code != http.StatusNotFound { - t.Fatalf("operational API after a failed start: status = %d, want %d", rec.Code, http.StatusNotFound) - } - if got := hit.count(); got != 0 { - t.Fatalf("operational handlers after a failed start = %d, want none", got) - } -} - func TestCancelledBeforeSetupStartsNothing(t *testing.T) { signal := governance.NewSetupSignal() ops := &fakeOperations{} @@ -351,7 +349,7 @@ func TestCancelledBeforeSetupStartsNothing(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) - go func() { done <- activate(ctx, testLog(), signal.Done(), false, ops, g) }() + go func() { done <- activate(ctx, testLog(), setUpRequired, signal.Done(), ops, g) }() cancel() if err := <-done; !errors.Is(err, context.Canceled) { @@ -373,7 +371,7 @@ func TestCompletedSetupStartsWithoutWaiting(t *testing.T) { ops := &fakeOperations{} served, hit, g := testServer(t) - if err := activate(context.Background(), testLog(), signal.Done(), true, ops, g); err != nil { + if err := activate(context.Background(), testLog(), setUpDone, signal.Done(), ops, g); err != nil { t.Fatalf("activate: %v", err) } if got := ops.starts.Load(); got != 1 { @@ -389,3 +387,168 @@ func TestCompletedSetupStartsWithoutWaiting(t *testing.T) { t.Fatalf("operational handlers = %d, want 1", got) } } + +// flakyLookup fails a set number of reads before answering, as a database that +// is briefly unreachable does. +type flakyLookup struct { + failures atomic.Int32 + required bool + reads atomic.Int32 +} + +// newFlakyLookup fails failures reads before answering required. +func newFlakyLookup(failures int32, required bool) *flakyLookup { + l := &flakyLookup{required: required} + l.failures.Store(failures) + return l +} + +func (f *flakyLookup) lookup(context.Context) (bool, error) { + f.reads.Add(1) + if f.failures.Load() > 0 { + f.failures.Add(-1) + return false, errors.New("connection refused") + } + return f.required, nil +} + +func TestUnreadableSetupStateIsRetried(t *testing.T) { + // A database that is briefly unavailable must not hold an instance shut + // until someone restarts it: setup state that can't be read is retried, and + // readiness reports the failure meanwhile. + lookup := newFlakyLookup(2, false) + ops := &fakeOperations{} + served, hit, g := testServer(t) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + done := make(chan error, 1) + go func() { done <- activate(ctx, testLog(), lookup.lookup, nil, ops, g) }() + + // While the read fails, nothing starts and readiness names the failure. + if got := ops.starts.Load(); got != 0 { + t.Fatalf("starts while setup state is unreadable = %d, want 0", got) + } + for range 100 { + if strings.Contains(readyzBody(t, served), stageSetupRead) { + break + } + time.Sleep(10 * time.Millisecond) + } + body := readyzBody(t, served) + if !strings.Contains(body, stageSetupRead) { + t.Fatalf("readiness while setup state is unreadable = %q, want %q", body, stageSetupRead) + } + if strings.Contains(body, "connection refused") { + t.Fatalf("readiness = %q, leaked the database's own error", body) + } + + if err := <-done; err != nil { + t.Fatalf("activate: %v", err) + } + if got := ops.starts.Load(); got != 1 { + t.Fatalf("starts after the read recovered = %d, want 1", got) + } + if got := lookup.reads.Load(); got < 3 { + t.Fatalf("reads = %d, want the failed read retried", got) + } + if rec := do(t, served, http.MethodGet, "/api/system/bootstrap", true); rec.Code != http.StatusOK { + t.Fatalf("operational API after the read recovered: status = %d, want %d", rec.Code, http.StatusOK) + } + if got := hit.count(); got != 1 { + t.Fatalf("operational handlers after the read recovered = %d, want 1", got) + } +} + +func TestUnreadableSetupStateWaitsForTheCommit(t *testing.T) { + // When the read keeps failing, a setup that commits in this process is + // still enough: the owner exists, so operations may start. + lookup := newFlakyLookup(1000, true) + ops := &fakeOperations{} + _, _, g := testServer(t) + signal := governance.NewSetupSignal() + + done := make(chan error, 1) + go func() { done <- activate(context.Background(), testLog(), lookup.lookup, signal.Done(), ops, g) }() + + time.Sleep(50 * time.Millisecond) + if got := ops.starts.Load(); got != 0 { + t.Fatalf("starts while setup state is unreadable = %d, want 0", got) + } + signal.Complete() + if err := <-done; err != nil { + t.Fatalf("activate: %v", err) + } + if got := ops.starts.Load(); got != 1 { + t.Fatalf("starts after the commit = %d, want 1", got) + } +} + +func readyzBody(t *testing.T, served http.Handler) string { + t.Helper() + return do(t, served, http.MethodGet, "/readyz", false).Body.String() +} + +func TestOperationsStartIsRetried(t *testing.T) { + // connectors.Module.Start returns an error only for the read of accepted + // connectors, before anything launches, so a failed Start leaves nothing + // running and the next attempt is safe. Without that retry a transient read + // error took the process down. + ops := &fakeOperations{startErr: errors.New("connection refused")} + ops.startFails.Store(2) + served, hit, g := testServer(t) + signal := governance.NewSetupSignal() + signal.Complete() + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- activate(ctx, testLog(), setUpDone, signal.Done(), ops, g) }() + + if err := <-done; err != nil { + t.Fatalf("activate: %v", err) + } + if got := ops.starts.Load(); got != 3 { + t.Fatalf("start calls = %d, want the two failures retried", got) + } + if !g.opened() { + t.Fatalf("the gate is shut after a retried start") + } + if rec := do(t, served, http.MethodGet, "/api/system/bootstrap", true); rec.Code != http.StatusOK { + t.Fatalf("operational API after a retried start: status = %d, want %d", rec.Code, http.StatusOK) + } + if got := hit.count(); got != 1 { + t.Fatalf("operational handlers = %d, want 1: one surface, not one per attempt", got) + } +} + +func TestReadinessNamesTheStageWithoutTheRawError(t *testing.T) { + // Readiness answers whoever can reach the port, so it names the stage that + // failed and keeps the database's own error text in the log. + ops := &fakeOperations{startErr: errors.New("pq: password authentication failed for user archie")} + ops.startFails.Store(1000) + served, _, g := testServer(t) + signal := governance.NewSetupSignal() + signal.Complete() + done := make(chan error, 1) + go func() { done <- activate(context.Background(), testLog(), setUpDone, signal.Done(), ops, g) }() + + for range 200 { + if strings.Contains(readyzBody(t, served), "starting operations failed") { + break + } + time.Sleep(10 * time.Millisecond) + } + body := readyzBody(t, served) + if !strings.Contains(body, "starting operations failed") { + t.Fatalf("readiness = %q, want it to name the stage", body) + } + for _, leak := range []string{"pq:", "password", "user archie", "connection refused"} { + if strings.Contains(body, leak) { + t.Fatalf("readiness = %q, leaked %q", body, leak) + } + } + if g.opened() { + t.Fatalf("the gate opened while a startup stage fails") + } +} diff --git a/internal/app/archie/operations.go b/internal/app/archie/operations.go index 321d3cb17..57b644dd8 100644 --- a/internal/app/archie/operations.go +++ b/internal/app/archie/operations.go @@ -39,27 +39,18 @@ type operations struct { } // Start brings up every part of the binary that does work, in the order the -// rest of it expects: connectors first, then the access chain they are checked -// against, then the shipped workflow, then the loops. A failure returns with -// what started still running for the caller to shut down; nothing is left to -// report ready. +// other parts depend on: the role policies and access chain, the workflow that +// ships with the binary, the connectors, then the loops. +// +// Calling it again after it returns an error is safe. Every step that can fail +// runs before a connector launches, and connectors.Module.Start fails only for +// the read of accepted connectors, before launching any. An error therefore +// means nothing is running. func (o *operations) Start(ctx context.Context) error { - if err := o.conns.Start(ctx); err != nil { - return fmt.Errorf("start connectors: %w", err) - } - o.checks.Register("connectors", o.conns.Ready) - access, err := startAccess(ctx, o.log, o.settings) if err != nil { return err } - // Administrative writes are checked against the same live policy chain as - // every other request. - o.settings.UseAuthorizer(access) - o.talk.Use(chat.Deps{ - People: o.settings, Access: access, Settings: o.settings, Toolbox: o.conns, Judge: o.judge, - Approver: o.allow, Memory: o.remember, Runs: o.engine, Notifier: o.tell, - }) // A workflow takes its context from the run handle, as a handler takes it // from the request. o.engine.Use(runs.Deps{ @@ -69,7 +60,19 @@ func (o *operations) Start(ctx context.Context) error { if err := installStarter(ctx, o.engine); err != nil { return err } + if err := o.conns.Start(ctx); err != nil { + return fmt.Errorf("read accepted connectors: %w", err) + } + // Nothing below can fail, so a retry cannot launch a connector or a loop twice. + o.checks.Register("connectors", o.conns.Ready) + // Administrative writes are checked against the same live policy chain as + // every other request. + o.settings.UseAuthorizer(access) + o.talk.Use(chat.Deps{ + People: o.settings, Access: access, Settings: o.settings, Toolbox: o.conns, Judge: o.judge, + Approver: o.allow, Memory: o.remember, Runs: o.engine, Notifier: o.tell, + }) for _, loop := range []func(context.Context){o.engine.Work, o.hear.Work, o.tell.Work, o.talk.Work} { o.workers.Go(func() { loop(ctx) }) } diff --git a/internal/app/archie/run.go b/internal/app/archie/run.go index b45d33672..124004cb2 100644 --- a/internal/app/archie/run.go +++ b/internal/app/archie/run.go @@ -46,8 +46,9 @@ const shutdownGrace = 10 * time.Second // // An instance whose owner setup has not committed serves the setup page and // nothing else: connectors, module routes, the shipped workflow and the worker -// loops all wait for that transaction. Durable state says whether it already -// happened, so a restart needs no setup form and serves operations at once. +// loops all wait for that transaction. Durable state says whether it has already +// happened, so a restart reads that state and starts serving operations +// immediately. func Run(ctx context.Context, configPath string) error { log := slog.New(slog.NewJSONHandler(os.Stdout, nil)).With("service", "archie", "version", buildinfo.Version) @@ -69,39 +70,22 @@ func Run(ctx context.Context, configPath string) error { } defer b.close() - // Durable state decides the first gate. A read that fails cannot rule setup - // out, so the gate stays closed: nothing operational runs on an instance - // that may have no owner. The failure is logged and readiness reports it, - // and a restart reads the state again. - required, setupErr := b.settings.AccountSetupRequired(ctx) - if setupErr != nil { - log.ErrorContext(ctx, "read whether owner setup is required", "err", setupErr) - } - setUp := setupErr == nil && !required - work, stop := context.WithCancel(ctx) defer stop() - started := make(chan error, 1) + started := make(chan struct{}) go func() { - err := activate(work, log, b.settings.SetupDone(), setUp, b.ops, b.gate) - if err != nil { - // Whatever half of the operations came up is no use without the rest, - // so the binary stops rather than serving a partial instance. - stop() - } - started <- err + // Startup failures are retried, so only shutdown ends activation. + _ = activate(work, log, b.settings.AccountSetupRequired, b.settings.SetupDone(), b.ops, b.gate) + close(started) }() serveErr := serve(work, log, cfg.Listen, b.served) stop() - startErr := <-started + <-started // Every loop activation began has returned before the caller closes the // connectors and the database. b.ops.Wait() - if startErr != nil && !errors.Is(startErr, context.Canceled) { - return startErr - } return serveErr } diff --git a/modules/connectors/module.go b/modules/connectors/module.go index 29461756b..13e90fa7d 100644 --- a/modules/connectors/module.go +++ b/modules/connectors/module.go @@ -81,6 +81,10 @@ func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, sealing Sealing) (*M // Start launches every accepted secret engine, then every accepted connector, // since a connector's credentials may be resolved through an engine. One that // fails to start is reported by Ready and logged; the others still run. +// +// Its only error is the read of the accepted connectors, which happens before +// anything launches, so a caller may call Start again after a failure without +// launching anything twice. func (m *Module) Start(ctx context.Context) error { accepted, err := m.store.List(ctx, m.orgID) if err != nil { diff --git a/modules/governance/access_test.go b/modules/governance/access_test.go index 719c9a8fe..cef110256 100644 --- a/modules/governance/access_test.go +++ b/modules/governance/access_test.go @@ -6,6 +6,8 @@ import ( "log/slog" "slices" "testing" + + "github.com/samcharles93/archie-core/modules/governance/internal/store" ) // TestInvalidInstancePolicy pins that an instance policy that no longer @@ -440,3 +442,44 @@ func TestCustomRoleAndMemberStatePolicy(t *testing.T) { }) } } + +// fakePolicySeed records the one atomic seed the module asks for. +type fakePolicySeed struct { + calls int + ids []string + rows []store.PolicyRow + err error +} + +func (f *fakePolicySeed) SeedOrgPolicies(_ context.Context, _ string, shippedIDs []string, rows []store.PolicyRow) error { + f.calls++ + f.ids, f.rows = shippedIDs, rows + return f.err +} + +// TestSeedShippedPolicies guards that the shipped set is written in one call +// rather than one policy at a time, which is what left an org holding only some +// of its role policies. The transaction, the zero-count check under the lock and +// how deletions and edits survive are proven against Postgres in the binary's +// smoke test. +func TestSeedShippedPolicies(t *testing.T) { + seeder := &fakePolicySeed{} + if err := seedShippedPolicies(t.Context(), seeder, DefaultOrgID); err != nil { + t.Fatalf("seed: %v", err) + } + shipped := ShippedOrgPolicies(DefaultOrgID) + if seeder.calls != 1 { + t.Fatalf("seed calls = %d, want 1", seeder.calls) + } + if len(seeder.rows) != len(shipped) || len(seeder.ids) != len(shipped) { + t.Fatalf("seeded %d rows and %d ids, want %d of each", len(seeder.rows), len(seeder.ids), len(shipped)) + } + for i, p := range shipped { + if seeder.rows[i].PolicyID != p.ID || seeder.ids[i] != p.ID { + t.Fatalf("row %d = %s, want %s", i, seeder.rows[i].PolicyID, p.ID) + } + } + if err := seedShippedPolicies(t.Context(), &fakePolicySeed{err: errors.New("connection reset by peer")}, DefaultOrgID); err == nil { + t.Fatal("a failed seed returned no error") + } +} diff --git a/modules/governance/internal/store/access.go b/modules/governance/internal/store/access.go index 6373436ff..5534852dc 100644 --- a/modules/governance/internal/store/access.go +++ b/modules/governance/internal/store/access.go @@ -192,17 +192,63 @@ func (s *Store) DeleteAccessPolicy(ctx context.Context, p PolicyRow) (found bool return found, nil } -// CountShippedOrgPolicies returns how many of policyIDs are present at org level for orgID. -func (s *Store) CountShippedOrgPolicies(ctx context.Context, orgID string, policyIDs []string) (int64, error) { - var count int64 - if err := s.db.GetContext(ctx, &count, ` - SELECT count(*) FROM governance.access_policies - WHERE level = 'org' AND org_id = $1 AND policy_id = ANY($2::text[])`, orgID, policyIDs); err != nil { - return 0, fmt.Errorf("count shipped org policies: %w", err) +// SeedOrgPolicies stores an org's initial policies, unless the org already has +// any of them. +// +// The check and the writes share one transaction, and the lock is taken before +// the check, so a second seed waits for the first to commit and then finds its +// rows: it writes nothing, and an edit or a deletion made in between stands. +func (s *Store) SeedOrgPolicies(ctx context.Context, orgID string, shippedIDs []string, rows []PolicyRow) error { + if len(rows) == 0 { + return nil } - return count, nil + err := s.withTx(ctx, func(tx *sqlx.Tx) error { + if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, seedLockKey(orgID)); err != nil { + return err + } + var present int64 + if err := tx.GetContext(ctx, &present, ` + SELECT count(*) FROM governance.access_policies + WHERE level = 'org' AND org_id = $1 AND policy_id = ANY($2::text[])`, + orgID, shippedIDs); err != nil { + return err + } + if present > 0 { + return nil + } + for _, p := range rows { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO governance.access_policies (level, org_id, workspace_id, object_kind, object_id, policy_id, text) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (level, org_id, workspace_id, object_kind, object_id, policy_id) + DO UPDATE SET text = EXCLUDED.text, updated_at = now()`, + p.Level, p.OrgID, p.WorkspaceID, p.ObjectKind, p.ObjectID, p.PolicyID, p.Text); err != nil { + return err + } + } + // One version for the whole seed: a reader sees none of the policies or + // all of them. + first := rows[0] + if _, err := tx.ExecContext(ctx, ` + INSERT INTO governance.access_policy_versions (level, org_id, workspace_id, version) + VALUES ($1, $2, $3, 2) + ON CONFLICT (level, org_id, workspace_id) + DO UPDATE SET version = governance.access_policy_versions.version + 1, updated_at = now()`, + first.Level, first.OrgID, first.WorkspaceID); err != nil { + return err + } + return bumpPolicyRevision(ctx, tx) + }) + if err != nil { + return fmt.Errorf("seed org policies: %w", err) + } + return nil } +// seedLockKey is the advisory lock key a seed takes before it reads whether the +// org has policies, distinct from the per-policy keys. +func seedLockKey(orgID string) string { return "seed/" + orgID } + // DeleteOrgPoliciesNotShipped removes non-shipped org policies and bumps revision. func (s *Store) DeleteOrgPoliciesNotShipped(ctx context.Context, orgID string, shippedIDs []string) error { return s.withTx(ctx, func(tx *sqlx.Tx) error { diff --git a/modules/governance/store_access.go b/modules/governance/store_access.go index 141edf738..bf433d681 100644 --- a/modules/governance/store_access.go +++ b/modules/governance/store_access.go @@ -113,21 +113,28 @@ func (m *Module) DeletePolicy(ctx context.Context, p Policy) error { return nil } -// EnsureShippedOrgPolicies seeds an org's shipped role policies when none exist yet. +// EnsureShippedOrgPolicies seeds the shipped role policies when the org has none +// of them. func (m *Module) EnsureShippedOrgPolicies(ctx context.Context, orgID OrgID) error { - count, err := m.store.CountShippedOrgPolicies(ctx, string(orgID), shippedOrgPolicyIDs()) - if err != nil { - return err + return seedShippedPolicies(ctx, m.store, orgID) +} + +// policySeedStore is the part of the store that seeds the shipped policies. +type policySeedStore interface { + SeedOrgPolicies(context.Context, string, []string, []store.PolicyRow) error +} + +// seedShippedPolicies writes the whole shipped set in one call, so a failure +// leaves none of it behind and the next attempt writes all of it. The store +// decides whether the seed already ran; an org that has any shipped policy is +// left alone, so a deletion or an edit an admin made stands across restarts. +func seedShippedPolicies(ctx context.Context, s policySeedStore, orgID OrgID) error { + shipped := ShippedOrgPolicies(orgID) + rows := make([]store.PolicyRow, 0, len(shipped)) + for _, p := range shipped { + rows = append(rows, policyToRow(p)) } - if count > 0 { - return nil - } - for _, p := range ShippedOrgPolicies(orgID) { - if _, err := m.store.PutAccessPolicy(ctx, policyToRow(p)); err != nil { - return err - } - } - return nil + return s.SeedOrgPolicies(ctx, string(orgID), shippedOrgPolicyIDs(), rows) } // ResetOrgPolicies removes non-shipped org policies and restores the shipped role policies. diff --git a/modules/governance/store_identities.go b/modules/governance/store_identities.go index f9be39242..f2418e7af 100644 --- a/modules/governance/store_identities.go +++ b/modules/governance/store_identities.go @@ -381,9 +381,9 @@ func (m *Module) RegisterAccount(ctx context.Context, r AccountRegistration, boo return Identity{}, mapIdentityErr(err) } if bootstrap { - // The owner is in durable state now. Whatever waits on this signal starts - // operations, so it is sent only after the transaction committed: a failed - // attempt returns above and leaves it closed. + // The owner is in durable state now. The composition root starts operations + // when this signal closes, so it is sent only after the transaction + // committed: a failed attempt returns above and leaves it closed. m.setup.Complete() } return identityFromRow(row), nil