fix(app): retry startup and atomically seed policies (#2081)
This commit is contained in:
9 files changed
+458
-122
No files matched your search
+110
-24
@@ -8,29 +8,44 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/platform/health"
|
||||
"github.com/samcharles93/archie-core/internal/platform/httpjson"
|
||||
)
|
||||
|
||||
// The two account operations the setup page calls. Everything else under
|
||||
// /api/ needs the instance it would be operating.
|
||||
// The two account operations the setup page calls. Every other /api/ route
|
||||
// needs an instance that already has an owner.
|
||||
const (
|
||||
authStatusPath = "/api/auth/status"
|
||||
setUpAccountPath = "/api/auth/setup"
|
||||
)
|
||||
|
||||
// What stopped operations from starting. Readiness reports the stage; the error
|
||||
// text stays in the log, since it can name hosts, roles and tables.
|
||||
const (
|
||||
stageSetupRead = "cannot read whether owner setup is required"
|
||||
stageStartup = "starting operations failed"
|
||||
)
|
||||
|
||||
// How long a failed boot step waits before it runs again. The delay doubles up
|
||||
// to a ceiling.
|
||||
const (
|
||||
startupRetryMin = 100 * time.Millisecond
|
||||
startupRetryMax = 5 * time.Second
|
||||
)
|
||||
|
||||
// gate is the surface the binary serves. Until the owner setup transaction has
|
||||
// committed it answers only what the setup page needs, and refuses every
|
||||
// operational route before its handler runs, whatever credential the request
|
||||
// carries. Once operations are started and wired it hands every request to
|
||||
// them.
|
||||
// committed it answers only what the setup page needs, and it refuses every
|
||||
// operational route before the handler runs, whatever credential the request
|
||||
// carries. Once the operations are running it passes every request to them.
|
||||
type gate struct {
|
||||
ops http.Handler
|
||||
setup http.Handler
|
||||
|
||||
mu sync.RWMutex
|
||||
open bool
|
||||
mu sync.RWMutex
|
||||
open bool
|
||||
problem string
|
||||
}
|
||||
|
||||
// newGate returns a closed gate over the operational handler and the dashboard
|
||||
@@ -53,6 +68,7 @@ func (g *gate) publish() {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.open = true
|
||||
g.problem = ""
|
||||
}
|
||||
|
||||
func (g *gate) opened() bool {
|
||||
@@ -61,11 +77,24 @@ func (g *gate) opened() bool {
|
||||
return g.open
|
||||
}
|
||||
|
||||
// stall records the stage that kept the gate closed, so readiness names it rather
|
||||
// than only the missing owner. An empty stage clears it.
|
||||
func (g *gate) stall(stage string) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.problem = stage
|
||||
}
|
||||
|
||||
// check is the readiness check for the gate. An instance still waiting for its
|
||||
// owner is alive but has nothing ready to serve, and that is not a crash.
|
||||
func (g *gate) check(context.Context) error {
|
||||
if g.opened() {
|
||||
g.mu.RLock()
|
||||
defer g.mu.RUnlock()
|
||||
switch {
|
||||
case g.open:
|
||||
return nil
|
||||
case g.problem != "":
|
||||
return errors.New(g.problem)
|
||||
}
|
||||
return errors.New("owner setup is incomplete")
|
||||
}
|
||||
@@ -91,10 +120,10 @@ func setupSurface(ops http.Handler, dist fs.FS) http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
// setupRequest is the whole routing decision while the gate is closed: the
|
||||
// setup page and the files it loads, and the two account operations it calls.
|
||||
// Nothing else passes, so a route inside the SPA is not a way in and neither is
|
||||
// the wrong method on a route that is.
|
||||
// setupRequest is the whole routing decision while the gate is closed: the setup
|
||||
// page, the files it loads, and the two account operations it calls. Nothing else
|
||||
// passes, so neither a deep link into the SPA nor a setup route called with the
|
||||
// wrong method reaches a handler.
|
||||
func setupRequest(r *http.Request, dist fs.FS) bool {
|
||||
switch r.URL.Path {
|
||||
case setUpAccountPath:
|
||||
@@ -108,9 +137,8 @@ func setupRequest(r *http.Request, dist fs.FS) bool {
|
||||
return r.URL.Path == "/" || builtFile(r.URL.Path, dist)
|
||||
}
|
||||
|
||||
// builtFile reports whether path names a file in the dashboard build. The
|
||||
// page's assets and icon are files; a route inside the SPA is not, so a deep
|
||||
// link cannot borrow the setup entry's page.
|
||||
// builtFile reports whether path names a file in the dashboard build. A route
|
||||
// inside the SPA does not, so a deep link cannot be served the setup page.
|
||||
func builtFile(path string, dist fs.FS) bool {
|
||||
if dist == nil {
|
||||
return false
|
||||
@@ -119,27 +147,85 @@ func builtFile(path string, dist fs.FS) bool {
|
||||
return err == nil && !info.IsDir()
|
||||
}
|
||||
|
||||
// starter is what activation brings up: the operations Run wired.
|
||||
// setupLookup reads whether owner setup is still required. Durable state is the
|
||||
// only source: nothing a request carries can answer it.
|
||||
type setupLookup func(context.Context) (bool, error)
|
||||
|
||||
// starter is what activation brings up: the operations Run wired. Start is safe
|
||||
// to call again after it returns an error.
|
||||
type starter interface {
|
||||
Start(context.Context) error
|
||||
}
|
||||
|
||||
// activate waits for the owner setup transaction to commit, starts the
|
||||
// operations, and then opens the gate. It runs on the process context, never
|
||||
// the setup request's: the request that triggers it is answered long before the
|
||||
// work it starts is done.
|
||||
func activate(ctx context.Context, log *slog.Logger, setup <-chan struct{}, setUp bool, ops starter, g *gate) error {
|
||||
if !setUp {
|
||||
// operations, and then opens the gate. It runs on the process context: the setup
|
||||
// request that triggered it has been answered by then.
|
||||
func activate(ctx context.Context, log *slog.Logger, lookup setupLookup, done <-chan struct{}, ops starter, g *gate) error {
|
||||
required, err := waitForSetup(ctx, log, lookup, done, g)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if required {
|
||||
select {
|
||||
case <-setup:
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
if err := ops.Start(ctx); err != nil {
|
||||
if err := retryStartup(ctx, log, g, stageStartup, ops.Start); err != nil {
|
||||
return err
|
||||
}
|
||||
g.publish()
|
||||
log.InfoContext(ctx, "owner setup is complete; serving operations")
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForSetup answers whether owner setup is still required. A failed read
|
||||
// cannot rule setup out, so it is retried rather than trusted.
|
||||
func waitForSetup(ctx context.Context, log *slog.Logger, lookup setupLookup, done <-chan struct{}, g *gate) (bool, error) {
|
||||
for delay := startupRetryMin; ; delay = min(delay*2, startupRetryMax) {
|
||||
required, err := lookup(ctx)
|
||||
if err == nil {
|
||||
g.stall("")
|
||||
return required, nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
g.stall(stageSetupRead)
|
||||
log.WarnContext(ctx, "startup step failed; retrying", "stage", stageSetupRead, "err", err, "in", delay)
|
||||
select {
|
||||
case <-time.After(delay):
|
||||
case <-done:
|
||||
// A setup has committed in this process while the reads failed, so an
|
||||
// owner exists whatever the database can answer.
|
||||
g.stall("")
|
||||
return false, nil
|
||||
case <-ctx.Done():
|
||||
return false, ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// retryStartup runs a boot step until it succeeds, recording the stage that
|
||||
// failed for readiness. A boot step that reads the database must not turn a brief
|
||||
// outage into a terminated process.
|
||||
func retryStartup(ctx context.Context, log *slog.Logger, g *gate, stage string, attempt func(context.Context) error) error {
|
||||
for delay := startupRetryMin; ; delay = min(delay*2, startupRetryMax) {
|
||||
err := attempt(ctx)
|
||||
if err == nil {
|
||||
g.stall("")
|
||||
return nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
g.stall(stage)
|
||||
log.WarnContext(ctx, "startup step failed; retrying", "stage", stage, "err", err, "in", delay)
|
||||
select {
|
||||
case <-time.After(delay):
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/platform/health"
|
||||
"github.com/samcharles93/archie-core/modules/dashboard"
|
||||
@@ -24,6 +25,12 @@ const testAdminToken = "Bearer a-bootstrap-admin-token"
|
||||
// testLog discards what the gate and activation log.
|
||||
func testLog() *slog.Logger { return slog.New(slog.DiscardHandler) }
|
||||
|
||||
// setUpRequired is durable state on an instance with no owner.
|
||||
func setUpRequired(context.Context) (bool, error) { return true, nil }
|
||||
|
||||
// setUpDone is durable state on an instance whose owner setup has committed.
|
||||
func setUpDone(context.Context) (bool, error) { return false, nil }
|
||||
|
||||
// reached records the requests that get past the gate, so a test can tell a
|
||||
// refusal from a handler that ran.
|
||||
type reached struct {
|
||||
@@ -105,13 +112,16 @@ func TestClosedGateServesOnlySetup(t *testing.T) {
|
||||
admin bool
|
||||
status int
|
||||
body string
|
||||
// servedAsPage marks a response the dashboard handler writes, which must
|
||||
// keep the CSP that stops chat-rendered Markdown loading a remote image.
|
||||
servedAsPage bool
|
||||
}{
|
||||
{name: "auth status", method: http.MethodGet, path: authStatusPath, status: http.StatusOK, body: `"setup_required":true`},
|
||||
{name: "auth status by head", method: http.MethodHead, path: authStatusPath, status: http.StatusOK},
|
||||
{name: "set up the owner", method: http.MethodPost, path: setUpAccountPath, status: http.StatusCreated},
|
||||
{name: "the setup page", method: http.MethodGet, path: "/", status: http.StatusOK, body: "archie setup"},
|
||||
{name: "a script the page loads", method: http.MethodGet, path: "/assets/app.js", status: http.StatusOK, body: "console.log(1)"},
|
||||
{name: "the page icon", method: http.MethodGet, path: "/favicon.svg", status: http.StatusOK, body: "<svg>"},
|
||||
{name: "the setup page", method: http.MethodGet, path: "/", status: http.StatusOK, body: "archie setup", servedAsPage: true},
|
||||
{name: "a script the page loads", method: http.MethodGet, path: "/assets/app.js", status: http.StatusOK, body: "console.log(1)", servedAsPage: true},
|
||||
{name: "the page icon", method: http.MethodGet, path: "/favicon.svg", status: http.StatusOK, body: "<svg>", servedAsPage: true},
|
||||
{name: "liveness", method: http.MethodGet, path: "/healthz", status: http.StatusOK, body: "ok\n"},
|
||||
|
||||
// From here down nothing reaches a handler, credential or not.
|
||||
@@ -147,6 +157,12 @@ func TestClosedGateServesOnlySetup(t *testing.T) {
|
||||
if got := hit.count(); got != 0 {
|
||||
t.Fatalf("%s %s: %d operational handlers ran, want none (reached %v)", tc.method, tc.path, got, hit.paths())
|
||||
}
|
||||
if tc.servedAsPage {
|
||||
// Mirrors the dashboard's own header: the setup page must not lose it.
|
||||
if got := rec.Header().Get("Content-Security-Policy"); got != "img-src 'self' data:" {
|
||||
t.Fatalf("%s %s: Content-Security-Policy = %q, want the dashboard's", tc.method, tc.path, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -247,16 +263,20 @@ func TestReadinessReportsSetupUntilOperationsStart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// fakeOperations stands in for the connectors, the access chain, the shipped
|
||||
// workflow and the worker loops, and counts how often they would start.
|
||||
// fakeOperations stands in for the startup work and counts how often it runs.
|
||||
type fakeOperations struct {
|
||||
starts atomic.Int32
|
||||
err error
|
||||
starts atomic.Int32
|
||||
startFails atomic.Int32
|
||||
startErr error
|
||||
}
|
||||
|
||||
func (f *fakeOperations) Start(context.Context) error {
|
||||
f.starts.Add(1)
|
||||
return f.err
|
||||
if f.startFails.Load() > 0 {
|
||||
f.startFails.Add(-1)
|
||||
return f.startErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestActivateStartsOperationsWhenSetupCommits(t *testing.T) {
|
||||
@@ -267,7 +287,7 @@ func TestActivateStartsOperationsWhenSetupCommits(t *testing.T) {
|
||||
served, hit, g := testServer(t)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- activate(context.Background(), testLog(), signal.Done(), false, ops, g) }()
|
||||
go func() { done <- activate(context.Background(), testLog(), setUpRequired, signal.Done(), ops, g) }()
|
||||
|
||||
if got := ops.starts.Load(); got != 0 {
|
||||
t.Fatalf("starts before the commit = %d, want 0", got)
|
||||
@@ -305,7 +325,7 @@ func TestConcurrentSetupCommitsStartOperationsOnce(t *testing.T) {
|
||||
_, _, g := testServer(t)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- activate(context.Background(), testLog(), signal.Done(), false, ops, g) }()
|
||||
go func() { done <- activate(context.Background(), testLog(), setUpRequired, signal.Done(), ops, g) }()
|
||||
|
||||
var completing sync.WaitGroup
|
||||
for range 8 {
|
||||
@@ -322,28 +342,6 @@ func TestConcurrentSetupCommitsStartOperationsOnce(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedStartLeavesTheGateClosed(t *testing.T) {
|
||||
// A connector or policy failure must not leave an instance answering
|
||||
// operational requests with half its wiring up.
|
||||
signal := governance.NewSetupSignal()
|
||||
ops := &fakeOperations{err: errors.New("connectors refused")}
|
||||
served, hit, g := testServer(t)
|
||||
signal.Complete()
|
||||
|
||||
if err := activate(context.Background(), testLog(), signal.Done(), false, ops, g); err == nil {
|
||||
t.Fatalf("activate returned no error, want the start failure")
|
||||
}
|
||||
if g.opened() {
|
||||
t.Fatalf("the gate opened after a failed start")
|
||||
}
|
||||
if rec := do(t, served, http.MethodGet, "/api/system/bootstrap", true); rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("operational API after a failed start: status = %d, want %d", rec.Code, http.StatusNotFound)
|
||||
}
|
||||
if got := hit.count(); got != 0 {
|
||||
t.Fatalf("operational handlers after a failed start = %d, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelledBeforeSetupStartsNothing(t *testing.T) {
|
||||
signal := governance.NewSetupSignal()
|
||||
ops := &fakeOperations{}
|
||||
@@ -351,7 +349,7 @@ func TestCancelledBeforeSetupStartsNothing(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- activate(ctx, testLog(), signal.Done(), false, ops, g) }()
|
||||
go func() { done <- activate(ctx, testLog(), setUpRequired, signal.Done(), ops, g) }()
|
||||
cancel()
|
||||
|
||||
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||
@@ -373,7 +371,7 @@ func TestCompletedSetupStartsWithoutWaiting(t *testing.T) {
|
||||
ops := &fakeOperations{}
|
||||
served, hit, g := testServer(t)
|
||||
|
||||
if err := activate(context.Background(), testLog(), signal.Done(), true, ops, g); err != nil {
|
||||
if err := activate(context.Background(), testLog(), setUpDone, signal.Done(), ops, g); err != nil {
|
||||
t.Fatalf("activate: %v", err)
|
||||
}
|
||||
if got := ops.starts.Load(); got != 1 {
|
||||
@@ -389,3 +387,168 @@ func TestCompletedSetupStartsWithoutWaiting(t *testing.T) {
|
||||
t.Fatalf("operational handlers = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// flakyLookup fails a set number of reads before answering, as a database that
|
||||
// is briefly unreachable does.
|
||||
type flakyLookup struct {
|
||||
failures atomic.Int32
|
||||
required bool
|
||||
reads atomic.Int32
|
||||
}
|
||||
|
||||
// newFlakyLookup fails failures reads before answering required.
|
||||
func newFlakyLookup(failures int32, required bool) *flakyLookup {
|
||||
l := &flakyLookup{required: required}
|
||||
l.failures.Store(failures)
|
||||
return l
|
||||
}
|
||||
|
||||
func (f *flakyLookup) lookup(context.Context) (bool, error) {
|
||||
f.reads.Add(1)
|
||||
if f.failures.Load() > 0 {
|
||||
f.failures.Add(-1)
|
||||
return false, errors.New("connection refused")
|
||||
}
|
||||
return f.required, nil
|
||||
}
|
||||
|
||||
func TestUnreadableSetupStateIsRetried(t *testing.T) {
|
||||
// A database that is briefly unavailable must not hold an instance shut
|
||||
// until someone restarts it: setup state that can't be read is retried, and
|
||||
// readiness reports the failure meanwhile.
|
||||
lookup := newFlakyLookup(2, false)
|
||||
ops := &fakeOperations{}
|
||||
served, hit, g := testServer(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- activate(ctx, testLog(), lookup.lookup, nil, ops, g) }()
|
||||
|
||||
// While the read fails, nothing starts and readiness names the failure.
|
||||
if got := ops.starts.Load(); got != 0 {
|
||||
t.Fatalf("starts while setup state is unreadable = %d, want 0", got)
|
||||
}
|
||||
for range 100 {
|
||||
if strings.Contains(readyzBody(t, served), stageSetupRead) {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
body := readyzBody(t, served)
|
||||
if !strings.Contains(body, stageSetupRead) {
|
||||
t.Fatalf("readiness while setup state is unreadable = %q, want %q", body, stageSetupRead)
|
||||
}
|
||||
if strings.Contains(body, "connection refused") {
|
||||
t.Fatalf("readiness = %q, leaked the database's own error", body)
|
||||
}
|
||||
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("activate: %v", err)
|
||||
}
|
||||
if got := ops.starts.Load(); got != 1 {
|
||||
t.Fatalf("starts after the read recovered = %d, want 1", got)
|
||||
}
|
||||
if got := lookup.reads.Load(); got < 3 {
|
||||
t.Fatalf("reads = %d, want the failed read retried", got)
|
||||
}
|
||||
if rec := do(t, served, http.MethodGet, "/api/system/bootstrap", true); rec.Code != http.StatusOK {
|
||||
t.Fatalf("operational API after the read recovered: status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
if got := hit.count(); got != 1 {
|
||||
t.Fatalf("operational handlers after the read recovered = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnreadableSetupStateWaitsForTheCommit(t *testing.T) {
|
||||
// When the read keeps failing, a setup that commits in this process is
|
||||
// still enough: the owner exists, so operations may start.
|
||||
lookup := newFlakyLookup(1000, true)
|
||||
ops := &fakeOperations{}
|
||||
_, _, g := testServer(t)
|
||||
signal := governance.NewSetupSignal()
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- activate(context.Background(), testLog(), lookup.lookup, signal.Done(), ops, g) }()
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if got := ops.starts.Load(); got != 0 {
|
||||
t.Fatalf("starts while setup state is unreadable = %d, want 0", got)
|
||||
}
|
||||
signal.Complete()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("activate: %v", err)
|
||||
}
|
||||
if got := ops.starts.Load(); got != 1 {
|
||||
t.Fatalf("starts after the commit = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func readyzBody(t *testing.T, served http.Handler) string {
|
||||
t.Helper()
|
||||
return do(t, served, http.MethodGet, "/readyz", false).Body.String()
|
||||
}
|
||||
|
||||
func TestOperationsStartIsRetried(t *testing.T) {
|
||||
// connectors.Module.Start returns an error only for the read of accepted
|
||||
// connectors, before anything launches, so a failed Start leaves nothing
|
||||
// running and the next attempt is safe. Without that retry a transient read
|
||||
// error took the process down.
|
||||
ops := &fakeOperations{startErr: errors.New("connection refused")}
|
||||
ops.startFails.Store(2)
|
||||
served, hit, g := testServer(t)
|
||||
signal := governance.NewSetupSignal()
|
||||
signal.Complete()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- activate(ctx, testLog(), setUpDone, signal.Done(), ops, g) }()
|
||||
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("activate: %v", err)
|
||||
}
|
||||
if got := ops.starts.Load(); got != 3 {
|
||||
t.Fatalf("start calls = %d, want the two failures retried", got)
|
||||
}
|
||||
if !g.opened() {
|
||||
t.Fatalf("the gate is shut after a retried start")
|
||||
}
|
||||
if rec := do(t, served, http.MethodGet, "/api/system/bootstrap", true); rec.Code != http.StatusOK {
|
||||
t.Fatalf("operational API after a retried start: status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
if got := hit.count(); got != 1 {
|
||||
t.Fatalf("operational handlers = %d, want 1: one surface, not one per attempt", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadinessNamesTheStageWithoutTheRawError(t *testing.T) {
|
||||
// Readiness answers whoever can reach the port, so it names the stage that
|
||||
// failed and keeps the database's own error text in the log.
|
||||
ops := &fakeOperations{startErr: errors.New("pq: password authentication failed for user archie")}
|
||||
ops.startFails.Store(1000)
|
||||
served, _, g := testServer(t)
|
||||
signal := governance.NewSetupSignal()
|
||||
signal.Complete()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- activate(context.Background(), testLog(), setUpDone, signal.Done(), ops, g) }()
|
||||
|
||||
for range 200 {
|
||||
if strings.Contains(readyzBody(t, served), "starting operations failed") {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
body := readyzBody(t, served)
|
||||
if !strings.Contains(body, "starting operations failed") {
|
||||
t.Fatalf("readiness = %q, want it to name the stage", body)
|
||||
}
|
||||
for _, leak := range []string{"pq:", "password", "user archie", "connection refused"} {
|
||||
if strings.Contains(body, leak) {
|
||||
t.Fatalf("readiness = %q, leaked %q", body, leak)
|
||||
}
|
||||
}
|
||||
if g.opened() {
|
||||
t.Fatalf("the gate opened while a startup stage fails")
|
||||
}
|
||||
}
|
||||
@@ -39,27 +39,18 @@ type operations struct {
|
||||
}
|
||||
|
||||
// Start brings up every part of the binary that does work, in the order the
|
||||
// rest of it expects: connectors first, then the access chain they are checked
|
||||
// against, then the shipped workflow, then the loops. A failure returns with
|
||||
// what started still running for the caller to shut down; nothing is left to
|
||||
// report ready.
|
||||
// other parts depend on: the role policies and access chain, the workflow that
|
||||
// ships with the binary, the connectors, then the loops.
|
||||
//
|
||||
// Calling it again after it returns an error is safe. Every step that can fail
|
||||
// runs before a connector launches, and connectors.Module.Start fails only for
|
||||
// the read of accepted connectors, before launching any. An error therefore
|
||||
// means nothing is running.
|
||||
func (o *operations) Start(ctx context.Context) error {
|
||||
if err := o.conns.Start(ctx); err != nil {
|
||||
return fmt.Errorf("start connectors: %w", err)
|
||||
}
|
||||
o.checks.Register("connectors", o.conns.Ready)
|
||||
|
||||
access, err := startAccess(ctx, o.log, o.settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Administrative writes are checked against the same live policy chain as
|
||||
// every other request.
|
||||
o.settings.UseAuthorizer(access)
|
||||
o.talk.Use(chat.Deps{
|
||||
People: o.settings, Access: access, Settings: o.settings, Toolbox: o.conns, Judge: o.judge,
|
||||
Approver: o.allow, Memory: o.remember, Runs: o.engine, Notifier: o.tell,
|
||||
})
|
||||
// A workflow takes its context from the run handle, as a handler takes it
|
||||
// from the request.
|
||||
o.engine.Use(runs.Deps{
|
||||
@@ -69,7 +60,19 @@ func (o *operations) Start(ctx context.Context) error {
|
||||
if err := installStarter(ctx, o.engine); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := o.conns.Start(ctx); err != nil {
|
||||
return fmt.Errorf("read accepted connectors: %w", err)
|
||||
}
|
||||
|
||||
// Nothing below can fail, so a retry cannot launch a connector or a loop twice.
|
||||
o.checks.Register("connectors", o.conns.Ready)
|
||||
// Administrative writes are checked against the same live policy chain as
|
||||
// every other request.
|
||||
o.settings.UseAuthorizer(access)
|
||||
o.talk.Use(chat.Deps{
|
||||
People: o.settings, Access: access, Settings: o.settings, Toolbox: o.conns, Judge: o.judge,
|
||||
Approver: o.allow, Memory: o.remember, Runs: o.engine, Notifier: o.tell,
|
||||
})
|
||||
for _, loop := range []func(context.Context){o.engine.Work, o.hear.Work, o.tell.Work, o.talk.Work} {
|
||||
o.workers.Go(func() { loop(ctx) })
|
||||
}
|
||||
|
||||
@@ -46,8 +46,9 @@ const shutdownGrace = 10 * time.Second
|
||||
//
|
||||
// An instance whose owner setup has not committed serves the setup page and
|
||||
// nothing else: connectors, module routes, the shipped workflow and the worker
|
||||
// loops all wait for that transaction. Durable state says whether it already
|
||||
// happened, so a restart needs no setup form and serves operations at once.
|
||||
// loops all wait for that transaction. Durable state says whether it has already
|
||||
// happened, so a restart reads that state and starts serving operations
|
||||
// immediately.
|
||||
func Run(ctx context.Context, configPath string) error {
|
||||
log := slog.New(slog.NewJSONHandler(os.Stdout, nil)).With("service", "archie", "version", buildinfo.Version)
|
||||
|
||||
@@ -69,39 +70,22 @@ func Run(ctx context.Context, configPath string) error {
|
||||
}
|
||||
defer b.close()
|
||||
|
||||
// Durable state decides the first gate. A read that fails cannot rule setup
|
||||
// out, so the gate stays closed: nothing operational runs on an instance
|
||||
// that may have no owner. The failure is logged and readiness reports it,
|
||||
// and a restart reads the state again.
|
||||
required, setupErr := b.settings.AccountSetupRequired(ctx)
|
||||
if setupErr != nil {
|
||||
log.ErrorContext(ctx, "read whether owner setup is required", "err", setupErr)
|
||||
}
|
||||
setUp := setupErr == nil && !required
|
||||
|
||||
work, stop := context.WithCancel(ctx)
|
||||
defer stop()
|
||||
|
||||
started := make(chan error, 1)
|
||||
started := make(chan struct{})
|
||||
go func() {
|
||||
err := activate(work, log, b.settings.SetupDone(), setUp, b.ops, b.gate)
|
||||
if err != nil {
|
||||
// Whatever half of the operations came up is no use without the rest,
|
||||
// so the binary stops rather than serving a partial instance.
|
||||
stop()
|
||||
}
|
||||
started <- err
|
||||
// Startup failures are retried, so only shutdown ends activation.
|
||||
_ = activate(work, log, b.settings.AccountSetupRequired, b.settings.SetupDone(), b.ops, b.gate)
|
||||
close(started)
|
||||
}()
|
||||
|
||||
serveErr := serve(work, log, cfg.Listen, b.served)
|
||||
stop()
|
||||
startErr := <-started
|
||||
<-started
|
||||
// Every loop activation began has returned before the caller closes the
|
||||
// connectors and the database.
|
||||
b.ops.Wait()
|
||||
if startErr != nil && !errors.Is(startErr, context.Canceled) {
|
||||
return startErr
|
||||
}
|
||||
return serveErr
|
||||
}
|
||||
|
||||
|
||||
@@ -81,6 +81,10 @@ func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, sealing Sealing) (*M
|
||||
// Start launches every accepted secret engine, then every accepted connector,
|
||||
// since a connector's credentials may be resolved through an engine. One that
|
||||
// fails to start is reported by Ready and logged; the others still run.
|
||||
//
|
||||
// Its only error is the read of the accepted connectors, which happens before
|
||||
// anything launches, so a caller may call Start again after a failure without
|
||||
// launching anything twice.
|
||||
func (m *Module) Start(ctx context.Context) error {
|
||||
accepted, err := m.store.List(ctx, m.orgID)
|
||||
if err != nil {
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"log/slog"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/modules/governance/internal/store"
|
||||
)
|
||||
|
||||
// TestInvalidInstancePolicy pins that an instance policy that no longer
|
||||
@@ -440,3 +442,44 @@ func TestCustomRoleAndMemberStatePolicy(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// fakePolicySeed records the one atomic seed the module asks for.
|
||||
type fakePolicySeed struct {
|
||||
calls int
|
||||
ids []string
|
||||
rows []store.PolicyRow
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakePolicySeed) SeedOrgPolicies(_ context.Context, _ string, shippedIDs []string, rows []store.PolicyRow) error {
|
||||
f.calls++
|
||||
f.ids, f.rows = shippedIDs, rows
|
||||
return f.err
|
||||
}
|
||||
|
||||
// TestSeedShippedPolicies guards that the shipped set is written in one call
|
||||
// rather than one policy at a time, which is what left an org holding only some
|
||||
// of its role policies. The transaction, the zero-count check under the lock and
|
||||
// how deletions and edits survive are proven against Postgres in the binary's
|
||||
// smoke test.
|
||||
func TestSeedShippedPolicies(t *testing.T) {
|
||||
seeder := &fakePolicySeed{}
|
||||
if err := seedShippedPolicies(t.Context(), seeder, DefaultOrgID); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
shipped := ShippedOrgPolicies(DefaultOrgID)
|
||||
if seeder.calls != 1 {
|
||||
t.Fatalf("seed calls = %d, want 1", seeder.calls)
|
||||
}
|
||||
if len(seeder.rows) != len(shipped) || len(seeder.ids) != len(shipped) {
|
||||
t.Fatalf("seeded %d rows and %d ids, want %d of each", len(seeder.rows), len(seeder.ids), len(shipped))
|
||||
}
|
||||
for i, p := range shipped {
|
||||
if seeder.rows[i].PolicyID != p.ID || seeder.ids[i] != p.ID {
|
||||
t.Fatalf("row %d = %s, want %s", i, seeder.rows[i].PolicyID, p.ID)
|
||||
}
|
||||
}
|
||||
if err := seedShippedPolicies(t.Context(), &fakePolicySeed{err: errors.New("connection reset by peer")}, DefaultOrgID); err == nil {
|
||||
t.Fatal("a failed seed returned no error")
|
||||
}
|
||||
}
|
||||
@@ -192,17 +192,63 @@ func (s *Store) DeleteAccessPolicy(ctx context.Context, p PolicyRow) (found bool
|
||||
return found, nil
|
||||
}
|
||||
|
||||
// CountShippedOrgPolicies returns how many of policyIDs are present at org level for orgID.
|
||||
func (s *Store) CountShippedOrgPolicies(ctx context.Context, orgID string, policyIDs []string) (int64, error) {
|
||||
var count int64
|
||||
if err := s.db.GetContext(ctx, &count, `
|
||||
SELECT count(*) FROM governance.access_policies
|
||||
WHERE level = 'org' AND org_id = $1 AND policy_id = ANY($2::text[])`, orgID, policyIDs); err != nil {
|
||||
return 0, fmt.Errorf("count shipped org policies: %w", err)
|
||||
// SeedOrgPolicies stores an org's initial policies, unless the org already has
|
||||
// any of them.
|
||||
//
|
||||
// The check and the writes share one transaction, and the lock is taken before
|
||||
// the check, so a second seed waits for the first to commit and then finds its
|
||||
// rows: it writes nothing, and an edit or a deletion made in between stands.
|
||||
func (s *Store) SeedOrgPolicies(ctx context.Context, orgID string, shippedIDs []string, rows []PolicyRow) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
return count, nil
|
||||
err := s.withTx(ctx, func(tx *sqlx.Tx) error {
|
||||
if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, seedLockKey(orgID)); err != nil {
|
||||
return err
|
||||
}
|
||||
var present int64
|
||||
if err := tx.GetContext(ctx, &present, `
|
||||
SELECT count(*) FROM governance.access_policies
|
||||
WHERE level = 'org' AND org_id = $1 AND policy_id = ANY($2::text[])`,
|
||||
orgID, shippedIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
if present > 0 {
|
||||
return nil
|
||||
}
|
||||
for _, p := range rows {
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO governance.access_policies (level, org_id, workspace_id, object_kind, object_id, policy_id, text)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (level, org_id, workspace_id, object_kind, object_id, policy_id)
|
||||
DO UPDATE SET text = EXCLUDED.text, updated_at = now()`,
|
||||
p.Level, p.OrgID, p.WorkspaceID, p.ObjectKind, p.ObjectID, p.PolicyID, p.Text); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// One version for the whole seed: a reader sees none of the policies or
|
||||
// all of them.
|
||||
first := rows[0]
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO governance.access_policy_versions (level, org_id, workspace_id, version)
|
||||
VALUES ($1, $2, $3, 2)
|
||||
ON CONFLICT (level, org_id, workspace_id)
|
||||
DO UPDATE SET version = governance.access_policy_versions.version + 1, updated_at = now()`,
|
||||
first.Level, first.OrgID, first.WorkspaceID); err != nil {
|
||||
return err
|
||||
}
|
||||
return bumpPolicyRevision(ctx, tx)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("seed org policies: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// seedLockKey is the advisory lock key a seed takes before it reads whether the
|
||||
// org has policies, distinct from the per-policy keys.
|
||||
func seedLockKey(orgID string) string { return "seed/" + orgID }
|
||||
|
||||
// DeleteOrgPoliciesNotShipped removes non-shipped org policies and bumps revision.
|
||||
func (s *Store) DeleteOrgPoliciesNotShipped(ctx context.Context, orgID string, shippedIDs []string) error {
|
||||
return s.withTx(ctx, func(tx *sqlx.Tx) error {
|
||||
|
||||
@@ -113,21 +113,28 @@ func (m *Module) DeletePolicy(ctx context.Context, p Policy) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnsureShippedOrgPolicies seeds an org's shipped role policies when none exist yet.
|
||||
// EnsureShippedOrgPolicies seeds the shipped role policies when the org has none
|
||||
// of them.
|
||||
func (m *Module) EnsureShippedOrgPolicies(ctx context.Context, orgID OrgID) error {
|
||||
count, err := m.store.CountShippedOrgPolicies(ctx, string(orgID), shippedOrgPolicyIDs())
|
||||
if err != nil {
|
||||
return err
|
||||
return seedShippedPolicies(ctx, m.store, orgID)
|
||||
}
|
||||
|
||||
// policySeedStore is the part of the store that seeds the shipped policies.
|
||||
type policySeedStore interface {
|
||||
SeedOrgPolicies(context.Context, string, []string, []store.PolicyRow) error
|
||||
}
|
||||
|
||||
// seedShippedPolicies writes the whole shipped set in one call, so a failure
|
||||
// leaves none of it behind and the next attempt writes all of it. The store
|
||||
// decides whether the seed already ran; an org that has any shipped policy is
|
||||
// left alone, so a deletion or an edit an admin made stands across restarts.
|
||||
func seedShippedPolicies(ctx context.Context, s policySeedStore, orgID OrgID) error {
|
||||
shipped := ShippedOrgPolicies(orgID)
|
||||
rows := make([]store.PolicyRow, 0, len(shipped))
|
||||
for _, p := range shipped {
|
||||
rows = append(rows, policyToRow(p))
|
||||
}
|
||||
if count > 0 {
|
||||
return nil
|
||||
}
|
||||
for _, p := range ShippedOrgPolicies(orgID) {
|
||||
if _, err := m.store.PutAccessPolicy(ctx, policyToRow(p)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return s.SeedOrgPolicies(ctx, string(orgID), shippedOrgPolicyIDs(), rows)
|
||||
}
|
||||
|
||||
// ResetOrgPolicies removes non-shipped org policies and restores the shipped role policies.
|
||||
|
||||
@@ -381,9 +381,9 @@ func (m *Module) RegisterAccount(ctx context.Context, r AccountRegistration, boo
|
||||
return Identity{}, mapIdentityErr(err)
|
||||
}
|
||||
if bootstrap {
|
||||
// The owner is in durable state now. Whatever waits on this signal starts
|
||||
// operations, so it is sent only after the transaction committed: a failed
|
||||
// attempt returns above and leaves it closed.
|
||||
// The owner is in durable state now. The composition root starts operations
|
||||
// when this signal closes, so it is sent only after the transaction
|
||||
// committed: a failed attempt returns above and leaves it closed.
|
||||
m.setup.Complete()
|
||||
}
|
||||
return identityFromRow(row), nil
|
||||
|
||||
Reference in new issue
Block a user