From cb77bc8dab53eecd39ccc9d75ff8135d955cc649 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 21 Sep 2026 23:09:23 +1000 Subject: [PATCH 1/2] test(archied): end-to-end restart test for a database-backed settings change archie-core-eju6: docs/prds/runtime-control-plane.md ("First implementation") lists an end-to-end restart test as a deliverable and there was none. One test drives the whole path against a real State Store: the dashboard's own HTTP adapter writes a task-limit change (attributed by the adapter to the System identity), the audit trail records it as an ordinary edit above the migration seed, the process that consumes it stops and starts again against the same store and the same file document, and the restarted process is running the stored limits rather than the file's. The apply-status row for that process reports the version it applied, and the settings page reads an aged record as not reporting instead of current. --- .../app/archied/control_plane_restart_test.go | 433 ++++++++++++++++++ 1 file changed, 433 insertions(+) create mode 100644 internal/app/archied/control_plane_restart_test.go diff --git a/internal/app/archied/control_plane_restart_test.go b/internal/app/archied/control_plane_restart_test.go new file mode 100644 index 000000000..92e196565 --- /dev/null +++ b/internal/app/archied/control_plane_restart_test.go @@ -0,0 +1,433 @@ +package archied + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + + "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/domain/applystatus" + "github.com/samcharles93/archie-core/internal/domain/identity" + "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/domain/workflow" + "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" + "github.com/samcharles93/archie-core/internal/secret" + "github.com/samcharles93/archie-core/internal/store" + "github.com/samcharles93/archie-core/internal/webui" +) + +// The task limits this test moves between the file document and the database. +// Every field differs between the two, so a process that came back on the +// file's value is distinguishable from one that came back on the stored one. +var ( + restartFileLimits = workflow.ExecutionSettings{MaxModelToolSteps: 10, MaxRuntime: 5 * time.Minute, MaxConsecutiveGateFailures: 3} + restartStoredLimits = workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4} +) + +// restartStoredDocument is what the dashboard's editor sends for +// restartStoredLimits: the resource's own document shape, spelled the way the +// control plane's executionSettingsDocument spells it. +const restartStoredDocument = `{"max_model_tool_steps": 25, "max_runtime_seconds": 3600, "max_consecutive_gate_failures": 4}` + +// settingsDocument mirrors the stored document, so an assertion can read a +// revision's value back as the limits it means rather than as bytes. +type settingsDocument struct { + MaxModelToolSteps int `json:"max_model_tool_steps"` + MaxRuntimeSeconds int64 `json:"max_runtime_seconds"` + MaxConsecutiveGateFailures int `json:"max_consecutive_gate_failures"` +} + +func (d settingsDocument) limits() workflow.ExecutionSettings { + return workflow.ExecutionSettings{ + MaxModelToolSteps: d.MaxModelToolSteps, + MaxRuntime: time.Duration(d.MaxRuntimeSeconds) * time.Second, + MaxConsecutiveGateFailures: d.MaxConsecutiveGateFailures, + } +} + +// TestControlPlaneSettingsChangeSurvivesProcessRestart is archie-core-eju6: the +// end-to-end restart check docs/prds/runtime-control-plane.md ("First +// implementation") lists as a deliverable. In one test it writes a task-limit +// change the way the dashboard writes one, sees the audit trail record it as an +// ordinary edit, stops the process that consumes it, and starts it again -- +// against the same State Store and the same file document -- and asserts the +// restarted process is running the stored limits rather than the file's, that +// the apply-status row for it reports the version it applied, and that a record +// older than the staleness window reads as not reporting rather than as current. +// +// Everything below the assertions is the production path: the State Store's own +// control-plane composition over the store it owns, seeded from the same file +// document the daemon reads (ImportConfig), served on the token-protected +// topology every deployment profile uses; the dashboard's HTTP adapter, whose +// attribution the test never supplies itself; and archied's own boot sequence. +func TestControlPlaneSettingsChangeSurvivesProcessRestart(t *testing.T) { + const adminToken = "state-store-admin-token" + kind := controlplane.WorkflowExecutionSettingsKind + + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.toml") + // The listener comes first so the file document can name the endpoint every + // process here dials: one document, read by the State Store and by both + // boots of the daemon. + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + t.Cleanup(func() { _ = listener.Close() }) + target := listener.Addr().String() + writeConfig(t, cfgPath, restartConfigTOML(dir, target, adminToken)) + + dashboard, admin := serveRestartControlPlane(t, listener, cfgPath, target, adminToken) + + // The process that consumes task limits is already running when the + // operator edits, on the limits the file document carries, and it has + // reported the version it applied. + first, stopFirst := bootRestartProcess(t, cfgPath) + if got, want := first.cfgHolder.Get().Budgets, budgetsFor(restartFileLimits); got != want { + t.Fatalf("running budgets before the edit = %+v, want the file document's %+v", got, want) + } + if row := reportedRow(t, admin, kind); row.AppliedVersion != 1 || row.Error != "" { + t.Fatalf("apply status before the edit = %+v, want version 1 and no error", row) + } + + // 1. The dashboard writes the change. The actor, source and request ID are + // the adapter's (webui's webAudit), not this test's: a request body cannot + // claim them. + replaceThroughDashboard(t, dashboard, kind, restartStoredDocument, 1) + + // 2. The audit trail carries it as an ordinary edit: the operator's + // identity, the dashboard as its source, and the value that was written. + revisions := historyThroughDashboard(t, dashboard, kind) + if len(revisions) != 2 { + t.Fatalf("audit trail holds %d revisions, want the migration seed and the edit", len(revisions)) + } + if edit := revisions[0]; edit.Version != 2 || edit.Actor != string(identity.SystemID) || edit.Source != "archie-ui" || !strings.HasPrefix(edit.RequestID, "ui-") { + t.Fatalf("the edit was recorded as %+v, want an ordinary archie-ui edit by the System identity at version 2", edit) + } + if got, err := documentLimits(revisions[0].Value); err != nil || got != restartStoredLimits { + t.Fatalf("the audit trail recorded %s, want %+v: %v", revisions[0].Value, restartStoredLimits, err) + } + // The revision under it is the migration seed. Without it the row above + // would be indistinguishable from a resource that had only ever been + // written by the importer. + if seed := revisions[1]; seed.Actor != "system:migration" || seed.Source != "legacy-config" { + t.Fatalf("the first revision = %+v, want the legacy-config migration seed", seed) + } + + // 3. The process stops, and comes back against the same State Store and the + // same file document. + stopFirst() + restarted, stopRestarted := bootRestartProcess(t, cfgPath) + defer stopRestarted() + + // 4. It came back on the stored value, not the file's. + if got, want := restarted.cfgHolder.Get().Budgets, budgetsFor(restartStoredLimits); got != want { + t.Fatalf("running budgets after the restart = %+v, want the stored %+v (the file document still says %+v)", got, want, budgetsFor(restartFileLimits)) + } + if got := restarted.executionSettings.Load(); got == nil || *got != restartStoredLimits { + t.Fatalf("the restarted process records %v as the settings it applied, want %+v", got, restartStoredLimits) + } + + // 5. The apply-status row for the restarted process reports the version it + // applied, and the settings page reads that row as current... + if row := reportedRow(t, admin, kind); row.AppliedVersion != 2 || row.Error != "" { + t.Fatalf("apply status after the restart = %+v, want version 2 and no error", row) + } + if state := applyStatusState(t, dashboard, kind); state != applyStateCurrentWire { + t.Fatalf("the settings page reads the freshly re-stamped row as %q, want %q", state, applyStateCurrentWire) + } + + // ...and reads a row older than the staleness window as not reporting + // rather than as current. The row is re-stamped every 30 seconds by the + // process that wrote it, so a row that stopped being re-stamped is a + // process that stopped -- the reader must not present its last report as + // live. Ageing it through the same administrative surface is what the + // clock would have done to it. + if err := admin.PutApplyStatus(t.Context(), storecontract.ApplyStatus{ + Process: applystatus.Daemon, Kind: kind, AppliedVersion: 2, + ReportedAt: time.Now().UTC().Add(-applystatus.StaleAfter - time.Second), + }); err != nil { + t.Fatalf("age the apply status row: %v", err) + } + if state := applyStatusState(t, dashboard, kind); state != applyStateNotReportingWire { + t.Fatalf("the settings page reads the aged row as %q, want %q (not reporting, never current)", state, applyStateNotReportingWire) + } +} + +// The two readings the dashboard renders for a row it holds. They are the +// wire's values, not this package's: webui's applyState returns them and +// ui/src/settings/ApplyStatusRows.vue shows "Not reporting" for the second, so +// asserting the wire value asserts what the operator is shown. +const ( + applyStateCurrentWire = "current" + applyStateNotReportingWire = "unknown" +) + +// restartConfigTOML is the operator's file document: the bootstrap settings a +// process reads before it can reach the State Store, the task limits the file +// still carries (the values the database replaces), the container image +// configuration.Validate requires, and the State Store endpoint. +func restartConfigTOML(dir, target, adminToken string) string { + return fmt.Sprintf(`bot_user = "widget" +db_path = %q + +[forge] +type = "github" +host = "https://github.example.com" + +[[repos]] +owner = "acme" +name = "app" + +[containers] +image = "archie:test" +pull_policy = "missing" + +[budgets] +max_steps = %d +wall_clock = %q +gate_max_failures = %d + +[services.state] +target = %q +target_token = %q +`, filepath.Join(dir, "archie"), restartFileLimits.MaxModelToolSteps, restartFileLimits.MaxRuntime, + restartFileLimits.MaxConsecutiveGateFailures, target, adminToken) +} + +// serveRestartControlPlane serves the State Store contract the way the +// standalone process serves it -- the production control-plane composition +// (openStateStoreControlPlane) over the store the process owns, seeded from the +// file document by the same ImportConfig call RunStateStore makes -- and +// returns the dashboard's own view of it plus the administratively dialed +// client. +// +// The credential is the administrative token, the only one PutApplyStatus and +// ListApplyStatus admit: a task-scoped grant is refused both of them by the +// deny-by-default arm of authorizesTaskScopedCall. The listener is the token +// interceptor pair stateStoreServerOpts installs for the non-loopback topology +// every container-mode deployment uses; the socket stays on loopback because +// this deployment is one host and the credential is what the harness needs. +func serveRestartControlPlane(t *testing.T, listener net.Listener, cfgPath, target, adminToken string) (*webui.Server, *staterpc.Client) { + t.Helper() + ctx := t.Context() + + st, err := store.Open(ctx, filepath.Join(filepath.Dir(cfgPath), "archie.db-tasks.sqlite")) + if err != nil { + t.Fatalf("open state store: %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + + storeBoot := newBootstrap() + storeBoot.stderrLog = true + storeBoot.log = slog.New(slog.DiscardHandler) + if err := storeBoot.loadConfig(ctx, cfgPath, ""); err != nil { + t.Fatalf("state store load config: %v", err) + } + + control, err := openStateStoreControlPlane(st) + if err != nil { + t.Fatalf("build the control plane server: %v", err) + } + versions, skipped, err := control.ImportConfig(ctx, storeBoot.cfg) + if err != nil { + t.Fatalf("seed control-plane resources: %v", err) + } + for _, skip := range skipped { + // Not fatal, by design: a kind the file config cannot seed stays absent + // and the file's value stays in effect. Reported so a kind this test + // cares about cannot be skipped silently. + t.Logf("control-plane resource not seeded from the file document: %s: %v", skip.Kind, skip.Err) + } + if got := versions[controlplane.WorkflowExecutionSettingsKind]; got != 1 { + t.Fatalf("the file document seeded workflow execution settings at version %d, want 1", got) + } + + grants := &staterpc.TaskGrants{} + opts := []grpc.ServerOption{ + grpc.ChainUnaryInterceptor(grants.UnaryInterceptor(adminToken)), + grpc.ChainStreamInterceptor(grants.StreamInterceptor(adminToken)), + } + storeBoot.st = st + deps := storeBoot.stateStoreDeps(grants) + deps.ControlPlane = control + go func() { _ = serveStateStore(ctx, listener, deps, opts) }() + + admin, cleanup, err := staterpc.Dial(target, adminToken) + if err != nil { + t.Fatalf("dial the state store: %v", err) + } + t.Cleanup(cleanup) + return &webui.Server{ControlPlane: admin.ControlPlane(), ApplyStatus: admin}, admin +} + +// bootRestartProcess boots one archied process against the State Store at +// cfgPath and returns it with the stop func that ends it. The phases are Run's, +// in Run's order, for the surfaces this feature runs through: resolve the file +// document, dial the State Store surfaces, layer the restart-required kinds +// over that document, then apply the settings that arrive on the watch and +// record what it applied. +// +// Run's later phases (forge, containers, NATS, chat) consume no task limit, so +// they stay out: this is the process the feature runs in, not a second +// composition. A stub would not do -- the point of the test is that the value +// survives a real boot reading a real store. +func bootRestartProcess(t *testing.T, cfgPath string) (*boot, func()) { + t.Helper() + ctx, cancel := context.WithCancel(t.Context()) + b := newBootstrap() + // The daemon's log destination is a deployment file this test does not + // create, so this boot logs where every other test's boot does. + b.stderrLog = true + b.log = slog.New(slog.DiscardHandler) + // The process name is what an apply-status record carries; a boot + // reporting under any other name would be invisible to the settings page. + b.processName = applystatus.Daemon + if err := b.loadConfig(ctx, cfgPath, ""); err != nil { + cancel() + t.Fatalf("load config: %v", err) + } + // openStores' own secret registry is not part of this feature: the file + // document carries the State Store token explicitly, so the dial resolves + // its credential without a forge credential this harness has no use for. + b.secrets = &secret.Registry{} + if err := b.openDaemonStateSurfaces(); err != nil { + cancel() + t.Fatalf("open the State Store surfaces: %v", err) + } + if err := b.loadRuntimeConfig(ctx); err != nil { + cancel() + t.Fatalf("layer the database settings over the file document: %v", err) + } + if err := b.startWorkflowExecutionSettings(ctx); err != nil { + cancel() + t.Fatalf("apply the stored workflow execution settings: %v", err) + } + var once sync.Once + stop := func() { + once.Do(func() { + // Cancel first, then release: a watch stream that fails on a live + // context is reported as a failed apply, and this process is being + // shut down, not failing. + cancel() + b.cleanup() + }) + } + t.Cleanup(stop) + return b, stop +} + +// documentLimits reads a stored or recorded document back as the limits it +// means. +func documentLimits(value []byte) (workflow.ExecutionSettings, error) { + var document settingsDocument + if err := json.Unmarshal(value, &document); err != nil { + return workflow.ExecutionSettings{}, err + } + return document.limits(), nil +} + +// dashboardRequest runs one request against the dashboard's own HTTP adapter. +func dashboardRequest(t *testing.T, dashboard *webui.Server, request *http.Request) *httptest.ResponseRecorder { + t.Helper() + recorder := httptest.NewRecorder() + dashboard.Handler().ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("%s %s = %d: %s", request.Method, request.URL.Path, recorder.Code, recorder.Body.String()) + } + return recorder +} + +// replaceThroughDashboard writes document as the dashboard's editor does, over +// the replace command, against the version the editor read. The attribution is +// the adapter's: the request body carries only the value and the version. +func replaceThroughDashboard(t *testing.T, dashboard *webui.Server, kind, document string, expectedVersion int64) { + t.Helper() + body := fmt.Sprintf(`{"value": %s, "expected_version": %d}`, document, expectedVersion) + request := httptest.NewRequestWithContext(t.Context(), http.MethodPost, + "/api/control-plane/resources/"+kind+"/commands/replace", strings.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("X-Archie-CSRF", "1") + dashboardRequest(t, dashboard, request) +} + +// revision is one row of the audit trail as the dashboard's history route +// renders it. +type revision struct { + Version int64 `json:"version"` + Value json.RawMessage `json:"value"` + Actor string `json:"actor"` + Source string `json:"source"` + RequestID string `json:"request_id"` +} + +func historyThroughDashboard(t *testing.T, dashboard *webui.Server, kind string) []revision { + t.Helper() + request := httptest.NewRequestWithContext(t.Context(), http.MethodGet, + "/api/control-plane/resources/"+kind+"/history", nil) + recorder := dashboardRequest(t, dashboard, request) + var response struct { + Revisions []revision `json:"revisions"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode the audit trail: %v (body %s)", err, recorder.Body.String()) + } + if len(response.Revisions) == 0 { + t.Fatalf("the audit trail for %s is empty", kind) + } + return response.Revisions +} + +// reportedRow is the apply-status row the settings page shows for the daemon +// and kind. It is read through the administrative token, which is the only +// credential that reaches ListApplyStatus. +func reportedRow(t *testing.T, admin *staterpc.Client, kind string) storecontract.ApplyStatus { + t.Helper() + rows, err := admin.ListApplyStatus(t.Context()) + if err != nil { + t.Fatalf("list apply status: %v", err) + } + for _, row := range rows { + if row.Process == applystatus.Daemon && row.Kind == kind { + return row + } + } + t.Fatalf("no apply-status row for %s/%s in %+v", applystatus.Daemon, kind, rows) + return storecontract.ApplyStatus{} +} + +// applyStatusState is the reading the settings page renders for the daemon's +// row for kind. +func applyStatusState(t *testing.T, dashboard *webui.Server, kind string) string { + t.Helper() + request := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/api/control-plane/apply-status", nil) + recorder := dashboardRequest(t, dashboard, request) + var page struct { + Records []struct { + Process string `json:"process"` + Kind string `json:"kind"` + State string `json:"state"` + } `json:"records"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &page); err != nil { + t.Fatalf("decode the apply-status page: %v (body %s)", err, recorder.Body.String()) + } + for _, record := range page.Records { + if record.Process == applystatus.Daemon && record.Kind == kind { + return record.State + } + } + t.Fatalf("the apply-status page carries no row for %s/%s: %+v", applystatus.Daemon, kind, page.Records) + return "" +} From 4f1f6473a3d155d7fd314a90aad4689bf0bcb04b Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 21 Sep 2026 23:12:24 +1000 Subject: [PATCH 2/2] test(archied): age the apply-status fixture by the documented window The clause-5 staleness reading aged its fixture by applystatus.StaleAfter, so the fixture moved with the very constant the reader is supposed to honour and the reading agreed with the reader by construction: mutating StaleAfter to 1000h left the test passing, which is a staleness assertion that never went stale. Age the row by the documented 90-second window (three 30-second re-stamps, docs/prds/control-plane-apply-status.md) held as a fixture literal instead, so a window that stopped expiring fails the reading, and say in the fresh half's assertion that it is the other end of that age pair. archie-core-eju6. --- .../app/archied/control_plane_restart_test.go | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/internal/app/archied/control_plane_restart_test.go b/internal/app/archied/control_plane_restart_test.go index 92e196565..a41dd8901 100644 --- a/internal/app/archied/control_plane_restart_test.go +++ b/internal/app/archied/control_plane_restart_test.go @@ -40,6 +40,17 @@ var ( // control plane's executionSettingsDocument spells it. const restartStoredDocument = `{"max_model_tool_steps": 25, "max_runtime_seconds": 3600, "max_consecutive_gate_failures": 4}` +// documentedStaleWindow is the window the operator-facing contract documents: +// every process re-stamps its records every 30 seconds, and "a record is stale +// once it is older than 90 seconds" (docs/prds/control-plane-apply-status.md, +// "What is reported"). It is a fixture literal on purpose. Reading +// applystatus.StaleAfter here would derive the aged row from the constant the +// reader is supposed to honour, so the fixture would move with any mutation of +// that constant and the reading below would agree with the reader by +// construction -- a test that passes for a window measured in hours is a test +// that never went stale at all. +const documentedStaleWindow = 90 * time.Second + // settingsDocument mirrors the stored document, so an assertion can read a // revision's value back as the limits it means rather than as bytes. type settingsDocument struct { @@ -140,23 +151,27 @@ func TestControlPlaneSettingsChangeSurvivesProcessRestart(t *testing.T) { } // 5. The apply-status row for the restarted process reports the version it - // applied, and the settings page reads that row as current... + // applied, and the settings page reads that row as current: it was reported + // at now, which is the fresh half of the age pair the aged read below + // completes. if row := reportedRow(t, admin, kind); row.AppliedVersion != 2 || row.Error != "" { t.Fatalf("apply status after the restart = %+v, want version 2 and no error", row) } if state := applyStatusState(t, dashboard, kind); state != applyStateCurrentWire { - t.Fatalf("the settings page reads the freshly re-stamped row as %q, want %q", state, applyStateCurrentWire) + t.Fatalf("the settings page reads the row reported at now as %q, want %q", state, applyStateCurrentWire) } - // ...and reads a row older than the staleness window as not reporting - // rather than as current. The row is re-stamped every 30 seconds by the - // process that wrote it, so a row that stopped being re-stamped is a + // ...and reads a row older than the documented staleness window as not + // reporting rather than as current. The row is re-stamped every 30 seconds + // by the process that wrote it, so a row that stopped being re-stamped is a // process that stopped -- the reader must not present its last report as - // live. Ageing it through the same administrative surface is what the - // clock would have done to it. + // live. Ageing it through the same administrative surface is what the clock + // would have done to it, and the age is the documented 90 seconds rather + // than whatever the reader's own constant happens to be, so a window that + // stopped expiring really does fail the reading. if err := admin.PutApplyStatus(t.Context(), storecontract.ApplyStatus{ Process: applystatus.Daemon, Kind: kind, AppliedVersion: 2, - ReportedAt: time.Now().UTC().Add(-applystatus.StaleAfter - time.Second), + ReportedAt: time.Now().UTC().Add(-documentedStaleWindow - time.Second), }); err != nil { t.Fatalf("age the apply status row: %v", err) }