test(archied): supersede the restart test with the broader one, staleness window made mutation-proof
Takes the wider of the two independently-written archie-core-eju6 tests: it also covers the queried half of boot.runtimeConfig (plugin_dir, not just the watched budgets), the SIGHUP reload re-applying the database layer, a task-scoped grant denied on PutApplyStatus/ListApplyStatus, and a real gRPC transport over bufconn behind TaskGrants. Its staleness leg had the same defect the narrower test did: the aged fixture was derived from applystatus.StaleAfter, the constant the reader honours, so the fixture moved with any mutation and the assertion held for a window measured in hours. The aged row now uses a documented-window literal (90s, the operator-facing contract) so the reader is constrained from both sides.
This commit is contained in:
1 file changed
+349
-359
@@ -1,44 +1,39 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/grpc/test/bufconn"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
"github.com/samcharles93/archie-core/internal/store"
|
||||
"github.com/samcharles93/archie-core/internal/webui"
|
||||
)
|
||||
|
||||
// The task limits this test moves between the file document and the database.
|
||||
// Every field differs between the two, so a process that came back on the
|
||||
// file's value is distinguishable from one that came back on the stored one.
|
||||
var (
|
||||
restartFileLimits = workflow.ExecutionSettings{MaxModelToolSteps: 10, MaxRuntime: 5 * time.Minute, MaxConsecutiveGateFailures: 3}
|
||||
restartStoredLimits = workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4}
|
||||
)
|
||||
|
||||
// restartStoredDocument is what the dashboard's editor sends for
|
||||
// restartStoredLimits: the resource's own document shape, spelled the way the
|
||||
// control plane's executionSettingsDocument spells it.
|
||||
const restartStoredDocument = `{"max_model_tool_steps": 25, "max_runtime_seconds": 3600, "max_consecutive_gate_failures": 4}`
|
||||
// restartAdminToken is the administrative credential the State Store's
|
||||
// TaskGrants interceptor accepts. PutApplyStatus and ListApplyStatus are
|
||||
// administrative (docs/prds/control-plane-apply-status.md): a task-scoped
|
||||
// grant is denied by authorizesTaskScopedCall's deny-by-default arm, so the
|
||||
// harness that reads and writes apply status must dial as the administrator.
|
||||
const restartAdminToken = "eju6-admin-token"
|
||||
|
||||
// documentedStaleWindow is the window the operator-facing contract documents:
|
||||
// every process re-stamps its records every 30 seconds, and "a record is stale
|
||||
@@ -51,335 +46,270 @@ const restartStoredDocument = `{"max_model_tool_steps": 25, "max_runtime_seconds
|
||||
// that never went stale at all.
|
||||
const documentedStaleWindow = 90 * time.Second
|
||||
|
||||
// settingsDocument mirrors the stored document, so an assertion can read a
|
||||
// revision's value back as the limits it means rather than as bytes.
|
||||
type settingsDocument struct {
|
||||
MaxModelToolSteps int `json:"max_model_tool_steps"`
|
||||
MaxRuntimeSeconds int64 `json:"max_runtime_seconds"`
|
||||
MaxConsecutiveGateFailures int `json:"max_consecutive_gate_failures"`
|
||||
// restartEditedSettings is the change the dashboard writes. It differs from
|
||||
// fileConfig()'s [budgets] on all three fields, so a process that comes back on
|
||||
// any file value is detectably not on the stored one.
|
||||
var restartEditedSettings = workflow.ExecutionSettings{
|
||||
MaxModelToolSteps: 40, MaxRuntime: 10 * time.Minute, MaxConsecutiveGateFailures: 2,
|
||||
}
|
||||
|
||||
func (d settingsDocument) limits() workflow.ExecutionSettings {
|
||||
return workflow.ExecutionSettings{
|
||||
MaxModelToolSteps: d.MaxModelToolSteps,
|
||||
MaxRuntime: time.Duration(d.MaxRuntimeSeconds) * time.Second,
|
||||
MaxConsecutiveGateFailures: d.MaxConsecutiveGateFailures,
|
||||
}
|
||||
// controlPlaneRestartFixture is a real State Store for the restart test: the
|
||||
// production control plane and apply-status surfaces registered on the real
|
||||
// gRPC service, behind TaskGrants' admin/task-grant interceptors, over a real
|
||||
// SQLite store. It is the composition state_store.go serves, not a stub.
|
||||
type controlPlaneRestartFixture struct {
|
||||
listener *bufconn.Listener
|
||||
admin *staterpc.Client
|
||||
}
|
||||
|
||||
// TestControlPlaneSettingsChangeSurvivesProcessRestart is archie-core-eju6: the
|
||||
// end-to-end restart check docs/prds/runtime-control-plane.md ("First
|
||||
// implementation") lists as a deliverable. In one test it writes a task-limit
|
||||
// change the way the dashboard writes one, sees the audit trail record it as an
|
||||
// ordinary edit, stops the process that consumes it, and starts it again --
|
||||
// against the same State Store and the same file document -- and asserts the
|
||||
// restarted process is running the stored limits rather than the file's, that
|
||||
// the apply-status row for it reports the version it applied, and that a record
|
||||
// older than the staleness window reads as not reporting rather than as current.
|
||||
//
|
||||
// Everything below the assertions is the production path: the State Store's own
|
||||
// control-plane composition over the store it owns, seeded from the same file
|
||||
// document the daemon reads (ImportConfig), served on the token-protected
|
||||
// topology every deployment profile uses; the dashboard's HTTP adapter, whose
|
||||
// attribution the test never supplies itself; and archied's own boot sequence.
|
||||
func TestControlPlaneSettingsChangeSurvivesProcessRestart(t *testing.T) {
|
||||
const adminToken = "state-store-admin-token"
|
||||
kind := controlplane.WorkflowExecutionSettingsKind
|
||||
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "config.toml")
|
||||
// The listener comes first so the file document can name the endpoint every
|
||||
// process here dials: one document, read by the State Store and by both
|
||||
// boots of the daemon.
|
||||
listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = listener.Close() })
|
||||
target := listener.Addr().String()
|
||||
writeConfig(t, cfgPath, restartConfigTOML(dir, target, adminToken))
|
||||
|
||||
dashboard, admin := serveRestartControlPlane(t, listener, cfgPath, target, adminToken)
|
||||
|
||||
// The process that consumes task limits is already running when the
|
||||
// operator edits, on the limits the file document carries, and it has
|
||||
// reported the version it applied.
|
||||
first, stopFirst := bootRestartProcess(t, cfgPath)
|
||||
if got, want := first.cfgHolder.Get().Budgets, budgetsFor(restartFileLimits); got != want {
|
||||
t.Fatalf("running budgets before the edit = %+v, want the file document's %+v", got, want)
|
||||
}
|
||||
if row := reportedRow(t, admin, kind); row.AppliedVersion != 1 || row.Error != "" {
|
||||
t.Fatalf("apply status before the edit = %+v, want version 1 and no error", row)
|
||||
}
|
||||
|
||||
// 1. The dashboard writes the change. The actor, source and request ID are
|
||||
// the adapter's (webui's webAudit), not this test's: a request body cannot
|
||||
// claim them.
|
||||
replaceThroughDashboard(t, dashboard, kind, restartStoredDocument, 1)
|
||||
|
||||
// 2. The audit trail carries it as an ordinary edit: the operator's
|
||||
// identity, the dashboard as its source, and the value that was written.
|
||||
revisions := historyThroughDashboard(t, dashboard, kind)
|
||||
if len(revisions) != 2 {
|
||||
t.Fatalf("audit trail holds %d revisions, want the migration seed and the edit", len(revisions))
|
||||
}
|
||||
if edit := revisions[0]; edit.Version != 2 || edit.Actor != string(identity.SystemID) || edit.Source != "archie-ui" || !strings.HasPrefix(edit.RequestID, "ui-") {
|
||||
t.Fatalf("the edit was recorded as %+v, want an ordinary archie-ui edit by the System identity at version 2", edit)
|
||||
}
|
||||
if got, err := documentLimits(revisions[0].Value); err != nil || got != restartStoredLimits {
|
||||
t.Fatalf("the audit trail recorded %s, want %+v: %v", revisions[0].Value, restartStoredLimits, err)
|
||||
}
|
||||
// The revision under it is the migration seed. Without it the row above
|
||||
// would be indistinguishable from a resource that had only ever been
|
||||
// written by the importer.
|
||||
if seed := revisions[1]; seed.Actor != "system:migration" || seed.Source != "legacy-config" {
|
||||
t.Fatalf("the first revision = %+v, want the legacy-config migration seed", seed)
|
||||
}
|
||||
|
||||
// 3. The process stops, and comes back against the same State Store and the
|
||||
// same file document.
|
||||
stopFirst()
|
||||
restarted, stopRestarted := bootRestartProcess(t, cfgPath)
|
||||
defer stopRestarted()
|
||||
|
||||
// 4. It came back on the stored value, not the file's.
|
||||
if got, want := restarted.cfgHolder.Get().Budgets, budgetsFor(restartStoredLimits); got != want {
|
||||
t.Fatalf("running budgets after the restart = %+v, want the stored %+v (the file document still says %+v)", got, want, budgetsFor(restartFileLimits))
|
||||
}
|
||||
if got := restarted.executionSettings.Load(); got == nil || *got != restartStoredLimits {
|
||||
t.Fatalf("the restarted process records %v as the settings it applied, want %+v", got, restartStoredLimits)
|
||||
}
|
||||
|
||||
// 5. The apply-status row for the restarted process reports the version it
|
||||
// applied, and the settings page reads that row as current: it was reported
|
||||
// at now, which is the fresh half of the age pair the aged read below
|
||||
// completes.
|
||||
if row := reportedRow(t, admin, kind); row.AppliedVersion != 2 || row.Error != "" {
|
||||
t.Fatalf("apply status after the restart = %+v, want version 2 and no error", row)
|
||||
}
|
||||
if state := applyStatusState(t, dashboard, kind); state != applyStateCurrentWire {
|
||||
t.Fatalf("the settings page reads the row reported at now as %q, want %q", state, applyStateCurrentWire)
|
||||
}
|
||||
|
||||
// ...and reads a row older than the documented staleness window as not
|
||||
// reporting rather than as current. The row is re-stamped every 30 seconds
|
||||
// by the process that wrote it, so a row that stopped being re-stamped is a
|
||||
// process that stopped -- the reader must not present its last report as
|
||||
// live. Ageing it through the same administrative surface is what the clock
|
||||
// would have done to it, and the age is the documented 90 seconds rather
|
||||
// than whatever the reader's own constant happens to be, so a window that
|
||||
// stopped expiring really does fail the reading.
|
||||
if err := admin.PutApplyStatus(t.Context(), storecontract.ApplyStatus{
|
||||
Process: applystatus.Daemon, Kind: kind, AppliedVersion: 2,
|
||||
ReportedAt: time.Now().UTC().Add(-documentedStaleWindow - time.Second),
|
||||
}); err != nil {
|
||||
t.Fatalf("age the apply status row: %v", err)
|
||||
}
|
||||
if state := applyStatusState(t, dashboard, kind); state != applyStateNotReportingWire {
|
||||
t.Fatalf("the settings page reads the aged row as %q, want %q (not reporting, never current)", state, applyStateNotReportingWire)
|
||||
}
|
||||
}
|
||||
|
||||
// The two readings the dashboard renders for a row it holds. They are the
|
||||
// wire's values, not this package's: webui's applyState returns them and
|
||||
// ui/src/settings/ApplyStatusRows.vue shows "Not reporting" for the second, so
|
||||
// asserting the wire value asserts what the operator is shown.
|
||||
const (
|
||||
applyStateCurrentWire = "current"
|
||||
applyStateNotReportingWire = "unknown"
|
||||
)
|
||||
|
||||
// restartConfigTOML is the operator's file document: the bootstrap settings a
|
||||
// process reads before it can reach the State Store, the task limits the file
|
||||
// still carries (the values the database replaces), the container image
|
||||
// configuration.Validate requires, and the State Store endpoint.
|
||||
func restartConfigTOML(dir, target, adminToken string) string {
|
||||
return fmt.Sprintf(`bot_user = "widget"
|
||||
db_path = %q
|
||||
|
||||
[forge]
|
||||
type = "github"
|
||||
host = "https://github.example.com"
|
||||
|
||||
[[repos]]
|
||||
owner = "acme"
|
||||
name = "app"
|
||||
|
||||
[containers]
|
||||
image = "archie:test"
|
||||
pull_policy = "missing"
|
||||
|
||||
[budgets]
|
||||
max_steps = %d
|
||||
wall_clock = %q
|
||||
gate_max_failures = %d
|
||||
|
||||
[services.state]
|
||||
target = %q
|
||||
target_token = %q
|
||||
`, filepath.Join(dir, "archie"), restartFileLimits.MaxModelToolSteps, restartFileLimits.MaxRuntime,
|
||||
restartFileLimits.MaxConsecutiveGateFailures, target, adminToken)
|
||||
}
|
||||
|
||||
// serveRestartControlPlane serves the State Store contract the way the
|
||||
// standalone process serves it -- the production control-plane composition
|
||||
// (openStateStoreControlPlane) over the store the process owns, seeded from the
|
||||
// file document by the same ImportConfig call RunStateStore makes -- and
|
||||
// returns the dashboard's own view of it plus the administratively dialed
|
||||
// client.
|
||||
//
|
||||
// The credential is the administrative token, the only one PutApplyStatus and
|
||||
// ListApplyStatus admit: a task-scoped grant is refused both of them by the
|
||||
// deny-by-default arm of authorizesTaskScopedCall. The listener is the token
|
||||
// interceptor pair stateStoreServerOpts installs for the non-loopback topology
|
||||
// every container-mode deployment uses; the socket stays on loopback because
|
||||
// this deployment is one host and the credential is what the harness needs.
|
||||
func serveRestartControlPlane(t *testing.T, listener net.Listener, cfgPath, target, adminToken string) (*webui.Server, *staterpc.Client) {
|
||||
// newControlPlaneRestartFixture opens the store, seeds it the way
|
||||
// archie-state-store does on a fresh database, serves the contract over
|
||||
// bufconn, and returns an administrative client for the dashboard and for
|
||||
// reading apply status back.
|
||||
func newControlPlaneRestartFixture(t *testing.T) *controlPlaneRestartFixture {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
local := store.OpenTest(t)
|
||||
t.Cleanup(func() { _ = local.Close() })
|
||||
|
||||
st, err := store.Open(ctx, filepath.Join(filepath.Dir(cfgPath), "archie.db-tasks.sqlite"))
|
||||
steps, err := stepVocabulary()
|
||||
if err != nil {
|
||||
t.Fatalf("open state store: %v", err)
|
||||
t.Fatalf("build step vocabulary: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = st.Close() })
|
||||
|
||||
storeBoot := newBootstrap()
|
||||
storeBoot.stderrLog = true
|
||||
storeBoot.log = slog.New(slog.DiscardHandler)
|
||||
if err := storeBoot.loadConfig(ctx, cfgPath, ""); err != nil {
|
||||
t.Fatalf("state store load config: %v", err)
|
||||
}
|
||||
|
||||
control, err := openStateStoreControlPlane(st)
|
||||
control, err := controlplane.NewServer(local, steps)
|
||||
if err != nil {
|
||||
t.Fatalf("build the control plane server: %v", err)
|
||||
t.Fatalf("build control plane server: %v", err)
|
||||
}
|
||||
versions, skipped, err := control.ImportConfig(ctx, storeBoot.cfg)
|
||||
// The seed the State Store runs before it serves: every kind lands at
|
||||
// version 1 holding the file document's value. A kind the validator refused
|
||||
// would be absent, and the assertions below would then be testing a missing
|
||||
// resource rather than a restart, so this test names the two kinds it needs.
|
||||
versions, _, err := control.ImportConfig(t.Context(), fileConfig())
|
||||
if err != nil {
|
||||
t.Fatalf("seed control-plane resources: %v", err)
|
||||
t.Fatalf("seed control plane resources: %v", err)
|
||||
}
|
||||
for _, skip := range skipped {
|
||||
// Not fatal, by design: a kind the file config cannot seed stays absent
|
||||
// and the file's value stays in effect. Reported so a kind this test
|
||||
// cares about cannot be skipped silently.
|
||||
t.Logf("control-plane resource not seeded from the file document: %s: %v", skip.Kind, skip.Err)
|
||||
}
|
||||
if got := versions[controlplane.WorkflowExecutionSettingsKind]; got != 1 {
|
||||
t.Fatalf("the file document seeded workflow execution settings at version %d, want 1", got)
|
||||
for _, kind := range []string{controlplane.WorkflowExecutionSettingsKind, controlplane.PluginSettingsKind} {
|
||||
if versions[kind] != 1 {
|
||||
t.Fatalf("%s seeded at version %d, want 1", kind, versions[kind])
|
||||
}
|
||||
}
|
||||
|
||||
grants := &staterpc.TaskGrants{}
|
||||
opts := []grpc.ServerOption{
|
||||
grpc.ChainUnaryInterceptor(grants.UnaryInterceptor(adminToken)),
|
||||
grpc.ChainStreamInterceptor(grants.StreamInterceptor(adminToken)),
|
||||
}
|
||||
storeBoot.st = st
|
||||
deps := storeBoot.stateStoreDeps(grants)
|
||||
deps.ControlPlane = control
|
||||
go func() { _ = serveStateStore(ctx, listener, deps, opts) }()
|
||||
listener := bufconn.Listen(1 << 20)
|
||||
server := grpc.NewServer(
|
||||
grpc.ChainUnaryInterceptor(grants.UnaryInterceptor(restartAdminToken)),
|
||||
grpc.ChainStreamInterceptor(grants.StreamInterceptor(restartAdminToken)),
|
||||
)
|
||||
staterpc.RegisterServer(server, staterpc.Deps{
|
||||
ControlPlane: control, Tasks: local, ApplyStatus: local, Grants: grants,
|
||||
})
|
||||
go func() { _ = server.Serve(listener) }()
|
||||
t.Cleanup(func() { server.Stop(); _ = listener.Close() })
|
||||
|
||||
admin, cleanup, err := staterpc.Dial(target, adminToken)
|
||||
return &controlPlaneRestartFixture{
|
||||
listener: listener,
|
||||
admin: dialStateStore(t, listener, restartAdminToken),
|
||||
}
|
||||
}
|
||||
|
||||
// dial dials a fresh administrative connection, what a restarted process would
|
||||
// open to the State Store it does not own.
|
||||
func (f *controlPlaneRestartFixture) dial(t *testing.T) *staterpc.Client {
|
||||
t.Helper()
|
||||
return dialStateStore(t, f.listener, restartAdminToken)
|
||||
}
|
||||
|
||||
// dialStateStore is the production client wiring (staterpc.Dial, the only dial
|
||||
// path the daemon and dashboard use) against the non-loopback bridge address
|
||||
// every container-mode profile uses, with the dialer redirected to bufconn.
|
||||
func dialStateStore(t *testing.T, listener *bufconn.Listener, token string) *staterpc.Client {
|
||||
t.Helper()
|
||||
client, cleanup, err := staterpc.Dial("172.17.0.1:9090", token, grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) {
|
||||
return listener.DialContext(ctx)
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("dial the state store: %v", err)
|
||||
t.Fatalf("dial state store: %v", err)
|
||||
}
|
||||
t.Cleanup(cleanup)
|
||||
return &webui.Server{ControlPlane: admin.ControlPlane(), ApplyStatus: admin}, admin
|
||||
return client
|
||||
}
|
||||
|
||||
// bootRestartProcess boots one archied process against the State Store at
|
||||
// cfgPath and returns it with the stop func that ends it. The phases are Run's,
|
||||
// in Run's order, for the surfaces this feature runs through: resolve the file
|
||||
// document, dial the State Store surfaces, layer the restart-required kinds
|
||||
// over that document, then apply the settings that arrive on the watch and
|
||||
// record what it applied.
|
||||
//
|
||||
// Run's later phases (forge, containers, NATS, chat) consume no task limit, so
|
||||
// they stay out: this is the process the feature runs in, not a second
|
||||
// composition. A stub would not do -- the point of the test is that the value
|
||||
// survives a real boot reading a real store.
|
||||
func bootRestartProcess(t *testing.T, cfgPath string) (*boot, func()) {
|
||||
// startRestartProcess composes the consuming half of archied the way run()
|
||||
// composes it: boot.runtimeConfig layers the stored restart-required kinds over
|
||||
// the file document, and the watched execution settings are applied and
|
||||
// recorded from there. Apply status is reported through the State Store
|
||||
// contract by the process's own reporter.
|
||||
func startRestartProcess(ctx context.Context, t *testing.T, client *staterpc.Client) *boot {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
b := newBootstrap()
|
||||
// The daemon's log destination is a deployment file this test does not
|
||||
// create, so this boot logs where every other test's boot does.
|
||||
b.stderrLog = true
|
||||
b.log = slog.New(slog.DiscardHandler)
|
||||
// The process name is what an apply-status record carries; a boot
|
||||
// reporting under any other name would be invisible to the settings page.
|
||||
b.processName = applystatus.Daemon
|
||||
if err := b.loadConfig(ctx, cfgPath, ""); err != nil {
|
||||
cancel()
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
// openStores' own secret registry is not part of this feature: the file
|
||||
// document carries the State Store token explicitly, so the dial resolves
|
||||
// its credential without a forge credential this harness has no use for.
|
||||
b.secrets = &secret.Registry{}
|
||||
if err := b.openDaemonStateSurfaces(); err != nil {
|
||||
cancel()
|
||||
t.Fatalf("open the State Store surfaces: %v", err)
|
||||
base := fileConfig()
|
||||
b := &boot{
|
||||
cfg: base,
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
processName: applystatus.Daemon,
|
||||
cfgHolder: config.NewHolder(base),
|
||||
controlPlane: controlplane.NewRPCClient(client.ControlPlane()),
|
||||
}
|
||||
b.applyStatus = applystatus.New(b.processName, client, b.log)
|
||||
if err := b.loadRuntimeConfig(ctx); err != nil {
|
||||
cancel()
|
||||
t.Fatalf("layer the database settings over the file document: %v", err)
|
||||
t.Fatalf("load runtime config: %v", err)
|
||||
}
|
||||
if err := b.startWorkflowExecutionSettings(ctx); err != nil {
|
||||
cancel()
|
||||
t.Fatalf("apply the stored workflow execution settings: %v", err)
|
||||
t.Fatalf("start workflow execution settings: %v", err)
|
||||
}
|
||||
var once sync.Once
|
||||
stop := func() {
|
||||
once.Do(func() {
|
||||
// Cancel first, then release: a watch stream that fails on a live
|
||||
// context is reported as a failed apply, and this process is being
|
||||
// shut down, not failing.
|
||||
cancel()
|
||||
b.cleanup()
|
||||
})
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
return b, stop
|
||||
return b
|
||||
}
|
||||
|
||||
// documentLimits reads a stored or recorded document back as the limits it
|
||||
// means.
|
||||
func documentLimits(value []byte) (workflow.ExecutionSettings, error) {
|
||||
var document settingsDocument
|
||||
if err := json.Unmarshal(value, &document); err != nil {
|
||||
return workflow.ExecutionSettings{}, err
|
||||
// TestRestartKeepsADatabaseOwnedSettingsChange is archie-core-eju6, the
|
||||
// end-to-end restart test docs/prds/runtime-control-plane.md ("First
|
||||
// implementation") lists as a deliverable. It drives settings changes through
|
||||
// the dashboard's own control-plane handler, reads one back from the audit
|
||||
// trail, restarts the process that consumes it, and asserts the restarted
|
||||
// process is running the stored values rather than the file's -- across both
|
||||
// halves of boot.runtimeConfig: the queried restart-required kinds and the
|
||||
// watched execution budgets (archie-core-ju85).
|
||||
//
|
||||
// The restart is a fresh boot composition dialing a fresh connection to the
|
||||
// State Store that holds the change. State Store persistence is what an OS
|
||||
// restart preserves; re-creating the consumer is what an OS restart does to
|
||||
// the consumer, so nothing in the property under test is substituted.
|
||||
func TestRestartKeepsADatabaseOwnedSettingsChange(t *testing.T) {
|
||||
fixture := newControlPlaneRestartFixture(t)
|
||||
dashboard := &webui.Server{ControlPlane: fixture.admin.ControlPlane(), ApplyStatus: fixture.admin}
|
||||
|
||||
// A process is already running on the seeded values. It is stopped before
|
||||
// the edit so the restart below -- not the live watch -- is what reads the
|
||||
// change back.
|
||||
firstCtx, stopFirst := context.WithCancel(t.Context())
|
||||
first := startRestartProcess(firstCtx, t, fixture.dial(t))
|
||||
stopFirst()
|
||||
if got := first.cfgHolder.Get().Budgets; got != budgetsFor(workflow.ExecutionSettings{MaxModelToolSteps: 1, MaxRuntime: time.Minute}) {
|
||||
t.Fatalf("first process budgets = %+v, want the seeded file budgets", got)
|
||||
}
|
||||
return document.limits(), nil
|
||||
}
|
||||
|
||||
// dashboardRequest runs one request against the dashboard's own HTTP adapter.
|
||||
func dashboardRequest(t *testing.T, dashboard *webui.Server, request *http.Request) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
recorder := httptest.NewRecorder()
|
||||
dashboard.Handler().ServeHTTP(recorder, request)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("%s %s = %d: %s", request.Method, request.URL.Path, recorder.Code, recorder.Body.String())
|
||||
if got := first.cfgHolder.Get().PluginDir; got != fileConfig().PluginDir {
|
||||
t.Fatalf("first process plugin dir = %q, want the file's", got)
|
||||
}
|
||||
|
||||
// 1. The dashboard writes the change. Its handler, not this test, supplies
|
||||
// the attribution: webAudit attributes to identity.SystemID.
|
||||
expectedVersion := int64(1)
|
||||
executionSettingsDocument := []byte(`{"max_model_tool_steps":40,"max_runtime_seconds":600,"max_consecutive_gate_failures":2}`)
|
||||
if version := dashboardReplace(t, dashboard, controlplane.WorkflowExecutionSettingsKind, executionSettingsDocument, expectedVersion); version != 2 {
|
||||
t.Fatalf("workflow execution settings after edit = version %d, want 2", version)
|
||||
}
|
||||
pluginDocument := []byte(`{"plugin_dir":"/db/plugins","module_dir":"/db/modules","secret_engine_dir":"/db/secrets","skills_dir":"/db/skills"}`)
|
||||
if version := dashboardReplace(t, dashboard, controlplane.PluginSettingsKind, pluginDocument, expectedVersion); version != 2 {
|
||||
t.Fatalf("plugin settings after edit = version %d, want 2", version)
|
||||
}
|
||||
|
||||
// 2. The change lands in the audit trail as an ordinary operator edit: the
|
||||
// newest revision names the dashboard and the System identity, and carries
|
||||
// the version it replaced.
|
||||
revisions := dashboardHistory(t, dashboard, controlplane.WorkflowExecutionSettingsKind)
|
||||
if len(revisions) != 2 {
|
||||
t.Fatalf("revisions = %d, want the seed and the edit", len(revisions))
|
||||
}
|
||||
edit := revisions[0]
|
||||
if edit.Version != 2 || edit.Actor != string(identity.SystemID) || edit.Source != "archie-ui" || edit.RequestID == "" {
|
||||
t.Fatalf("newest revision audit = %+v, want the dashboard's System-attributed edit at version 2", edit)
|
||||
}
|
||||
if revisions[1].Actor != "system:migration" || revisions[1].Version != 1 {
|
||||
t.Fatalf("older revision = %+v, want the migration seed it replaced", revisions[1])
|
||||
}
|
||||
// The value rides along, so a restore is an ordinary replace of it.
|
||||
if got := decodeExecutionSettingsRevision(t, edit.Value); got != restartEditedSettings {
|
||||
t.Fatalf("revision value = %+v, want the edited %+v", got, restartEditedSettings)
|
||||
}
|
||||
|
||||
// 3-4. Restart the consumer. It re-resolves the same file document and
|
||||
// must come back on the stored values for both the queried kind and the
|
||||
// watched budgets.
|
||||
restartCtx, stopRestart := context.WithCancel(t.Context())
|
||||
defer stopRestart()
|
||||
restarted := startRestartProcess(restartCtx, t, fixture.dial(t))
|
||||
|
||||
settings := restarted.executionSettings.Load()
|
||||
if settings == nil {
|
||||
t.Fatal("restarted process recorded no workflow execution settings")
|
||||
}
|
||||
if *settings != restartEditedSettings {
|
||||
t.Errorf("restarted settings = %+v, want the stored %+v", *settings, restartEditedSettings)
|
||||
}
|
||||
if got := restarted.cfgHolder.Get().Budgets; got != budgetsFor(restartEditedSettings) {
|
||||
t.Errorf("restarted budgets = %+v, want the stored %+v", got, budgetsFor(restartEditedSettings))
|
||||
}
|
||||
if got := restarted.cfgHolder.Get().PluginDir; got != "/db/plugins" {
|
||||
t.Errorf("restarted plugin dir = %q, want the stored /db/plugins, not the file's %q", got, fileConfig().PluginDir)
|
||||
}
|
||||
|
||||
// The SIGHUP reload re-resolves the file document alone. It runs the same
|
||||
// boot.runtimeConfig, so it must not revert either half to its file value
|
||||
// (archie-core-ju85).
|
||||
if err := restarted.reloadConfig(restartCtx, &configuration.Document{Config: fileConfig()}); err != nil {
|
||||
t.Fatalf("reload: %v", err)
|
||||
}
|
||||
if got := restarted.cfgHolder.Get().Budgets; got != budgetsFor(restartEditedSettings) {
|
||||
t.Errorf("budgets after reload = %+v, want the stored %+v re-applied", got, budgetsFor(restartEditedSettings))
|
||||
}
|
||||
if got := restarted.cfgHolder.Get().PluginDir; got != "/db/plugins" {
|
||||
t.Errorf("plugin dir after reload = %q, want the stored /db/plugins", got)
|
||||
}
|
||||
|
||||
// 5. The restarted process reports the version it applied, for both halves,
|
||||
// and a record that has stopped being re-stamped reads as not reporting.
|
||||
reported, err := fixture.admin.ListApplyStatus(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("list apply status: %v", err)
|
||||
}
|
||||
for _, kind := range []string{controlplane.WorkflowExecutionSettingsKind, controlplane.PluginSettingsKind} {
|
||||
record := applyStatusFor(t, reported, applystatus.Daemon, kind)
|
||||
if record.AppliedVersion != 2 {
|
||||
t.Errorf("%s apply status version = %d, want the applied 2", kind, record.AppliedVersion)
|
||||
}
|
||||
if record.Error != "" {
|
||||
t.Errorf("%s apply status error = %q, want none for an applied version", kind, record.Error)
|
||||
}
|
||||
}
|
||||
|
||||
statusView := dashboardApplyStatus(t, dashboard)
|
||||
live := applyStatusViewFor(t, statusView, applystatus.Daemon, controlplane.WorkflowExecutionSettingsKind)
|
||||
if live.State != "current" {
|
||||
t.Errorf("a record re-stamped at restart reads %q, want current", live.State)
|
||||
}
|
||||
|
||||
// One record per (process, kind), replaced in place: an old report for the
|
||||
// same key replaces the live one rather than sitting beside it.
|
||||
stale := time.Now().Add(-documentedStaleWindow - time.Second)
|
||||
if err := fixture.admin.PutApplyStatus(t.Context(), storecontract.ApplyStatus{
|
||||
Process: applystatus.Daemon, Kind: controlplane.WorkflowExecutionSettingsKind,
|
||||
AppliedVersion: 2, ReportedAt: stale,
|
||||
}); err != nil {
|
||||
t.Fatalf("put stale apply status: %v", err)
|
||||
}
|
||||
statusView = dashboardApplyStatus(t, dashboard)
|
||||
aged := applyStatusViewFor(t, statusView, applystatus.Daemon, controlplane.WorkflowExecutionSettingsKind)
|
||||
if aged.State != "unknown" {
|
||||
t.Errorf("a record older than %v reads %q, want unknown rather than current", documentedStaleWindow, aged.State)
|
||||
}
|
||||
if count := countApplyStatusViews(statusView, applystatus.Daemon, controlplane.WorkflowExecutionSettingsKind); count != 1 {
|
||||
t.Errorf("rows for (%s, %s) = %d, want the report replaced in place", applystatus.Daemon, controlplane.WorkflowExecutionSettingsKind, count)
|
||||
}
|
||||
|
||||
// The administrative credential is what admits the report: a task-scoped
|
||||
// grant, which the daemon issues to agent containers, is refused by both
|
||||
// apply-status RPCs.
|
||||
grant, err := fixture.admin.RegisterTaskGrant(t.Context(), 1, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("register task grant: %v", err)
|
||||
}
|
||||
taskScoped := dialStateStore(t, fixture.listener, grant)
|
||||
if _, err := taskScoped.ListApplyStatus(t.Context()); status.Code(err) != codes.PermissionDenied {
|
||||
t.Errorf("ListApplyStatus with a task-scoped grant = %v, want PermissionDenied", err)
|
||||
}
|
||||
if err := taskScoped.PutApplyStatus(t.Context(), storecontract.ApplyStatus{Process: applystatus.Daemon, Kind: controlplane.PluginSettingsKind, AppliedVersion: 9}); status.Code(err) != codes.PermissionDenied {
|
||||
t.Errorf("PutApplyStatus with a task-scoped grant = %v, want PermissionDenied", err)
|
||||
}
|
||||
return recorder
|
||||
}
|
||||
|
||||
// replaceThroughDashboard writes document as the dashboard's editor does, over
|
||||
// the replace command, against the version the editor read. The attribution is
|
||||
// the adapter's: the request body carries only the value and the version.
|
||||
func replaceThroughDashboard(t *testing.T, dashboard *webui.Server, kind, document string, expectedVersion int64) {
|
||||
t.Helper()
|
||||
body := fmt.Sprintf(`{"value": %s, "expected_version": %d}`, document, expectedVersion)
|
||||
request := httptest.NewRequestWithContext(t.Context(), http.MethodPost,
|
||||
"/api/control-plane/resources/"+kind+"/commands/replace", strings.NewReader(body))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("X-Archie-CSRF", "1")
|
||||
dashboardRequest(t, dashboard, request)
|
||||
}
|
||||
|
||||
// revision is one row of the audit trail as the dashboard's history route
|
||||
// renders it.
|
||||
type revision struct {
|
||||
type controlPlaneRevision struct {
|
||||
Version int64 `json:"version"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Actor string `json:"actor"`
|
||||
@@ -387,62 +317,122 @@ type revision struct {
|
||||
RequestID string `json:"request_id"`
|
||||
}
|
||||
|
||||
func historyThroughDashboard(t *testing.T, dashboard *webui.Server, kind string) []revision {
|
||||
t.Helper()
|
||||
request := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||
"/api/control-plane/resources/"+kind+"/history", nil)
|
||||
recorder := dashboardRequest(t, dashboard, request)
|
||||
var response struct {
|
||||
Revisions []revision `json:"revisions"`
|
||||
}
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("decode the audit trail: %v (body %s)", err, recorder.Body.String())
|
||||
}
|
||||
if len(response.Revisions) == 0 {
|
||||
t.Fatalf("the audit trail for %s is empty", kind)
|
||||
}
|
||||
return response.Revisions
|
||||
type applyStatusRecord struct {
|
||||
Process string `json:"process"`
|
||||
Kind string `json:"kind"`
|
||||
AppliedVersion int64 `json:"applied_version"`
|
||||
Error string `json:"error"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
// reportedRow is the apply-status row the settings page shows for the daemon
|
||||
// and kind. It is read through the administrative token, which is the only
|
||||
// credential that reaches ListApplyStatus.
|
||||
func reportedRow(t *testing.T, admin *staterpc.Client, kind string) storecontract.ApplyStatus {
|
||||
// decodeExecutionSettingsRevision reads a revision's value in the same shape
|
||||
// the stored document uses, so the audit assertion compares values rather than
|
||||
// just versions.
|
||||
func decodeExecutionSettingsRevision(t *testing.T, value json.RawMessage) workflow.ExecutionSettings {
|
||||
t.Helper()
|
||||
rows, err := admin.ListApplyStatus(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("list apply status: %v", err)
|
||||
var document struct {
|
||||
MaxModelToolSteps int `json:"max_model_tool_steps"`
|
||||
MaxRuntimeSeconds int64 `json:"max_runtime_seconds"`
|
||||
MaxConsecutiveGateFailures int `json:"max_consecutive_gate_failures"`
|
||||
}
|
||||
for _, row := range rows {
|
||||
if row.Process == applystatus.Daemon && row.Kind == kind {
|
||||
return row
|
||||
if err := json.Unmarshal(value, &document); err != nil {
|
||||
t.Fatalf("decode revision value: %v (%s)", err, value)
|
||||
}
|
||||
return workflow.ExecutionSettings{
|
||||
MaxModelToolSteps: document.MaxModelToolSteps,
|
||||
MaxRuntime: time.Duration(document.MaxRuntimeSeconds) * time.Second,
|
||||
MaxConsecutiveGateFailures: document.MaxConsecutiveGateFailures,
|
||||
}
|
||||
}
|
||||
|
||||
// dashboardReplace writes a resource the way the dashboard does -- through the
|
||||
// HTTP handler, which supplies the operator attribution the request body never
|
||||
// carries -- and returns the new version.
|
||||
func dashboardReplace(t *testing.T, server *webui.Server, kind string, value []byte, expectedVersion int64) int64 {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(map[string]any{"value": json.RawMessage(value), "expected_version": expectedVersion})
|
||||
if err != nil {
|
||||
t.Fatalf("encode %s request: %v", kind, err)
|
||||
}
|
||||
request := httptest.NewRequestWithContext(t.Context(), http.MethodPost,
|
||||
"/api/control-plane/resources/"+kind+"/commands/replace", bytes.NewReader(body))
|
||||
recorder := httptest.NewRecorder()
|
||||
server.Handler().ServeHTTP(recorder, request)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("dashboard replace %s = %d, want 200: %s", kind, recorder.Code, recorder.Body.String())
|
||||
}
|
||||
var response struct {
|
||||
Resource struct {
|
||||
Version int64 `json:"version"`
|
||||
} `json:"resource"`
|
||||
}
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("decode %s replace response: %v (%s)", kind, err, recorder.Body.String())
|
||||
}
|
||||
return response.Resource.Version
|
||||
}
|
||||
|
||||
// dashboardHistory reads a resource's audit trail through the dashboard route,
|
||||
// so the edit observed is the one the UI would show.
|
||||
func dashboardHistory(t *testing.T, server *webui.Server, kind string) []controlPlaneRevision {
|
||||
t.Helper()
|
||||
var view struct {
|
||||
Revisions []controlPlaneRevision `json:"revisions"`
|
||||
}
|
||||
dashboardGET(t, server, "/api/control-plane/resources/"+kind+"/history", &view)
|
||||
return view.Revisions
|
||||
}
|
||||
|
||||
// dashboardApplyStatus reads the apply-status page the dashboard renders.
|
||||
func dashboardApplyStatus(t *testing.T, server *webui.Server) []applyStatusRecord {
|
||||
t.Helper()
|
||||
var view struct {
|
||||
Records []applyStatusRecord `json:"records"`
|
||||
}
|
||||
dashboardGET(t, server, "/api/control-plane/apply-status", &view)
|
||||
return view.Records
|
||||
}
|
||||
|
||||
func dashboardGET(t *testing.T, server *webui.Server, path string, target any) {
|
||||
t.Helper()
|
||||
recorder := httptest.NewRecorder()
|
||||
server.Handler().ServeHTTP(recorder, httptest.NewRequestWithContext(t.Context(), http.MethodGet, path, nil))
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("GET %s = %d, want 200: %s", path, recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), target); err != nil {
|
||||
t.Fatalf("decode %s: %v (%s)", path, err, recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func applyStatusFor(t *testing.T, statuses []storecontract.ApplyStatus, process, kind string) storecontract.ApplyStatus {
|
||||
t.Helper()
|
||||
for _, record := range statuses {
|
||||
if record.Process == process && record.Kind == kind {
|
||||
return record
|
||||
}
|
||||
}
|
||||
t.Fatalf("no apply-status row for %s/%s in %+v", applystatus.Daemon, kind, rows)
|
||||
t.Fatalf("no apply status for (%s, %s) among %+v", process, kind, statuses)
|
||||
return storecontract.ApplyStatus{}
|
||||
}
|
||||
|
||||
// applyStatusState is the reading the settings page renders for the daemon's
|
||||
// row for kind.
|
||||
func applyStatusState(t *testing.T, dashboard *webui.Server, kind string) string {
|
||||
func applyStatusViewFor(t *testing.T, records []applyStatusRecord, process, kind string) applyStatusRecord {
|
||||
t.Helper()
|
||||
request := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/api/control-plane/apply-status", nil)
|
||||
recorder := dashboardRequest(t, dashboard, request)
|
||||
var page struct {
|
||||
Records []struct {
|
||||
Process string `json:"process"`
|
||||
Kind string `json:"kind"`
|
||||
State string `json:"state"`
|
||||
} `json:"records"`
|
||||
}
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &page); err != nil {
|
||||
t.Fatalf("decode the apply-status page: %v (body %s)", err, recorder.Body.String())
|
||||
}
|
||||
for _, record := range page.Records {
|
||||
if record.Process == applystatus.Daemon && record.Kind == kind {
|
||||
return record.State
|
||||
for _, record := range records {
|
||||
if record.Process == process && record.Kind == kind {
|
||||
return record
|
||||
}
|
||||
}
|
||||
t.Fatalf("the apply-status page carries no row for %s/%s: %+v", applystatus.Daemon, kind, page.Records)
|
||||
return ""
|
||||
t.Fatalf("no apply-status row for (%s, %s) among %+v", process, kind, records)
|
||||
return applyStatusRecord{}
|
||||
}
|
||||
|
||||
func countApplyStatusViews(records []applyStatusRecord, process, kind string) int {
|
||||
count := 0
|
||||
for _, record := range records {
|
||||
if record.Process == process && record.Kind == kind {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
Reference in new issue
Block a user