Merge crew/codex-1184: provider secret resolution test, watch outage reporting, projection parity guard
This commit is contained in:
commit
8bcced0f47
8 files changed
+353
-151
No files matched your search
@@ -232,7 +232,7 @@
|
||||
"url": "/dev/prds/control-plane-apply-status/",
|
||||
"title": "Control-plane apply status",
|
||||
"section": "PRDs",
|
||||
"body": "# Control-plane apply status\n\n**Status:** Approved\n**Date:** 2026-09-21\n**Tracking:** `archie-core-pskb`, and `archie-core-nwa0` which depends on it\n\n## Outcome\n\nAn operator who edits a control-plane resource can see which processes are\nrunning that version, which are still on an older one, which rejected it, and\nwhich have stopped reporting.\n\n`docs/prds/runtime-control-plane.md` requires this and does not say where the\nreport goes. This decides that.\n\n## Where the report goes\n\nApply status is a State Store contract surface. `ControlPlaneService` is the\nsurface for editing a resource, and a status report is not an edit. The\nprecedent is the config snapshot pair: a process publishes what it knows, the\nUI process reads it, and both RPCs are administrative, so a task-scoped grant\nreaches neither. That denial needs no new rule: `authorizesTaskScopedCall`\nnames three methods and denies everything else.\n\n`docs/prds/state-store-contract.md` gains the pair in its inventory, and its\nRPC count at two places moves from 45 to 47.\n\nThe read returns all records as a list, including an empty one. The\n`found=false` convention for absent single rows does not apply.\n\n## What is reported\n\nOne record per process and resource kind, replaced in place. It holds the\nprocess name, the resource kind, the version applied, the error if the process\ncould not apply it, and the time it reported.\n\nEach process re-stamps its records every 30 seconds. A record is stale once it\nis older than 90 seconds, and a stale record means the process that wrote it is\nno longer running. The report time is therefore a liveness signal, not only a\nwrite timestamp.\n\nA process reports at the point it applies a resource, so the record says what\nhappened rather than what was intended. Nothing probes.\n\nProcess names come from one shared constant naming `archied`, `archie-gateway`\nand `archie-messaging`. A deployment runs at most one of each. Writer and\nreader are bound to that constant by test, because a drifted name is\nindistinguishable from a process that never reported.\n\n## Which processes report\n\n- `archied` and `archie-gateway` apply every restart-required kind through\n `boot.runtimeConfig`, at boot and on SIGHUP.\n- `archied` also applies the live kinds it watches, and reports each update it\n applies.\n- `archie-messaging` applies channel settings at startup and reports that kind.\n\n## What can be reported, and what cannot\n\nA process reaches its apply point only after the State Store has answered it.\nWhen the State Store is unreachable, the process exits or refuses to start and\nwrites no record. An unreachable store is never reported as a failure by the\nprocess it affected. A record carries the other failure: the store answered,\nand the value would not validate.\n\nAbsence has two meanings and the UI never collapses them: no process applies\nthis kind, or a process that applies other kinds has not reported this one.\n\n## Relationship to the reload status\n\nApply status covers control-plane resource kinds. `ReloadStatus` covers the\nfile document and stays the only report for a SIGHUP that failed to resolve or\nvalidate it. Neither is derived from the other.\n\nWriting a record is best-effort. On the SIGHUP path it takes its own bounded\ncontext, separate from the budget the reload spends reading the control plane,\nand a failed write is logged rather than turned into a failed reload.\n\n## What the UI shows\n\nThe settings page shows, per resource, the stored version beside each process's\napplied version, error, and report time. A process on an older version of a\nrestart-required resource is pending a restart. A process with an error is\nfailed, with the error beside it. A process whose record is stale is unknown,\nnever current.\n\n## A record outlives the process that wrote it\n\nRecords are replaced in place, so a process that applies a version and then\nstops would otherwise leave a record asserting that version indefinitely, and\nthe settings page would show a dead process as current.\n`docs/prds/status-health-surface.md` faced the same choice and stamped the start\nof a poll pass so a wedged poller reads as stale rather than healthy.\n\nSettled: the re-stamp above. The alternatives considered were\ncarrying the instance identity generated at process start, which detects a\nrestart but not a death, and accepting the limitation with the page saying so.\n\n## Verification\n\n- A process that applies a resource writes a record carrying that version.\n- A process whose apply fails validation writes a record carrying the error and\n no new version.\n- A reload that fails leaves the previously applied version recorded.\n- A record written under a process name outside the shared constant fails the\n test binding writer to reader.\n- A task-scoped credential is refused by both RPCs.\n- A failed report does not fail the reload that produced it.\n- A record that stops being re-Line truncated
|
||||
"body": "# Control-plane apply status\n\n**Status:** Approved\n**Date:** 2026-09-21\n**Tracking:** `archie-core-pskb`, and `archie-core-nwa0` which depends on it\n\n## Outcome\n\nAn operator who edits a control-plane resource can see which processes are\nrunning that version, which are still on an older one, which rejected it, and\nwhich have stopped reporting.\n\n`docs/prds/runtime-control-plane.md` requires this and does not say where the\nreport goes. This decides that.\n\n## Where the report goes\n\nApply status is a State Store contract surface. `ControlPlaneService` is the\nsurface for editing a resource, and a status report is not an edit. The\nprecedent is the config snapshot pair: a process publishes what it knows, the\nUI process reads it, and both RPCs are administrative, so a task-scoped grant\nreaches neither. That denial needs no new rule: `authorizesTaskScopedCall`\nnames three methods and denies everything else.\n\n`docs/prds/state-store-contract.md` gains the pair in its inventory, and its\nRPC count at two places moves from 45 to 47.\n\nThe read returns all records as a list, including an empty one. The\n`found=false` convention for absent single rows does not apply.\n\n## What is reported\n\nOne record per process and resource kind, replaced in place. It holds the\nprocess name, the resource kind, the version applied, the error if the process\ncould not apply it, and the time it reported.\n\nEach process re-stamps its records every 30 seconds. A record is stale once it\nis older than 90 seconds, and a stale record means the process that wrote it is\nno longer running. The report time is therefore a liveness signal, not only a\nwrite timestamp.\n\nA process reports at the point it applies a resource, so the record says what\nhappened rather than what was intended. Nothing probes.\n\nProcess names come from one shared constant naming `archied`, `archie-gateway`\nand `archie-messaging`. A deployment runs at most one of each. Writer and\nreader are bound to that constant by test, because a drifted name is\nindistinguishable from a process that never reported.\n\n## Which processes report\n\n- `archied` and `archie-gateway` apply every restart-required kind through\n `boot.runtimeConfig`, at boot and on SIGHUP.\n- `archied` also applies the live kinds it watches, and reports each update it\n applies.\n- `archie-messaging` applies channel settings at startup and reports that kind.\n\n## What can be reported, and what cannot\n\nA process reaches its apply point only after the State Store has answered it.\nAt startup, when the State Store is unreachable, the process exits or refuses\nto start and writes no record. A live watcher that loses its established\ncontrol-plane stream reports the connection failure against its last applied\nversion while it retries. Re-establishing the stream clears that outage record\nwithout clearing a versioned document rejection. A transport failure is not a\ndocument rejection; it reports that live delivery is unavailable.\n\nAbsence has two meanings and the UI never collapses them: no process applies\nthis kind, or a process that applies other kinds has not reported this one.\n\n## Relationship to the reload status\n\nApply status covers control-plane resource kinds. `ReloadStatus` covers the\nfile document and stays the only report for a SIGHUP that failed to resolve or\nvalidate it. Neither is derived from the other.\n\nWriting a record is best-effort. On the SIGHUP path it takes its own bounded\ncontext, separate from the budget the reload spends reading the control plane,\nand a failed write is logged rather than turned into a failed reload.\n\n## What the UI shows\n\nThe settings page shows, per resource, the stored version beside each process's\napplied version, error, and report time. A process on an older version of a\nrestart-required resource is pending a restart. A process with an error is\nfailed, with the error beside it. A process whose record is stale is unknown,\nnever current.\n\n## A record outlives the process that wrote it\n\nRecords are replaced in place, so a process that applies a version and then\nstops would otherwise leave a record asserting that version indefinitely, and\nthe settings page would show a dead process as current.\n`docs/prds/status-health-surface.md` faced the same choice and stamped the start\nof a poll pass so a wedged poller reads as stale rather than healthy.\n\nSettled: the re-stamp above. The alternatives considered were\ncarrying the instance identity generated at process start, which detects a\nrestart but not a death, and accepting the limitation with the page saying so.\n\n## Verification\n\n- A process that applies a resource writes a record carrying that version.\n- A process whose apply fails validation writes a record carrying the error and\n no new version.\n- A live watch outage is reported against the last applied version until the\n stream is re-established; re-establishment preserves any outstanLine truncated
|
||||
},
|
||||
{
|
||||
"url": "/dev/prds/curator-sampler-wave1/",
|
||||
|
||||
@@ -58,10 +58,12 @@ indistinguishable from a process that never reported.
|
||||
## What can be reported, and what cannot
|
||||
|
||||
A process reaches its apply point only after the State Store has answered it.
|
||||
When the State Store is unreachable, the process exits or refuses to start and
|
||||
writes no record. An unreachable store is never reported as a failure by the
|
||||
process it affected. A record carries the other failure: the store answered,
|
||||
and the value would not validate.
|
||||
At startup, when the State Store is unreachable, the process exits or refuses
|
||||
to start and writes no record. A live watcher that loses its established
|
||||
control-plane stream reports the connection failure against its last applied
|
||||
version while it retries. Re-establishing the stream clears that outage record
|
||||
without clearing a versioned document rejection. A transport failure is not a
|
||||
document rejection; it reports that live delivery is unavailable.
|
||||
|
||||
Absence has two meanings and the UI never collapses them: no process applies
|
||||
this kind, or a process that applies other kinds has not reported this one.
|
||||
@@ -101,6 +103,9 @@ restart but not a death, and accepting the limitation with the page saying so.
|
||||
- A process that applies a resource writes a record carrying that version.
|
||||
- A process whose apply fails validation writes a record carrying the error and
|
||||
no new version.
|
||||
- A live watch outage is reported against the last applied version until the
|
||||
stream is re-established; re-establishment preserves any outstanding
|
||||
versioned rejection.
|
||||
- A reload that fails leaves the previously applied version recorded.
|
||||
- A record written under a process name outside the shared constant fails the
|
||||
test binding writer to reader.
|
||||
|
||||
@@ -115,41 +115,45 @@ func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var lastApplyErr, streamErr error
|
||||
open := func(ctx context.Context, after int64) (<-chan controlplane.AppliedSettings, error) {
|
||||
updates, err := b.controlPlane.WatchWorkflowExecutionSettings(ctx, after)
|
||||
if err == nil {
|
||||
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, after, lastApplyErr)
|
||||
}
|
||||
return updates, err
|
||||
}
|
||||
go keepWatch(ctx, b.log, controlplane.WorkflowExecutionSettingsKind, version, updates,
|
||||
b.controlPlane.WatchWorkflowExecutionSettings,
|
||||
open,
|
||||
waitFor,
|
||||
func(update controlplane.AppliedSettings) int64 { return update.Version },
|
||||
func(update controlplane.AppliedSettings) {
|
||||
if update.Err != nil {
|
||||
// A refused update arrives here, not through
|
||||
// applyWorkflowExecutionSettings: controlplane.Client decodes every
|
||||
// document it is handed, so a stored value this process cannot run
|
||||
// never reaches the apply call. Report it against the version it
|
||||
// came from -- the reporter keeps the version it last applied --
|
||||
// so the settings page shows the refusal instead of a version this
|
||||
// process never ran (docs/prds/control-plane-apply-status.md).
|
||||
//
|
||||
// A stream failure arrives the same way and must not be reported:
|
||||
// it carries no version at all, because a Recv error is the store
|
||||
// not answering rather than a document, and versions start at 1
|
||||
// (store.PutResource). The rule is
|
||||
// docs/prds/control-plane-apply-status.md, "What can be reported,
|
||||
// and what cannot": a process reaches its apply point only after
|
||||
// the State Store has answered it, and an unreachable store is
|
||||
// never reported as a failure by the process it affected. The
|
||||
// record would not stick of itself -- Report overwrites it on the
|
||||
// next applied version -- but a store that never changes again
|
||||
// leaves the one stamp standing, and the process must not write it
|
||||
// at all. Reporting it would also overwrite the text of a standing
|
||||
// refusal, which is the one message that says what to fix. The
|
||||
// reconnect is the loop's business, not the record's.
|
||||
// A refused document carries its version, while a stream failure
|
||||
// does not. Keep a refusal visible across a transport outage; when
|
||||
// there is no outstanding refusal, the end callback records the
|
||||
// outage until a watch is re-established.
|
||||
if update.Version > 0 {
|
||||
lastApplyErr = update.Err
|
||||
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, update.Version, update.Err)
|
||||
} else {
|
||||
streamErr = update.Err
|
||||
}
|
||||
b.log.Error("workflow execution settings watch failed", "err", update.Err)
|
||||
return
|
||||
}
|
||||
_ = b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version)
|
||||
lastApplyErr = b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version)
|
||||
},
|
||||
func() {
|
||||
if lastApplyErr != nil {
|
||||
return
|
||||
}
|
||||
if streamErr == nil {
|
||||
streamErr = fmt.Errorf("control-plane watch stream ended")
|
||||
}
|
||||
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, 0,
|
||||
fmt.Errorf("workflow execution settings watch unavailable: %w", streamErr))
|
||||
streamErr = nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -132,7 +132,8 @@ func (*settingsWatchStub) Command(context.Context, *pb.CommandRequest, ...grpc.C
|
||||
panic("unexpected Command")
|
||||
}
|
||||
|
||||
func (s *settingsWatchStub) Watch(context.Context, *pb.WatchRequest, ...grpc.CallOption) (grpc.ServerStreamingClient[pb.WatchResponse], error) {
|
||||
func (s *settingsWatchStub) Watch(ctx context.Context, request *pb.WatchRequest, _ ...grpc.CallOption) (grpc.ServerStreamingClient[pb.WatchResponse], error) {
|
||||
_ = request
|
||||
responses := make([]*pb.WatchResponse, 0, len(s.documents))
|
||||
for i, document := range s.documents {
|
||||
responses = append(responses, &pb.WatchResponse{Resource: &pb.Resource{
|
||||
@@ -141,7 +142,7 @@ func (s *settingsWatchStub) Watch(context.Context, *pb.WatchRequest, ...grpc.Cal
|
||||
ValueJson: []byte(document),
|
||||
}})
|
||||
}
|
||||
return &watchStreamStub{responses: responses}, nil
|
||||
return &watchStreamStub{responses: responses, done: ctx.Done()}, nil
|
||||
}
|
||||
|
||||
// watchStreamStub is the client end of a Watch stream. Recv is the only method
|
||||
@@ -150,6 +151,7 @@ func (s *settingsWatchStub) Watch(context.Context, *pb.WatchRequest, ...grpc.Cal
|
||||
type watchStreamStub struct {
|
||||
grpc.ClientStream
|
||||
responses []*pb.WatchResponse
|
||||
done <-chan struct{}
|
||||
next int
|
||||
// onDeliver, when set, runs as the stream delivers each response, so a
|
||||
// stub standing in for the store can store what it just delivered.
|
||||
@@ -158,7 +160,11 @@ type watchStreamStub struct {
|
||||
|
||||
func (s *watchStreamStub) Recv() (*pb.WatchResponse, error) {
|
||||
if s.next >= len(s.responses) {
|
||||
return nil, io.EOF
|
||||
if s.done == nil {
|
||||
return nil, io.EOF
|
||||
}
|
||||
<-s.done
|
||||
return nil, context.Canceled
|
||||
}
|
||||
response := s.responses[s.next]
|
||||
s.next++
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -113,22 +114,27 @@ func newReloadBoot(t *testing.T, stub *controlPlaneStub) *boot {
|
||||
return b
|
||||
}
|
||||
|
||||
func TestRuntimeConfigResolvesStoredProviderReference(t *testing.T) {
|
||||
t.Setenv("ARCHIE_TEST_STORED_PROVIDER", "provider-secret")
|
||||
resources := databaseOwnedResources()
|
||||
resources[controlplane.ProviderSettingsKind] = map[string]any{"main": map[string]any{
|
||||
"class": "openai", "api_key_ref": map[string]any{"engine": "env", "key": "ARCHIE_TEST_STORED_PROVIDER"},
|
||||
}}
|
||||
b := newReloadBoot(t, &controlPlaneStub{values: resources})
|
||||
b.secrets = secret.NewRegistry()
|
||||
if err := b.loadRuntimeConfig(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
func TestRuntimeConfigResolvesStoredProviderReferences(t *testing.T) {
|
||||
for _, providerID := range []string{"main", "secondary"} {
|
||||
t.Run(providerID, func(t *testing.T) {
|
||||
refKey := "ARCHIE_TEST_STORED_PROVIDER_" + strings.ToUpper(providerID)
|
||||
t.Setenv(refKey, "provider-secret")
|
||||
resources := databaseOwnedResources()
|
||||
resources[controlplane.ProviderSettingsKind] = map[string]any{providerID: map[string]any{
|
||||
"class": "openai", "api_key_ref": map[string]any{"engine": "env", "key": refKey},
|
||||
}}
|
||||
b := newReloadBoot(t, &controlPlaneStub{values: resources})
|
||||
b.secrets = secret.NewRegistry()
|
||||
if err := b.loadRuntimeConfig(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := b.cfgHolder.Get().Providers[providerID]
|
||||
if got.APIKey != (secret.SecretRef{}) || got.APIKeyEnv != providerSecretEnvName("root", providerID) {
|
||||
t.Fatalf("effective provider credential = %+v, want a resolved process-local env name", got)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Unsetenv(got.APIKeyEnv) })
|
||||
})
|
||||
}
|
||||
got := b.cfgHolder.Get().Providers["main"]
|
||||
if got.APIKey != (secret.SecretRef{}) || got.APIKeyEnv != providerSecretEnvName("root", "main") {
|
||||
t.Fatalf("effective provider credential = %+v, want a resolved process-local env name", got)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Unsetenv(got.APIKeyEnv) })
|
||||
}
|
||||
|
||||
// TestReloadConfigKeepsDatabaseOwnedSettings is archie-core-ju85. The SIGHUP
|
||||
|
||||
@@ -52,12 +52,10 @@ const controlPlaneWatchHealthyMultiple = 2
|
||||
// re-establishes it in this same goroutine until ctx ends.
|
||||
//
|
||||
// A control-plane watch stream is not permanent. The State Store restarts, the
|
||||
// connection drops, the server closes the stream -- and nothing on the client
|
||||
// side reopens it. A watch that ends on its first end stops applying that kind
|
||||
// for the rest of the process's life: every later stored change is silently
|
||||
// never applied while the last apply-status record keeps re-stamping, so the
|
||||
// settings page reads the process as current, and the process recovers only by
|
||||
// restarting (archie-core-yrmr).
|
||||
// connection drops, or the server closes the stream. keepWatch reopens it in
|
||||
// place; workflow-execution-settings also reports the gap so apply status does
|
||||
// not keep re-stamping the previous successful version as current while no
|
||||
// live updates can arrive (archie-core-1050).
|
||||
//
|
||||
// One goroutine per kind, reconnecting in place: a long outage does not stack
|
||||
// watchers. Reopening resumes after the last version an update carried, so a
|
||||
@@ -129,6 +127,7 @@ func keepWatch[T any](
|
||||
wait func(ctx context.Context, d time.Duration) bool,
|
||||
versionOf func(update T) int64,
|
||||
deliver func(update T),
|
||||
onEnded ...func(),
|
||||
) {
|
||||
stream, backoff := first, controlPlaneWatchRetryMin
|
||||
for {
|
||||
@@ -146,6 +145,9 @@ func keepWatch[T any](
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if len(onEnded) > 0 {
|
||||
onEnded[0]()
|
||||
}
|
||||
backoff = retryDelay(backoff, attemptHealthy(progressed, time.Since(started), backoff))
|
||||
log.Warn("control plane watch stream ended; reconnecting",
|
||||
"kind", kind, "after_version", after, "retry_in", backoff)
|
||||
|
||||
@@ -219,22 +219,22 @@ func TestWorkflowExecutionSettingsWatchIsReEstablishedAfterTheStreamEnds(t *test
|
||||
t.Errorf("watch resumed after %v, want the boot version then the last version handled", got)
|
||||
}
|
||||
|
||||
// Boot's own apply, then the update the re-established stream
|
||||
// delivered. Neither way the stream ends writes a record of its own:
|
||||
// a stream the server closed emits no update at all (controlplane
|
||||
// .Client sends one only for a Recv error), and a Recv failure
|
||||
// carries no version, which is the unreachable store the process must
|
||||
// never report as a failure of its own
|
||||
// (docs/prds/control-plane-apply-status.md, "What can be reported, and
|
||||
// what cannot").
|
||||
records := status.awaitCount(t, controlplane.WorkflowExecutionSettingsKind, 2)
|
||||
if len(records) != 2 {
|
||||
t.Fatalf("apply status = %+v, want the boot apply and the recovered one alone", records)
|
||||
// Boot applies version 1, the ended stream reports the outage, the
|
||||
// reopened watch clears it, and the delivered version 2 is applied.
|
||||
records := status.awaitCount(t, controlplane.WorkflowExecutionSettingsKind, 4)
|
||||
if len(records) != 4 {
|
||||
t.Fatalf("apply status = %+v, want boot, outage, reconnect, and update reports", records)
|
||||
}
|
||||
if records[0].AppliedVersion != 1 || records[0].Error != "" {
|
||||
t.Errorf("boot report = %+v, want a clean apply of version 1", records[0])
|
||||
}
|
||||
last := records[len(records)-1]
|
||||
if records[1].AppliedVersion != 1 || !strings.Contains(records[1].Error, "watch unavailable") {
|
||||
t.Errorf("outage report = %+v, want version 1 with a watch outage", records[1])
|
||||
}
|
||||
if records[2].AppliedVersion != 1 || records[2].Error != "" {
|
||||
t.Errorf("reconnect report = %+v, want the outage cleared at version 1", records[2])
|
||||
}
|
||||
last := records[3]
|
||||
if last.AppliedVersion != 2 || last.Error != "" {
|
||||
t.Errorf("last apply status = %+v, want the recovered apply of version 2 with the failure cleared", last)
|
||||
}
|
||||
@@ -256,7 +256,7 @@ func TestWorkflowExecutionSettingsWatchIsReEstablishedAfterTheStreamEnds(t *test
|
||||
// point), applystatus.Reporter keeps a standing error and re-stamps it forever,
|
||||
// and the dashboard renders any non-empty error as failed. The process reads as
|
||||
// failed for the rest of its life, after one blip.
|
||||
func TestWatchTransportFailureIsNotReportedAsAFailedApply(t *testing.T) {
|
||||
func TestWatchTransportFailureIsReportedUntilTheWatchRecovers(t *testing.T) {
|
||||
t.Parallel()
|
||||
b, recorder := newLiveApplyBoot(t, fileConfig())
|
||||
stub := &watchStub{
|
||||
@@ -285,23 +285,25 @@ func TestWatchTransportFailureIsNotReportedAsAFailedApply(t *testing.T) {
|
||||
// the loop delivers what ended the stream, then reopens.
|
||||
stub.awaitWatchCalls(t, controlplane.WorkflowExecutionSettingsKind, 2)
|
||||
|
||||
records := recorder.awaitCount(t, controlplane.WorkflowExecutionSettingsKind, 1)
|
||||
if len(records) != 1 {
|
||||
t.Fatalf("apply status = %+v, want the boot apply alone: an unreachable store writes no record", records)
|
||||
records := recorder.awaitCount(t, controlplane.WorkflowExecutionSettingsKind, 3)
|
||||
if len(records) != 3 {
|
||||
t.Fatalf("apply status = %+v, want boot, outage, and recovery reports", records)
|
||||
}
|
||||
if last := records[0]; last.AppliedVersion != 1 || last.Error != "" {
|
||||
t.Errorf("apply status = %+v, want the clean apply of version 1 still standing", last)
|
||||
if records[0].AppliedVersion != 1 || records[0].Error != "" {
|
||||
t.Errorf("boot apply status = %+v, want the clean apply of version 1", records[0])
|
||||
}
|
||||
if records[1].AppliedVersion != 1 || !strings.Contains(records[1].Error, "watch unavailable") {
|
||||
t.Errorf("outage apply status = %+v, want version 1 with a watch outage", records[1])
|
||||
}
|
||||
if records[2].AppliedVersion != 1 || records[2].Error != "" {
|
||||
t.Errorf("recovered apply status = %+v, want version 1 with the outage cleared", records[2])
|
||||
}
|
||||
}
|
||||
|
||||
// TestWatchTransportFailureDoesNotOverwriteARefusal is the other half of the
|
||||
// same rule, and the reason the branch is on the version rather than on the
|
||||
// error: a refused document arrives carrying the version it came from, so it is
|
||||
// the store answering and must be reported; a transport failure carries no
|
||||
// version and must not. Reporting the transport failure against the version
|
||||
// still live overwrites the refusal's text, so a validation error against
|
||||
// version 2 reads "control-plane unavailable: ..." after any blip, and the
|
||||
// operator loses the one message that says what to fix.
|
||||
// TestWatchTransportFailureDoesNotOverwriteARefusal keeps the more useful
|
||||
// versioned rejection visible when a stream drops afterward. Reconnection
|
||||
// restores that rejection instead of clearing it as though a valid update had
|
||||
// arrived.
|
||||
func TestWatchTransportFailureDoesNotOverwriteARefusal(t *testing.T) {
|
||||
t.Parallel()
|
||||
b, recorder := newLiveApplyBoot(t, fileConfig())
|
||||
@@ -332,18 +334,18 @@ func TestWatchTransportFailureDoesNotOverwriteARefusal(t *testing.T) {
|
||||
|
||||
stub.awaitWatchCalls(t, controlplane.WorkflowExecutionSettingsKind, 2)
|
||||
|
||||
records := recorder.awaitCount(t, controlplane.WorkflowExecutionSettingsKind, 2)
|
||||
if len(records) != 2 {
|
||||
t.Fatalf("apply status = %+v, want the boot apply and the refusal alone", records)
|
||||
records := recorder.awaitCount(t, controlplane.WorkflowExecutionSettingsKind, 3)
|
||||
if len(records) != 3 {
|
||||
t.Fatalf("apply status = %+v, want boot, refusal, and the preserved refusal after reconnect", records)
|
||||
}
|
||||
last := records[len(records)-1]
|
||||
last := records[2]
|
||||
if last.AppliedVersion != 1 {
|
||||
t.Errorf("refusal reported against version %d, want the version still live (1)", last.AppliedVersion)
|
||||
}
|
||||
if !strings.Contains(last.Error, "max model/tool steps must not be negative") {
|
||||
t.Errorf("apply status error = %q, want the refusal's own text", last.Error)
|
||||
}
|
||||
if strings.Contains(last.Error, "control-plane unavailable") {
|
||||
if strings.Contains(last.Error, "watch unavailable") {
|
||||
t.Errorf("apply status error = %q, want the transport blip not to have overwritten the refusal", last.Error)
|
||||
}
|
||||
// The refused version is unusable to this build, so the resume point moves
|
||||
@@ -394,13 +396,16 @@ func TestWatchReconnectDoesNotReplayWhatItAlreadyApplied(t *testing.T) {
|
||||
t.Errorf("watch resumed after %v, want the boot version then the last version handled", got)
|
||||
}
|
||||
|
||||
// One report for the boot apply and one for the update: a third would be
|
||||
// the re-served version 1 applied over itself.
|
||||
records := recorder.awaitCount(t, controlplane.WorkflowExecutionSettingsKind, 2)
|
||||
if len(records) != 2 {
|
||||
t.Fatalf("apply status = %+v, want the boot apply and the update alone", records)
|
||||
// The outage and reconnection records sit between boot and version 2; the
|
||||
// re-served version 1 is still filtered out.
|
||||
records := recorder.awaitCount(t, controlplane.WorkflowExecutionSettingsKind, 4)
|
||||
if len(records) != 4 {
|
||||
t.Fatalf("apply status = %+v, want boot, outage, reconnect, and version 2", records)
|
||||
}
|
||||
last := records[len(records)-1]
|
||||
if records[1].AppliedVersion != 1 || !strings.Contains(records[1].Error, "watch unavailable") {
|
||||
t.Errorf("outage report = %+v, want an outage against version 1", records[1])
|
||||
}
|
||||
last := records[3]
|
||||
if last.AppliedVersion != 2 || last.Error != "" {
|
||||
t.Errorf("last apply status = %+v, want the recovered apply of version 2", last)
|
||||
}
|
||||
|
||||
@@ -194,74 +194,248 @@ func TestRuntimeConfigCarriesParallelToolCallsThroughTheProjection(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
// TestToolSettingsProjectionCarriesEveryMCPServerField is the guard the field
|
||||
// itself cannot be. mcpServerSettings mirrors config.MCPServer field by field,
|
||||
// so the next field added to MCPServer -- internal/config's Sandboxed, when
|
||||
// mcp-tool-completion's #178 lands -- silently falls out of the control-plane
|
||||
// path unless something fails when it does. This test is that something: it
|
||||
// populates every MCPServer field by reflection, so a field added later is
|
||||
// populated without editing any fixture here, then requires the seed to name
|
||||
// the field and the layering to hand it back.
|
||||
//
|
||||
// mcpServerFieldsLeftToTheFile is the only way past the check, and every entry
|
||||
// in it has to say what the projection carries in place of the field.
|
||||
func TestToolSettingsProjectionCarriesEveryMCPServerField(t *testing.T) {
|
||||
var server config.MCPServer
|
||||
populateEveryField(t, reflect.ValueOf(&server).Elem(), "config.MCPServer")
|
||||
// The resource validator knows a fixed transport vocabulary, so the round
|
||||
// trip needs one of them. It is the only value here the fixture takes from
|
||||
// the validator rather than from the field's type; stdio requires a command,
|
||||
// which the fixture has already set.
|
||||
server.Transport = "stdio"
|
||||
base := config.Config{Tools: config.ToolsConfig{MCPServers: []config.MCPServer{server}}}
|
||||
// settingsProjectionSpec describes a source config struct and the explicit
|
||||
// control-plane document struct that projects it. Overrides cover renamed or
|
||||
// transformed keys; the two allowlists require a reason for one-way fields.
|
||||
type settingsProjectionSpec struct {
|
||||
name string
|
||||
source any
|
||||
projection any
|
||||
keyOverrides map[string]string
|
||||
fileSurrogates map[string]string
|
||||
leftToFile map[string]string
|
||||
projectionOnly map[string]string
|
||||
}
|
||||
|
||||
seed := seedToolSettings(t, base)
|
||||
seeded := seededServer(t, seed)
|
||||
got := layerToolSettings(t, seed, base).Tools.MCPServers[0]
|
||||
// settingsProjectionSpecs includes the custom config mirrors in this package:
|
||||
// provider and tool resources, plus the channel document types aliased from
|
||||
// controlplanerpc. Repo and container resources carry config types directly.
|
||||
var settingsProjectionSpecs = []settingsProjectionSpec{
|
||||
{
|
||||
name: "tools", source: config.ToolsConfig{}, projection: toolSettings{},
|
||||
keyOverrides: map[string]string{"Policy": "policy"},
|
||||
},
|
||||
{
|
||||
name: "MCP server", source: config.MCPServer{}, projection: mcpServerSettings{},
|
||||
fileSurrogates: map[string]string{"Headers": "headers_configured"},
|
||||
leftToFile: map[string]string{"Headers": "HTTP credentials remain file-owned"},
|
||||
},
|
||||
{
|
||||
name: "MiniMax", source: config.MinimaxConfig{}, projection: minimaxSettings{},
|
||||
keyOverrides: map[string]string{"APIKey": "api_key_ref"},
|
||||
projectionOnly: map[string]string{"CredentialConfigured": "derived from APIKey"},
|
||||
},
|
||||
{
|
||||
name: "provider", source: config.Provider{}, projection: providerDocument{},
|
||||
keyOverrides: map[string]string{"APIKey": "api_key_ref"},
|
||||
projectionOnly: map[string]string{"HasCredential": "derived from APIKey and APIKeyEnv"},
|
||||
},
|
||||
{name: "chat", source: config.ChatConfig{}, projection: channelSettings{}},
|
||||
{
|
||||
name: "Telegram", source: config.TelegramConfig{}, projection: telegramSettings{},
|
||||
keyOverrides: map[string]string{"Token": "token_ref"},
|
||||
leftToFile: map[string]string{
|
||||
"UpdateCheckCommand": "update commands stay file-owned",
|
||||
"UpdateInstallCommand": "update commands stay file-owned",
|
||||
},
|
||||
projectionOnly: map[string]string{"CredentialConfigured": "derived from Token"},
|
||||
},
|
||||
{
|
||||
name: "webhook route", source: config.WebhookRoute{}, projection: webhookChannelSettings{},
|
||||
keyOverrides: map[string]string{"Secret": "secret_ref"},
|
||||
projectionOnly: map[string]string{"CredentialConfigured": "derived from Secret"},
|
||||
},
|
||||
{name: "email", source: config.EmailConfig{}, projection: emailSettings{}},
|
||||
{name: "rate limit", source: config.RateLimitConfig{}, projection: rateLimitSettings{}},
|
||||
}
|
||||
|
||||
typ := reflect.TypeOf(server)
|
||||
for i := range typ.NumField() {
|
||||
field := typ.Field(i)
|
||||
name := jsonFieldName(field)
|
||||
surrogate, leftToTheFile := mcpServerFieldsLeftToTheFile[field.Name]
|
||||
if leftToTheFile {
|
||||
// Deliberately not carried: the stored value must not hold the
|
||||
// field's own value, and whatever the projection carries instead
|
||||
// must be there.
|
||||
if _, carried := seeded[name]; carried {
|
||||
t.Errorf("the %s seed carries MCPServer.%s (%q), which mcpServerFieldsLeftToTheFile leaves to the file document", ToolSettingsKind, field.Name, name)
|
||||
func TestSettingsProjectionTypesStayInParity(t *testing.T) {
|
||||
for _, spec := range settingsProjectionSpecs {
|
||||
t.Run(spec.name, func(t *testing.T) {
|
||||
sourceType, projectionType := reflect.TypeOf(spec.source), reflect.TypeOf(spec.projection)
|
||||
projectionKeys := make(map[string]string, projectionType.NumField())
|
||||
for field := range projectionType.Fields() {
|
||||
projectionKeys[jsonFieldName(field)] = field.Name
|
||||
}
|
||||
if _, ok := seeded[surrogate]; !ok {
|
||||
t.Errorf("the %s seed carries neither MCPServer.%s nor the %q the projection carries in its place", ToolSettingsKind, field.Name, surrogate)
|
||||
|
||||
mappedKeys := map[string]string{}
|
||||
for field := range sourceType.Fields() {
|
||||
key := spec.keyOverrides[field.Name]
|
||||
if key == "" {
|
||||
key = configFieldName(field)
|
||||
}
|
||||
if reason, allowed := spec.leftToFile[field.Name]; allowed {
|
||||
if reason == "" {
|
||||
t.Errorf("%s.%s has an empty left-to-file reason", spec.name, field.Name)
|
||||
}
|
||||
if surrogate := spec.fileSurrogates[field.Name]; surrogate != "" {
|
||||
if _, exists := projectionKeys[surrogate]; !exists {
|
||||
t.Errorf("%s.%s is left to the file but projection has no surrogate key %q", spec.name, field.Name, surrogate)
|
||||
}
|
||||
mappedKeys[surrogate] = field.Name
|
||||
}
|
||||
continue
|
||||
}
|
||||
if key == "" {
|
||||
t.Errorf("%s.%s has no config or projection key; add an explicit key override or left-to-file reason", spec.name, field.Name)
|
||||
continue
|
||||
}
|
||||
if _, exists := projectionKeys[key]; !exists {
|
||||
t.Errorf("%s.%s maps to %q, which %s does not project", spec.name, field.Name, key, projectionType)
|
||||
}
|
||||
mappedKeys[key] = field.Name
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, carried := seeded[name]; !carried {
|
||||
t.Errorf("the %s seed has no %q key, so seedTools never projects MCPServer.%s and the layering can only restore its zero value", ToolSettingsKind, name, field.Name)
|
||||
continue
|
||||
}
|
||||
wantField := reflect.ValueOf(server).Field(i).Interface()
|
||||
if gotField := reflect.ValueOf(got).Field(i).Interface(); !reflect.DeepEqual(gotField, wantField) {
|
||||
t.Errorf("MCPServer.%s = %#v after the control-plane round trip, want %#v: carry it through mcpServerSettings, seedTools, and runtimeToolConfigFrom, or add it to mcpServerFieldsLeftToTheFile with what the projection carries in its place", field.Name, gotField, wantField)
|
||||
}
|
||||
}
|
||||
// An allowlist entry for a field MCPServer no longer has is a dead pass.
|
||||
for name := range mcpServerFieldsLeftToTheFile {
|
||||
if _, ok := typ.FieldByName(name); !ok {
|
||||
t.Errorf("mcpServerFieldsLeftToTheFile names MCPServer.%s, which the struct no longer has", name)
|
||||
}
|
||||
|
||||
for name, reason := range spec.leftToFile {
|
||||
if _, exists := sourceType.FieldByName(name); !exists {
|
||||
t.Errorf("%s left-to-file allowlist names missing source field %s", spec.name, name)
|
||||
}
|
||||
if reason == "" {
|
||||
t.Errorf("%s left-to-file allowlist for %s has no reason", spec.name, name)
|
||||
}
|
||||
}
|
||||
for key, fieldName := range spec.keyOverrides {
|
||||
if _, exists := sourceType.FieldByName(key); !exists {
|
||||
t.Errorf("%s key override names missing source field %s", spec.name, key)
|
||||
}
|
||||
if _, exists := projectionKeys[fieldName]; !exists {
|
||||
t.Errorf("%s key override names missing projection key %q", spec.name, fieldName)
|
||||
}
|
||||
}
|
||||
for fieldName, key := range spec.fileSurrogates {
|
||||
if _, exists := sourceType.FieldByName(fieldName); !exists {
|
||||
t.Errorf("%s file-surrogate mapping names missing source field %s", spec.name, fieldName)
|
||||
}
|
||||
if _, exists := projectionKeys[key]; !exists {
|
||||
t.Errorf("%s file-surrogate mapping names missing projection key %q", spec.name, key)
|
||||
}
|
||||
if _, allowed := spec.leftToFile[fieldName]; !allowed {
|
||||
t.Errorf("%s.%s has file surrogate %q but is not allowed to remain file-owned", spec.name, fieldName, key)
|
||||
}
|
||||
}
|
||||
for field := range projectionType.Fields() {
|
||||
key := jsonFieldName(field)
|
||||
if _, exists := mappedKeys[key]; exists {
|
||||
continue
|
||||
}
|
||||
if reason, allowed := spec.projectionOnly[field.Name]; !allowed || reason == "" {
|
||||
t.Errorf("%s.%s projects JSON key %q with no source field or derived-field reason", spec.name, field.Name, key)
|
||||
}
|
||||
}
|
||||
for name := range spec.projectionOnly {
|
||||
if _, exists := projectionType.FieldByName(name); !exists {
|
||||
t.Errorf("%s projection-only allowlist names missing projection field %s", spec.name, name)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// mcpServerFieldsLeftToTheFile names the MCPServer fields the control plane's
|
||||
// projection deliberately does not carry, with the projection key that stands
|
||||
// in for each. It is deliberately short: a field belongs here only when the
|
||||
// file document has to stay its owner. Headers hold credentials, so the stored
|
||||
// resource records only that some are configured and the layering restores the
|
||||
// real map from the base config by server name (runtime_config.go), which
|
||||
// TestRuntimeConfigUsesDatabaseResourcesAndPreservesBootstrapOnlySecrets pins.
|
||||
var mcpServerFieldsLeftToTheFile = map[string]string{
|
||||
"Headers": "headers_configured",
|
||||
func configFieldName(field reflect.StructField) string {
|
||||
for _, tag := range []string{"toml", "json"} {
|
||||
name, _, _ := strings.Cut(field.Tag.Get(tag), ",")
|
||||
if name == "-" {
|
||||
return ""
|
||||
}
|
||||
if name != "" {
|
||||
return name
|
||||
}
|
||||
}
|
||||
return field.Name
|
||||
}
|
||||
|
||||
func TestToolSettingsProjectionCarriesEveryMirroredField(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
source any
|
||||
configure func(*config.Config, reflect.Value)
|
||||
projectedItem func(*testing.T, []byte) map[string]any
|
||||
result func(config.Config) reflect.Value
|
||||
}{
|
||||
{
|
||||
name: "MCP server",
|
||||
source: config.MCPServer{},
|
||||
configure: func(cfg *config.Config, value reflect.Value) {
|
||||
server, ok := value.Interface().(config.MCPServer)
|
||||
if !ok {
|
||||
t.Fatalf("source type = %T, want config.MCPServer", value.Interface())
|
||||
}
|
||||
cfg.Tools.MCPServers = []config.MCPServer{server}
|
||||
},
|
||||
projectedItem: seededServer,
|
||||
result: func(cfg config.Config) reflect.Value { return reflect.ValueOf(cfg.Tools.MCPServers[0]) },
|
||||
},
|
||||
{
|
||||
name: "MiniMax",
|
||||
source: config.MinimaxConfig{},
|
||||
configure: func(cfg *config.Config, value reflect.Value) {
|
||||
minimax, ok := value.Interface().(config.MinimaxConfig)
|
||||
if !ok {
|
||||
t.Fatalf("source type = %T, want config.MinimaxConfig", value.Interface())
|
||||
}
|
||||
cfg.Tools.Minimax = minimax
|
||||
},
|
||||
projectedItem: func(t *testing.T, seed []byte) map[string]any {
|
||||
t.Helper()
|
||||
var document struct {
|
||||
Minimax map[string]any `json:"minimax"`
|
||||
}
|
||||
if err := json.Unmarshal(seed, &document); err != nil {
|
||||
t.Fatalf("decode %s seed: %v", ToolSettingsKind, err)
|
||||
}
|
||||
return document.Minimax
|
||||
},
|
||||
result: func(cfg config.Config) reflect.Value { return reflect.ValueOf(cfg.Tools.Minimax) },
|
||||
},
|
||||
}
|
||||
|
||||
specs := make(map[string]settingsProjectionSpec, len(settingsProjectionSpecs))
|
||||
for _, spec := range settingsProjectionSpecs {
|
||||
specs[spec.name] = spec
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
source := reflect.New(reflect.TypeOf(tt.source)).Elem()
|
||||
populateEveryField(t, source, "config."+source.Type().Name())
|
||||
if tt.name == "MCP server" {
|
||||
source.FieldByName("Transport").SetString("stdio") // the validator's supported value
|
||||
}
|
||||
base := config.Config{}
|
||||
tt.configure(&base, source)
|
||||
seed := seedToolSettings(t, base)
|
||||
seeded := tt.projectedItem(t, seed)
|
||||
got := tt.result(layerToolSettings(t, seed, base))
|
||||
spec := specs[tt.name]
|
||||
for i := range source.NumField() {
|
||||
field := source.Type().Field(i)
|
||||
key := spec.keyOverrides[field.Name]
|
||||
if key == "" {
|
||||
key = configFieldName(field)
|
||||
}
|
||||
surrogate := spec.fileSurrogates[field.Name]
|
||||
_, leftToFile := spec.leftToFile[field.Name]
|
||||
switch {
|
||||
case surrogate != "":
|
||||
if _, ok := seeded[surrogate]; !ok {
|
||||
t.Errorf("%s seed has no %q surrogate for file-owned %s.%s", ToolSettingsKind, surrogate, source.Type(), field.Name)
|
||||
}
|
||||
case leftToFile:
|
||||
if key != "" {
|
||||
if _, ok := seeded[key]; ok {
|
||||
t.Errorf("%s seed unexpectedly carries file-owned %s.%s as %q", ToolSettingsKind, source.Type(), field.Name, key)
|
||||
}
|
||||
}
|
||||
default:
|
||||
if _, ok := seeded[key]; !ok {
|
||||
t.Errorf("%s seed has no %q key for %s.%s", ToolSettingsKind, key, source.Type(), field.Name)
|
||||
}
|
||||
}
|
||||
if gotField, wantField := got.Field(i).Interface(), source.Field(i).Interface(); !reflect.DeepEqual(gotField, wantField) {
|
||||
t.Errorf("%s.%s after round trip = %#v, want %#v", source.Type(), field.Name, gotField, wantField)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// seedToolSettings is the value a State Store writes for the tool settings when
|
||||
|
||||
Reference in new issue
Block a user