fix(controlplane): repair a stored workflow collection that no longer validates (#1622)

This commit is contained in:
sam committed 2026-11-08 05:49:40 +11:00
1 parent cc8428643a
commit cf1be16763
8 files changed
+275 -16

No files matched your search

+4
View File
@@ -9,6 +9,10 @@ release's per-component sections are labelled beneath its heading.
- An image or voice attachment that arrives as a link is now checked against what the link serves, the same as an uploaded one. A link whose content is not the declared type, or that cannot be read, is refused.
### archie-state-store
- A workflow that no longer validates after an upgrade (for example it names a step type that was removed) is repaired at startup instead of breaking every workflow. A shipped workflow is restored from this release's copy; a workflow you created is removed and logged, and its earlier revision stays in the resource history. Valid workflows you created or edited are kept.
## [1.61.0] - 2026-11-07
### archied
+5
View File
@@ -202,6 +202,11 @@ func (b *server) seedAndValidatePolicies(ctx context.Context) error {
// not to the sequence that triggers it.
func (b *server) reportUnseededResources(skipped []controlplanestore.SeedSkip) {
for _, skip := range skipped {
if len(skip.Removed) > 0 {
b.log.Warn("removed stored entries that no longer validate; their earlier revision is in the resource history",
"kind", skip.Kind, "removed", skip.Removed)
continue
}
b.log.Error("control-plane resource not seeded; the kind stays absent and the file's value is the one in effect",
"kind", skip.Kind, "err", skip.Err)
}
+34
View File
@@ -387,3 +387,37 @@ func hasStepType(node any, types []string) bool {
}
return false
}
// RepairDefinitionCollection rebuilds a stored collection that no longer
// validates, keeping every entry it can: a valid entry is kept, an invalid
// shipped entry is replaced by this build's copy, and any other invalid or
// duplicate entry is removed and its id returned. An unreadable document or a
// repair that still fails validation is an error.
func RepairDefinitionCollection(value []byte, registry StepRegistry) (WorkflowDefinitionCollection, []string, error) {
var stored WorkflowDefinitionCollection
if err := json.Unmarshal(value, &stored); err != nil {
return WorkflowDefinitionCollection{}, nil, err
}
shipped := ShippedDefinitions()
seen := make(map[string]struct{}, len(stored.Definitions))
var repaired WorkflowDefinitionCollection
var removed []string
for _, entry := range stored.Definitions {
if _, dup := seen[entry.ID]; dup {
removed = append(removed, entry.ID)
continue
}
seen[entry.ID] = struct{}{}
if definition, err := ParseDefinition(entry.YAML, registry); err == nil && definition.ID == entry.ID {
repaired.Definitions = append(repaired.Definitions, entry)
} else if factory, ok := shipped.DefinitionByID(entry.ID); ok {
repaired.Definitions = append(repaired.Definitions, factory)
} else {
removed = append(removed, entry.ID)
}
}
if err := ValidateDefinitionCollection(repaired, registry); err != nil {
return WorkflowDefinitionCollection{}, nil, err
}
return repaired, removed, nil
}
@@ -0,0 +1,82 @@
package workflow
import (
"encoding/json"
"slices"
"strings"
"testing"
)
// A stored collection that stopped validating keeps every entry it can: valid
// entries stay, a broken shipped entry is restored, a broken custom one goes.
func TestRepairDefinitionCollection(t *testing.T) {
shipped, _ := ShippedDefinitions().DefinitionByID("triage")
custom := strings.Replace(shipped.YAML, "id: triage", "id: mine", 1)
edited := shipped.YAML + "\n# edited by the operator\n"
broken := "id: broken\nsteps:\n - id: a\n type: removed.step\n"
// with is the shipped set with triage's YAML swapped and extra appended.
with := func(triage string, extra ...WorkflowDefinitionEntry) ([]WorkflowDefinitionEntry, map[string]string) {
entries, byID := []WorkflowDefinitionEntry{}, map[string]string{}
for _, entry := range ShippedDefinitions().Definitions {
if entry.ID == "triage" {
entry.YAML = triage
}
entries = append(entries, entry)
byID[entry.ID] = entry.YAML
}
for _, entry := range extra {
entries = append(entries, entry)
byID[entry.ID] = entry.YAML
}
return entries, byID
}
restoredStored, _ := with(broken, WorkflowDefinitionEntry{ID: "mine", YAML: custom})
_, restoredWant := with(shipped.YAML, WorkflowDefinitionEntry{ID: "mine", YAML: custom})
editedStored, _ := with(edited, WorkflowDefinitionEntry{ID: "broken", YAML: broken})
_, editedWant := with(edited)
tests := []struct {
name string
stored []WorkflowDefinitionEntry
want map[string]string
wantRemoved []string
}{
{
name: "broken shipped entry is restored, custom kept",
stored: restoredStored,
want: restoredWant,
},
{
name: "broken custom entry is removed, edited shipped kept",
stored: editedStored,
want: editedWant,
wantRemoved: []string{"broken"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
value, err := json.Marshal(WorkflowDefinitionCollection{Definitions: tt.stored})
if err != nil {
t.Fatal(err)
}
got, removed, err := RepairDefinitionCollection(value, triageRegistry())
if err != nil {
t.Fatal(err)
}
byID := map[string]string{}
for _, entry := range got.Definitions {
byID[entry.ID] = entry.YAML
}
if len(byID) != len(tt.want) {
t.Fatalf("kept %d entries, want %d", len(byID), len(tt.want))
}
for id, yaml := range tt.want {
if byID[id] != yaml {
t.Errorf("entry %q not kept as expected", id)
}
}
if !slices.Equal(removed, tt.wantRemoved) {
t.Errorf("removed %v, want %v", removed, tt.wantRemoved)
}
})
}
}
@@ -221,7 +221,7 @@ func (s *Server) resource(ctx context.Context, orgID, kind string) (storecontrac
}
return resource, err
}
if _, err := s.seedKind(ctx, orgID, s.definitions[kind], config.Config{}); err != nil && !errors.Is(err, storecontract.ErrResourceVersionConflict) {
if _, _, err := s.seedKind(ctx, orgID, s.definitions[kind], config.Config{}); err != nil && !errors.Is(err, storecontract.ErrResourceVersionConflict) {
return storecontract.Resource{}, err
}
return s.store.Resource(ctx, orgID, kind)
@@ -28,6 +28,10 @@ type Definition struct {
// Defaults is the factory value clients offer as "restore shipped". Only
// resources that ship with content set it.
Defaults func() any
// Repair rebuilds a stored value this build no longer accepts, keeping what
// it can, and names what it removed. Nil means an invalid stored value of a
// shipped kind is replaced by the seed.
Repair func(stored []byte) (value []byte, removed []string, err error)
// SchemaExtensions are merged into the derived schema's top level, for
// vocabulary the document type alone cannot express.
SchemaExtensions map[string]any
@@ -73,6 +77,9 @@ func (d Definition) Decode(input []byte) ([]byte, error) {
type SeedSkip struct {
Kind string
Err error
// Removed names entries a repair dropped from a stored value; Err is nil
// then, because the kind was written.
Removed []string
}
// seedActor is the audit identity seeds write under.
@@ -89,7 +96,10 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin
versions := make(map[string]int64, len(s.ordered))
var skipped []SeedSkip
for _, definition := range s.ordered {
version, err := s.seedKind(ctx, storecontract.DefaultOrgID, definition, cfg)
version, removed, err := s.seedKind(ctx, storecontract.DefaultOrgID, definition, cfg)
if len(removed) > 0 {
skipped = append(skipped, SeedSkip{Kind: definition.Kind, Removed: removed})
}
var refusal *seedRefusal
switch {
case errors.As(err, &refusal):
@@ -108,9 +118,13 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin
if resource.OrgID == storecontract.DefaultOrgID || resource.Kind != WorkflowDefinitionsKind {
continue
}
if _, err := s.seedKind(ctx, resource.OrgID, s.definitions[resource.Kind], config.Config{}); err != nil {
_, removed, err := s.seedKind(ctx, resource.OrgID, s.definitions[resource.Kind], config.Config{})
if err != nil {
return nil, nil, err
}
if len(removed) > 0 {
skipped = append(skipped, SeedSkip{Kind: resource.Kind, Removed: removed})
}
}
return versions, skipped, nil
}
@@ -127,30 +141,32 @@ func (r *seedRefusal) Unwrap() error { return r.err }
// seedKind seeds, refreshes or leaves one kind and returns its version. A
// *seedRefusal means the seed failed validation and nothing was written.
func (s *Server) seedKind(ctx context.Context, orgID string, definition Definition, cfg config.Config) (int64, error) {
func (s *Server) seedKind(ctx context.Context, orgID string, definition Definition, cfg config.Config) (int64, []string, error) {
stored, storedErr := s.store.Resource(ctx, orgID, definition.Kind)
absent := errors.Is(storedErr, storecontract.ErrResourceNotFound)
if storedErr != nil && !absent {
return 0, storedErr
return 0, nil, storedErr
}
value, err := definition.seededValue(cfg)
if err != nil {
// Skipped, not fatal.
return 0, &seedRefusal{err: err}
return 0, nil, &seedRefusal{err: err}
}
if !absent && bytes.Equal(stored.Value, value) {
// The store already holds what this build seeds. Comparing the values
// rather than re-writing them is what keeps a boot from adding a
// revision for a document that did not change.
return stored.Version, nil
return stored.Version, nil, nil
}
var removed []string
if !absent {
refresh, err := s.shippedValueIsStale(ctx, orgID, definition)
var keep bool
value, removed, keep, err = s.replacement(ctx, orgID, definition, stored.Value, value)
if err != nil {
return 0, err
return 0, nil, err
}
if !refresh {
return stored.Version, nil
if keep {
return stored.Version, nil, nil
}
}
write := storecontract.ResourceWrite{
@@ -161,9 +177,29 @@ func (s *Server) seedKind(ctx context.Context, orgID string, definition Definiti
}
resource, err := s.store.PutResource(ctx, write)
if err != nil {
return 0, fmt.Errorf("seed %s: %w", definition.Kind, err)
return 0, nil, fmt.Errorf("seed %s: %w", definition.Kind, err)
}
return resource.Version, nil
return resource.Version, removed, nil
}
// replacement decides what a seed writes over a stored value: nothing (keep),
// a repair of it, or the seeded value. A failed repair falls back to the seed:
// an invalid stored value breaks every reader, so keeping it protects nothing.
func (s *Server) replacement(ctx context.Context, orgID string, definition Definition, stored, seeded []byte) ([]byte, []string, bool, error) {
replace, err := s.storedValueIsReplaced(ctx, orgID, definition, stored)
if err != nil {
return nil, nil, false, err
}
if !replace {
return nil, nil, true, nil
}
if definition.Repair == nil || definition.accepts(stored) {
return seeded, nil, false, nil
}
if repaired, removed, err := definition.Repair(stored); err == nil {
return repaired, removed, false, nil
}
return seeded, nil, false, nil
}
// expectedVersion is the optimistic-concurrency guard a seed writes under: the
@@ -175,15 +211,24 @@ func expectedVersion(stored storecontract.Resource, absent bool) int64 {
return stored.Version
}
// shippedValueIsStale reports whether the newest stored revision of a
// shipped document was written by the seed.
func (s *Server) shippedValueIsStale(ctx context.Context, orgID string, definition Definition) (bool, error) {
// storedValueIsReplaced reports whether a stored shipped document is rewritten
// at seed: the seed wrote it, or this build no longer accepts it.
func (s *Server) storedValueIsReplaced(ctx context.Context, orgID string, definition Definition, stored []byte) (bool, error) {
if definition.Defaults == nil {
return false, nil
}
if !definition.accepts(stored) {
return true, nil
}
return s.seedWroteNewest(ctx, orgID, definition.Kind)
}
// accepts reports whether input decodes under this build's validator.
func (d Definition) accepts(input []byte) bool {
_, err := d.Decode(input)
return err == nil
}
// seedWroteNewest reports whether the newest revision of kind was written by a
// seed. It reads the ledger rather than the current row because the ledger is
// what records who wrote a value, and the row does not.
@@ -0,0 +1,81 @@
package controlplanestore
import (
"context"
"errors"
"slices"
"testing"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
)
type memStore struct {
resource storecontract.Resource
history []storecontract.Resource
}
func (m *memStore) ListResources(context.Context) ([]storecontract.Resource, error) {
return []storecontract.Resource{m.resource}, nil
}
func (m *memStore) Resource(context.Context, string, string) (storecontract.Resource, error) {
return m.resource, nil
}
func (m *memStore) ResourceHistory(context.Context, string, string, int) ([]storecontract.Resource, error) {
return m.history, nil
}
func (m *memStore) PutResource(_ context.Context, w storecontract.ResourceWrite) (storecontract.Resource, error) {
m.resource = storecontract.Resource{OrgID: w.OrgID, Kind: w.Kind, Value: w.Value, Version: m.resource.Version + 1}
return m.resource, nil
}
// A shipped document an operator edited is kept, unless it no longer validates:
// then it is repaired, or replaced by the build's copy when it has no repair.
func TestSeedRefreshesStoredShippedDocumentOnlyWhenInvalid(t *testing.T) {
definition := Definition{
Kind: "shipped",
Seed: func(config.Config) any { return "current" },
Defaults: func() any { return "current" },
Validate: func(in []byte) error {
if string(in) != `"current"` && string(in) != `"edited"` {
return errors.New("names a deleted step type")
}
return nil
},
}
repair := func([]byte) ([]byte, []string, error) { return []byte(`"edited"`), []string{"gone"}, nil }
tests := []struct {
name string
stored string
repair func([]byte) ([]byte, []string, error)
want string
wantRemoved []string
}{
{"edited and valid is kept", `"edited"`, repair, `"edited"`, nil},
{"invalid is repaired", `"deleted-step"`, repair, `"edited"`, []string{"gone"}},
{"invalid without a repair is replaced", `"deleted-step"`, nil, `"current"`, nil},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
store := &memStore{
resource: storecontract.Resource{Kind: "shipped", Value: []byte(tt.stored), Version: 3},
history: []storecontract.Resource{{Actor: "operator", Source: "dashboard"}},
}
s := &Server{store: store}
definition.Repair = tt.repair
_, removed, err := s.seedKind(context.Background(), storecontract.DefaultOrgID, definition, config.Config{})
if err != nil {
t.Fatal(err)
}
if !slices.Equal(removed, tt.wantRemoved) {
t.Fatalf("removed %v, want %v", removed, tt.wantRemoved)
}
if got := string(store.resource.Value); got != tt.want {
t.Fatalf("stored %s, want %s", got, tt.want)
}
})
}
}
@@ -37,6 +37,14 @@ func workflowDefinitionsDefinition(steps workflow.StepRegistry, settings map[str
_, err := workflow.DecodeDefinitionCollection(input, steps)
return err
},
Repair: func(stored []byte) ([]byte, []string, error) {
collection, removed, err := workflow.RepairDefinitionCollection(stored, steps)
if err != nil {
return nil, nil, err
}
value, err := json.Marshal(collection)
return value, removed, err
},
SchemaExtensions: map[string]any{"x-step-types": stepTypeVocabulary(steps, settings), "x-intake-routes": workflow.IntakeRoutes()},
}
}