diff --git a/CHANGELOG.md b/CHANGELOG.md index f670a43e5..43e286184 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ release's per-component sections are labelled beneath its heading. - An image or voice attachment that arrives as a link is now checked against what the link serves, the same as an uploaded one. A link whose content is not the declared type, or that cannot be read, is refused. +### archie-state-store + +- A workflow that no longer validates after an upgrade (for example it names a step type that was removed) is repaired at startup instead of breaking every workflow. A shipped workflow is restored from this release's copy; a workflow you created is removed and logged, and its earlier revision stays in the resource history. Valid workflows you created or edited are kept. + ## [1.61.0] - 2026-11-07 ### archied diff --git a/internal/app/statestore/server.go b/internal/app/statestore/server.go index d1a0818c7..d70343cb2 100644 --- a/internal/app/statestore/server.go +++ b/internal/app/statestore/server.go @@ -202,6 +202,11 @@ func (b *server) seedAndValidatePolicies(ctx context.Context) error { // not to the sequence that triggers it. func (b *server) reportUnseededResources(skipped []controlplanestore.SeedSkip) { for _, skip := range skipped { + if len(skip.Removed) > 0 { + b.log.Warn("removed stored entries that no longer validate; their earlier revision is in the resource history", + "kind", skip.Kind, "removed", skip.Removed) + continue + } b.log.Error("control-plane resource not seeded; the kind stays absent and the file's value is the one in effect", "kind", skip.Kind, "err", skip.Err) } diff --git a/internal/domain/workflow/definition.go b/internal/domain/workflow/definition.go index 3dfc99f40..db87860cc 100644 --- a/internal/domain/workflow/definition.go +++ b/internal/domain/workflow/definition.go @@ -387,3 +387,37 @@ func hasStepType(node any, types []string) bool { } return false } + +// RepairDefinitionCollection rebuilds a stored collection that no longer +// validates, keeping every entry it can: a valid entry is kept, an invalid +// shipped entry is replaced by this build's copy, and any other invalid or +// duplicate entry is removed and its id returned. An unreadable document or a +// repair that still fails validation is an error. +func RepairDefinitionCollection(value []byte, registry StepRegistry) (WorkflowDefinitionCollection, []string, error) { + var stored WorkflowDefinitionCollection + if err := json.Unmarshal(value, &stored); err != nil { + return WorkflowDefinitionCollection{}, nil, err + } + shipped := ShippedDefinitions() + seen := make(map[string]struct{}, len(stored.Definitions)) + var repaired WorkflowDefinitionCollection + var removed []string + for _, entry := range stored.Definitions { + if _, dup := seen[entry.ID]; dup { + removed = append(removed, entry.ID) + continue + } + seen[entry.ID] = struct{}{} + if definition, err := ParseDefinition(entry.YAML, registry); err == nil && definition.ID == entry.ID { + repaired.Definitions = append(repaired.Definitions, entry) + } else if factory, ok := shipped.DefinitionByID(entry.ID); ok { + repaired.Definitions = append(repaired.Definitions, factory) + } else { + removed = append(removed, entry.ID) + } + } + if err := ValidateDefinitionCollection(repaired, registry); err != nil { + return WorkflowDefinitionCollection{}, nil, err + } + return repaired, removed, nil +} diff --git a/internal/domain/workflow/definition_repair_test.go b/internal/domain/workflow/definition_repair_test.go new file mode 100644 index 000000000..ee5fc74a2 --- /dev/null +++ b/internal/domain/workflow/definition_repair_test.go @@ -0,0 +1,82 @@ +package workflow + +import ( + "encoding/json" + "slices" + "strings" + "testing" +) + +// A stored collection that stopped validating keeps every entry it can: valid +// entries stay, a broken shipped entry is restored, a broken custom one goes. +func TestRepairDefinitionCollection(t *testing.T) { + shipped, _ := ShippedDefinitions().DefinitionByID("triage") + custom := strings.Replace(shipped.YAML, "id: triage", "id: mine", 1) + edited := shipped.YAML + "\n# edited by the operator\n" + broken := "id: broken\nsteps:\n - id: a\n type: removed.step\n" + // with is the shipped set with triage's YAML swapped and extra appended. + with := func(triage string, extra ...WorkflowDefinitionEntry) ([]WorkflowDefinitionEntry, map[string]string) { + entries, byID := []WorkflowDefinitionEntry{}, map[string]string{} + for _, entry := range ShippedDefinitions().Definitions { + if entry.ID == "triage" { + entry.YAML = triage + } + entries = append(entries, entry) + byID[entry.ID] = entry.YAML + } + for _, entry := range extra { + entries = append(entries, entry) + byID[entry.ID] = entry.YAML + } + return entries, byID + } + restoredStored, _ := with(broken, WorkflowDefinitionEntry{ID: "mine", YAML: custom}) + _, restoredWant := with(shipped.YAML, WorkflowDefinitionEntry{ID: "mine", YAML: custom}) + editedStored, _ := with(edited, WorkflowDefinitionEntry{ID: "broken", YAML: broken}) + _, editedWant := with(edited) + tests := []struct { + name string + stored []WorkflowDefinitionEntry + want map[string]string + wantRemoved []string + }{ + { + name: "broken shipped entry is restored, custom kept", + stored: restoredStored, + want: restoredWant, + }, + { + name: "broken custom entry is removed, edited shipped kept", + stored: editedStored, + want: editedWant, + wantRemoved: []string{"broken"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + value, err := json.Marshal(WorkflowDefinitionCollection{Definitions: tt.stored}) + if err != nil { + t.Fatal(err) + } + got, removed, err := RepairDefinitionCollection(value, triageRegistry()) + if err != nil { + t.Fatal(err) + } + byID := map[string]string{} + for _, entry := range got.Definitions { + byID[entry.ID] = entry.YAML + } + if len(byID) != len(tt.want) { + t.Fatalf("kept %d entries, want %d", len(byID), len(tt.want)) + } + for id, yaml := range tt.want { + if byID[id] != yaml { + t.Errorf("entry %q not kept as expected", id) + } + } + if !slices.Equal(removed, tt.wantRemoved) { + t.Errorf("removed %v, want %v", removed, tt.wantRemoved) + } + }) + } +} diff --git a/internal/infrastructure/controlplanestore/controlplane.go b/internal/infrastructure/controlplanestore/controlplane.go index 469afec48..897487f67 100644 --- a/internal/infrastructure/controlplanestore/controlplane.go +++ b/internal/infrastructure/controlplanestore/controlplane.go @@ -221,7 +221,7 @@ func (s *Server) resource(ctx context.Context, orgID, kind string) (storecontrac } return resource, err } - if _, err := s.seedKind(ctx, orgID, s.definitions[kind], config.Config{}); err != nil && !errors.Is(err, storecontract.ErrResourceVersionConflict) { + if _, _, err := s.seedKind(ctx, orgID, s.definitions[kind], config.Config{}); err != nil && !errors.Is(err, storecontract.ErrResourceVersionConflict) { return storecontract.Resource{}, err } return s.store.Resource(ctx, orgID, kind) diff --git a/internal/infrastructure/controlplanestore/registry.go b/internal/infrastructure/controlplanestore/registry.go index 4485ecd8a..10e98b4b8 100644 --- a/internal/infrastructure/controlplanestore/registry.go +++ b/internal/infrastructure/controlplanestore/registry.go @@ -28,6 +28,10 @@ type Definition struct { // Defaults is the factory value clients offer as "restore shipped". Only // resources that ship with content set it. Defaults func() any + // Repair rebuilds a stored value this build no longer accepts, keeping what + // it can, and names what it removed. Nil means an invalid stored value of a + // shipped kind is replaced by the seed. + Repair func(stored []byte) (value []byte, removed []string, err error) // SchemaExtensions are merged into the derived schema's top level, for // vocabulary the document type alone cannot express. SchemaExtensions map[string]any @@ -73,6 +77,9 @@ func (d Definition) Decode(input []byte) ([]byte, error) { type SeedSkip struct { Kind string Err error + // Removed names entries a repair dropped from a stored value; Err is nil + // then, because the kind was written. + Removed []string } // seedActor is the audit identity seeds write under. @@ -89,7 +96,10 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin versions := make(map[string]int64, len(s.ordered)) var skipped []SeedSkip for _, definition := range s.ordered { - version, err := s.seedKind(ctx, storecontract.DefaultOrgID, definition, cfg) + version, removed, err := s.seedKind(ctx, storecontract.DefaultOrgID, definition, cfg) + if len(removed) > 0 { + skipped = append(skipped, SeedSkip{Kind: definition.Kind, Removed: removed}) + } var refusal *seedRefusal switch { case errors.As(err, &refusal): @@ -108,9 +118,13 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin if resource.OrgID == storecontract.DefaultOrgID || resource.Kind != WorkflowDefinitionsKind { continue } - if _, err := s.seedKind(ctx, resource.OrgID, s.definitions[resource.Kind], config.Config{}); err != nil { + _, removed, err := s.seedKind(ctx, resource.OrgID, s.definitions[resource.Kind], config.Config{}) + if err != nil { return nil, nil, err } + if len(removed) > 0 { + skipped = append(skipped, SeedSkip{Kind: resource.Kind, Removed: removed}) + } } return versions, skipped, nil } @@ -127,30 +141,32 @@ func (r *seedRefusal) Unwrap() error { return r.err } // seedKind seeds, refreshes or leaves one kind and returns its version. A // *seedRefusal means the seed failed validation and nothing was written. -func (s *Server) seedKind(ctx context.Context, orgID string, definition Definition, cfg config.Config) (int64, error) { +func (s *Server) seedKind(ctx context.Context, orgID string, definition Definition, cfg config.Config) (int64, []string, error) { stored, storedErr := s.store.Resource(ctx, orgID, definition.Kind) absent := errors.Is(storedErr, storecontract.ErrResourceNotFound) if storedErr != nil && !absent { - return 0, storedErr + return 0, nil, storedErr } value, err := definition.seededValue(cfg) if err != nil { // Skipped, not fatal. - return 0, &seedRefusal{err: err} + return 0, nil, &seedRefusal{err: err} } if !absent && bytes.Equal(stored.Value, value) { // The store already holds what this build seeds. Comparing the values // rather than re-writing them is what keeps a boot from adding a // revision for a document that did not change. - return stored.Version, nil + return stored.Version, nil, nil } + var removed []string if !absent { - refresh, err := s.shippedValueIsStale(ctx, orgID, definition) + var keep bool + value, removed, keep, err = s.replacement(ctx, orgID, definition, stored.Value, value) if err != nil { - return 0, err + return 0, nil, err } - if !refresh { - return stored.Version, nil + if keep { + return stored.Version, nil, nil } } write := storecontract.ResourceWrite{ @@ -161,9 +177,29 @@ func (s *Server) seedKind(ctx context.Context, orgID string, definition Definiti } resource, err := s.store.PutResource(ctx, write) if err != nil { - return 0, fmt.Errorf("seed %s: %w", definition.Kind, err) + return 0, nil, fmt.Errorf("seed %s: %w", definition.Kind, err) } - return resource.Version, nil + return resource.Version, removed, nil +} + +// replacement decides what a seed writes over a stored value: nothing (keep), +// a repair of it, or the seeded value. A failed repair falls back to the seed: +// an invalid stored value breaks every reader, so keeping it protects nothing. +func (s *Server) replacement(ctx context.Context, orgID string, definition Definition, stored, seeded []byte) ([]byte, []string, bool, error) { + replace, err := s.storedValueIsReplaced(ctx, orgID, definition, stored) + if err != nil { + return nil, nil, false, err + } + if !replace { + return nil, nil, true, nil + } + if definition.Repair == nil || definition.accepts(stored) { + return seeded, nil, false, nil + } + if repaired, removed, err := definition.Repair(stored); err == nil { + return repaired, removed, false, nil + } + return seeded, nil, false, nil } // expectedVersion is the optimistic-concurrency guard a seed writes under: the @@ -175,15 +211,24 @@ func expectedVersion(stored storecontract.Resource, absent bool) int64 { return stored.Version } -// shippedValueIsStale reports whether the newest stored revision of a -// shipped document was written by the seed. -func (s *Server) shippedValueIsStale(ctx context.Context, orgID string, definition Definition) (bool, error) { +// storedValueIsReplaced reports whether a stored shipped document is rewritten +// at seed: the seed wrote it, or this build no longer accepts it. +func (s *Server) storedValueIsReplaced(ctx context.Context, orgID string, definition Definition, stored []byte) (bool, error) { if definition.Defaults == nil { return false, nil } + if !definition.accepts(stored) { + return true, nil + } return s.seedWroteNewest(ctx, orgID, definition.Kind) } +// accepts reports whether input decodes under this build's validator. +func (d Definition) accepts(input []byte) bool { + _, err := d.Decode(input) + return err == nil +} + // seedWroteNewest reports whether the newest revision of kind was written by a // seed. It reads the ledger rather than the current row because the ledger is // what records who wrote a value, and the row does not. diff --git a/internal/infrastructure/controlplanestore/seed_refresh_test.go b/internal/infrastructure/controlplanestore/seed_refresh_test.go new file mode 100644 index 000000000..a110d0899 --- /dev/null +++ b/internal/infrastructure/controlplanestore/seed_refresh_test.go @@ -0,0 +1,81 @@ +package controlplanestore + +import ( + "context" + "errors" + "slices" + "testing" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/storecontract" +) + +type memStore struct { + resource storecontract.Resource + history []storecontract.Resource +} + +func (m *memStore) ListResources(context.Context) ([]storecontract.Resource, error) { + return []storecontract.Resource{m.resource}, nil +} + +func (m *memStore) Resource(context.Context, string, string) (storecontract.Resource, error) { + return m.resource, nil +} + +func (m *memStore) ResourceHistory(context.Context, string, string, int) ([]storecontract.Resource, error) { + return m.history, nil +} + +func (m *memStore) PutResource(_ context.Context, w storecontract.ResourceWrite) (storecontract.Resource, error) { + m.resource = storecontract.Resource{OrgID: w.OrgID, Kind: w.Kind, Value: w.Value, Version: m.resource.Version + 1} + return m.resource, nil +} + +// A shipped document an operator edited is kept, unless it no longer validates: +// then it is repaired, or replaced by the build's copy when it has no repair. +func TestSeedRefreshesStoredShippedDocumentOnlyWhenInvalid(t *testing.T) { + definition := Definition{ + Kind: "shipped", + Seed: func(config.Config) any { return "current" }, + Defaults: func() any { return "current" }, + Validate: func(in []byte) error { + if string(in) != `"current"` && string(in) != `"edited"` { + return errors.New("names a deleted step type") + } + return nil + }, + } + repair := func([]byte) ([]byte, []string, error) { return []byte(`"edited"`), []string{"gone"}, nil } + tests := []struct { + name string + stored string + repair func([]byte) ([]byte, []string, error) + want string + wantRemoved []string + }{ + {"edited and valid is kept", `"edited"`, repair, `"edited"`, nil}, + {"invalid is repaired", `"deleted-step"`, repair, `"edited"`, []string{"gone"}}, + {"invalid without a repair is replaced", `"deleted-step"`, nil, `"current"`, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + store := &memStore{ + resource: storecontract.Resource{Kind: "shipped", Value: []byte(tt.stored), Version: 3}, + history: []storecontract.Resource{{Actor: "operator", Source: "dashboard"}}, + } + s := &Server{store: store} + definition.Repair = tt.repair + _, removed, err := s.seedKind(context.Background(), storecontract.DefaultOrgID, definition, config.Config{}) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(removed, tt.wantRemoved) { + t.Fatalf("removed %v, want %v", removed, tt.wantRemoved) + } + if got := string(store.resource.Value); got != tt.want { + t.Fatalf("stored %s, want %s", got, tt.want) + } + }) + } +} diff --git a/internal/infrastructure/controlplanestore/workflow_definitions.go b/internal/infrastructure/controlplanestore/workflow_definitions.go index 0f4130910..cc524b97e 100644 --- a/internal/infrastructure/controlplanestore/workflow_definitions.go +++ b/internal/infrastructure/controlplanestore/workflow_definitions.go @@ -37,6 +37,14 @@ func workflowDefinitionsDefinition(steps workflow.StepRegistry, settings map[str _, err := workflow.DecodeDefinitionCollection(input, steps) return err }, + Repair: func(stored []byte) ([]byte, []string, error) { + collection, removed, err := workflow.RepairDefinitionCollection(stored, steps) + if err != nil { + return nil, nil, err + } + value, err := json.Marshal(collection) + return value, removed, err + }, SchemaExtensions: map[string]any{"x-step-types": stepTypeVocabulary(steps, settings), "x-intake-routes": workflow.IntakeRoutes()}, } }