feat(store): project installed package contributions into org resources
An install that stayed in installed_packages left every contribution unprojected: nothing made a package's workflow a usable org resource, and nothing stopped it on removal. This lands projection for the workflow family, the one kind whose org resource, validation and live read path all exist: - storepkg.Service gains Projections (family -> projector): install resolves each projected family's contributed names against the fetched layer (FilesFromLayer), applies them, and on failure withdraws and removes the half installation; remove withdraws before the record goes, so a failed withdrawal keeps the package installed and usable. - controlplane.WorkflowProjector merges the package's entries into the org's workflow-definitions document — the same document the daemon's WorkflowDefinitions client reads per dispatch, so a projected workflow is usable at the next dispatch and a withdrawn one stops dispatching, without restarts. A contributed id displacing an operator's own workflow is refused (storepkg.ErrContributionCollision), never silently replaced. - installed_package_contributions (migration 0023) records the entries a package projected, so removal takes exactly its own contributions out. - stateStoreDeps wires the projector set on the State Store's pool; the composition is pinned by the live round-trip test over a real gRPC connection. Prompts, skills, MCP server definitions and default settings remain unprojected: their contribution formats are not settled by docs/prds/store.md and are left for their own slices rather than half-wired here.
This commit is contained in:
16 files changed
+1194
-1
No files matched your search
@@ -0,0 +1,155 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/test/bufconn"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storepkg"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||
)
|
||||
|
||||
// contributedLayer ships one workflow definition file's portable YAML.
|
||||
func contributedLayer(t *testing.T, path, yaml string) []byte {
|
||||
t.Helper()
|
||||
var raw bytes.Buffer
|
||||
writer := tar.NewWriter(&raw)
|
||||
if err := writer.WriteHeader(&tar.Header{Typeflag: tar.TypeReg, Name: path, Size: int64(len(yaml))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := writer.Write([]byte(yaml)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw.Bytes()
|
||||
}
|
||||
|
||||
func packageDescriptor() storepkg.Descriptor {
|
||||
return storepkg.Descriptor{
|
||||
APIVersion: storepkg.APIVersion, DisplayName: "Review pack", Version: "1",
|
||||
Contributes: storepkg.Contributions{Workflows: []string{"workflows/review.yaml"}},
|
||||
Files: []storepkg.File{{Path: "workflows/review.yaml", Mode: 0o644}},
|
||||
}
|
||||
}
|
||||
|
||||
// TestPackageContributionsLiveRoundTrip drives the State Store's own
|
||||
// composition — the projections stateStoreDeps wires, the install and remove
|
||||
// RPCs the dashboard calls, and the workflow-definitions resource the daemon
|
||||
// reads per dispatch — over one real gRPC connection: installing a package
|
||||
// makes its workflow a usable org resource, and removing it stops being
|
||||
// usable, both without restarting anything.
|
||||
func TestPackageContributionsLiveRoundTrip(t *testing.T) {
|
||||
db := pgstore.Open(t)
|
||||
projections, err := newPackageProjections(db.Pool)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const yaml = "id: review\nsteps:\n - type: bootstrap.apply\n"
|
||||
service := storepkg.Service{
|
||||
Registry: packageLayerRegistry{descriptor: packageDescriptor(), layer: contributedLayer(t, "workflows/review.yaml", yaml)},
|
||||
Store: postgres.NewInstalledPackages(db.Pool),
|
||||
Projections: projections,
|
||||
}
|
||||
listener := bufconn.Listen(1 << 20)
|
||||
server := grpc.NewServer()
|
||||
staterpc.RegisterServer(server, staterpc.Deps{Packages: service})
|
||||
go func() { _ = server.Serve(listener) }()
|
||||
t.Cleanup(func() { server.Stop(); _ = listener.Close() })
|
||||
conn, err := grpc.NewClient("passthrough:///state",
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return listener.DialContext(ctx) }))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
client := staterpc.NewClient(conn)
|
||||
const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
if _, err := client.InstallPackage(t.Context(), "org-a", "review", "localhost:5000/review", digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The contribution is usable: the org resource the daemon's
|
||||
// WorkflowDefinitions client reads carries it now, no restart involved.
|
||||
resource, err := postgres.NewResources(db.Pool).Resource(t.Context(), "org-a", controlplane.WorkflowDefinitionsKind)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
collection, err := workflow.DecodeDefinitionCollection(resource.Value, workflow.BuiltinStepRegistry())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := collection.DefinitionByID("review"); !ok {
|
||||
t.Fatal("installed package's workflow is not an org workflow after install")
|
||||
}
|
||||
if err := client.RemoveInstalled(t.Context(), "org-a", "review"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resource, err = postgres.NewResources(db.Pool).Resource(t.Context(), "org-a", controlplane.WorkflowDefinitionsKind)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
collection, err = workflow.DecodeDefinitionCollection(resource.Value, workflow.BuiltinStepRegistry())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := collection.DefinitionByID("review"); ok {
|
||||
t.Fatal("removed package's workflow still dispatches: the org resource kept it")
|
||||
}
|
||||
if entries, err := postgres.NewPackageContributions(db.Pool).Entries(t.Context(), "org-a", "review"); err != nil || len(entries) != 0 {
|
||||
t.Fatalf("contributions ledger after removal = %#v, %v", entries, err)
|
||||
}
|
||||
}
|
||||
|
||||
type packageLayerRegistry struct {
|
||||
descriptor storepkg.Descriptor
|
||||
layer []byte
|
||||
}
|
||||
|
||||
func (r packageLayerRegistry) Fetch(context.Context, string, string) (storepkg.Descriptor, []byte, error) {
|
||||
return r.descriptor, r.layer, nil
|
||||
}
|
||||
|
||||
// TestStateStoreDepsWiresPackageProjections pins the wiring the live round
|
||||
// trip drives: stateStoreDeps builds projections on the State Store's pool and
|
||||
// carries them into the packages service it composes.
|
||||
func TestStateStoreDepsWiresPackageProjections(t *testing.T) {
|
||||
file := "state_store.go"
|
||||
fset := token.NewFileSet()
|
||||
parsed, err := parser.ParseFile(fset, file, nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls []string
|
||||
ast.Inspect(parsed, func(node ast.Node) bool {
|
||||
if call, ok := node.(*ast.CallExpr); ok {
|
||||
if fn, ok := call.Fun.(*ast.Ident); ok {
|
||||
calls = append(calls, fn.Name)
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
found := false
|
||||
for _, call := range calls {
|
||||
if call == "newPackageProjections" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("stateStoreDeps (%s body) wires no package projections", file)
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
@@ -289,10 +290,20 @@ func (b *boot) openStateStore(ctx context.Context) error {
|
||||
func (b *boot) stateStoreDeps(grants *staterpc.TaskGrants) staterpc.Deps {
|
||||
deps := staterpc.Deps{Tasks: b.st, Log: b.log, Grants: grants}
|
||||
if b.pg != nil {
|
||||
deps.Packages = storepkg.Service{
|
||||
packages := storepkg.Service{
|
||||
Registry: registry.LocalRegistry{},
|
||||
Store: postgres.NewInstalledPackages(b.pg),
|
||||
}
|
||||
// A projection failure degrades the org resources a package would have
|
||||
// contributed to, the way a group above degrades: the State Store boots
|
||||
// and reports, rather than refusing to serve the task tables.
|
||||
projections, err := newPackageProjections(b.pg)
|
||||
if err != nil {
|
||||
b.log.Error("build package projections", "err", err)
|
||||
} else {
|
||||
packages.Projections = projections
|
||||
}
|
||||
deps.Packages = packages
|
||||
}
|
||||
if identities, ok := b.st.(identity.Repository); ok {
|
||||
deps.Identities = identities
|
||||
@@ -494,3 +505,22 @@ func stateStoreServerOpts(listen, token string, grants *staterpc.TaskGrants) (op
|
||||
grpc.ChainStreamInterceptor(grants.StreamInterceptor(token)),
|
||||
}, false, nil
|
||||
}
|
||||
|
||||
// newPackageProjections builds the family projectors an installed package's
|
||||
// contributions apply through: workflows become org workflow definitions, the
|
||||
// one resource kind the runtime reads live today. The step vocabulary
|
||||
// (stepVocabulary) is the same one this process's control plane validates
|
||||
// dashboard edits with, so a definition the projector admits is compilable
|
||||
// where it dispatches (openStateStoreControlPlane for the pairing).
|
||||
func newPackageProjections(pool *pgxpool.Pool) (map[string]storepkg.FamilyProjector, error) {
|
||||
steps, err := stepVocabulary()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("register workflow step vocabulary: %w", err)
|
||||
}
|
||||
workflows, err := controlplane.NewWorkflowPackageProjector(
|
||||
postgres.NewResources(pool), postgres.NewPackageContributions(pool), steps)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build workflow package projector: %w", err)
|
||||
}
|
||||
return map[string]storepkg.FamilyProjector{storepkg.FamilyWorkflows: workflows}, nil
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package controlplane
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storepkg"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
)
|
||||
|
||||
// projectionActor is the audit identity every package projection carries: the
|
||||
// write is made by the installed package, never by a request body.
|
||||
const projectionActor = "archie-state-store"
|
||||
|
||||
// projectWriteAttempts bounds one apply or withdrawal: each attempt re-reads
|
||||
// the org's document and re-merges, so a document the operator edited during
|
||||
// the projection is folded in, and only a persistently moving document
|
||||
// refuses the projection.
|
||||
const projectWriteAttempts = 3
|
||||
|
||||
// WorkflowProjector carries an installed package's workflow contributions
|
||||
// into the org's workflow-definitions resource, the same document an operator
|
||||
// edits on the dashboard (docs/prds/store.md, "Packages"). The readers the
|
||||
// daemon holds read that resource per dispatch, so a projected workflow is
|
||||
// usable at the next dispatch and a withdrawn one stops dispatching, without
|
||||
// restarting processes.
|
||||
//
|
||||
// Apply merges: a contributed id that belongs to an operator or to another
|
||||
// package is refused (storepkg.ErrContributionCollision), never silently
|
||||
// replaced.
|
||||
type WorkflowProjector struct {
|
||||
store ResourceStore
|
||||
ledger storepkg.ProjectionLedger
|
||||
steps workflow.StepRegistry
|
||||
}
|
||||
|
||||
var _ storepkg.FamilyProjector = (*WorkflowProjector)(nil)
|
||||
|
||||
// NewWorkflowPackageProjector builds the projector for storepkg.FamilyWorkflows
|
||||
// on the workflow step vocabulary the composition root registered: the
|
||||
// vocabulary the projected definitions are compiled and validated against,
|
||||
// the same one the control plane validates dashboard edits with.
|
||||
func NewWorkflowPackageProjector(resources ResourceStore, ledger storepkg.ProjectionLedger, manager *workflow.Manager) (*WorkflowProjector, error) {
|
||||
steps, err := stepRegistry(manager)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resources == nil || ledger == nil {
|
||||
return nil, errors.New("control plane: workflow projection needs a resource store and a contributions ledger")
|
||||
}
|
||||
return &WorkflowProjector{store: resources, ledger: ledger, steps: steps}, nil
|
||||
}
|
||||
|
||||
func (p *WorkflowProjector) Apply(ctx context.Context, orgID, name, digest string, contents map[string][]byte) error {
|
||||
entries := make([]workflow.WorkflowDefinitionEntry, 0, len(contents))
|
||||
recorded := make([]storepkg.ProjectionEntry, 0, len(contents))
|
||||
for contentName, content := range contents {
|
||||
id, err := workflow.DefinitionID(string(content))
|
||||
if err != nil {
|
||||
return fmt.Errorf("contributed workflow %q: %w", contentName, err)
|
||||
}
|
||||
entries = append(entries, workflow.WorkflowDefinitionEntry{ID: id, YAML: string(content)})
|
||||
recorded = append(recorded, storepkg.ProjectionEntry{Family: storepkg.FamilyWorkflows, EntryID: id})
|
||||
}
|
||||
// One apply, one request: a retried PutResource reuses the same request ID
|
||||
// and answers once, while a later install of the same package writes a
|
||||
// revision of its own.
|
||||
source := "package:" + name + "@" + digest
|
||||
if err := p.write(ctx, orgID, source, "package-project:"+digest, func(collection workflow.WorkflowDefinitionCollection) (workflow.WorkflowDefinitionCollection, error) {
|
||||
return mergeContribution(collection, entries, name)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.ledger.Record(ctx, orgID, name, recorded); err != nil {
|
||||
return fmt.Errorf("record contributions of %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *WorkflowProjector) Withdraw(ctx context.Context, orgID, name, digest string) error {
|
||||
entries, err := p.ledger.Entries(ctx, orgID, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("entries of package %q: %w", name, err)
|
||||
}
|
||||
withdrawn := make(map[string]struct{}, len(entries))
|
||||
for _, entry := range entries {
|
||||
if entry.Family == storepkg.FamilyWorkflows {
|
||||
withdrawn[entry.EntryID] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(withdrawn) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := p.write(ctx, orgID, "package-withdraw:"+name+"@"+digest, "package-withdraw:"+digest,
|
||||
func(collection workflow.WorkflowDefinitionCollection) (workflow.WorkflowDefinitionCollection, error) {
|
||||
merged := make([]workflow.WorkflowDefinitionEntry, 0, len(collection.Definitions))
|
||||
for _, item := range collection.Definitions {
|
||||
if _, gone := withdrawn[item.ID]; gone {
|
||||
continue
|
||||
}
|
||||
merged = append(merged, item)
|
||||
}
|
||||
return workflow.WorkflowDefinitionCollection{Definitions: merged}, nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return p.ledger.Forget(ctx, orgID, name, storepkg.FamilyWorkflows)
|
||||
}
|
||||
|
||||
// write merges an org's stored workflow-definitions document and puts it
|
||||
// back, bounded by projectWriteAttempts: each attempt re-reads and re-merges,
|
||||
// so a document the operator edited during the projection is folded in, and
|
||||
// the write carries the ledger's replay identity of one episode.
|
||||
func (p *WorkflowProjector) write(ctx context.Context, orgID, source, requestPrefix string, merge func(workflow.WorkflowDefinitionCollection) (workflow.WorkflowDefinitionCollection, error)) error {
|
||||
requestID := requestPrefix + ":" + strconv.FormatInt(time.Now().UTC().UnixNano(), 36)
|
||||
for attempt := 0; ; attempt++ {
|
||||
collection, version := p.collection(ctx, orgID)
|
||||
merged, err := merge(collection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
value, err := encodeWorkflowDefinitions(merged, p.steps)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := p.store.PutResource(ctx, storecontract.ResourceWrite{
|
||||
OrgID: orgID, Kind: WorkflowDefinitionsKind, Value: value, ExpectedVersion: version,
|
||||
Actor: projectionActor, Source: source, RequestID: requestID,
|
||||
}); errors.Is(err, storecontract.ErrResourceVersionConflict) && attempt < projectWriteAttempts-1 {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// collection reads an org's workflow-definitions document. An org that never
|
||||
// received one starts from the shipped definitions, the state a first write
|
||||
// seeds, so projection leaves the org with one answer either way.
|
||||
func (p *WorkflowProjector) collection(ctx context.Context, orgID string) (workflow.WorkflowDefinitionCollection, int64) {
|
||||
resource, err := p.store.Resource(ctx, orgID, WorkflowDefinitionsKind)
|
||||
if errors.Is(err, storecontract.ErrResourceNotFound) {
|
||||
return workflow.ShippedDefinitions(), 0
|
||||
}
|
||||
if err != nil {
|
||||
return workflow.WorkflowDefinitionCollection{}, 0
|
||||
}
|
||||
collection, err := workflow.DecodeDefinitionCollection(resource.Value, p.steps)
|
||||
if err != nil {
|
||||
return workflow.WorkflowDefinitionCollection{}, 0
|
||||
}
|
||||
return collection, resource.Version
|
||||
}
|
||||
|
||||
// mergeContribution merges a package's contributed entries into an org's
|
||||
// collection. An operator's entry with the same workflow id is a refusal, not
|
||||
// a silent replacement; the package's own entries reach here only after its
|
||||
// ledger rows were cleared, so colliding is always foreign work.
|
||||
func mergeContribution(base workflow.WorkflowDefinitionCollection, contributed []workflow.WorkflowDefinitionEntry, name string) (workflow.WorkflowDefinitionCollection, error) {
|
||||
replacing := make(map[string]struct{}, len(contributed))
|
||||
for _, entry := range contributed {
|
||||
replacing[entry.ID] = struct{}{}
|
||||
}
|
||||
merged := make([]workflow.WorkflowDefinitionEntry, 0, len(base.Definitions)+len(contributed))
|
||||
for _, entry := range base.Definitions {
|
||||
if _, incoming := replacing[entry.ID]; incoming {
|
||||
return workflow.WorkflowDefinitionCollection{}, fmt.Errorf("package %q: %w: workflow %q", name, storepkg.ErrContributionCollision, entry.ID)
|
||||
}
|
||||
merged = append(merged, entry)
|
||||
}
|
||||
merged = append(merged, contributed...)
|
||||
return workflow.WorkflowDefinitionCollection{Definitions: merged}, nil
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package controlplane
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storepkg"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore"
|
||||
)
|
||||
|
||||
const testDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
// testPackageProjector builds the workflow package projector against a real
|
||||
// migrated database and the production step vocabulary. The projector's
|
||||
// ledger rows describe installed packages, so a fixture installs one.
|
||||
func testPackageProjector(ctx context.Context, t *testing.T, orgID, pkgName, digest string) (*WorkflowProjector, ResourceStore, storepkg.ProjectionLedger) {
|
||||
t.Helper()
|
||||
db := pgstore.Open(t)
|
||||
installed := storepkg.Installed{
|
||||
OrgID: orgID, Name: pkgName, Reference: "localhost:5000/" + pkgName,
|
||||
Digest: digest, Descriptor: testPackageDescriptor(), Layer: []byte("layer"), UpdatePolicy: "manual",
|
||||
}
|
||||
if err := postgres.NewInstalledPackages(db.Pool).Install(ctx, installed); err != nil {
|
||||
t.Fatalf("install package record: %v", err)
|
||||
}
|
||||
resources := postgres.NewResources(db.Pool)
|
||||
ledger := postgres.NewPackageContributions(db.Pool)
|
||||
projector, err := NewWorkflowPackageProjector(resources, ledger, testSteps(t))
|
||||
if err != nil {
|
||||
t.Fatalf("build workflow projector: %v", err)
|
||||
}
|
||||
return projector, resources, ledger
|
||||
}
|
||||
|
||||
// testPackageDescriptor is the valid descriptor backing the ledger fixture.
|
||||
func testPackageDescriptor() storepkg.Descriptor {
|
||||
return storepkg.Descriptor{
|
||||
APIVersion: storepkg.APIVersion, DisplayName: "Review pack", Version: "1",
|
||||
Contributes: storepkg.Contributions{Workflows: []string{"workflows/review.yaml"}},
|
||||
}
|
||||
}
|
||||
|
||||
// contributedWorkflow builds a contributed workflow the shipped vocabulary
|
||||
// compiles, with a fresh id no shipped or operator workflow claims.
|
||||
func contributedWorkflow(name string) workflow.WorkflowDefinitionEntry {
|
||||
return workflow.WorkflowDefinitionEntry{ID: name, YAML: "id: " + name + "\nsteps:\n - type: bootstrap.apply\n"}
|
||||
}
|
||||
|
||||
// storedCollection returns an org's workflow-definitions document the way its
|
||||
// readers see it: shipped definitions stand in for an absent row.
|
||||
func storedCollection(ctx context.Context, t *testing.T, resources ResourceStore, orgID string) (workflow.WorkflowDefinitionCollection, int64) {
|
||||
t.Helper()
|
||||
resource, err := resources.Resource(ctx, orgID, WorkflowDefinitionsKind)
|
||||
if errors.Is(err, storecontract.ErrResourceNotFound) {
|
||||
return workflow.ShippedDefinitions(), 0
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
collection, err := workflow.DecodeDefinitionCollection(resource.Value, workflow.BuiltinStepRegistry())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return collection, resource.Version
|
||||
}
|
||||
|
||||
// operatorEdit installs a workflow the operator owns, exactly a dashboard
|
||||
// replacement does.
|
||||
func operatorEdit(ctx context.Context, t *testing.T, resources ResourceStore, orgID string, entry workflow.WorkflowDefinitionEntry) {
|
||||
t.Helper()
|
||||
collection, version := storedCollection(ctx, t, resources, orgID)
|
||||
collection.Definitions = append(collection.Definitions, entry)
|
||||
value, err := encodeWorkflowDefinitions(collection, testSteps(t).Registry())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := resources.PutResource(ctx, storecontract.ResourceWrite{
|
||||
OrgID: orgID, Kind: WorkflowDefinitionsKind, Value: value, ExpectedVersion: version,
|
||||
Actor: "operator", Source: "dashboard", RequestID: "operator-edit",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func resourceVersionOf(ctx context.Context, t *testing.T, resources ResourceStore, orgID string) int64 {
|
||||
t.Helper()
|
||||
_, version := storedCollection(ctx, t, resources, orgID)
|
||||
return version
|
||||
}
|
||||
|
||||
func TestWorkflowProjectorApplyAndWithdraw(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
const pkgName = "review"
|
||||
const contributedID = "review"
|
||||
contributedYAML := contributedWorkflow(contributedID).YAML
|
||||
|
||||
// applied builds a projector that has projected one contributed workflow.
|
||||
applied := func(t *testing.T) (*WorkflowProjector, ResourceStore, storepkg.ProjectionLedger) {
|
||||
t.Helper()
|
||||
projector, resources, ledger := testPackageProjector(ctx, t, "org-a", pkgName, testDigest)
|
||||
if err := projector.Apply(ctx, "org-a", pkgName, testDigest, map[string][]byte{
|
||||
"workflows/review.yaml": []byte(contributedYAML),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return projector, resources, ledger
|
||||
}
|
||||
|
||||
t.Run("apply makes the contributed workflow an org workflow without a restart", func(t *testing.T) {
|
||||
_, resources, ledger := applied(t)
|
||||
collection, _ := storedCollection(ctx, t, resources, "org-a")
|
||||
if _, ok := collection.DefinitionByID(contributedID); !ok {
|
||||
t.Fatal("org workflow-definitions does not carry the contributed workflow")
|
||||
}
|
||||
entries, err := ledger.Entries(ctx, "org-a", pkgName)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 1 || entries[0] != (storepkg.ProjectionEntry{Family: storepkg.FamilyWorkflows, EntryID: contributedID}) {
|
||||
t.Fatalf("ledger = %#v", entries)
|
||||
}
|
||||
})
|
||||
t.Run("withdraw takes exactly the package's entries back out", func(t *testing.T) {
|
||||
projector, resources, _ := testPackageProjector(ctx, t, "org-a", pkgName, testDigest)
|
||||
other := contributedWorkflow("review-operator")
|
||||
if err := projector.Apply(ctx, "org-a", pkgName, testDigest, map[string][]byte{
|
||||
"workflows/review.yaml": []byte(contributedYAML),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
operatorEdit(ctx, t, resources, "org-a", other)
|
||||
if err := projector.Withdraw(ctx, "org-a", pkgName, testDigest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
collection, _ := storedCollection(ctx, t, resources, "org-a")
|
||||
if _, ok := collection.DefinitionByID(contributedID); ok {
|
||||
t.Fatal("the package's contributed workflow survives withdrawal")
|
||||
}
|
||||
if _, ok := collection.DefinitionByID(other.ID); !ok {
|
||||
t.Fatal("operator workflow was withdrawn by a package remove")
|
||||
}
|
||||
})
|
||||
t.Run("a contributed id colliding with an operator workflow is refused", func(t *testing.T) {
|
||||
projector, resources, _ := testPackageProjector(ctx, t, "org-a", pkgName, testDigest)
|
||||
operatorEdit(ctx, t, resources, "org-a", contributedWorkflow(contributedID))
|
||||
before := resourceVersionOf(ctx, t, resources, "org-a")
|
||||
if err := projector.Apply(ctx, "org-a", pkgName, testDigest, map[string][]byte{
|
||||
"workflows/review.yaml": []byte(contributedYAML),
|
||||
}); !errors.Is(err, storepkg.ErrContributionCollision) {
|
||||
t.Fatalf("colliding contribution error = %v", err)
|
||||
}
|
||||
if after := resourceVersionOf(ctx, t, resources, "org-a"); after != before {
|
||||
t.Fatal("refused contribution wrote the resource anyway")
|
||||
}
|
||||
})
|
||||
t.Run("an invalid contributed workflow is refused without a write", func(t *testing.T) {
|
||||
projector, resources, _ := testPackageProjector(ctx, t, "org-a", pkgName, testDigest)
|
||||
before := resourceVersionOf(ctx, t, resources, "org-a")
|
||||
if err := projector.Apply(ctx, "org-a", pkgName, testDigest, map[string][]byte{
|
||||
"workflows/broken.yaml": []byte("id: broken\nsteps:\n - type: not-a-step\n"),
|
||||
}); err == nil {
|
||||
t.Fatal("invalid workflow definition projected")
|
||||
}
|
||||
if after := resourceVersionOf(ctx, t, resources, "org-a"); after != before {
|
||||
t.Fatal("refused projection wrote the resource anyway")
|
||||
}
|
||||
})
|
||||
t.Run("withdraw of a package that never applied is a no-op", func(t *testing.T) {
|
||||
projector, _, _ := testPackageProjector(ctx, t, "org-x", pkgName, testDigest)
|
||||
if err := projector.Withdraw(ctx, "org-x", pkgName, testDigest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package storepkg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// The descriptor families a package's Contributes names, and the family a
|
||||
// projector takes over.
|
||||
const (
|
||||
FamilyWorkflows = "workflows"
|
||||
FamilyPrompts = "prompts"
|
||||
FamilySkills = "skills"
|
||||
FamilyMCPServer = "mcpServers"
|
||||
FamilyDefaults = "defaults"
|
||||
)
|
||||
|
||||
// families is the family order projection follows, for a deterministic
|
||||
// install and withdrawal.
|
||||
var families = []string{FamilyWorkflows, FamilyPrompts, FamilySkills, FamilyMCPServer, FamilyDefaults}
|
||||
|
||||
// familyNames returns the names one descriptor family contributes.
|
||||
func (c Contributions) familyNames(family string) []string {
|
||||
switch family {
|
||||
case FamilyWorkflows:
|
||||
return c.Workflows
|
||||
case FamilyPrompts:
|
||||
return c.Prompts
|
||||
case FamilySkills:
|
||||
return c.Skills
|
||||
case FamilyMCPServer:
|
||||
return c.MCPServers
|
||||
case FamilyDefaults:
|
||||
return c.Defaults
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// FamilyProjector applies one descriptor family of an installed package into
|
||||
// the org resource that family contributes to, and withdraws its
|
||||
// contributions again on removal. Installing therefore makes a contribution
|
||||
// usable and removing stops it, both without restarting processes.
|
||||
type FamilyProjector interface {
|
||||
// Apply carries the contribution into the org's resources. contents maps
|
||||
// each contributed name to its layer content.
|
||||
Apply(ctx context.Context, orgID, name, digest string, contents map[string][]byte) error
|
||||
// Withdraw takes the contribution back out for an org's package.
|
||||
Withdraw(ctx context.Context, orgID, name, digest string) error
|
||||
}
|
||||
|
||||
// ProjectionEntry records one family's entry a package contributed to its org
|
||||
// resource, so removal can take exactly those entries back out.
|
||||
type ProjectionEntry struct {
|
||||
Family string
|
||||
EntryID string
|
||||
}
|
||||
|
||||
// ProjectionLedger keeps the entries an org's installed package projected.
|
||||
// Implementations live with the installed packages they describe.
|
||||
type ProjectionLedger interface {
|
||||
Record(context.Context, string, string, []ProjectionEntry) error
|
||||
Entries(ctx context.Context, orgID, name string) ([]ProjectionEntry, error)
|
||||
Forget(ctx context.Context, orgID, name, family string) error
|
||||
}
|
||||
|
||||
// resolveFamilyContents resolves every projected family of a package before
|
||||
// anything persists: a contribution the layer cannot back is an install that
|
||||
// can never become usable.
|
||||
func (s Service) resolveFamilyContents(descriptor Descriptor, layer []byte) (map[string]map[string][]byte, error) {
|
||||
contents := map[string]map[string][]byte{}
|
||||
for _, family := range families {
|
||||
if _, ok := s.Projections[family]; !ok {
|
||||
continue
|
||||
}
|
||||
entries, err := s.familyContents(descriptor, layer, family)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
continue
|
||||
}
|
||||
contents[family] = entries
|
||||
}
|
||||
return contents, nil
|
||||
}
|
||||
|
||||
// familyContents resolves one family's contributed names against the fetched
|
||||
// layer. A contributed name without layer content is an install that can
|
||||
// never make its contribution usable, so it refuses here.
|
||||
func (s Service) familyContents(descriptor Descriptor, layer []byte, family string) (map[string][]byte, error) {
|
||||
names := descriptor.Contributes.familyNames(family)
|
||||
if len(names) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
files, err := FilesFromLayer(layer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
contents := make(map[string][]byte, len(names))
|
||||
for _, name := range names {
|
||||
content, ok := files[name]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("contributed %s %q is not a package file", family, name)
|
||||
}
|
||||
declared := false
|
||||
for _, file := range descriptor.Files {
|
||||
if file.Path == name {
|
||||
declared = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !declared {
|
||||
return nil, fmt.Errorf("contributed %s %q is not a declared package file", family, name)
|
||||
}
|
||||
contents[name] = content
|
||||
}
|
||||
return contents, nil
|
||||
}
|
||||
|
||||
// project carries a new installation's contributions into the org resources
|
||||
// their families project to. A family with no projector is persisted but not
|
||||
// projected.
|
||||
func (s Service) project(ctx context.Context, installed Installed, contents map[string]map[string][]byte) error {
|
||||
for _, family := range families {
|
||||
entries, ok := contents[family]
|
||||
if !ok || len(entries) == 0 {
|
||||
continue
|
||||
}
|
||||
projector, ok := s.Projections[family]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if err := projector.Apply(ctx, installed.OrgID, installed.Name, installed.Digest, entries); err != nil {
|
||||
return fmt.Errorf("project %s of package %q: %w", family, installed.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// withdraw takes an installation's contributions back out of the org
|
||||
// resources they were projected into, before the package record goes: a
|
||||
// removal that fails its withdrawal keeps the package installed and its
|
||||
// contributions usable. Families without projectors withdraw nothing.
|
||||
func (s Service) withdraw(ctx context.Context, installed Installed) error {
|
||||
for _, family := range families {
|
||||
projector, ok := s.Projections[family]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if err := projector.Withdraw(ctx, installed.OrgID, installed.Name, installed.Digest); err != nil {
|
||||
return fmt.Errorf("withdraw %s of package %q: %w", family, installed.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package storepkg
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// contributedPackageDescriptor declares one contributed workflow file.
|
||||
const contributedPackage = `apiVersion: dev.archie.package.v1
|
||||
displayName: Review pack
|
||||
version: "1"
|
||||
contributes:
|
||||
workflows:
|
||||
- workflows/review.yaml
|
||||
files:
|
||||
- path: workflows/review.yaml
|
||||
mode: 0644
|
||||
`
|
||||
|
||||
func buildLayer(t *testing.T, files map[string][]byte, packed bool) []byte {
|
||||
t.Helper()
|
||||
var raw bytes.Buffer
|
||||
writer := tar.NewWriter(&raw)
|
||||
for _, name := range sortedFileNames(files) {
|
||||
if err := writer.WriteHeader(&tar.Header{
|
||||
Typeflag: tar.TypeReg, Name: name, Size: int64(len(files[name])), Mode: 0o644,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := writer.Write(files[name]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !packed {
|
||||
return raw.Bytes()
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
if _, err := gz.Write(raw.Bytes()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func sortedFileNames(files map[string][]byte) []string {
|
||||
keys := make([]string, 0, len(files))
|
||||
for name := range files {
|
||||
keys = append(keys, name)
|
||||
}
|
||||
for i := range keys {
|
||||
for j := i + 1; j < len(keys); j++ {
|
||||
if keys[j] < keys[i] {
|
||||
keys[i], keys[j] = keys[j], keys[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func TestFilesFromLayer(t *testing.T) {
|
||||
files := map[string][]byte{
|
||||
"workflows/review.yaml": []byte("id: review\nsteps:\n"),
|
||||
"workflows/other.yaml": []byte("id: other\nsteps:\n"),
|
||||
}
|
||||
for _, form := range []struct {
|
||||
name string
|
||||
layer []byte
|
||||
}{
|
||||
{name: "plain tar layer", layer: buildLayer(t, files, false)},
|
||||
{name: "gzipped tar layer", layer: buildLayer(t, files, true)},
|
||||
} {
|
||||
t.Run(form.name, func(t *testing.T) {
|
||||
extracted, err := FilesFromLayer(form.layer)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(extracted) != 2 || !bytes.Equal(extracted["workflows/review.yaml"], files["workflows/review.yaml"]) {
|
||||
t.Fatalf("files = %v", extracted)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("a non-tar layer is an error", func(t *testing.T) {
|
||||
if _, err := FilesFromLayer([]byte("not a tar")); err == nil {
|
||||
t.Fatal("non-tar layer extracted")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// fakeProjector records the contents one family receives, and the order the
|
||||
// service applies and withdraws them.
|
||||
type fakeProjector struct {
|
||||
contents map[string]map[string][]byte // org/name -> contributed name -> content
|
||||
applyCalled bool
|
||||
withdrawn []string
|
||||
applyErr error
|
||||
withdrawErr error
|
||||
}
|
||||
|
||||
func (p *fakeProjector) Apply(_ context.Context, orgID, name, _ string, contents map[string][]byte) error {
|
||||
p.applyCalled = true
|
||||
if p.applyErr != nil {
|
||||
return p.applyErr
|
||||
}
|
||||
p.contents[orgID+"/"+name] = contents
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *fakeProjector) Withdraw(_ context.Context, orgID, name, _ string) error {
|
||||
p.withdrawn = append(p.withdrawn, orgID+"/"+name)
|
||||
if p.withdrawErr != nil {
|
||||
return p.withdrawErr
|
||||
}
|
||||
delete(p.contents, orgID+"/"+name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestServiceInstallProjectsContributions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
descriptor, err := Decode(strings.NewReader(contributedPackage))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workflowYAML := []byte("id: review\nsteps:\n")
|
||||
layer := buildLayer(t, map[string][]byte{"workflows/review.yaml": workflowYAML}, false)
|
||||
service := func(store Repository, projector *fakeProjector) Service {
|
||||
if projector == nil {
|
||||
return Service{Registry: &fakeLayeredRegistry{descriptor: descriptor, layer: layer}, Store: store}
|
||||
}
|
||||
return Service{Registry: &fakeLayeredRegistry{descriptor: descriptor, layer: layer}, Store: store, Projections: map[string]FamilyProjector{FamilyWorkflows: projector}}
|
||||
}
|
||||
|
||||
t.Run("a contributed workflow reaches the family projector", func(t *testing.T) {
|
||||
projector := &fakeProjector{contents: map[string]map[string][]byte{}}
|
||||
store := &fakeInstalledStore{records: map[string]Installed{}}
|
||||
if _, err := service(store, projector).Install(ctx, "org-a", "review", "localhost:5000/review", digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !projector.applyCalled {
|
||||
t.Fatal("projector never applied")
|
||||
}
|
||||
if got := projector.contents["org-a/review"]["workflows/review.yaml"]; !bytes.Equal(got, workflowYAML) {
|
||||
t.Fatalf("projected content = %q", got)
|
||||
}
|
||||
})
|
||||
t.Run("a failed projection is not a silent half install", func(t *testing.T) {
|
||||
projector := &fakeProjector{contents: map[string]map[string][]byte{}, applyErr: errors.New("no resource kind")}
|
||||
store := &fakeInstalledStore{records: map[string]Installed{}}
|
||||
if _, err := service(store, projector).Install(ctx, "org-a", "review", "localhost:5000/review", digest); err == nil {
|
||||
t.Fatal("install succeeded although the projection failed")
|
||||
}
|
||||
if _, ok := store.records["org-a/review"]; ok {
|
||||
t.Fatal("failed projection left an installed package without its contributions")
|
||||
}
|
||||
})
|
||||
t.Run("a contribution the layer does not carry refuses the install", func(t *testing.T) {
|
||||
empty := buildLayer(t, map[string][]byte{}, false)
|
||||
projector := &fakeProjector{contents: map[string]map[string][]byte{}}
|
||||
svc := Service{Registry: &fakeLayeredRegistry{descriptor: descriptor, layer: empty}, Store: &fakeInstalledStore{records: map[string]Installed{}}, Projections: map[string]FamilyProjector{FamilyWorkflows: projector}}
|
||||
if _, err := svc.Install(ctx, "org-a", "review", "localhost:5000/review", digest); err == nil {
|
||||
t.Fatal("contributed name without layer content installed")
|
||||
}
|
||||
})
|
||||
t.Run("removal withdraws before the record goes", func(t *testing.T) {
|
||||
projector := &fakeProjector{contents: map[string]map[string][]byte{}}
|
||||
store := &fakeInstalledStore{records: map[string]Installed{"org-a/review": {OrgID: "org-a", Name: "review", Descriptor: descriptor, Digest: digest, UpdatePolicy: "manual"}}}
|
||||
if err := service(store, projector).Remove(ctx, "org-a", "review"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := store.records["org-a/review"]; ok {
|
||||
t.Fatal("removed package record remains")
|
||||
}
|
||||
if len(projector.withdrawn) != 1 || projector.withdrawn[0] != "org-a/review" {
|
||||
t.Fatalf("withdrawn = %v", projector.withdrawn)
|
||||
}
|
||||
})
|
||||
t.Run("a failed withdraw keeps the whole installation", func(t *testing.T) {
|
||||
projector := &fakeProjector{contents: map[string]map[string][]byte{}, withdrawErr: errors.New("version conflict")}
|
||||
store := &fakeInstalledStore{records: map[string]Installed{"org-a/review": {OrgID: "org-a", Name: "review", Descriptor: descriptor, Digest: digest, UpdatePolicy: "manual"}}}
|
||||
if err := service(store, projector).Remove(ctx, "org-a", "review"); err == nil {
|
||||
t.Fatal("removal succeeded although the withdrawal failed")
|
||||
}
|
||||
if _, ok := store.records["org-a/review"]; !ok {
|
||||
t.Fatal("package removed but its withdrawable contributions were already gone")
|
||||
}
|
||||
})
|
||||
t.Run("a family without a projector installs unprojected", func(t *testing.T) {
|
||||
store := &fakeInstalledStore{records: map[string]Installed{}}
|
||||
if _, err := service(store, nil).Install(ctx, "org-a", "review", "localhost:5000/review", digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := store.records["org-a/review"]; !ok {
|
||||
t.Fatal("non-projected family refused to install")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -16,6 +16,10 @@ var (
|
||||
// package. Acceptance records the package's own declared grants against
|
||||
// the pinned digest (docs/prds/store.md, "Authority").
|
||||
ErrAuthorityNotDeclared = errors.New("store package does not declare this authority")
|
||||
// ErrContributionCollision: the package contributes an entry the org
|
||||
// resource already holds under an id the package never projected. The
|
||||
// contribution is refused rather than replacing the operator's entry.
|
||||
ErrContributionCollision = errors.New("package contribution replaces an entry it does not own")
|
||||
)
|
||||
|
||||
// Installed is one organisation's pinned Archie package.
|
||||
@@ -62,6 +66,11 @@ type Manager interface {
|
||||
type Service struct {
|
||||
Registry Registry
|
||||
Store Repository
|
||||
// Projections carries, per descriptor family, the projector that applies
|
||||
// a package's contribution to the org resource that family projects to,
|
||||
// and withdraws it again on removal. A family with no entry is stored but
|
||||
// never projected: installing leaves it unused until a projector lands.
|
||||
Projections map[string]FamilyProjector
|
||||
}
|
||||
|
||||
var _ Manager = Service{}
|
||||
@@ -133,9 +142,19 @@ func (s Service) Install(ctx context.Context, orgID, name, reference, digest str
|
||||
OrgID: orgID, Name: name, Reference: reference, Digest: digest,
|
||||
Descriptor: descriptor, Layer: layer, UpdatePolicy: "manual",
|
||||
}
|
||||
contents, err := s.resolveFamilyContents(descriptor, layer)
|
||||
if err != nil {
|
||||
return Installed{}, err
|
||||
}
|
||||
if err := s.Store.Install(ctx, installed); err != nil {
|
||||
return Installed{}, err
|
||||
}
|
||||
if err := s.project(ctx, installed, contents); err != nil {
|
||||
// An install without its projected contributions is not usable. Take
|
||||
// whatever was projected back out, then the installation itself, so the
|
||||
// org is left with no package at all rather than a half-usable one.
|
||||
return Installed{}, errors.Join(err, s.withdraw(ctx, installed), s.Store.Remove(ctx, orgID, name))
|
||||
}
|
||||
return installed, nil
|
||||
}
|
||||
|
||||
@@ -143,5 +162,15 @@ func (s Service) Remove(ctx context.Context, orgID, name string) error {
|
||||
if strings.TrimSpace(orgID) == "" || strings.TrimSpace(name) == "" {
|
||||
return errors.New("org and name are required")
|
||||
}
|
||||
installed, err := s.Store.Get(ctx, orgID, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("installed package %q: %w", name, err)
|
||||
}
|
||||
// Withdraw before the record goes: the package's contributions must stop
|
||||
// being usable only once its installation stops being true. A withdrawal
|
||||
// that fails keeps the package installed and its record kept.
|
||||
if err := s.withdraw(ctx, installed); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Store.Remove(ctx, orgID, name)
|
||||
}
|
||||
@@ -224,3 +224,16 @@ func TestServiceRemove(t *testing.T) {
|
||||
t.Fatalf("other org package removed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeLayeredRegistry is a registry returning a descriptor with its layer
|
||||
// bytes, the way a fetched package carries both.
|
||||
type fakeLayeredRegistry struct {
|
||||
descriptor Descriptor
|
||||
layer []byte
|
||||
}
|
||||
|
||||
func (r *fakeLayeredRegistry) Fetch(context.Context, string, string) (Descriptor, []byte, error) {
|
||||
return r.descriptor, r.layer, nil
|
||||
}
|
||||
|
||||
var _ Registry = (*fakeLayeredRegistry)(nil)
|
||||
@@ -0,0 +1,45 @@
|
||||
package storepkg
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// FilesFromLayer reads an installed package's layer into its files, keyed by
|
||||
// path. The fetch already validated the layer against Descriptor.Files and
|
||||
// held it within the package size limit; this recovers the content the
|
||||
// contributed names point at, plain or gzipped, whichever the package shipped.
|
||||
func FilesFromLayer(layer []byte) (map[string][]byte, error) {
|
||||
reader := io.Reader(bytes.NewReader(layer))
|
||||
if len(layer) >= 2 && layer[0] == 0x1f && layer[1] == 0x8b {
|
||||
gz, err := gzip.NewReader(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read package layer: %w", err)
|
||||
}
|
||||
defer func() { _ = gz.Close() }()
|
||||
reader = gz
|
||||
}
|
||||
files := map[string][]byte{}
|
||||
tarReader := tar.NewReader(reader)
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return files, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read package layer: %w", err)
|
||||
}
|
||||
if header.Typeflag != tar.TypeReg {
|
||||
continue
|
||||
}
|
||||
content, err := io.ReadAll(tarReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read package file %q: %w", header.Name, err)
|
||||
}
|
||||
files[header.Name] = content
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
-- +goose Up
|
||||
-- The entries an org's installed package projected into control-plane
|
||||
-- resources. Removal reads this ledger to take exactly the package's own
|
||||
-- contributions back out, leaving every entry the operator or another package
|
||||
-- owns. Rows cascade with the installed package they describe.
|
||||
CREATE TABLE installed_package_contributions (
|
||||
org_id text NOT NULL,
|
||||
package_name text NOT NULL,
|
||||
family text NOT NULL,
|
||||
entry_id text NOT NULL,
|
||||
PRIMARY KEY (org_id, package_name, family, entry_id),
|
||||
FOREIGN KEY (org_id, package_name) REFERENCES installed_packages (org_id, name) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE installed_package_contributions;
|
||||
@@ -0,0 +1,64 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/storepkg"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb"
|
||||
)
|
||||
|
||||
// PackageContributions records the control-plane resource entries an org's
|
||||
// installed package projected. Removal consults this ledger to take exactly
|
||||
// the package's own contributions back out, so a remove that fails midway
|
||||
// keeps the ledger for the next attempt and another package's entries stay.
|
||||
type PackageContributions struct{ pool *pgxpool.Pool }
|
||||
|
||||
var _ storepkg.ProjectionLedger = (*PackageContributions)(nil)
|
||||
|
||||
func NewPackageContributions(pool *pgxpool.Pool) *PackageContributions {
|
||||
return &PackageContributions{pool: pool}
|
||||
}
|
||||
|
||||
func (s *PackageContributions) Record(ctx context.Context, orgID, name string, entries []storepkg.ProjectionEntry) error {
|
||||
if orgID == "" || name == "" {
|
||||
return errors.New("org and package name are required")
|
||||
}
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
queries := postgresdb.New(tx)
|
||||
if err := queries.LockInstalledPackageContributions(ctx, postgresdb.LockInstalledPackageContributionsParams{Column1: orgID, Column2: name}); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if err := queries.InsertInstalledPackageContribution(ctx, postgresdb.InsertInstalledPackageContributionParams{
|
||||
OrgID: orgID, PackageName: name, Family: entry.Family, EntryID: entry.EntryID,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (s *PackageContributions) Entries(ctx context.Context, orgID, name string) ([]storepkg.ProjectionEntry, error) {
|
||||
rows, err := postgresdb.New(s.pool).ListInstalledPackageContributions(ctx, postgresdb.ListInstalledPackageContributionsParams{OrgID: orgID, PackageName: name})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries := make([]storepkg.ProjectionEntry, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
entries = append(entries, storepkg.ProjectionEntry{Family: row.Family, EntryID: row.EntryID})
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func (s *PackageContributions) Forget(ctx context.Context, orgID, name, family string) error {
|
||||
return postgresdb.New(s.pool).DeleteInstalledPackageContributions(ctx, postgresdb.DeleteInstalledPackageContributionsParams{
|
||||
OrgID: orgID, PackageName: name, Family: family,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: installed_package_contributions.sql
|
||||
|
||||
package postgresdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
const deleteInstalledPackageContributions = `-- name: DeleteInstalledPackageContributions :exec
|
||||
DELETE FROM installed_package_contributions
|
||||
WHERE org_id = $1 AND package_name = $2 AND family = $3
|
||||
`
|
||||
|
||||
type DeleteInstalledPackageContributionsParams struct {
|
||||
OrgID string
|
||||
PackageName string
|
||||
Family string
|
||||
}
|
||||
|
||||
func (q *Queries) DeleteInstalledPackageContributions(ctx context.Context, arg DeleteInstalledPackageContributionsParams) error {
|
||||
_, err := q.db.Exec(ctx, deleteInstalledPackageContributions, arg.OrgID, arg.PackageName, arg.Family)
|
||||
return err
|
||||
}
|
||||
|
||||
const insertInstalledPackageContribution = `-- name: InsertInstalledPackageContribution :exec
|
||||
INSERT INTO installed_package_contributions (org_id, package_name, family, entry_id)
|
||||
VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING
|
||||
`
|
||||
|
||||
type InsertInstalledPackageContributionParams struct {
|
||||
OrgID string
|
||||
PackageName string
|
||||
Family string
|
||||
EntryID string
|
||||
}
|
||||
|
||||
func (q *Queries) InsertInstalledPackageContribution(ctx context.Context, arg InsertInstalledPackageContributionParams) error {
|
||||
_, err := q.db.Exec(ctx, insertInstalledPackageContribution,
|
||||
arg.OrgID,
|
||||
arg.PackageName,
|
||||
arg.Family,
|
||||
arg.EntryID,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const listInstalledPackageContributions = `-- name: ListInstalledPackageContributions :many
|
||||
SELECT org_id, package_name, family, entry_id FROM installed_package_contributions
|
||||
WHERE org_id = $1 AND package_name = $2
|
||||
`
|
||||
|
||||
type ListInstalledPackageContributionsParams struct {
|
||||
OrgID string
|
||||
PackageName string
|
||||
}
|
||||
|
||||
func (q *Queries) ListInstalledPackageContributions(ctx context.Context, arg ListInstalledPackageContributionsParams) ([]InstalledPackageContribution, error) {
|
||||
rows, err := q.db.Query(ctx, listInstalledPackageContributions, arg.OrgID, arg.PackageName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []InstalledPackageContribution
|
||||
for rows.Next() {
|
||||
var i InstalledPackageContribution
|
||||
if err := rows.Scan(
|
||||
&i.OrgID,
|
||||
&i.PackageName,
|
||||
&i.Family,
|
||||
&i.EntryID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const lockInstalledPackageContributions = `-- name: LockInstalledPackageContributions :exec
|
||||
SELECT pg_advisory_xact_lock(hashtextextended('installed-package-contributions:' || $1::text || ':' || $2::text, 0))
|
||||
`
|
||||
|
||||
type LockInstalledPackageContributionsParams struct {
|
||||
Column1 string
|
||||
Column2 string
|
||||
}
|
||||
|
||||
func (q *Queries) LockInstalledPackageContributions(ctx context.Context, arg LockInstalledPackageContributionsParams) error {
|
||||
_, err := q.db.Exec(ctx, lockInstalledPackageContributions, arg.Column1, arg.Column2)
|
||||
return err
|
||||
}
|
||||
@@ -197,6 +197,13 @@ type InstalledPackage struct {
|
||||
AcceptedAuthority []byte
|
||||
}
|
||||
|
||||
type InstalledPackageContribution struct {
|
||||
OrgID string
|
||||
PackageName string
|
||||
Family string
|
||||
EntryID string
|
||||
}
|
||||
|
||||
type InstalledPackageRequirement struct {
|
||||
OrgID string
|
||||
PackageName string
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
-- name: LockInstalledPackageContributions :exec
|
||||
SELECT pg_advisory_xact_lock(hashtextextended('installed-package-contributions:' || $1::text || ':' || $2::text, 0));
|
||||
|
||||
-- name: InsertInstalledPackageContribution :exec
|
||||
INSERT INTO installed_package_contributions (org_id, package_name, family, entry_id)
|
||||
VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING;
|
||||
|
||||
-- name: ListInstalledPackageContributions :many
|
||||
SELECT org_id, package_name, family, entry_id FROM installed_package_contributions
|
||||
WHERE org_id = $1 AND package_name = $2;
|
||||
|
||||
-- name: DeleteInstalledPackageContributions :exec
|
||||
DELETE FROM installed_package_contributions
|
||||
WHERE org_id = $1 AND package_name = $2 AND family = $3;
|
||||
@@ -88,6 +88,7 @@ func TestUnmapErrorSentinelFidelity(t *testing.T) {
|
||||
{name: "source not found", store: storecontract.ErrSourceNotFound, rehyd: storecontract.ErrSourceNotFound},
|
||||
{name: "source path taken", store: storecontract.ErrSourcePathTaken, rehyd: storecontract.ErrSourcePathTaken},
|
||||
{name: "package authority not declared", store: storepkg.ErrAuthorityNotDeclared, rehyd: storepkg.ErrAuthorityNotDeclared},
|
||||
{name: "package contribution collision", store: storepkg.ErrContributionCollision, rehyd: storepkg.ErrContributionCollision},
|
||||
{name: "source signing stale", store: storecontract.ErrSourceSigningStale, rehyd: storecontract.ErrSourceSigningStale},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -511,6 +511,7 @@ var wireErrors = []struct {
|
||||
{storepkg.ErrInstalled, codes.AlreadyExists, storepkg.ErrInstalled.Error()},
|
||||
{storepkg.ErrRequired, codes.FailedPrecondition, storepkg.ErrRequired.Error()},
|
||||
{storepkg.ErrAuthorityNotDeclared, codes.FailedPrecondition, storepkg.ErrAuthorityNotDeclared.Error()},
|
||||
{storepkg.ErrContributionCollision, codes.InvalidArgument, storepkg.ErrContributionCollision.Error()},
|
||||
{logging.ErrTaskLogsUnavailable, codes.Unavailable, msgTaskLogsUnavailable},
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user