Merge pull request #1236 from samcharles93/fix/1235-repin-workflow-when-requeue-changes-it
fix(daemon): re-pin a task's workflow definition when a requeue names another
This commit is contained in:
commit
e904f341b8
4 files changed
+365
-4
No files matched your search
@@ -1842,11 +1842,33 @@ func (d *Daemon) publicationGrant(ctx context.Context, task *workflow.Task) (str
|
||||
return grant, revoke, true
|
||||
}
|
||||
|
||||
// pinWorkflowDefinition puts the task's workflow definition pin in the state
|
||||
// the worker's compile requires before it runs a dispatch: a stored definition
|
||||
// that names the workflow the task row names
|
||||
// (agentworker.CompilePinnedWorkflow).
|
||||
//
|
||||
// A pin that is present, hash-valid and id-consistent is reused unchanged, so
|
||||
// the definition an operator had active when the task was pinned survives every
|
||||
// later dispatch even after the active definition moved on. Anything else -- no
|
||||
// pin, or a pin naming a workflow this task no longer runs -- is resolved again
|
||||
// from the active collection for the workflow the task names.
|
||||
//
|
||||
// The re-pin is what makes the waiting_human -> approved requeue dispatchable:
|
||||
// RequeueTask writes the new workflow column and leaves the previous run's
|
||||
// definition pinned, so the requeued row carries the two half-state values the
|
||||
// worker refuses to run. Resolving here, at the write, is where the row
|
||||
// becomes valid; a consumer that repaired it would only be tolerating a state
|
||||
// this producer is responsible for.
|
||||
//
|
||||
// An already-pinned definition is deliberately NOT refreshed to the operator's
|
||||
// current active one: a run's definition is pinned once, so a mid-flight
|
||||
// change to a definition never alters work already in progress.
|
||||
func (d *Daemon) pinWorkflowDefinition(ctx context.Context, task *workflow.Task) error {
|
||||
if task.WorkflowDefinitionYAML != "" {
|
||||
if workflow.DigestDefinition(task.WorkflowDefinitionYAML) != task.WorkflowDefinitionDigest {
|
||||
return fmt.Errorf("stored workflow definition digest mismatch")
|
||||
}
|
||||
pinned, ok, err := pinnedDefinitionID(task)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok && pinned == task.Workflow {
|
||||
return nil
|
||||
}
|
||||
d.runActionPlaybooks(ctx, task)
|
||||
@@ -1860,6 +1882,25 @@ func (d *Daemon) pinWorkflowDefinition(ctx context.Context, task *workflow.Task)
|
||||
return d.pinWorkflowFromCollection(ctx, task, collection, version)
|
||||
}
|
||||
|
||||
// pinnedDefinitionID reports the workflow a task's stored definition pin names,
|
||||
// with ok false for a task carrying no pin. It enforces what this package owns
|
||||
// about a stored pin: the YAML hashes to the digest recorded beside it, and it
|
||||
// declares an id at all. A pin failing either is a row an operator has to
|
||||
// resolve, not one to silently overwrite.
|
||||
func pinnedDefinitionID(task *workflow.Task) (string, bool, error) {
|
||||
if task.WorkflowDefinitionYAML == "" {
|
||||
return "", false, nil
|
||||
}
|
||||
if workflow.DigestDefinition(task.WorkflowDefinitionYAML) != task.WorkflowDefinitionDigest {
|
||||
return "", false, fmt.Errorf("stored workflow definition digest mismatch")
|
||||
}
|
||||
id, err := workflow.DefinitionID(task.WorkflowDefinitionYAML)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("stored workflow definition: %w", err)
|
||||
}
|
||||
return id, true, nil
|
||||
}
|
||||
|
||||
// runActionPlaybooks runs the action playbooks matching a task that is about
|
||||
// to be pinned, which happens once per task (docs/prds/action-playbook-run.md).
|
||||
// A run never affects routing and its failure never fails the task. A task
|
||||
|
||||
@@ -2,6 +2,8 @@ package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
@@ -46,3 +48,247 @@ func TestWorkflowDefinitionPinSurvivesActiveOverride(t *testing.T) {
|
||||
t.Fatal("persisted workflow digest does not identify the pinned YAML")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPinWorkflowDefinitionRepinsWhenTheTaskNamesAnotherWorkflow covers the
|
||||
// waiting_human -> approved requeue: RequeueTask writes the new workflow name
|
||||
// and leaves the previous run's definition pinned, so the requeued row carries
|
||||
// a workflow column and a stored definition that disagree.
|
||||
//
|
||||
// Those two are what the worker's compile refuses at dispatch
|
||||
// (agentworker.CompilePinnedWorkflow: "pinned workflow definition does not
|
||||
// match task identity"), so a pin that names a different workflow than the
|
||||
// task is a dispatch that can never run, and the daemon -- the producer that
|
||||
// wrote the pin -- re-pins it from the collection the task's workflow is
|
||||
// resolved out of. A pin that is present, digest-valid AND id-consistent stays
|
||||
// a strict no-op, so an operator's newer active definition does not overwrite
|
||||
// a task's existing pin on the next dispatch.
|
||||
func TestPinWorkflowDefinitionRepinsWhenTheTaskNamesAnotherWorkflow(t *testing.T) {
|
||||
const (
|
||||
pinnedVersion = 4
|
||||
activeVersion = 9
|
||||
)
|
||||
feasibilityPin := "id: feasibility\nsteps:\n - type: feasibility.assess\n"
|
||||
customPin := "id: custom\nsteps:\n - type: bootstrap.apply\n"
|
||||
// The active collection is the one the daemon resolves a named workflow
|
||||
// out of. Its "custom" entry is deliberately different YAML from customPin:
|
||||
// that is the operator's newer active definition, and the only way to tell
|
||||
// a reuse from a re-pin is that a reused pin keeps the older YAML.
|
||||
active := &workflowDefinitionsStub{
|
||||
collection: workflow.WorkflowDefinitionCollection{Definitions: []workflow.WorkflowDefinitionEntry{
|
||||
{ID: "feasibility", YAML: feasibilityPin},
|
||||
{ID: "implement", YAML: "id: implement\nsteps:\n - type: implement.prepare\n"},
|
||||
{ID: "custom", YAML: "id: custom\nsteps:\n - type: bootstrap.prepare\n"},
|
||||
}},
|
||||
version: activeVersion,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
// taskWorkflow is the row's workflow column: what the requeue wrote,
|
||||
// and the workflow the pinned definition must end up naming.
|
||||
taskWorkflow string
|
||||
// pinnedYAML is the definition the row carries into the dispatch --
|
||||
// for a requeue, the workflow the task ran before its workflow column
|
||||
// changed.
|
||||
pinnedYAML string
|
||||
// shipped pins from the shipped definitions, the way a daemon with no
|
||||
// definitions provider does.
|
||||
shipped bool
|
||||
wantWorkflow string
|
||||
wantYAML string
|
||||
wantVersion int64
|
||||
}{
|
||||
{
|
||||
name: "a requeue onto another workflow is re-pinned from the active collection",
|
||||
taskWorkflow: "implement",
|
||||
pinnedYAML: feasibilityPin,
|
||||
wantWorkflow: "implement",
|
||||
wantYAML: "id: implement\nsteps:\n - type: implement.prepare\n",
|
||||
wantVersion: activeVersion,
|
||||
},
|
||||
{
|
||||
name: "a daemon with no definitions provider re-pins from the shipped set",
|
||||
taskWorkflow: "implement",
|
||||
pinnedYAML: feasibilityPin,
|
||||
shipped: true,
|
||||
wantWorkflow: "implement",
|
||||
wantVersion: 0,
|
||||
},
|
||||
{
|
||||
name: "an id-consistent pin is reused, so an active override does not displace it",
|
||||
taskWorkflow: "custom",
|
||||
pinnedYAML: customPin,
|
||||
wantWorkflow: "custom",
|
||||
wantYAML: customPin,
|
||||
wantVersion: pinnedVersion,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
wantYAML := tt.wantYAML
|
||||
if tt.shipped {
|
||||
entry, ok := workflow.ShippedDefinitions().DefinitionByID(tt.wantWorkflow)
|
||||
if !ok {
|
||||
t.Fatalf("shipped definitions have no %q", tt.wantWorkflow)
|
||||
}
|
||||
wantYAML = entry.YAML
|
||||
}
|
||||
|
||||
resources := pgstore.Open(t)
|
||||
defer resources.Close()
|
||||
task, err := resources.EnqueueChatTask(t.Context(), "acme", "widget", tt.name, "", tt.taskWorkflow, "operator", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// RequeueTask sets status, workflow and park_reason, and never
|
||||
// touches the workflow_definition_* columns, so this is the row a
|
||||
// requeue hands to the next dispatch: the new workflow name over
|
||||
// the previous run's definition.
|
||||
task.WorkflowDefinitionYAML = tt.pinnedYAML
|
||||
task.WorkflowDefinitionDigest = workflow.DigestDefinition(tt.pinnedYAML)
|
||||
task.WorkflowDefinitionVersion = pinnedVersion
|
||||
if err := resources.Update(t.Context(), task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
d := &Daemon{Store: resources, Log: slog.New(slog.DiscardHandler)}
|
||||
if !tt.shipped {
|
||||
d.WorkflowDefinitions = active
|
||||
}
|
||||
if err := d.pinWorkflowDefinition(t.Context(), task); err != nil {
|
||||
t.Fatalf("pinWorkflowDefinition: %v", err)
|
||||
}
|
||||
if task.Workflow != tt.wantWorkflow || task.WorkflowDefinitionYAML != wantYAML || task.WorkflowDefinitionVersion != tt.wantVersion {
|
||||
t.Fatalf("pin = (%q, version %d, %q),\nwant (%q, version %d, %q)",
|
||||
task.Workflow, task.WorkflowDefinitionVersion, task.WorkflowDefinitionYAML,
|
||||
tt.wantWorkflow, tt.wantVersion, wantYAML)
|
||||
}
|
||||
|
||||
stored, err := resources.TaskByID(t.Context(), task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stored.Workflow != tt.wantWorkflow || stored.WorkflowDefinitionYAML != wantYAML || stored.WorkflowDefinitionVersion != tt.wantVersion {
|
||||
t.Fatalf("persisted pin = (%q, version %d, %q),\nwant (%q, version %d, %q)",
|
||||
stored.Workflow, stored.WorkflowDefinitionVersion, stored.WorkflowDefinitionYAML,
|
||||
tt.wantWorkflow, tt.wantVersion, wantYAML)
|
||||
}
|
||||
requirePinMatchesTaskIdentity(t, stored)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestApproveRequeueRepinsTheWorkflowDefinition reproduces the operator-visible
|
||||
// defect end to end through the real store writes: a task runs feasibility,
|
||||
// parks waiting_human with the feasibility definition pinned, and the approval
|
||||
// requeue moves it to the implement workflow. RequeueTask sets status, workflow
|
||||
// and park_reason and leaves the workflow_definition_* columns, so the next
|
||||
// dispatch met an id mismatch and the worker's compile parked the task again --
|
||||
// identically on every further approval, leaving decline the only working
|
||||
// action.
|
||||
//
|
||||
// What the task row must satisfy before that dispatch is the worker's own
|
||||
// precondition (agentworker.CompilePinnedWorkflow), asserted here through
|
||||
// requirePinMatchesTaskIdentity.
|
||||
func TestApproveRequeueRepinsTheWorkflowDefinition(t *testing.T) {
|
||||
resources := pgstore.Open(t)
|
||||
defer resources.Close()
|
||||
if _, err := resources.EnqueueChatTask(t.Context(), "acme", "widget", "feature request", "", "feasibility", "operator", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claim, err := resources.ClaimNext(t.Context())
|
||||
if err != nil || claim == nil {
|
||||
t.Fatalf("claim: (%v, %v)", claim, err)
|
||||
}
|
||||
|
||||
// The run pins the feasibility definition and parks for the human decision.
|
||||
provider := &workflowDefinitionsStub{collection: workflow.WorkflowDefinitionCollection{Definitions: []workflow.WorkflowDefinitionEntry{
|
||||
{ID: "feasibility", YAML: "id: feasibility\nsteps:\n - type: feasibility.assess\n"},
|
||||
{ID: "implement", YAML: "id: implement\nsteps:\n - type: implement.prepare\n"},
|
||||
}}, version: 2}
|
||||
d := &Daemon{Store: resources, Log: slog.New(slog.DiscardHandler), WorkflowDefinitions: provider}
|
||||
if err := d.pinWorkflowDefinition(t.Context(), claim); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := resources.Transition(t.Context(), claim.ID, "running", "waiting_human", "PRD delivered, awaiting go/no-go"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The operator approves: the same write taskactions.Service.apply performs.
|
||||
if err := resources.Requeue(t.Context(), claim.ID, "waiting_human", "implement"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requeued, err := resources.TaskByID(t.Context(), claim.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if requeued.Workflow != "implement" {
|
||||
t.Fatalf("requeued workflow = %q, want implement", requeued.Workflow)
|
||||
}
|
||||
if got, err := workflow.DefinitionID(requeued.WorkflowDefinitionYAML); err != nil || got != "feasibility" {
|
||||
t.Fatalf("the requeue changed the stored pin to (%q, %v); this test only means anything while the pin is still the previous workflow's", got, err)
|
||||
}
|
||||
|
||||
// The next dispatch re-pins before the worker's compile sees the row.
|
||||
if err := d.pinWorkflowDefinition(t.Context(), requeued); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stored, err := resources.TaskByID(t.Context(), requeued.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requirePinMatchesTaskIdentity(t, stored)
|
||||
}
|
||||
|
||||
// TestPinWorkflowDefinitionRefusesATamperedPin keeps the digest half of the
|
||||
// guard intact: a stored definition that does not hash to the digest recorded
|
||||
// with it is a corrupt row, and repairing it here would silently run a
|
||||
// definition nobody wrote. It is refused, and the row is left for an operator.
|
||||
func TestPinWorkflowDefinitionRefusesATamperedPin(t *testing.T) {
|
||||
resources := pgstore.Open(t)
|
||||
defer resources.Close()
|
||||
task, err := resources.EnqueueChatTask(t.Context(), "acme", "widget", "tampered", "", "implement", "operator", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task.WorkflowDefinitionYAML = "id: implement\nsteps:\n - type: implement.prepare\n"
|
||||
task.WorkflowDefinitionDigest = workflow.DigestDefinition("id: implement\nsteps:\n - type: implement.build\n")
|
||||
if err := resources.Update(t.Context(), task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
d := &Daemon{Store: resources, Log: slog.New(slog.DiscardHandler), WorkflowDefinitions: &workflowDefinitionsStub{
|
||||
collection: workflow.WorkflowDefinitionCollection{Definitions: []workflow.WorkflowDefinitionEntry{
|
||||
{ID: "implement", YAML: "id: implement\nsteps:\n - type: implement.prepare\n"},
|
||||
}},
|
||||
version: 3,
|
||||
}}
|
||||
err = d.pinWorkflowDefinition(t.Context(), task)
|
||||
if err == nil || !strings.Contains(err.Error(), "digest mismatch") {
|
||||
t.Fatalf("pinWorkflowDefinition error = %v, want a digest mismatch refusal", err)
|
||||
}
|
||||
if task.WorkflowDefinitionVersion != 0 {
|
||||
t.Fatalf("refused pin wrote version %d; a refused pin must leave the row alone", task.WorkflowDefinitionVersion)
|
||||
}
|
||||
}
|
||||
|
||||
// requirePinMatchesTaskIdentity holds a persisted pin to the invariant the
|
||||
// worker's compile enforces before it runs a dispatch
|
||||
// (agentworker.CompilePinnedWorkflow): the pinned definition compiles against
|
||||
// the process vocabulary, names the workflow the task row names, and hashes to
|
||||
// the digest stored beside it. A producer that leaves a pin failing this is
|
||||
// the defect that parks a requeued task forever.
|
||||
func requirePinMatchesTaskIdentity(t *testing.T, stored *workflow.Task) {
|
||||
t.Helper()
|
||||
compiled, err := workflow.ParseAndCompile(stored.WorkflowDefinitionYAML, workflow.BuiltinStepRegistry())
|
||||
if err != nil {
|
||||
t.Fatalf("pinned definition does not compile: %v", err)
|
||||
}
|
||||
if compiled.Name != stored.Workflow {
|
||||
t.Fatalf("pinned definition names %q, the task row names %q: the worker's compile refuses this pair", compiled.Name, stored.Workflow)
|
||||
}
|
||||
if got := workflow.DigestDefinition(stored.WorkflowDefinitionYAML); got != stored.WorkflowDefinitionDigest {
|
||||
t.Fatalf("pinned definition digest = %s, the row records %s", got, stored.WorkflowDefinitionDigest)
|
||||
}
|
||||
}
|
||||
@@ -105,6 +105,28 @@ func DigestDefinition(src string) string {
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// DefinitionID returns the workflow id a definition declares without compiling
|
||||
// it. It is the identity half of DigestDefinition, for the readers that hold
|
||||
// only stored YAML: a pin's id and the task's workflow column have to agree
|
||||
// before a dispatch runs, and DigestDefinition cannot answer that.
|
||||
//
|
||||
// It reads the single declared id and ignores the rest, so it neither needs a
|
||||
// step registry nor rejects a definition written against a vocabulary the
|
||||
// caller does not have. ParseDefinition stays the validator; this is only the
|
||||
// name.
|
||||
func DefinitionID(src string) (string, error) {
|
||||
var declared struct {
|
||||
ID string `yaml:"id"`
|
||||
}
|
||||
if err := yaml.Unmarshal([]byte(src), &declared); err != nil {
|
||||
return "", fmt.Errorf("parse workflow YAML: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(declared.ID) == "" {
|
||||
return "", fmt.Errorf("workflow id is required")
|
||||
}
|
||||
return declared.ID, nil
|
||||
}
|
||||
|
||||
// WorkflowDefinitionEntry and WorkflowDefinitionCollection are defined in
|
||||
// package task so dashboards can decode the stored projection without linking
|
||||
// the engine. Validation and compilation stay here.
|
||||
|
||||
@@ -33,6 +33,58 @@ func TestParseDefinitionAllowsRepeatingAnOperation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefinitionIDReadsTheDeclaredWorkflowName covers the identity a stored
|
||||
// definition carries: DefinitionID answers "which workflow is this" for a
|
||||
// reader holding only YAML, without a step registry and without rejecting a
|
||||
// definition written against a vocabulary this process does not have (which is
|
||||
// why a daemon cannot answer it by compiling).
|
||||
func TestDefinitionIDReadsTheDeclaredWorkflowName(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
src string
|
||||
want string
|
||||
err string
|
||||
}{
|
||||
{
|
||||
name: "the shipped feasibility definition names itself",
|
||||
src: "id: feasibility\nsteps:\n - type: feasibility.assess\n",
|
||||
want: "feasibility",
|
||||
},
|
||||
{
|
||||
name: "an unregistered step type is still an answerable id",
|
||||
src: "id: operator-local\nsteps:\n - type: plugin.step\n",
|
||||
want: "operator-local",
|
||||
},
|
||||
{
|
||||
name: "a definition with no id is refused",
|
||||
src: "steps:\n - type: feasibility.assess\n",
|
||||
err: "id is required",
|
||||
},
|
||||
{
|
||||
name: "malformed YAML is refused",
|
||||
src: "id: [unterminated\n",
|
||||
err: "parse workflow YAML",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := DefinitionID(tt.src)
|
||||
if tt.err != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tt.err) {
|
||||
t.Fatalf("DefinitionID() error = %v, want %q", err, tt.err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("DefinitionID: %v", err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("DefinitionID() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShippedDefinitionsEmitDeclaredInputsForPRReview(t *testing.T) {
|
||||
entry, ok := ShippedDefinitions().DefinitionByID("pr-review")
|
||||
if !ok {
|
||||
|
||||
Reference in new issue
Block a user