fix(daemon): classify workflow-pin parks by cause, not a blanket transient

pinTaskProfile parked every pinWorkflowDefinition failure as
taskstate.ParkTransient. That class documents that a requeue can succeed,
but the dominant pin failures are durable: a stored definition whose YAML
does not hash to the digest beside it, a definition that does not parse,
and a workflow or playbook binding resolving to an undefined workflow all
recur on every requeue until an operator repairs the row or the
configuration. A future retry pass reading transient as its input would
loop on them forever.

pinFailure now carries the class its cause needs, decided where the cause
is known, and pinParkClass reads it at the park site. A durable pin defect
parks operator-actionable; only a State Store read or persist failure --
the genuinely environmental case -- stays transient. The safe default for
an unclassified cause is ParkNeedsHuman, never a requeue that will fix it.

Red-first: TestDurablePinDefectIsNotParkedTransient failed at its
assertion before the fix (parked "transient"); TestTransientPinFailureStaysTransient
keeps the environmental case transient so the rule is per-cause, not a
blanket flip.
This commit is contained in:
sam committed 2026-10-02 12:57:53 +10:00
1 parent f69ce1db56
commit de24485e10
2 files changed
+114 -8

No files matched your search

+36 -8
View File
@@ -1913,12 +1913,14 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo conf
}
// pinTaskProfile pins the task's workflow definition and resolves the agent
// profile it names, parking the task when either fails. An unconfigured
// profile needs an operator: the fix is configuration, then a retry.
// profile it names, parking the task when either fails. The park class is the
// failing cause's own: an unusable profile, a corrupt stored pin and a
// definition set that no longer offers the task's workflow all need an
// operator, while a State Store read or persist failure is environmental.
func (d *Daemon) pinTaskProfile(ctx context.Context, task *workflow.Task) (config.AgentProfile, bool) {
if err := d.pinWorkflowDefinition(ctx, task); err != nil {
d.Log.Error("pin workflow definition failed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task.ID, "pin workflow definition: "+err.Error(), taskstate.ParkTransient)
d.parkRunningTask(ctx, task.ID, "pin workflow definition: "+err.Error(), pinParkClass(err))
return config.AgentProfile{}, false
}
iface, err := workflowtask.ParseWorkflowInterface(task.WorkflowDefinitionYAML)
@@ -2004,7 +2006,7 @@ func (d *Daemon) publicationGrant(ctx context.Context, task *workflow.Task) (str
func (d *Daemon) pinWorkflowDefinition(ctx context.Context, task *workflow.Task) error {
pinned, ok, err := pinnedDefinitionID(task)
if err != nil {
return err
return pinFailure{taskstate.ParkNeedsHuman, err}
}
if ok && pinned == task.Workflow {
return nil
@@ -2015,11 +2017,37 @@ func (d *Daemon) pinWorkflowDefinition(ctx context.Context, task *workflow.Task)
}
collection, version, err := d.WorkflowDefinitions.WorkflowDefinitions(ctx)
if err != nil {
return err
return pinFailure{taskstate.ParkTransient, err}
}
return d.pinWorkflowFromCollection(ctx, task, collection, version)
}
// pinFailure carries the park class a workflow-definition pin failure needs.
// The class is decided where the cause is known -- inside
// pinWorkflowDefinition and pinWorkflowFromCollection -- never inferred from
// the reason text at the park site. A defect in the task's own stored pin, or a
// definition set that no longer offers the workflow the task names, is durable:
// the same dispatch fails again on every requeue until an operator repairs the
// row or the configuration, so it parks operator-actionable. Only a State
// Store read or persist failure is environmental, where a requeue can succeed.
type pinFailure struct {
class taskstate.ParkClass
err error
}
func (e pinFailure) Error() string { return e.err.Error() }
func (e pinFailure) Unwrap() error { return e.err }
// pinParkClass reports the class a pin failure carries, defaulting to
// taskstate.ParkNeedsHuman for a cause that did not classify itself: the safe
// misread is "an operator should look at this", never "a requeue will fix it".
func pinParkClass(err error) taskstate.ParkClass {
if failure, ok := errors.AsType[pinFailure](err); ok {
return failure.class
}
return taskstate.ParkNeedsHuman
}
// pinnedDefinitionID reports the workflow a task's stored definition pin names,
// with ok false for a task carrying no pin. It enforces what this package owns
// about a stored pin: the YAML hashes to the digest recorded beside it, and it
@@ -2112,18 +2140,18 @@ func (d *Daemon) pinWorkflowFromCollection(ctx context.Context, task *workflow.T
}
id, err := d.resolveWorkflowID(task, available)
if err != nil {
return err
return pinFailure{taskstate.ParkNeedsHuman, err}
}
definition, ok := collection.DefinitionByID(id)
if !ok {
return fmt.Errorf("workflow definition %q disappeared", id)
return pinFailure{taskstate.ParkNeedsHuman, fmt.Errorf("workflow definition %q disappeared", id)}
}
task.Workflow = id
task.WorkflowDefinitionVersion = version
task.WorkflowDefinitionYAML = definition.YAML
task.WorkflowDefinitionDigest = workflow.DigestDefinition(definition.YAML)
if err := d.Store.Update(ctx, task); err != nil {
return fmt.Errorf("persist workflow definition pin: %w", err)
return pinFailure{taskstate.ParkTransient, fmt.Errorf("persist workflow definition pin: %w", err)}
}
return nil
}
+78
View File
@@ -0,0 +1,78 @@
package daemon
import (
"context"
"errors"
"log/slog"
"testing"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/taskstate"
)
// parkClassRecorder captures the class of the one park a test drives, without
// a live store. The embedded TaskStore is nil: only ParkTask is reachable from
// the code under test, and any other store call panics loudly rather than
// silently succeeding.
type parkClassRecorder struct {
storecontract.TaskStore
class taskstate.ParkClass
}
func (r *parkClassRecorder) ParkTask(_ context.Context, _ int64, _, _, class string) error {
r.class = class
return nil
}
// TestDurablePinDefectIsNotParkedTransient pins the classification of a pin the
// task itself carries wrong. A stored definition whose YAML does not hash to
// the digest recorded beside it is a durable defect: the same row fails the
// same way on every requeue until an operator repairs it, so ParkTransient --
// whose own contract says "a requeue can succeed" -- would hand a future retry
// pass an endless loop.
func TestDurablePinDefectIsNotParkedTransient(t *testing.T) {
store := &parkClassRecorder{}
d := &Daemon{Store: store, Log: slog.New(slog.DiscardHandler)}
task := &workflow.Task{
ID: 1,
Workflow: "implement",
WorkflowDefinitionYAML: "id: implement\nsteps:\n - type: implement.prepare\n",
WorkflowDefinitionDigest: workflow.DigestDefinition("id: implement\nsteps:\n - type: implement.build\n"),
}
if _, ok := d.pinTaskProfile(context.Background(), task); ok {
t.Fatal("pinTaskProfile accepted a tampered pin")
}
if store.class == "" {
t.Fatal("a durable pin defect was not parked at all")
}
if store.class == taskstate.ParkTransient {
t.Fatalf("durable pin defect parked as %q: a requeue recurs on the same defect, so a retry pass cannot make progress", store.class)
}
}
// failingWorkflowDefinitions stands in for a State Store read failure.
type failingWorkflowDefinitions struct{ err error }
func (f failingWorkflowDefinitions) WorkflowDefinitions(context.Context) (workflow.WorkflowDefinitionCollection, int64, error) {
return workflow.WorkflowDefinitionCollection{}, 0, f.err
}
// TestTransientPinFailureStaysTransient keeps the other half of the cause-based
// rule: a State Store read failure is environmental, so it must still park
// transient rather than being folded in with the durable defects.
func TestTransientPinFailureStaysTransient(t *testing.T) {
store := &parkClassRecorder{}
d := &Daemon{
Store: store,
Log: slog.New(slog.DiscardHandler),
WorkflowDefinitions: failingWorkflowDefinitions{err: errors.New("state store unavailable")},
}
task := &workflow.Task{ID: 1, Workflow: "implement"}
if _, ok := d.pinTaskProfile(context.Background(), task); ok {
t.Fatal("pinTaskProfile succeeded despite a definition read failure")
}
if store.class != taskstate.ParkTransient {
t.Fatalf("pin class = %q, want %q for a State Store read failure", store.class, taskstate.ParkTransient)
}
}