Also fix repository-free message.send validation (#2046) and add a single-binary development smoke path (#1977). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
494 lines
17 KiB
Go
494 lines
17 KiB
Go
package workflow
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"embed"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/fs"
|
|
"path"
|
|
"slices"
|
|
"strings"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
|
|
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
|
)
|
|
|
|
// YAMLDefinition is the portable, versioned workflow representation.
|
|
type YAMLDefinition struct {
|
|
ID string `yaml:"id" json:"id"`
|
|
// Package names the installed package that contributed this workflow. The
|
|
// package projector sets it; a run of the workflow is bounded by the
|
|
// authority accepted for that package.
|
|
Package string `yaml:"package,omitempty" json:"package,omitempty"`
|
|
Steps []StepRecord `yaml:"steps" json:"steps"`
|
|
// WorkflowInterface declares the workflow's inputs, repository mode and
|
|
// agent profile.
|
|
task.WorkflowInterface `yaml:",inline" json:",inline"`
|
|
}
|
|
|
|
// repoFreeStepTypes are the step types that run without a repository. A
|
|
// workflow whose repository is not required may use only these, since it may
|
|
// run with no worktree at all. workflow.call needs none of the caller's own:
|
|
// the callee's own repository mode decides whether its run clones.
|
|
var repoFreeStepTypes = map[string]bool{
|
|
AgentRunStepName: true, WorkflowCallStepName: true, FinishStepName: true,
|
|
HandoffStepName: true, ApproveStepName: true, CloseIssueStepName: true, CommentStepName: true,
|
|
MessageSendStepName: true,
|
|
}
|
|
|
|
// NeedsRepository reports whether a step type can run only with a worktree.
|
|
func NeedsRepository(stepType string) bool { return !repoFreeStepTypes[stepType] }
|
|
|
|
// StepRecord selects one registered step type and supplies its typed settings.
|
|
type StepRecord struct {
|
|
// ID names the step so later steps can reference its result.
|
|
ID string `yaml:"id,omitempty" json:"id,omitempty"`
|
|
Type string `yaml:"type,omitempty" json:"type,omitempty"`
|
|
Settings yaml.Node `yaml:"settings,omitempty" json:"-"`
|
|
// When skips the step unless the condition holds: a reference that must
|
|
// be truthy, negated with !, or compared with == or != to a literal.
|
|
When string `yaml:"when,omitempty" json:"when,omitempty"`
|
|
// OnFailure is "park" (the default) or "continue".
|
|
OnFailure string `yaml:"on_failure,omitempty" json:"on_failure,omitempty"`
|
|
// Retry runs a failed step again before on_failure applies.
|
|
Retry *RetryPolicy `yaml:"retry,omitempty" json:"retry,omitempty"`
|
|
// Parallel makes this step a set of named branches that run at the same
|
|
// time instead of a typed step. Each branch is a list of steps.
|
|
Parallel map[string][]StepRecord `yaml:"parallel,omitempty" json:"parallel,omitempty"`
|
|
// Switch makes this step a choice: the case named by a reference's value
|
|
// runs, else the default, else nothing.
|
|
Switch *SwitchStep `yaml:"switch,omitempty" json:"switch,omitempty"`
|
|
}
|
|
|
|
// SwitchStep runs the steps of the case whose key equals the value On
|
|
// references, else the case named default, else nothing.
|
|
type SwitchStep struct {
|
|
On string `yaml:"on" json:"on"`
|
|
Cases map[string][]StepRecord `yaml:"cases" json:"cases"`
|
|
}
|
|
|
|
const switchDefault = "default"
|
|
|
|
// RetryPolicy is how often a failed step runs again, and how long it waits
|
|
// between attempts.
|
|
type RetryPolicy struct {
|
|
Attempts int `yaml:"attempts" json:"attempts"`
|
|
Backoff string `yaml:"backoff,omitempty" json:"backoff,omitempty"`
|
|
}
|
|
|
|
const onFailureContinue = "continue"
|
|
|
|
type StepFactory func(yaml.Node) (Stage, error)
|
|
|
|
// StepRegistry is the closed set of safe workflow operations available to YAML.
|
|
type StepRegistry map[string]StepFactory
|
|
|
|
// dropNeeds removes a needs: block a definition written before the field went
|
|
// away may still carry. The key is deleted from the document's root mapping,
|
|
// so the caller's strict decode sees no trace of it while every other key,
|
|
// including those inside steps, keeps its original wording.
|
|
func dropNeeds(src string) ([]byte, error) {
|
|
decoder := yaml.NewDecoder(strings.NewReader(src))
|
|
var document yaml.Node
|
|
if err := decoder.Decode(&document); err != nil {
|
|
return nil, fmt.Errorf("parse workflow YAML: %w", err)
|
|
}
|
|
if document.Kind == yaml.DocumentNode && len(document.Content) > 0 {
|
|
if root := document.Content[0]; root.Kind == yaml.MappingNode {
|
|
for i := 0; i+1 < len(root.Content); i += 2 {
|
|
if root.Content[i].Value == "needs" {
|
|
root.Content = slices.Delete(root.Content, i, i+2)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
var encoded bytes.Buffer
|
|
encoder := yaml.NewEncoder(&encoded)
|
|
if err := encoder.Encode(&document); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := encoder.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
return encoded.Bytes(), nil
|
|
}
|
|
|
|
// ParseDefinition strictly parses and validates one YAML workflow.
|
|
func ParseDefinition(src string, registry StepRegistry) (YAMLDefinition, error) {
|
|
document, err := dropNeeds(src)
|
|
if err != nil {
|
|
return YAMLDefinition{}, err
|
|
}
|
|
var definition YAMLDefinition
|
|
decoder := yaml.NewDecoder(bytes.NewReader(document))
|
|
decoder.KnownFields(true)
|
|
if err := decoder.Decode(&definition); err != nil {
|
|
return YAMLDefinition{}, fmt.Errorf("parse workflow YAML: %w", err)
|
|
}
|
|
if strings.TrimSpace(definition.ID) == "" {
|
|
return YAMLDefinition{}, fmt.Errorf("workflow id is required")
|
|
}
|
|
if len(definition.Steps) == 0 {
|
|
return YAMLDefinition{}, fmt.Errorf("workflow %q has no steps", definition.ID)
|
|
}
|
|
if err := definition.Validate(); err != nil {
|
|
return YAMLDefinition{}, fmt.Errorf("workflow %q: %w", definition.ID, err)
|
|
}
|
|
mode := definition.RepositoryMode()
|
|
earlier := newRefScope(definition.Inputs)
|
|
for i, step := range definition.Steps {
|
|
if err := checkStep(step, mode, earlier, registry, false); err != nil {
|
|
return YAMLDefinition{}, fmt.Errorf("workflow %q step %d: %w", definition.ID, i+1, err)
|
|
}
|
|
}
|
|
if err := checkOutputCaptureCollisions(definition.ID, definition.Outputs, definition.Steps); err != nil {
|
|
return YAMLDefinition{}, err
|
|
}
|
|
return definition, nil
|
|
}
|
|
|
|
// checkOutputCaptureCollisions refuses a definition that declares an output
|
|
// named result beside an agent.run step capturing its structured result
|
|
// through the capture tool of the same name. appendOutputTools refuses that
|
|
// clash when the step runs, so a definition carrying it must not save.
|
|
func checkOutputCaptureCollisions(workflowID string, outputs map[string]task.OutputSpec, steps []StepRecord) error {
|
|
if _, declared := outputs[resultToolName]; !declared {
|
|
return nil
|
|
}
|
|
for _, step := range steps {
|
|
if step.Type == AgentRunStepName {
|
|
var s agentRunSettings
|
|
if err := step.Settings.Decode(&s); err == nil && s.Result != nil {
|
|
return fmt.Errorf("workflow %q: output %q collides with agent.run step %q's result capture tool %q; rename the output or remove settings.result", workflowID, resultToolName, step.StageName(), resultToolName)
|
|
}
|
|
}
|
|
for _, branch := range step.Parallel {
|
|
if err := checkOutputCaptureCollisions(workflowID, outputs, branch); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if step.Switch != nil {
|
|
for _, branch := range step.Switch.Cases {
|
|
if err := checkOutputCaptureCollisions(workflowID, outputs, branch); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// PackageOf returns the package a pinned definition was projected from, or
|
|
// empty for a workflow the operator wrote.
|
|
func PackageOf(src string) string {
|
|
var declared struct {
|
|
Package string `yaml:"package"`
|
|
}
|
|
if err := yaml.Unmarshal([]byte(src), &declared); err != nil {
|
|
return ""
|
|
}
|
|
return declared.Package
|
|
}
|
|
|
|
// StageNames returns the names a definition's steps run under, in order. It
|
|
// reads a definition already pinned, so it decodes without validating.
|
|
func StageNames(src string) ([]string, error) {
|
|
var definition YAMLDefinition
|
|
if err := yaml.Unmarshal([]byte(src), &definition); err != nil {
|
|
return nil, fmt.Errorf("parse workflow YAML: %w", err)
|
|
}
|
|
names := make([]string, len(definition.Steps))
|
|
for i, step := range definition.Steps {
|
|
names[i] = step.StageName()
|
|
}
|
|
return names, nil
|
|
}
|
|
|
|
// ParseAndCompile validates and compiles one definition in a single operation.
|
|
func ParseAndCompile(src string, registry StepRegistry) (Workflow, error) {
|
|
definition, err := ParseDefinition(src, registry)
|
|
if err != nil {
|
|
return Workflow{}, err
|
|
}
|
|
return Compile(definition, registry)
|
|
}
|
|
|
|
// DigestDefinition returns the immutable content identity stored with a run.
|
|
func DigestDefinition(src string) string {
|
|
sum := sha256.Sum256([]byte(src))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
// DefinitionID returns the workflow id a definition declares without
|
|
// compiling or validating it.
|
|
func DefinitionID(src string) (string, error) {
|
|
var declared struct {
|
|
ID string `yaml:"id"`
|
|
}
|
|
if err := yaml.Unmarshal([]byte(src), &declared); err != nil {
|
|
return "", fmt.Errorf("parse workflow YAML: %w", err)
|
|
}
|
|
if strings.TrimSpace(declared.ID) == "" {
|
|
return "", fmt.Errorf("workflow id is required")
|
|
}
|
|
return declared.ID, nil
|
|
}
|
|
|
|
// PinnedDefinitionID returns the workflow id a task's definition pin declares,
|
|
// checking the pin against its digest. ok is false when the task carries no
|
|
// pin. This is the workflow the task's current run was compiled from, which is
|
|
// what the run's own log names; the task's workflow column names the workflow
|
|
// it runs next.
|
|
func PinnedDefinitionID(t *Task) (string, bool, error) {
|
|
if t.WorkflowDefinitionYAML == "" {
|
|
return "", false, nil
|
|
}
|
|
if DigestDefinition(t.WorkflowDefinitionYAML) != t.WorkflowDefinitionDigest {
|
|
return "", false, fmt.Errorf("stored workflow definition digest mismatch")
|
|
}
|
|
id, err := DefinitionID(t.WorkflowDefinitionYAML)
|
|
if err != nil {
|
|
return "", false, fmt.Errorf("stored workflow definition: %w", err)
|
|
}
|
|
return id, true, nil
|
|
}
|
|
|
|
// WorkflowDefinitionEntry and WorkflowDefinitionCollection are defined in
|
|
// package task so dashboards can decode the stored projection without linking
|
|
// the engine. Validation and compilation stay here.
|
|
type (
|
|
WorkflowDefinitionEntry = task.WorkflowDefinitionEntry
|
|
WorkflowDefinitionCollection = task.WorkflowDefinitionCollection
|
|
)
|
|
|
|
// ValidateDefinitionCollection rejects malformed, duplicate, or mismatched
|
|
// definitions, and every workflow.call step a definition carries: the callee
|
|
// must exist, its inputs must be satisfiable, and the call graph must be
|
|
// acyclic.
|
|
func ValidateDefinitionCollection(collection WorkflowDefinitionCollection, registry StepRegistry) error {
|
|
seen := make(map[string]struct{}, len(collection.Definitions))
|
|
parsed := make(map[string]YAMLDefinition, len(collection.Definitions))
|
|
for _, entry := range collection.Definitions {
|
|
if _, ok := seen[entry.ID]; ok {
|
|
return fmt.Errorf("duplicate workflow id %q", entry.ID)
|
|
}
|
|
seen[entry.ID] = struct{}{}
|
|
definition, err := ParseDefinition(entry.YAML, registry)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if definition.ID != entry.ID {
|
|
return fmt.Errorf("workflow entry id %q does not match YAML id %q", entry.ID, definition.ID)
|
|
}
|
|
parsed[entry.ID] = definition
|
|
}
|
|
if err := validateWorkflowCalls(parsed); err != nil {
|
|
return err
|
|
}
|
|
return validateControlTargets(parsed)
|
|
}
|
|
|
|
// DecodeDefinitionCollection strictly decodes the control-plane projection.
|
|
func DecodeDefinitionCollection(value []byte, registry StepRegistry) (WorkflowDefinitionCollection, error) {
|
|
var collection WorkflowDefinitionCollection
|
|
decoder := json.NewDecoder(bytes.NewReader(value))
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(&collection); err != nil {
|
|
return WorkflowDefinitionCollection{}, err
|
|
}
|
|
if err := ValidateDefinitionCollection(collection, registry); err != nil {
|
|
return WorkflowDefinitionCollection{}, err
|
|
}
|
|
return collection, nil
|
|
}
|
|
|
|
// ShippedDefinitions returns the restorable factory definitions.
|
|
func ShippedDefinitions() WorkflowDefinitionCollection {
|
|
return WorkflowDefinitionCollection{Definitions: shippedYAML()}
|
|
}
|
|
|
|
//go:embed shipped/*.yaml
|
|
var shippedFiles embed.FS
|
|
|
|
// shippedYAML is the shipped workflows written in the general step types.
|
|
func shippedYAML() []WorkflowDefinitionEntry {
|
|
files, _ := fs.Glob(shippedFiles, "shipped/*.yaml")
|
|
entries := make([]WorkflowDefinitionEntry, 0, len(files))
|
|
for _, name := range files {
|
|
data, err := shippedFiles.ReadFile(name)
|
|
if err != nil {
|
|
panic(err) // embedded at build time; unreadable means a broken binary
|
|
}
|
|
id := strings.TrimSuffix(path.Base(name), ".yaml")
|
|
entries = append(entries, WorkflowDefinitionEntry{ID: id, YAML: string(data)})
|
|
}
|
|
return entries
|
|
}
|
|
|
|
// branches is every list of steps nested in a step: its parallel branches or
|
|
// its switch cases.
|
|
func (step StepRecord) branches() map[string][]StepRecord {
|
|
if step.Switch != nil {
|
|
return step.Switch.Cases
|
|
}
|
|
return step.Parallel
|
|
}
|
|
|
|
// CheckModelAliases reports the first agent.run step in collection that names
|
|
// a model alias aliases does not define. An empty alias means the default and
|
|
// is resolved at dispatch.
|
|
func CheckModelAliases(collection WorkflowDefinitionCollection, registry StepRegistry, aliases map[string]string) error {
|
|
for _, entry := range collection.Definitions {
|
|
definition, err := ParseDefinition(entry.YAML, registry)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := checkStepAliases(entry.ID, definition.Steps, aliases); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func checkStepAliases(workflowID string, steps []StepRecord, aliases map[string]string) error {
|
|
for _, step := range steps {
|
|
if step.Type == AgentRunStepName {
|
|
var s agentRunSettings
|
|
if err := step.Settings.Decode(&s); err != nil {
|
|
return fmt.Errorf("workflow %q step %q: %w", workflowID, step.ID, err)
|
|
}
|
|
if s.Model != "" && aliases[s.Model] == "" {
|
|
return fmt.Errorf("workflow %q step %q: model alias %q is not configured", workflowID, step.ID, s.Model)
|
|
}
|
|
}
|
|
for _, branch := range step.Parallel {
|
|
if err := checkStepAliases(workflowID, branch, aliases); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if step.Switch != nil {
|
|
for _, branch := range step.Switch.Cases {
|
|
if err := checkStepAliases(workflowID, branch, aliases); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// modelFreeStepTypes are the step types that never call a model. Any other
|
|
// type counts as model use, so an unknown or new step keeps the credential
|
|
// boundary.
|
|
var modelFreeStepTypes = map[string]bool{
|
|
FinishStepName: true, HandoffStepName: true, ApproveStepName: true,
|
|
CloseIssueStepName: true, CommentStepName: true, CommandRunStepName: true,
|
|
WorkflowCallStepName: true, RepoPrepareStepName: true, RepoToolingStepName: true,
|
|
RepoCommitStepName: true,
|
|
RepoOpenPRStepName: true, GateRepositoryName: true, DiffRulesStepName: true,
|
|
}
|
|
|
|
// UsesModel reports whether a pinned definition has any step that may call a
|
|
// model. A missing or undecodable definition counts as using one.
|
|
func UsesModel(src string) bool {
|
|
var definition YAMLDefinition
|
|
if err := yaml.Unmarshal([]byte(src), &definition); err != nil || len(definition.Steps) == 0 {
|
|
return true
|
|
}
|
|
return stepsUseModel(definition.Steps)
|
|
}
|
|
|
|
func stepsUseModel(steps []StepRecord) bool {
|
|
for _, step := range steps {
|
|
for _, branch := range step.Parallel {
|
|
if stepsUseModel(branch) {
|
|
return true
|
|
}
|
|
}
|
|
if step.Switch != nil {
|
|
for _, branch := range step.Switch.Cases {
|
|
if stepsUseModel(branch) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
if step.Parallel == nil && step.Switch == nil && !modelFreeStepTypes[step.Type] {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// pushStepTypes are the steps that publish to the repository's forge.
|
|
var pushStepTypes = []string{RepoCommitStepName, RepoOpenPRStepName}
|
|
|
|
// DefinitionPushes reports whether a workflow definition has a step, at any
|
|
// depth, that pushes to the repository.
|
|
func DefinitionPushes(src string) bool {
|
|
var doc any
|
|
if yaml.Unmarshal([]byte(src), &doc) != nil {
|
|
return false
|
|
}
|
|
return hasStepType(doc, pushStepTypes)
|
|
}
|
|
|
|
func hasStepType(node any, types []string) bool {
|
|
switch n := node.(type) {
|
|
case map[string]any:
|
|
if t, ok := n["type"].(string); ok && slices.Contains(types, t) {
|
|
return true
|
|
}
|
|
for _, v := range n {
|
|
if hasStepType(v, types) {
|
|
return true
|
|
}
|
|
}
|
|
case []any:
|
|
for _, v := range n {
|
|
if hasStepType(v, types) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// RepairDefinitionCollection rebuilds a stored collection that no longer
|
|
// validates, keeping every entry it can: a valid entry is kept, an invalid
|
|
// shipped entry is replaced by this build's copy, and any other invalid or
|
|
// duplicate entry is removed and its id returned. An unreadable document or a
|
|
// repair that still fails validation is an error.
|
|
func RepairDefinitionCollection(value []byte, registry StepRegistry) (WorkflowDefinitionCollection, []string, error) {
|
|
var stored WorkflowDefinitionCollection
|
|
if err := json.Unmarshal(value, &stored); err != nil {
|
|
return WorkflowDefinitionCollection{}, nil, err
|
|
}
|
|
shipped := ShippedDefinitions()
|
|
seen := make(map[string]struct{}, len(stored.Definitions))
|
|
var repaired WorkflowDefinitionCollection
|
|
var removed []string
|
|
for _, entry := range stored.Definitions {
|
|
if _, dup := seen[entry.ID]; dup {
|
|
removed = append(removed, entry.ID)
|
|
continue
|
|
}
|
|
seen[entry.ID] = struct{}{}
|
|
if definition, err := ParseDefinition(entry.YAML, registry); err == nil && definition.ID == entry.ID {
|
|
repaired.Definitions = append(repaired.Definitions, entry)
|
|
} else if factory, ok := shipped.DefinitionByID(entry.ID); ok {
|
|
repaired.Definitions = append(repaired.Definitions, factory)
|
|
} else {
|
|
removed = append(removed, entry.ID)
|
|
}
|
|
}
|
|
if err := ValidateDefinitionCollection(repaired, registry); err != nil {
|
|
return WorkflowDefinitionCollection{}, nil, err
|
|
}
|
|
return repaired, removed, nil
|
|
}
|