From 4afefe087ef8b0ee6c74146b10ab1eeed81c60bf Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Fri, 2 Oct 2026 02:07:27 +1000 Subject: [PATCH] feat(workflow): declared workflow outputs for wait:true callers A workflow declares named, typed outputs beside its inputs (task.WorkflowInterface.Outputs). A run writes each one once through a capture tool named after the output, or from a deterministic Go stage via TaskContext.WriteOutput; the finish write persists them on tasks.outputs before the outcome transition. A wait:true caller receives its successful callee's outputs through WorkflowCallStatus and publishes one as its own through a call step's outputs assignment. Build authority: docs/prds/workflow-call-outputs.md. - save time: WorkflowInterface.Validate checks output names and types; checkCallOutputs checks a call's assignment against both declarations and refuses outputs over a wait:false call. - run time: undeclared, mistyped and required-but-missing values park the run; published callee values are re-checked against the caller's type. - storage/wire: tasks.outputs (goose 0024), reset by each claim so the set is attempt-scoped; an encoded set past the structured-payload bound is refused, not clipped. Task.outputs_json and WorkflowCallStatusResponse.outputs_json carry it; task.Caller keeps its two methods. - examples: firewall-contain calls firewall-investigate and publishes its summary as its own output. --- examples/workflows/firewall-contain.yaml | 28 ++ examples/workflows/firewall-investigate.yaml | 9 +- internal/contracts/state/v1/state.pb.go | 41 ++- internal/domain/workflow/agent.go | 16 +- internal/domain/workflow/changes_test.go | 14 +- internal/domain/workflow/definition.go | 2 +- internal/domain/workflow/outputs.go | 226 +++++++++++++++ internal/domain/workflow/outputs_test.go | 269 ++++++++++++++++++ internal/domain/workflow/task/caller.go | 11 +- internal/domain/workflow/task/interface.go | 79 ++++- .../domain/workflow/task/interface_test.go | 109 +++++++ internal/domain/workflow/task/task.go | 6 + internal/domain/workflow/workflow.go | 22 ++ internal/domain/workflow/workflow_call.go | 77 ++++- .../domain/workflow/workflow_call_test.go | 200 ++++++++++++- .../agenttransport/nats/transport.go | 2 +- .../migrations/0024_workflow_outputs.sql | 17 ++ .../postgres/postgresdb/call_tasks.sql.go | 5 +- .../postgres/postgresdb/models.go | 1 + .../postgres/postgresdb/tasks.sql.go | 43 ++- .../infrastructure/postgres/queries/tasks.sql | 19 +- internal/infrastructure/postgres/store.go | 57 +++- .../postgres/tasks_outputs_test.go | 129 +++++++++ internal/infrastructure/staterpc/client.go | 6 +- internal/infrastructure/staterpc/server.go | 4 +- .../staterpc/task_grants_call_test.go | 6 +- internal/infrastructure/staterpc/values.go | 18 +- .../workflow_call_conformance_test.go | 35 ++- proto/state/v1/state.proto | 12 +- 29 files changed, 1381 insertions(+), 82 deletions(-) create mode 100644 examples/workflows/firewall-contain.yaml create mode 100644 internal/domain/workflow/outputs.go create mode 100644 internal/domain/workflow/outputs_test.go create mode 100644 internal/infrastructure/postgres/migrations/0024_workflow_outputs.sql create mode 100644 internal/infrastructure/postgres/tasks_outputs_test.go diff --git a/examples/workflows/firewall-contain.yaml b/examples/workflows/firewall-contain.yaml new file mode 100644 index 000000000..23507bd4c --- /dev/null +++ b/examples/workflows/firewall-contain.yaml @@ -0,0 +1,28 @@ +# The containment workflow a firewall-investigate run calls: +# firewall-investigate (workflow.call, wait: true) hands its findings upward +# from its declared summary, and this workflow's own agent stage decides and +# records the containment as its own declared outputs. Paste it into the +# dashboard's workflow definitions alongside firewall-investigate +# (docs/prds/workflow-call-outputs.md). +id: firewall-contain +repository: none +profile: network-investigator +inputs: + src_ip: { type: string, required: true } +outputs: + contained: { type: bool, required: true } + summary: { type: string } +steps: + - type: workflow.call + settings: + workflow: firewall-investigate + wait: true + inputs: { src_ip: "inputs.src_ip" } + outputs: { summary: "outputs.summary" } + - type: agent.run + settings: + mission: >- + Using the investigation summary the called workflow returned, decide + whether the source address in the inputs should be contained. Write + your decision with the contained tool, and a one-line rationale with + the summary tool. \ No newline at end of file diff --git a/examples/workflows/firewall-investigate.yaml b/examples/workflows/firewall-investigate.yaml index e0e4d2df8..81d579ae0 100644 --- a/examples/workflows/firewall-investigate.yaml +++ b/examples/workflows/firewall-investigate.yaml @@ -2,17 +2,24 @@ # src_ip from a mapped parameter and runs one agent mission in a scratch # workspace under the network-investigator profile. Paste it into the # dashboard's workflow definitions. +# +# Its declared output (summary) is offered to the agent stages as a capture +# tool named after it; a wait:true caller publishes it as one of its own +# outputs (docs/prds/workflow-call-outputs.md). id: firewall-investigate repository: none profile: network-investigator inputs: src_ip: { type: string, required: true } severity: { type: string } +outputs: + summary: { type: string } steps: - type: agent.run settings: mission: >- Investigate the source address in the inputs: who owns it, what it resolves to, and whether it is reachable. Report what you found and - how serious it looks given the severity. + how serious it looks given the severity, and write your one-line + conclusion with the summary tool. read_only: true diff --git a/internal/contracts/state/v1/state.pb.go b/internal/contracts/state/v1/state.pb.go index a6adc5d91..34a0121a2 100644 --- a/internal/contracts/state/v1/state.pb.go +++ b/internal/contracts/state/v1/state.pb.go @@ -1336,8 +1336,14 @@ type Task struct { // writer reads as a depth-0 root run. CallParentTaskId int64 `protobuf:"varint,36,opt,name=call_parent_task_id,json=callParentTaskId,proto3" json:"call_parent_task_id,omitempty"` CallDepth int32 `protobuf:"varint,37,opt,name=call_depth,json=callDepth,proto3" json:"call_depth,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // outputs_json is the JSON object of declared-output values this run wrote + // (task.EncodeOutputs), empty for a run that wrote none. The container's + // Update sends the whole row, so a column the message cannot carry is a + // column that write drops (docs/prds/workflow-call-outputs.md, "Storage and + // wire"). + OutputsJson string `protobuf:"bytes,38,opt,name=outputs_json,json=outputsJson,proto3" json:"outputs_json,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Task) Reset() { @@ -1630,6 +1636,13 @@ func (x *Task) GetCallDepth() int32 { return 0 } +func (x *Task) GetOutputsJson() string { + if x != nil { + return x.OutputsJson + } + return "" +} + // Event mirrors internal/events.Event. Data is carried as a JSON object // string (matching the store's own on-disk representation) rather than // google.protobuf.Struct, to stay a direct mirror of the existing column. @@ -10221,9 +10234,13 @@ func (x *WorkflowCallStatusRequest) GetCallTaskId() int64 { type WorkflowCallStatusResponse struct { state protoimpl.MessageState `protogen:"open.v1"` // status is the callee's task status and detail its latest transition - // detail (empty before the callee first moves). + // detail (empty before the callee first moves). outputs_json is the + // callee's written outputs (task.EncodeOutputs), so a wait:true caller can + // publish one as its own (docs/prds/workflow-call-outputs.md, "Storage and + // wire"). Status string `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` Detail string `protobuf:"bytes,2,opt,name=detail,proto3" json:"detail,omitempty"` + OutputsJson string `protobuf:"bytes,3,opt,name=outputs_json,json=outputsJson,proto3" json:"outputs_json,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -10272,6 +10289,13 @@ func (x *WorkflowCallStatusResponse) GetDetail() string { return "" } +func (x *WorkflowCallStatusResponse) GetOutputsJson() string { + if x != nil { + return x.OutputsJson + } + return "" +} + type InstalledPackage struct { state protoimpl.MessageState `protogen:"open.v1"` OrgId string `protobuf:"bytes,1,opt,name=org_id,json=orgId,proto3" json:"org_id,omitempty"` @@ -11939,7 +11963,8 @@ const file_state_v1_state_proto_rawDesc = "" + "\asubject\x18\x03 \x01(\tR\asubject\x12-\n" + "\x05audit\x18\x04 \x01(\v2\x17.state.v1.IdentityAuditR\x05audit\"M\n" + "\x1bBindIdentitySubjectResponse\x12.\n" + - "\bidentity\x18\x01 \x01(\v2\x12.state.v1.IdentityR\bidentity\"\xde\t\n" + + "\bidentity\x18\x01 \x01(\v2\x12.state.v1.IdentityR\bidentity\"\x81\n" + + "\n" + "\x04Task\x12\x0e\n" + "\x02id\x18\x01 \x01(\x03R\x02id\x12\x14\n" + "\x05owner\x18\x02 \x01(\tR\x05owner\x12\x12\n" + @@ -11989,7 +12014,8 @@ const file_state_v1_state_proto_rawDesc = "" + "\x03org\x18# \x01(\tR\x03org\x12-\n" + "\x13call_parent_task_id\x18$ \x01(\x03R\x10callParentTaskId\x12\x1d\n" + "\n" + - "call_depth\x18% \x01(\x05R\tcallDepth\"\xf8\x02\n" + + "call_depth\x18% \x01(\x05R\tcallDepth\x12!\n" + + "\foutputs_json\x18& \x01(\tR\voutputsJson\"\xf8\x02\n" + "\x05Event\x12\x0e\n" + "\x02id\x18\x01 \x01(\x03R\x02id\x12*\n" + "\x02at\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x02at\x12\x12\n" + @@ -12541,10 +12567,11 @@ const file_state_v1_state_proto_rawDesc = "" + "\x19WorkflowCallStatusRequest\x12$\n" + "\x0ecaller_task_id\x18\x01 \x01(\x03R\fcallerTaskId\x12 \n" + "\fcall_task_id\x18\x02 \x01(\x03R\n" + - "callTaskId\"L\n" + + "callTaskId\"o\n" + "\x1aWorkflowCallStatusResponse\x12\x16\n" + "\x06status\x18\x01 \x01(\tR\x06status\x12\x16\n" + - "\x06detail\x18\x02 \x01(\tR\x06detail\"\xa2\x02\n" + + "\x06detail\x18\x02 \x01(\tR\x06detail\x12!\n" + + "\foutputs_json\x18\x03 \x01(\tR\voutputsJson\"\xa2\x02\n" + "\x10InstalledPackage\x12\x15\n" + "\x06org_id\x18\x01 \x01(\tR\x05orgId\x12\x12\n" + "\x04name\x18\x02 \x01(\tR\x04name\x12\x1c\n" + diff --git a/internal/domain/workflow/agent.go b/internal/domain/workflow/agent.go index 797332d91..06c12cea5 100644 --- a/internal/domain/workflow/agent.go +++ b/internal/domain/workflow/agent.go @@ -70,7 +70,10 @@ func (a AgentStage) Stage() Stage { if err != nil { return err } - req := a.buildRequest(tc, modelRef) + req, buildErr := a.buildRequest(tc, modelRef) + if buildErr != nil { + return buildErr + } // The agent call is a child of the stage's recorded step: the store // says what the stage's runtime did, not just what the stage did. res, runErr := tc.RunAgentChild(ctx, a.Name, func() (agentexec.Result, error) { @@ -94,7 +97,7 @@ func (a AgentStage) resolveModel(tc *TaskContext) (string, error) { } // buildRequest assembles the agentexec.Request for this stage's run. -func (a AgentStage) buildRequest(tc *TaskContext, modelRef string) agentexec.Request { +func (a AgentStage) buildRequest(tc *TaskContext, modelRef string) (agentexec.Request, error) { budget := agentexec.Budget{ MaxSteps: tc.Cfg.Budgets.MaxSteps, WallClock: tc.Cfg.Budgets.WallClock.Std(), @@ -111,6 +114,10 @@ func (a AgentStage) buildRequest(tc *TaskContext, modelRef string) agentexec.Req if a.CaptureTools != nil { captureTools = a.CaptureTools(tc) } + captureTools, err := tc.appendOutputTools(captureTools) + if err != nil { + return agentexec.Request{}, err + } protection := agentexec.Protection{Suffixes: append([]string(nil), tc.Repo.Protect...)} if a.ProtectGlobs != nil { @@ -146,7 +153,7 @@ func (a AgentStage) buildRequest(tc *TaskContext, modelRef string) agentexec.Req Notes: tc.Task.Notes, CaptureTools: captureTools, Plugins: pluginSpecs(tc.SkillPlugins), - } + }, nil } // buildExtraRules prepends memory context, if the daemon wired a memory @@ -236,6 +243,9 @@ func (a AgentStage) deliverResult(tc *TaskContext, res agentexec.Result) error { return fmt.Errorf("review: %w", err) } } + if err := tc.applyOutputCaptures(res); err != nil { + return err + } if a.OnResult != nil { return a.OnResult(tc, res) } diff --git a/internal/domain/workflow/changes_test.go b/internal/domain/workflow/changes_test.go index a003c8f1a..83493e3cb 100644 --- a/internal/domain/workflow/changes_test.go +++ b/internal/domain/workflow/changes_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "log/slog" + "maps" "os" "path/filepath" "strings" @@ -25,6 +26,8 @@ type recordingStore struct { transitions []transition steps []StepStart finishes []StepFinish + updates []Task + sequence []string insertEr error stepEr error finishEr error @@ -37,10 +40,19 @@ type recordingStore struct { // never parks the task" a test rather than a reading of the code. type transition struct{ from, to, detail string } -func (s *recordingStore) Update(context.Context, *Task) error { return nil } +func (s *recordingStore) Update(_ context.Context, t *Task) error { + clone := *t + outputs := make(map[string]any, len(t.Outputs)) + maps.Copy(outputs, t.Outputs) + clone.Outputs = outputs + s.updates = append(s.updates, clone) + s.sequence = append(s.sequence, "update") + return nil +} func (s *recordingStore) Transition(_ context.Context, _ int64, from, to, detail string) error { s.transitions = append(s.transitions, transition{from: from, to: to, detail: detail}) + s.sequence = append(s.sequence, "transition:"+to) return nil } diff --git a/internal/domain/workflow/definition.go b/internal/domain/workflow/definition.go index 538de9b35..402329efb 100644 --- a/internal/domain/workflow/definition.go +++ b/internal/domain/workflow/definition.go @@ -87,7 +87,7 @@ func Compile(definition YAMLDefinition, registry StepRegistry) (Workflow, error) } stages = append(stages, stage) } - return Workflow{Name: definition.ID, Stages: stages}, nil + return Workflow{Name: definition.ID, Stages: stages, Interface: definition.WorkflowInterface}, nil } // ParseAndCompile validates and compiles one definition in a single operation. diff --git a/internal/domain/workflow/outputs.go b/internal/domain/workflow/outputs.go new file mode 100644 index 000000000..bd8aed7ce --- /dev/null +++ b/internal/domain/workflow/outputs.go @@ -0,0 +1,226 @@ +package workflow + +import ( + "encoding/json" + "fmt" + "maps" + "slices" + "strings" + + "github.com/samcharles93/archie-core/internal/agentexec" + "github.com/samcharles93/archie-core/internal/domain/workflow/task" +) + +// Declared workflow outputs: how a run writes one and how a wait:true caller +// publishes one (docs/prds/workflow-call-outputs.md). A run's only structured +// results are capture calls, so declared outputs ride the capture path: each +// is offered to the run's agent stages as a capture tool named after it, and +// the engine applies, validates, publishes and persists the accepted values. + +// workflowInterface is the declaration the run's compiled workflow carries +// (Workflow.Interface, set by Run); a stage body invoked outside Run falls +// back to the pinned definition's YAML, which the pin guarantees matches the +// compile the request came from. +func (tc *TaskContext) workflowInterface() task.WorkflowInterface { + if tc.runInterfaceSet { + return tc.runInterface + } + if tc.ifaceParsed == nil { + parsed, err := task.ParseWorkflowInterface(tc.Task.WorkflowDefinitionYAML) + if err != nil { + // The pinned YAML was validated when the definition was saved, so + // a parse failure here cannot be a definition the engine accepts; + // an empty interface keeps such a run output-less. + parsed = task.WorkflowInterface{} + tc.Log.Warn("pinned workflow definition does not parse for outputs; ignoring declared outputs", "err", err) + } + tc.ifaceParsed = &parsed + } + return *tc.ifaceParsed +} + +// WriteOutput records one declared output's value for this attempt: the path +// a deterministic Go stage takes. An undeclared key, a mistyped value or a +// second write is refused, and the refusing error parks the run before its +// outcome transition. +func (tc *TaskContext) WriteOutput(name string, value any) error { + spec, ok := tc.workflowInterface().Outputs[name] + if !ok { + return fmt.Errorf("output %q is not declared by the workflow", name) + } + if _, written := tc.Task.Outputs[name]; written { + return fmt.Errorf("output %q has already been written; each output is written once per attempt", name) + } + if value != nil { + if got := task.ValueType(value); !task.TypeAccepts(spec.Type, got) { + return fmt.Errorf("output %q is %s, want %s", name, got, spec.Type) + } + } + if tc.Task.Outputs == nil { + tc.Task.Outputs = map[string]any{} + } + tc.Task.Outputs[name] = value + return nil +} + +// outputCaptureTool builds the capture tool one declared output is offered +// as: named after the output, whose arguments are {"value": ...}, with the +// declared type as the value's schema. The schema guides the agent; the +// run-time re-check is what enforces. +func outputCaptureTool(name string, spec task.OutputSpec) agentexec.CaptureTool { + value := map[string]any{"description": fmt.Sprintf("The value of the %q output.", name)} + if typ := jsonSchemaType(spec.Type); typ != "" { + value["type"] = typ + } + params, err := json.Marshal(map[string]any{ + "type": "object", + "properties": map[string]any{"value": value}, + "required": []string{"value"}, + }) + if err != nil { + // A fixed-shape map cannot fail to marshal. Keep the call acceptable + // through RequiredFields alone if it ever did. + params = []byte(`{"type":"object"}`) + } + return agentexec.CaptureTool{ + Name: name, + Description: fmt.Sprintf("Write the workflow output %q. Call exactly once, with the value as the value argument.", name), + Parameters: params, + // The value member is required, but a null value counts as not written + // (there is nothing to reject: the run simply does not write it). + RequiredFields: []string{"value"}, + MaxCalls: 1, + } +} + +// jsonSchemaType maps the workflow value vocabulary onto JSON Schema's type +// names. "any" has no constraint, so it yields no type at all -- an empty +// string, never a null that is not a valid JSON Schema type. +func jsonSchemaType(vocab string) string { + switch vocab { + case "string", "number", "object", "array": + return vocab + case "bool": + return "boolean" + default: // "any" + return "" + } +} + +// appendOutputTools offers the declared outputs the run has not written yet +// to this stage as capture tools, refusing a stage capture tool that shares a +// declared output's name: the value would otherwise be ambiguous. +func (tc *TaskContext) appendOutputTools(tools []agentexec.CaptureTool) ([]agentexec.CaptureTool, error) { + iface := tc.workflowInterface() + for _, tool := range tools { + if _, clash := iface.Outputs[tool.Name]; clash { + return nil, fmt.Errorf("output %q is also a stage capture tool's name; a declared output is written through its own tool", tool.Name) + } + } + for _, name := range slices.Sorted(maps.Keys(iface.Outputs)) { + // A stage is not offered an output the run has written. + if _, already := tc.Task.Outputs[name]; already { + continue + } + tools = append(tools, outputCaptureTool(name, iface.Outputs[name])) + } + return tools, nil +} + +// applyOutputCaptures applies the declared-output capture calls a passed +// agent result carries: the accepted call's value member is the output's +// value, re-validated against the declared type the way readCaptures +// re-checks every record, before it enters the run's set. +func (tc *TaskContext) applyOutputCaptures(res agentexec.Result) error { + iface := tc.workflowInterface() + for name, calls := range res.Captures { + spec, ok := iface.Outputs[name] + if !ok { + // Every other capture tool is the stage's own; its OnResult owns + // its records. + continue + } + for _, call := range calls { + value, written, err := decodeCapturedOutput(call) + if err != nil { + return fmt.Errorf("output %q: %w", name, err) + } + if !written { + continue + } + if got := task.ValueType(value); !task.TypeAccepts(spec.Type, got) { + return fmt.Errorf("output %q is %s, want %s", name, got, spec.Type) + } + if _, written := tc.Task.Outputs[name]; written { + return fmt.Errorf("output %q has already been written; each output is written once per attempt", name) + } + if tc.Task.Outputs == nil { + tc.Task.Outputs = map[string]any{} + } + tc.Task.Outputs[name] = value + } + } + return nil +} + +// decodeCapturedOutput reads one accepted capture call's value member. A call +// with no value, or a null value, counts as not written, as CheckInputs +// treats a null input. +func decodeCapturedOutput(call json.RawMessage) (value any, written bool, err error) { + var args struct { + Value json.RawMessage `json:"value"` + } + if err := json.Unmarshal(call, &args); err != nil { + return nil, false, err + } + if len(args.Value) == 0 || string(args.Value) == "null" { + return nil, false, nil + } + decoder := json.NewDecoder(strings.NewReader(string(args.Value))) + decoder.UseNumber() + if err := decoder.Decode(&value); err != nil { + return nil, false, err + } + return value, true, nil +} + +// validateFinishOutputs checks the attempt's written outputs before the +// outcome transition: required, undeclared and mistyped values park here, so +// no caller observes a terminal state that breaks the run's own promise. +func (tc *TaskContext) validateFinishOutputs() error { + return tc.workflowInterface().CheckOutputs(tc.Task.Outputs) +} + +// publishCalleeOutputs publishes a successful callee's outputs the call step +// assigns into the caller's own set. An output the callee never wrote leaves +// the caller's output absent -- the caller's own required judgment decides; a +// written null counts as not written; a value whose type the caller's declared +// output cannot accept is refused, naming the output. +func (tc *TaskContext) publishCalleeOutputs(assignments map[string]string, calleeOutputs map[string]any) error { + if len(assignments) == 0 { + return nil + } + iface := tc.workflowInterface() + for _, calleeName := range slices.Sorted(maps.Keys(assignments)) { + ref := assignments[calleeName] + callerName := strings.TrimPrefix(ref, outputReference) + spec, ok := iface.Outputs[callerName] + if !ok { + // The save check refuses an undeclared reference; this re-check + // keeps a swapped callee from publishing into nothing. + return fmt.Errorf("output %q references %q, which this workflow does not declare", calleeName, ref) + } + value, ok := calleeOutputs[calleeName] + if !ok || value == nil { + // Declared but never written: the caller's output stays absent. + continue + } + if got := task.ValueType(value); !task.TypeAccepts(spec.Type, got) { + return fmt.Errorf("callee output %q is %s, which this workflow's output %q (%s) cannot accept", calleeName, got, callerName, spec.Type) + } + if err := tc.WriteOutput(callerName, value); err != nil { + return err + } + } + return nil +} diff --git a/internal/domain/workflow/outputs_test.go b/internal/domain/workflow/outputs_test.go new file mode 100644 index 000000000..a3af66b8c --- /dev/null +++ b/internal/domain/workflow/outputs_test.go @@ -0,0 +1,269 @@ +package workflow + +import ( + "context" + "encoding/json" + "log/slog" + "slices" + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/agentexec" + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/workflow/task" +) + +// outputsRunYAML is the smallest workflow with a declared output: one agent +// stage under repository none. +const outputsRunYAML = "id: w\nrepository: none\noutputs:\n verdict: {type: bool, required: true}\n summary: {type: string}\nsteps:\n - type: agent.run\n settings:\n mission: m\n" + +// outputsTC returns a TaskContext wired like the runtime tests, with the +// workflow's own YAML pinned on the task row. +func outputsTC(t *testing.T, yaml string, store Store, agent agentexec.Runner) *TaskContext { + t.Helper() + return &TaskContext{ + Task: &Task{ID: 3, Attempt: 1, WorkflowDefinitionYAML: yaml}, + Store: store, Trees: &fakeTrees{dir: "/scratch/3"}, Agent: agent, + Cfg: config.Config{Models: map[string]string{"builder": "p/m"}}, + Log: slog.New(slog.DiscardHandler), + } +} + +// outputCaptureResult builds a passed agent result carrying one accepted capture +// call for the output tool named name. +func outputCaptureResult(name string, args ...string) agentexec.Result { + captures := map[string][]json.RawMessage{} + for _, raw := range args { + captures[name] = append(captures[name], json.RawMessage(raw)) + } + return agentexec.Result{Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed, Summary: "done", Captures: captures} +} + +// TestRunOffersOutputsAsCaptureToolsAndAppliesThem pins the agent write path +// (docs/prds/workflow-call-outputs.md, "How a run writes one"): each declared +// output is a capture tool named after it with a {"value": ...} argument, and +// an accepted call enters the run's output set, which the finish write +// persists into the row before the outcome transition. +func TestRunOffersOutputsAsCaptureToolsAndAppliesThem(t *testing.T) { + wf := compileCaller(t, outputsRunYAML) + runner := &fakeAgentRunner{result: outputCaptureResult("verdict", `{"value": true}`)} + store := &recordingStore{} + tc := outputsTC(t, outputsRunYAML, store, runner) + Run(context.Background(), wf, tc) + + var offered bool + for _, tool := range runner.request.CaptureTools { + if tool.Name == "verdict" { + offered = true + if tool.MaxCalls != 1 { + t.Errorf("verdict tool MaxCalls = %d, want 1: each output is written once per attempt", tool.MaxCalls) + } + if !slicesContains(tool.RequiredFields, "value") { + t.Errorf("verdict tool RequiredFields = %v, want [value]", tool.RequiredFields) + } + if !strings.Contains(string(tool.Parameters), `"value"`) { + t.Errorf("verdict tool parameters = %s, want a value argument", tool.Parameters) + } + } + } + if !offered { + t.Fatalf("CaptureTools = %+v, want a tool named for the declared output", runner.request.CaptureTools) + } + if got := tc.Task.Outputs["verdict"]; got != true { + t.Fatalf("run outputs = %+v, want {verdict: true} with bool restored", tc.Task.Outputs) + } + // The finish write replaces the row's set wholesale before the outcome + // transition carries the value to any caller. + if len(store.updates) == 0 { + t.Fatal("the run never persisted its row") + } + lastUpdate := store.updates[len(store.updates)-1] + if lastUpdate.Outputs["verdict"] != true { + t.Fatalf("row outputs = %+v, want the written verdict", lastUpdate.Outputs) + } + if seq := updatesBeforeTransition(store.sequence, "completed"); seq == -1 { + t.Fatalf("sequence = %v, want an update before the completed transition", store.sequence) + } else if store.updates[seq].Outputs["verdict"] != true { + t.Fatalf("update %d outputs = %+v, want the written verdict persisted before the transition", seq, store.updates[seq].Outputs) + } +} + +// TestWrittenOutputIsNotOfferedToALaterStage pins the once-per-attempt rule: +// a stage is not offered an output the run has written, so a later stage +// cannot overwrite a value. +func TestWrittenOutputIsNotOfferedToALaterStage(t *testing.T) { + yaml := string(append([]byte("id: w\nrepository: none\noutputs:\n verdict: {type: bool}\nsteps:\n"), []byte(" - type: agent.run\n settings:\n mission: first\n - type: agent.run\n settings:\n mission: second\n")...)) + wf := compileCaller(t, yaml) + var offered []bool + agent := agentRunnerFunc(func(_ context.Context, _ string, req agentexec.Request, _ agentexec.ToolCallReporter) (agentexec.Result, error) { + for _, tool := range req.CaptureTools { + if tool.Name == "verdict" { + offered = append(offered, true) + } + } + return agentexec.Result{Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed, Summary: "ok"}, nil + }) + tc := outputsTC(t, yaml, &recordingStore{}, agent) + Run(context.Background(), wf, tc) + if len(offered) != 1 { + t.Fatalf("verdict tool offered %d times (%v), want exactly once across the run's stages", len(offered), offered) + } +} + +// TestCapturedOutputIsReValidated pins the run-time type check: a captured +// value is re-validated against the declared type when the run reads it +// back, and a null counts as not written. +func TestCapturedOutputIsReValidated(t *testing.T) { + t.Run("mistyped capture parks naming the output", func(t *testing.T) { + wf := compileCaller(t, outputsRunYAML) + store := &recordingStore{} + agent := &fakeAgentRunner{result: outputCaptureResult("verdict", `{"value": "yes"}`)} + tc := outputsTC(t, outputsRunYAML, store, agent) + Run(context.Background(), wf, tc) + if tc.Outcome.Status == StatusCompleted && !strings.Contains(tc.Task.ParkReason, `output "verdict" is string, want bool`) { + t.Fatalf("park reason %q, want the mistyped output named; outcome %+v", tc.Task.ParkReason, tc.Outcome) + } + }) + t.Run("null capture counts as not written", func(t *testing.T) { + yaml := "id: w\nrepository: none\noutputs:\n summary: {type: string}\nsteps:\n - type: agent.run\n settings:\n mission: m\n" + wf := compileCaller(t, yaml) + store := &recordingStore{} + agent := &fakeAgentRunner{result: outputCaptureResult("summary", `{"value": null}`)} + tc := outputsTC(t, yaml, store, agent) + Run(context.Background(), wf, tc) + if tc.Outcome.Status != StatusCompleted { + t.Fatalf("outcome = %+v, want completed (the optional output stayed absent)", tc.Outcome) + } + if _, written := tc.Task.Outputs["summary"]; written { + t.Fatalf("run outputs = %+v, want the null write to count as not written", tc.Task.Outputs) + } + }) +} + +// TestRequiredOutputNeverWrittenParks pins the promise rule: a run whose +// required output is never written parks naming the output, reaching no +// successful terminal state (docs/prds/workflow-call-outputs.md, "Failure +// rules"). +func TestRequiredOutputNeverWrittenParks(t *testing.T) { + wf := compileCaller(t, outputsRunYAML) + store := &recordingStore{} + agent := &fakeAgentRunner{result: agentexec.Result{Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed, Summary: "done"}} + tc := outputsTC(t, outputsRunYAML, store, agent) + Run(context.Background(), wf, tc) + if !strings.Contains(tc.Task.ParkReason, `output "verdict" is required`) { + t.Fatalf("park reason %q, want the missing required output named", tc.Task.ParkReason) + } + if len(store.transitions) == 0 || store.transitions[len(store.transitions)-1].to != StatusParked { + t.Fatalf("transitions = %+v, want the run parked rather than reaching a terminal state", store.transitions) + } +} + +// TestGoStageOutputRules pins the deterministic Go path: a stage sets the +// same values through WriteOutput, which refuses an undeclared key, a +// mistyped value and a second write; a key a stage bypasses it with parks at +// the finish check, identically to a captured one. +func TestGoStageOutputRules(t *testing.T) { + iface := task.WorkflowInterface{Outputs: map[string]task.OutputSpec{ + "verdict": {Type: "bool"}, + "note": {Type: "string"}, + }} + tc := &TaskContext{Task: &Task{ID: 1, WorkflowDefinitionYAML: "id: w\noutputs:\n verdict: {type: bool}\n"}} + tc.runInterface, tc.runInterfaceSet = iface, true + + if err := tc.WriteOutput("verdict", true); err != nil { + t.Fatalf("WriteOutput(verdict) = %v, want nil", err) + } + if got := tc.Task.Outputs["verdict"]; got != true { + t.Fatalf("outputs = %+v, want the written value", got) + } + if err := tc.WriteOutput("verdict", false); err == nil || !strings.Contains(err.Error(), "verdict") { + t.Fatalf("second WriteOutput(verdict) = %v, want a refusal naming the output", err) + } + if err := tc.WriteOutput("bogus", 7); err == nil || !strings.Contains(err.Error(), "not declared") { + t.Fatalf("WriteOutput(bogus) = %v, want an undeclared-output refusal", err) + } + if err := tc.WriteOutput("note", json.Number("3")); err == nil || !strings.Contains(err.Error(), `is number, want string`) { + t.Fatalf("WriteOutput(note, number) = %v, want a type refusal", err) + } + + // A value set around WriteOutput is judged at finish, so a bypass cannot + // smuggle an undeclared key into the row. + wf := Workflow{Name: "w", Stages: []Stage{{Name: "set", Run: func(_ context.Context, tc *TaskContext) error { + tc.Task.Outputs = map[string]any{"bogus": 7} + tc.Outcome = Outcome{Status: StatusCompleted, Detail: "done"} + return nil + }}}, Interface: iface} + store := &recordingStore{} + tc = &TaskContext{Task: &Task{ID: 9, WorkflowDefinitionYAML: "id: w\n"}, Store: store, Trees: &fakeTrees{}, Log: slog.New(slog.DiscardHandler)} + Run(context.Background(), wf, tc) + if !strings.Contains(tc.Task.ParkReason, `"bogus" is not declared`) { + t.Fatalf("park reason %q, want the undeclared key named; outcome %+v", tc.Task.ParkReason, tc.Outcome) + } +} + +// TestStageCaptureToolSharingOutputNameRefused pins the ambiguity refusal: a +// stage's own capture tool may not share a declared output's name, because +// the value would otherwise be ambiguous +// (docs/prds/workflow-call-outputs.md, "Failure rules"). +func TestStageCaptureToolSharingOutputNameRefused(t *testing.T) { + stage := AgentStage{ + Name: "decide", + Role: "builder", + Mission: func(*TaskContext) string { return "m" }, + CaptureTools: func(*TaskContext) []agentexec.CaptureTool { + return []agentexec.CaptureTool{{Name: "verdict", MaxCalls: 1}} + }, + }.Stage() + tc := &TaskContext{ + Task: &Task{ID: 3, WorkflowDefinitionYAML: outputsRunYAML}, + Agent: &fakeAgentRunner{result: outputCaptureResult("verdict", `{"value": true}`)}, + Cfg: config.Config{Models: map[string]string{"builder": "p/m"}}, + Log: slog.New(slog.DiscardHandler), + } + err := stage.Run(context.Background(), tc) + if err == nil || !strings.Contains(err.Error(), "verdict") { + t.Fatalf("stage error = %v, want a refusal naming the shared tool name", err) + } +} + +func slicesContains(list []string, want string) bool { + return slices.Contains(list, want) +} + +// updatesBeforeTransition returns the index of the last update recorded +// before the transition to to_ was asked for, or -1 when there is none. +func updatesBeforeTransition(sequence []string, to string) int { + last := -1 + for i, entry := range sequence { + if entry == "update" { + last = i + continue + } + if entry == "transition:"+to { + return last + } + } + return -1 +} + +// TestOutputCaptureToolSchemaOmitsUnconstrainedType pins the "any" output's +// tool schema: no valid JSON Schema type exists for an unconstrained value, so +// the property carries none (a null type is not a valid schema, and would +// misguide a strict adapter). +func TestOutputCaptureToolSchemaOmitsUnconstrainedType(t *testing.T) { + for _, test := range []struct { + spec task.OutputSpec + want string + }{ + {spec: task.OutputSpec{Type: "bool"}, want: `"type":"boolean"`}, + {spec: task.OutputSpec{Type: "any"}, want: ""}, + } { + tool := outputCaptureTool("out", test.spec) + if strings.Contains(string(tool.Parameters), `"type":null`) { + t.Fatalf("output type %q parameters = %s, want no null schema type", test.spec.Type, tool.Parameters) + } + if test.want != "" && !strings.Contains(string(tool.Parameters), test.want) { + t.Fatalf("output type %q parameters = %s, want containing %s", test.spec.Type, tool.Parameters, test.want) + } + } +} diff --git a/internal/domain/workflow/task/caller.go b/internal/domain/workflow/task/caller.go index a849505c1..6dd4d650e 100644 --- a/internal/domain/workflow/task/caller.go +++ b/internal/domain/workflow/task/caller.go @@ -14,8 +14,11 @@ type Caller interface { // and the callee's depth is the caller's + 1. The store refuses a // caller that is not running and a call past the depth limit. StartCall(ctx context.Context, callerTaskID int64, workflow string, inputs map[string]any) (*Task, error) - // CallStatus reads one call's callee: its status and latest transition - // detail. A store only answers for a task whose call_parent_task_id is - // callerTaskID, so a caller reads only its own callees. - CallStatus(ctx context.Context, callerTaskID, callTaskID int64) (status, detail string, err error) + // CallStatus reads one call's callee: its status, latest transition + // detail and written outputs (decoded from the callee's row; + // docs/prds/workflow-call-outputs.md, "Storage and wire" — the callee + // writes them and the caller only receives them). A store only answers + // for a task whose call_parent_task_id is callerTaskID, so a caller + // reads only its own callees. + CallStatus(ctx context.Context, callerTaskID, callTaskID int64) (status, detail string, outputs map[string]any, err error) } diff --git a/internal/domain/workflow/task/interface.go b/internal/domain/workflow/task/interface.go index 651d67882..12a023caa 100644 --- a/internal/domain/workflow/task/interface.go +++ b/internal/domain/workflow/task/interface.go @@ -32,6 +32,15 @@ type InputSpec struct { Required bool `yaml:"required,omitempty" json:"required,omitempty"` } +// OutputSpec declares one workflow output: the same type vocabulary an +// input uses. A required output the run never writes parks it before any +// terminal state (docs/prds/workflow-call-outputs.md, "Failure rules"); +// required defaults to false. +type OutputSpec struct { + Type string `yaml:"type" json:"type"` + Required bool `yaml:"required,omitempty" json:"required,omitempty"` +} + // WorkflowInterface is what a workflow declares to whatever starts it: the // inputs it takes, whether it works on a repository, and the agent profile it // runs under. It lives beside the definition entry so a process that does not @@ -39,6 +48,11 @@ type InputSpec struct { type WorkflowInterface struct { Inputs map[string]InputSpec `yaml:"inputs,omitempty" json:"inputs,omitempty"` Repository RepositoryMode `yaml:"repository,omitempty" json:"repository,omitempty"` + // Outputs declares the named, typed structured results a run writes + // once each (docs/prds/workflow-call-outputs.md). A wait:true caller + // reads them through WorkflowCallStatus; a call step publishes one as + // one of the caller's own outputs. + Outputs map[string]OutputSpec `yaml:"outputs,omitempty" json:"outputs,omitempty"` // Profile names a [containers.profiles] entry. It is resolved when a task // is dispatched, not when the workflow is saved. Profile string `yaml:"profile,omitempty" json:"profile,omitempty"` @@ -78,8 +92,8 @@ func ParseWorkflowInterface(src string) (WorkflowInterface, error) { return w, w.Validate() } -// Validate rejects an unknown repository mode, a malformed input name and an -// unknown input type. +// Validate rejects an unknown repository mode, a malformed input or output +// name and an unknown input or output type. func (w WorkflowInterface) Validate() error { switch w.Repository { case "", RepositoryRequired, RepositoryOptional, RepositoryNone: @@ -94,6 +108,14 @@ func (w WorkflowInterface) Validate() error { return fmt.Errorf("workflow input %q type %q must be one of %s", name, spec.Type, strings.Join(inputTypes, ", ")) } } + for name, spec := range w.Outputs { + if !inputName.MatchString(name) { + return fmt.Errorf("workflow output %q must be a letter or underscore followed by letters, digits or underscores", name) + } + if !slices.Contains(inputTypes, spec.Type) { + return fmt.Errorf("workflow output %q type %q must be one of %s", name, spec.Type, strings.Join(inputTypes, ", ")) + } + } if w.Needs.GateRetries < 0 { return fmt.Errorf("workflow needs.gate_retries must not be negative") } @@ -130,6 +152,31 @@ func (w WorkflowInterface) CheckInputs(values map[string]any) error { return nil } +// CheckOutputs rejects declared-output values that do not satisfy the +// declared outputs: the same rules CheckInputs applies to inputs (required, +// undeclared, wrong type), read from the other side of the interface +// (docs/prds/workflow-call-outputs.md, "Failure rules"). A written null +// counts as not written, as CheckInputs treats a null input; an unwritten +// optional output leaves its key absent rather than null. +func (w WorkflowInterface) CheckOutputs(values map[string]any) error { + for name, spec := range w.Outputs { + v, ok := values[name] + if spec.Required && (!ok || v == nil) { + return fmt.Errorf("output %q is required", name) + } + } + for name, v := range values { + spec, ok := w.Outputs[name] + if !ok { + return fmt.Errorf("output %q is not declared by the workflow", name) + } + if got := ValueType(v); v != nil && !TypeAccepts(spec.Type, got) { + return fmt.Errorf("output %q is %s, want %s", name, got, spec.Type) + } + } + return nil +} + // TypeAccepts reports whether a value or parameter of type got may feed an // input declared as want. func TypeAccepts(want, got string) bool { @@ -183,3 +230,31 @@ func DecodeInputs(s string) (map[string]any, error) { } return inputs, nil } + +// EncodeOutputs is the stored and wire form of a task's written outputs: a +// JSON object, or empty for none (docs/prds/workflow-call-outputs.md, +// "Storage and wire"). +func EncodeOutputs(outputs map[string]any) (string, error) { + if len(outputs) == 0 { + return "", nil + } + data, err := json.Marshal(outputs) + if err != nil { + return "", fmt.Errorf("encode task outputs: %w", err) + } + return string(data), nil +} + +// DecodeOutputs reverses EncodeOutputs, keeping numbers exact. +func DecodeOutputs(s string) (map[string]any, error) { + if s == "" { + return nil, nil + } + decoder := json.NewDecoder(strings.NewReader(s)) + decoder.UseNumber() + var outputs map[string]any + if err := decoder.Decode(&outputs); err != nil { + return nil, fmt.Errorf("decode task outputs: %w", err) + } + return outputs, nil +} diff --git a/internal/domain/workflow/task/interface_test.go b/internal/domain/workflow/task/interface_test.go index 4a7f3c44f..677252412 100644 --- a/internal/domain/workflow/task/interface_test.go +++ b/internal/domain/workflow/task/interface_test.go @@ -86,3 +86,112 @@ func TestCheckInputs(t *testing.T) { }) } } + +// TestParseWorkflowInterfaceOutputs pins the declared-output grammar beside +// the declared-input grammar (docs/prds/workflow-call-outputs.md, "Where +// outputs are declared"): the same identifier shape, the same type +// vocabulary, the same Validate both ParseDefinition and +// ParseWorkflowInterface call. +func TestParseWorkflowInterfaceOutputs(t *testing.T) { + tests := []struct { + name string + src string + wantErr string + }{ + { + name: "outputs parse beside inputs", + src: "id: a\ninputs:\n src_ip: {type: string}\noutputs:\n contained: {type: bool, required: true}\n summary: {type: string}\n", + }, + {name: "unknown output type", src: "id: a\noutputs:\n contained: {type: boolean}\n", wantErr: `output "contained" type "boolean" must be one of`}, + {name: "malformed output name", src: "id: a\noutputs:\n con-tained: {type: bool}\n", wantErr: `output "con-tained" must be a letter or underscore`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w, err := ParseWorkflowInterface(tt.src) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("err = %v, want containing %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + contained := w.Outputs["contained"] + if contained.Type != "bool" || !contained.Required { + t.Errorf("outputs.contained = %+v, want {bool required}", contained) + } + // required defaults to false. + if summary := w.Outputs["summary"]; summary.Type != "string" || summary.Required { + t.Errorf("outputs.summary = %+v, want {string not required}", summary) + } + }) + } +} + +// TestCheckOutputs pins the run-time and finish-time refusal the engine +// applies to a written output set: required, undeclared, wrong type -- with +// a written null counting as not written, exactly as CheckInputs judges a +// null input (docs/prds/workflow-call-outputs.md, "Failure rules"). +func TestCheckOutputs(t *testing.T) { + w := WorkflowInterface{Outputs: map[string]OutputSpec{ + "contained": {Type: "bool", Required: true}, + "summary": {Type: "string"}, + "extra": {Type: "any"}, + }} + tests := []struct { + name string + values map[string]any + wantErr string + }{ + {name: "required and typed", values: map[string]any{"contained": true, "summary": "done"}}, + {name: "optional null is absent", values: map[string]any{"contained": true, "summary": nil}}, + {name: "any takes an object", values: map[string]any{"contained": true, "extra": map[string]any{"a": 1.0}}}, + {name: "missing required", values: map[string]any{"summary": "done"}, wantErr: `"contained" is required`}, + {name: "null required", values: map[string]any{"contained": nil}, wantErr: `"contained" is required`}, + {name: "undeclared", values: map[string]any{"contained": true, "port": 22}, wantErr: `"port" is not declared`}, + {name: "wrong type", values: map[string]any{"contained": true, "summary": json.Number("3")}, wantErr: `"summary" is number, want string`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := w.CheckOutputs(tt.values) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("CheckOutputs() = %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("CheckOutputs() = %v, want containing %q", err, tt.wantErr) + } + }) + } +} + +// TestEncodeDecodeOutputs pins outputs' stored form: the shape EncodeInputs +// already has, with numbers exact across the round trip and empty for none. +func TestEncodeDecodeOutputs(t *testing.T) { + if s, err := EncodeOutputs(nil); err != nil || s != "" { + t.Fatalf("EncodeOutputs(nil) = (%q, %v), want (\"\", nil)", s, err) + } + encoded, err := EncodeOutputs(map[string]any{"contained": true, "n": json.Number("3")}) + if err != nil { + t.Fatal(err) + } + decoded, err := DecodeOutputs(encoded) + if err != nil { + t.Fatal(err) + } + if decoded["contained"] != true { + t.Errorf("contained = %v, want true", decoded["contained"]) + } + if _, ok := decoded["n"].(json.Number); !ok { + t.Errorf("n = %T, want json.Number", decoded["n"]) + } + if outputs, err := DecodeOutputs(""); err != nil || outputs != nil { + t.Fatalf("DecodeOutputs(\"\") = (%+v, %v), want (nil, nil)", outputs, err) + } + if _, err := DecodeOutputs("{"); err == nil { + t.Fatal("DecodeOutputs({) = nil, want an error") + } +} diff --git a/internal/domain/workflow/task/task.go b/internal/domain/workflow/task/task.go index 225ea41a5..1fe918c47 100644 --- a/internal/domain/workflow/task/task.go +++ b/internal/domain/workflow/task/task.go @@ -102,6 +102,12 @@ type Task struct { // the workflow's declared inputs at dispatch. They reach the agent as // structured data, never as body text. Inputs map[string]any `json:"inputs,omitempty"` + // Outputs are the structured results this run wrote to the workflow's + // declared outputs (docs/prds/workflow-call-outputs.md). The set is + // attempt-scoped: a claim starts the next attempt with an empty set and + // the finish write replaces the row's set wholesale. Callers read these + // only from a successful terminal state, through WorkflowCallStatus. + Outputs map[string]any `json:"outputs,omitempty"` // ReviewPayload is the JSON-encoded review unit (the forge review's // actionable comments) the remediate workflow's current run must // address. The daemon's reaction consumer injects it before queuing a diff --git a/internal/domain/workflow/workflow.go b/internal/domain/workflow/workflow.go index e9a0f7a72..8ee0ccd10 100644 --- a/internal/domain/workflow/workflow.go +++ b/internal/domain/workflow/workflow.go @@ -161,6 +161,15 @@ type TaskContext struct { // Outcome describes where the task ended up; the engine applies it. Outcome Outcome + // runInterface is the declaration the run's compiled workflow carries + // (Workflow.Interface, set by Run), so the declared-outputs pieces + // (outputs.go) read one source: the engine's own compile of the pinned + // YAML. A stage body invoked outside Run falls back to parsing the + // pinned YAML once, memoized here. + runInterface task.WorkflowInterface + runInterfaceSet bool + ifaceParsed *task.WorkflowInterface + // SystemPrompt, when non-nil, returns additional context to inject // before the agent's mission in every agent stage request. Wired by // the composition root (cmd/archied) from the MemoryManager. @@ -320,6 +329,10 @@ func Route(t *Task, reg Registry) Workflow { func Run(ctx context.Context, wf Workflow, tc *TaskContext) { t := tc.Task t.Workflow = wf.Name + // The run's declared interface (outputs.go) is the compiled definition's + // own, so every write/publish/finish check judges the YAML the task row + // pinned. + tc.runInterface, tc.runInterfaceSet = wf.Interface, true log := tc.Log.With("workflow", wf.Name, "repo", tc.Repo.FullName(), "issue", t.IssueNumber) for _, stage := range wf.Stages { @@ -437,6 +450,15 @@ func finish(ctx context.Context, tc *TaskContext, log *slog.Logger) { park(ctx, tc, tc.Outcome.Detail) return } + // The attempt's declared outputs are validated before the outcome + // transition: an undeclared, mistyped or required-but-missing value + // parks the run here, so no caller observes a terminal state that + // breaks the workflow's own promise + // (docs/prds/workflow-call-outputs.md, "Failure rules"). + if err := tc.validateFinishOutputs(); err != nil { + park(ctx, tc, err.Error()) + return + } if err := tc.Store.Update(ctx, t); err != nil { park(ctx, tc, fmt.Sprintf("persist workflow outcome: %v", err)) return diff --git a/internal/domain/workflow/workflow_call.go b/internal/domain/workflow/workflow_call.go index f26f3c374..6c769a9e6 100644 --- a/internal/domain/workflow/workflow_call.go +++ b/internal/domain/workflow/workflow_call.go @@ -40,12 +40,23 @@ type workflowCallSettings struct { Workflow string `yaml:"workflow"` Inputs map[string]any `yaml:"inputs,omitempty"` Wait bool `yaml:"wait,omitempty"` + // Outputs publishes a callee output as one of the caller's own: the key + // names the callee's declared output, the value an outputs. + // reference naming the caller's declared output + // (docs/prds/workflow-call-outputs.md, "How a caller receives one"). + Outputs map[string]string `yaml:"outputs,omitempty"` } // callReference is the prefix an inputs value must carry to be read as a // reference to the calling workflow's declared input. const callReference = "inputs." +// outputReference is the prefix a call step's outputs value must carry to be +// read as a reference to the caller's own declared output (the mirror image +// of callReference: there the prefixed side is the caller's, here the map +// key is the callee's). +const outputReference = "outputs." + // refName is the identifier grammar a reference suffix must match, shared // with the workflow interface's input names. var refName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) @@ -74,6 +85,14 @@ func newWorkflowCallStage(settings yaml.Node) (Stage, error) { return Stage{}, fmt.Errorf("%s: input %q reference %q must name an input as inputs.", WorkflowCallStepName, name, ref) } } + for name, ref := range s.Outputs { + if !strings.HasPrefix(ref, outputReference) || !refName.MatchString(strings.TrimPrefix(ref, outputReference)) { + return Stage{}, fmt.Errorf("%s: output %q must name a declared output as outputs.", WorkflowCallStepName, name) + } + } + if !s.Wait && len(s.Outputs) > 0 { + return Stage{}, fmt.Errorf("%s: outputs on a wait:false call can never be read: nothing has finished when the caller moves on", WorkflowCallStepName) + } return Stage{Name: WorkflowCallStepName, Run: func(ctx context.Context, tc *TaskContext) error { return runWorkflowCall(ctx, s, tc) }}, nil @@ -126,7 +145,7 @@ func runWorkflowCall(ctx context.Context, s workflowCallSettings, tc *TaskContex // caller itself being cancelled -- its CancelExecution already swept this // step with the other non-terminal ones, so a late write here must not // pretend the wait is still the caller's business. - waitErr := awaitCallee(ctx, s, tc, callee.ID) + calleeOutputs, waitErr := awaitCallee(ctx, s, tc, callee.ID) if waitErr != nil { if ctx.Err() == nil { if ferr := tc.finishChildStep(ctx, callStep, taskstate.StepFailed, waitErr.Error(), 0); ferr != nil { @@ -135,6 +154,16 @@ func runWorkflowCall(ctx context.Context, s workflowCallSettings, tc *TaskContex } return waitErr } + // The callee succeeded: the outputs its finished run wrote are what + // this call publishes into the caller's own output set. A publish + // failure is the caller's failure, not the call's, so the call step + // closes failed and the run parks naming the output. + if err := tc.publishCalleeOutputs(s.Outputs, calleeOutputs); err != nil { + if ferr := tc.finishChildStep(ctx, callStep, taskstate.StepFailed, err.Error(), 0); ferr != nil { + return fmt.Errorf("%s: could not be closed: %w", WorkflowCallStepName, ferr) + } + return err + } if err := tc.finishChildStep(ctx, callStep, taskstate.StepSucceeded, fmt.Sprintf("%q (task %d) finished", s.Workflow, callee.ID), 0); err != nil { return fmt.Errorf("%s: could not be closed: %w", WorkflowCallStepName, err) @@ -199,14 +228,17 @@ func callEnded(status string) bool { } // awaitCallee polls the callee until it is terminal, failing the stage with -// the callee's detail when the callee did not succeed. The caller's own wall -// clock bounds the wait. -func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, callTaskID int64) error { +// the callee's detail when the callee did not succeed, and returning the +// callee's written outputs when it did. The caller's own wall clock bounds +// the wait. +func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, callTaskID int64) (map[string]any, error) { + var outputs map[string]any for { - status, detail, err := tc.Calls.CallStatus(ctx, tc.Task.ID, callTaskID) + status, detail, latestOutputs, err := tc.Calls.CallStatus(ctx, tc.Task.ID, callTaskID) if err != nil { - return fmt.Errorf("%s: read task %d: %w", WorkflowCallStepName, callTaskID, err) + return nil, fmt.Errorf("%s: read task %d: %w", WorkflowCallStepName, callTaskID, err) } + outputs = latestOutputs if callEnded(status) { if callSucceeded(status) { if err := tc.EmitDurable(ctx, events.KindWorkflowCallFinished, WorkflowCallStepName, @@ -214,13 +246,13 @@ func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, c map[string]any{"callee_task_id": callTaskID, "workflow": s.Workflow, "status": status, "detail": detail}); err != nil { tc.Log.Warn("workflow call finish not persisted", "err", err) } - return nil + return outputs, nil } - return fmt.Errorf("%s: callee %q (task %d) ended %s: %s", WorkflowCallStepName, s.Workflow, callTaskID, status, detail) + return nil, fmt.Errorf("%s: callee %q (task %d) ended %s: %s", WorkflowCallStepName, s.Workflow, callTaskID, status, detail) } select { case <-ctx.Done(): - return ctx.Err() + return nil, ctx.Err() case <-time.After(callPollInterval): } } @@ -260,6 +292,9 @@ func validateWorkflowCalls(parsed map[string]YAMLDefinition) error { if err := checkCallInputs(id, i, d.WorkflowInterface, callee, s.Inputs); err != nil { return err } + if err := checkCallOutputs(id, i, d.WorkflowInterface, callee, s.Outputs); err != nil { + return err + } calls[id] = append(calls[id], s.Workflow) } } @@ -304,6 +339,30 @@ func checkCallInputs(callerID string, index int, caller task.WorkflowInterface, return nil } +// checkCallOutputs validates one call's saved outputs assignment: the key +// must be declared by the callee, the reference must name an output the +// caller declares, and the callee's declared type must satisfy the caller's +// (docs/prds/workflow-call-outputs.md, "Save time"). The shape of the value +// (a well-formed outputs. reference) is the factory's refusal; only +// the declarations are the collection's to judge, exactly as for inputs. +func checkCallOutputs(callerID string, index int, caller task.WorkflowInterface, callee YAMLDefinition, outputs map[string]string) error { + for calleeName, ref := range outputs { + calleeSpec, ok := callee.Outputs[calleeName] + if !ok { + return fmt.Errorf("workflow %q step %d: output %q is not declared by %q", callerID, index+1, calleeName, callee.ID) + } + callerName := strings.TrimPrefix(ref, outputReference) + callerSpec, ok := caller.Outputs[callerName] + if !ok { + return fmt.Errorf("workflow %q step %d: output %q references %q, which the calling workflow does not declare", callerID, index+1, calleeName, ref) + } + if !task.TypeAccepts(callerSpec.Type, calleeSpec.Type) { + return fmt.Errorf("workflow %q step %d: output %q is %s, want %s", callerID, index+1, calleeName, calleeSpec.Type, callerSpec.Type) + } + } + return nil +} + // callInputType names the type a saved call value carries at run time: a // reference carries the calling workflow's declared input type, a literal its // own JSON type. An error means the reference names an input the caller does diff --git a/internal/domain/workflow/workflow_call_test.go b/internal/domain/workflow/workflow_call_test.go index d66f9fa06..db397289e 100644 --- a/internal/domain/workflow/workflow_call_test.go +++ b/internal/domain/workflow/workflow_call_test.go @@ -37,8 +37,9 @@ type startCall struct { } type statusAt struct { - status string - detail string + status string + detail string + outputs map[string]any } func (f *fakeCaller) StartCall(_ context.Context, callerID int64, workflow string, inputs map[string]any) (*task.Task, error) { @@ -50,19 +51,19 @@ func (f *fakeCaller) StartCall(_ context.Context, callerID int64, workflow strin return callee, nil } -func (f *fakeCaller) CallStatus(_ context.Context, callerID, callID int64) (string, string, error) { +func (f *fakeCaller) CallStatus(_ context.Context, callerID, callID int64) (string, string, map[string]any, error) { if f.statusErr != nil { - return "", "", f.statusErr + return "", "", nil, f.statusErr } seq := f.statuses[callID] if len(seq) == 0 { - return "", "", nil + return "", "", nil, nil } at := seq[0] if len(seq) > 1 { f.statuses[callID] = seq[1:] } - return at.status, at.detail, nil + return at.status, at.detail, at.outputs, nil } func callerYAML(callee string, extra ...string) string { @@ -252,7 +253,7 @@ func TestWorkflowCallWaitFalseStartsAndContinues(t *testing.T) { func TestWorkflowCallWaitTrueWaitsForTerminalCallee(t *testing.T) { wf := compileCaller(t, "id: caller\nrepository: none\nsteps:\n - type: workflow.call\n settings:\n workflow: callee\n wait: true\n - type: agent.run\n settings:\n mission: report\n") caller := &fakeCaller{statuses: map[int64][]statusAt{ - 101: {{StatusQueued, ""}, {StatusRunning, ""}, {StatusCompleted, "contained 10.0.0.9"}}, + 101: {{status: StatusQueued}, {status: StatusRunning}, {status: StatusCompleted, detail: "contained 10.0.0.9"}}, }} store := &recordingStore{} tc := &TaskContext{ @@ -276,7 +277,7 @@ func TestWorkflowCallWaitTrueWaitsForTerminalCallee(t *testing.T) { func TestWorkflowCallWaitTrueFailsWhenCalleeFails(t *testing.T) { wf := compileCaller(t, "id: caller\nrepository: none\nsteps:\n - type: workflow.call\n settings:\n workflow: callee\n wait: true\n") caller := &fakeCaller{statuses: map[int64][]statusAt{ - 101: {{StatusParked, "needs an operator"}}, + 101: {{status: StatusParked, detail: "needs an operator"}}, }} store := &recordingStore{} tc := &TaskContext{ @@ -327,3 +328,186 @@ func TestWorkflowCallSettingsValidation(t *testing.T) { }) } } + +// ---- Save-time: declared outputs ---- + +// TestValidateCollectionWorkflowCallOutputs pins checkCallOutputs beside +// checkCallInputs (docs/prds/workflow-call-outputs.md, "Save time"): the key +// must be declared by the callee, the value must name an output the caller +// declares, and the callee's declared type must satisfy the caller's. +func TestValidateCollectionWorkflowCallOutputs(t *testing.T) { + const callee = "id: callee\noutputs:\n contained: {type: bool, required: true}\n summary: {type: string}\nsteps:\n - type: agent.run\n settings:\n mission: m\n" + tests := []struct { + name string + yamls []string + want string + notViaCall bool // the refusal is the definition's own, not a call check + }{ + { + name: "callee does not declare the published output", + yamls: []string{ + "id: caller\noutputs:\n contained: {type: bool}\nsteps:\n - type: workflow.call\n settings:\n workflow: callee\n wait: true\n outputs: {contained: \"outputs.contained\"}\n", + "id: callee\nsteps:\n - type: agent.run\n settings:\n mission: m\n", + }, + want: `output "contained" is not declared by "callee"`, + }, + { + name: "caller does not declare the referenced output", + yamls: []string{ + "id: caller\nsteps:\n - type: workflow.call\n settings:\n workflow: callee\n wait: true\n outputs: {contained: \"outputs.contained\"}\n", + callee, + }, + want: `output "contained" references "outputs.contained", which the calling workflow does not declare`, + }, + { + name: "callee type cannot satisfy the caller's", + yamls: []string{ + "id: caller\noutputs:\n contained: {type: string}\nsteps:\n - type: workflow.call\n settings:\n workflow: callee\n wait: true\n outputs: {contained: \"outputs.contained\"}\n", + callee, + }, + want: `output "contained" is bool, want string`, + }, + { + name: "outputs over a wait:false call", + yamls: []string{"id: caller\nsteps:\n - type: workflow.call\n settings:\n workflow: callee\n outputs: {contained: \"outputs.contained\"}\n", callee}, + want: "outputs on a wait:false call", + notViaCall: true, + }, + { + name: "output value is not an outputs. reference", + yamls: []string{"id: caller\noutputs:\n contained: {type: bool}\nsteps:\n - type: workflow.call\n settings:\n workflow: callee\n wait: true\n outputs: {contained: \"inputs.src_ip\"}\n", callee}, + want: `must name a declared output as outputs.`, + notViaCall: true, + }, + { + name: "output reference is malformed", + yamls: []string{"id: caller\noutputs:\n contained: {type: bool}\nsteps:\n - type: workflow.call\n settings:\n workflow: callee\n wait: true\n outputs: {contained: \"outputs.con-tained\"}\n", callee}, + want: `must name a declared output as outputs.`, + notViaCall: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.notViaCall { + _, err := ParseDefinition(test.yamls[0], callRegistry()) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("ParseDefinition() error = %v, want containing %q", err, test.want) + } + return + } + err := ValidateDefinitionCollection(collectionOf(append([]string{test.yamls[0]}, test.yamls[1:]...)...), callRegistry()) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("ValidateDefinitionCollection() error = %v, want containing %q", err, test.want) + } + }) + } + // The valid publication: both sides declare, the types agree. + valid := collectionOf( + "id: caller\nrepository: none\noutputs:\n contained: {type: bool}\n summary: {type: string}\nsteps:\n - type: workflow.call\n settings:\n workflow: callee\n wait: true\n inputs: {}\n outputs: {contained: \"outputs.contained\", summary: \"outputs.summary\"}\n", + callee, + ) + if err := ValidateDefinitionCollection(valid, callRegistry()); err != nil { + t.Fatalf("ValidateDefinitionCollection(matching publication) = %v, want nil", err) + } +} + +// outputsCallerYAML declares the caller's own outputs and publishes a +// callee output as one of them (docs/prds/workflow-call-outputs.md, "How a +// caller receives one"). The trailing agent.run gives the caller an outcome +// of its own, so its finish check judges its declared outputs. +const outputsCallerYAML = "id: caller\nrepository: none\ninputs:\n src_ip: {type: string}\noutputs:\n contained: {type: bool, required: true}\n summary: {type: string}\nsteps:\n - type: workflow.call\n settings:\n workflow: callee\n wait: true\n inputs: {src_ip: \"inputs.src_ip\"}\n outputs: {contained: \"outputs.contained\", summary: \"outputs.summary\"}\n - type: agent.run\n settings:\n mission: report\n" + +// TestWorkflowCallWaitTruePublishesCalleeOutputs pins the receive-and-publish +// path: a wait:true caller reads the outputs its successful callee wrote +// through CallStatus, the published ones enter the caller's own set, and the +// caller's finish write persists them into the caller's row before its own +// outcome transition. +func TestWorkflowCallWaitTruePublishesCalleeOutputs(t *testing.T) { + wf := compileCaller(t, outputsCallerYAML) + caller := &fakeCaller{statuses: map[int64][]statusAt{ + 101: {{status: StatusCompleted, detail: "contained 10.0.0.9", outputs: map[string]any{ + "contained": true, "summary": "blocked the address", "extra": "not assigned", + }}}, + }} + store := &recordingStore{} + tc := &TaskContext{ + Task: &Task{ID: 3, Attempt: 1, WorkflowDefinitionYAML: outputsCallerYAML, Inputs: map[string]any{"src_ip": "10.0.0.9"}}, + Calls: caller, Store: store, Trees: &fakeTrees{dir: "/scratch/3"}, Log: slog.New(slog.DiscardHandler), + Agent: &fakeAgentRunner{result: agentexec.Result{Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed, Summary: "reported"}}, + Cfg: config.Config{Models: map[string]string{"builder": "p/m"}}, + } + Run(context.Background(), wf, tc) + if tc.Outcome.Status != StatusCompleted { + t.Fatalf("outcome = %+v, want completed after the callee succeeded", tc.Outcome) + } + if tc.Task.Outputs["contained"] != true { + t.Fatalf("caller outputs = %+v, want the callee's bool restored as its own", tc.Task.Outputs) + } + if got := tc.Task.Outputs["summary"]; got != "blocked the address" { + t.Fatalf("caller outputs.summary = %v, want the callee's value", got) + } + if _, undeclared := tc.Task.Outputs["extra"]; undeclared { + t.Fatalf("caller outputs = %+v, want the assignment only, not the callee's whole set", tc.Task.Outputs) + } + // The caller's row carries the published values before its outcome + // transition, so the status its own caller sees when its wait ends + // already carries them. + if len(store.updates) == 0 || store.updates[len(store.updates)-1].Outputs["contained"] != true { + t.Fatalf("row updates = %+v, want the published outputs written", store.updates) + } + if seq := updatesBeforeTransition(store.sequence, "completed"); seq == -1 { + t.Fatalf("sequence = %v, want an update before the completed transition", store.sequence) + } else if store.updates[seq].Outputs["contained"] != true { + t.Fatal("the row write carrying the outputs did not precede the outcome transition") + } +} + +// TestCallPublishSkipsUnwrittenOutputAndRequiredParks pins both halves of +// "declared but never written" on the caller side: a callee output the +// callee never wrote leaves the caller's output absent, and the caller's own +// required judgment then parks it naming the output. +func TestCallPublishSkipsUnwrittenOutputAndRequiredParks(t *testing.T) { + wf := compileCaller(t, outputsCallerYAML) + caller := &fakeCaller{statuses: map[int64][]statusAt{ + 101: {{status: StatusCompleted, detail: "done", outputs: map[string]any{"summary": "x"}}}, + }} + store := &recordingStore{} + tc := &TaskContext{ + Task: &Task{ID: 3, Attempt: 1, WorkflowDefinitionYAML: outputsCallerYAML, Inputs: map[string]any{"src_ip": "10.0.0.9"}}, + Calls: caller, Store: store, Trees: &fakeTrees{}, Log: slog.New(slog.DiscardHandler), + Agent: &fakeAgentRunner{result: agentexec.Result{Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed, Summary: "reported"}}, + Cfg: config.Config{Models: map[string]string{"builder": "p/m"}}, + } + Run(context.Background(), wf, tc) + if got := tc.Task.Outputs["summary"]; got != "x" { + t.Fatalf("caller outputs.summary = %v, want the one value the callee wrote", got) + } + if _, absent := tc.Task.Outputs["contained"]; absent { + t.Fatal("the unwritten callee output was published anyway") + } + if !strings.Contains(tc.Task.ParkReason, `output "contained" is required`) { + t.Fatalf("park reason %q, want the caller's missing required output named", tc.Task.ParkReason) + } + if len(store.transitions) == 0 || store.transitions[len(store.transitions)-1].to != StatusParked { + t.Fatalf("transitions = %+v, want the caller parked rather than reaching a terminal state", store.transitions) + } +} + +// TestCallPublishTypeMismatchParks pins the run-time re-check: a published +// callee value is re-checked against the caller's declared type before it +// enters the caller's set; a mismatch parks the caller naming the output. +func TestCallPublishTypeMismatchParks(t *testing.T) { + wf := compileCaller(t, outputsCallerYAML) + caller := &fakeCaller{statuses: map[int64][]statusAt{ + 101: {{status: StatusCompleted, detail: "done", outputs: map[string]any{"contained": "yes"}}}, + }} + store := &recordingStore{} + tc := &TaskContext{ + Task: &Task{ID: 3, Attempt: 1, WorkflowDefinitionYAML: outputsCallerYAML, Inputs: map[string]any{"src_ip": "10.0.0.9"}}, + Calls: caller, Store: store, Trees: &fakeTrees{}, Log: slog.New(slog.DiscardHandler), + } + Run(context.Background(), wf, tc) + if !strings.Contains(tc.Task.ParkReason, "contained") { + t.Fatalf("park reason %q, want the mismatched output named", tc.Task.ParkReason) + } +} diff --git a/internal/infrastructure/agenttransport/nats/transport.go b/internal/infrastructure/agenttransport/nats/transport.go index f3478b008..7a460a898 100644 --- a/internal/infrastructure/agenttransport/nats/transport.go +++ b/internal/infrastructure/agenttransport/nats/transport.go @@ -182,7 +182,7 @@ func (d deadlineCaller) StartCall(ctx context.Context, callerTaskID int64, wf st return d.Caller.StartCall(ctx, callerTaskID, wf, inputs) } -func (d deadlineCaller) CallStatus(ctx context.Context, callerTaskID, callTaskID int64) (string, string, error) { +func (d deadlineCaller) CallStatus(ctx context.Context, callerTaskID, callTaskID int64) (string, string, map[string]any, error) { ctx, cancel := d.withDeadline(ctx) defer cancel() return d.Caller.CallStatus(ctx, callerTaskID, callTaskID) diff --git a/internal/infrastructure/postgres/migrations/0024_workflow_outputs.sql b/internal/infrastructure/postgres/migrations/0024_workflow_outputs.sql new file mode 100644 index 000000000..de709c429 --- /dev/null +++ b/internal/infrastructure/postgres/migrations/0024_workflow_outputs.sql @@ -0,0 +1,17 @@ +-- Declared workflow outputs on the task row +-- (docs/prds/workflow-call-outputs.md, "Storage and wire"). +-- +-- tasks.outputs is the JSON object of declared-output values a run wrote, +-- written by the run's own row write exactly as tasks.inputs is. The column +-- defaults to the empty string, so every existing row and writer reads as a +-- run that wrote nothing (the empty set). It is attempt-scoped: the claim +-- statements reset it alongside the attempt increment, so nothing a previous +-- attempt wrote survives into the next. + +-- +goose Up + +ALTER TABLE tasks ADD COLUMN outputs text NOT NULL DEFAULT ''; + +-- +goose Down + +ALTER TABLE tasks DROP COLUMN outputs; diff --git a/internal/infrastructure/postgres/postgresdb/call_tasks.sql.go b/internal/infrastructure/postgres/postgresdb/call_tasks.sql.go index 634148908..acbab0915 100644 --- a/internal/infrastructure/postgres/postgresdb/call_tasks.sql.go +++ b/internal/infrastructure/postgres/postgresdb/call_tasks.sql.go @@ -24,7 +24,7 @@ func (q *Queries) CallStatusDetail(ctx context.Context, taskID int64) (string, e const enqueueCallTask = `-- name: EnqueueCallTask :one WITH caller AS ( - SELECT t.id, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, t.status, t.workflow, t.branch, t.plan, t.notes, t.pr_number, t.tokens_used, t.iterations, t.attempt, t.park_reason, t.watch_comment_id, t.park_class, t.remediation_rounds, t.retry_count, t.source, t.identity, t.binding_id, t.binding_version, t.review_payload, t.workflow_definition_version, t.workflow_definition_digest, t.workflow_definition_yaml, t.created_at, t.updated_at, t.review_cursor, t.inputs, t.org_id, t.workspace_id, t.call_parent_task_id, t.call_depth FROM tasks t WHERE t.id = $1 FOR UPDATE + SELECT t.id, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, t.status, t.workflow, t.branch, t.plan, t.notes, t.pr_number, t.tokens_used, t.iterations, t.attempt, t.park_reason, t.watch_comment_id, t.park_class, t.remediation_rounds, t.retry_count, t.source, t.identity, t.binding_id, t.binding_version, t.review_payload, t.workflow_definition_version, t.workflow_definition_digest, t.workflow_definition_yaml, t.created_at, t.updated_at, t.review_cursor, t.inputs, t.org_id, t.workspace_id, t.call_parent_task_id, t.call_depth, t.outputs FROM tasks t WHERE t.id = $1 FOR UPDATE ) INSERT INTO tasks (owner, repo, issue_number, title, body, labels, workflow, source, identity, org_id, workspace_id, inputs, call_parent_task_id, call_depth) SELECT @@ -41,7 +41,7 @@ SELECT caller.id, caller.call_depth + 1 FROM caller WHERE caller.status = 'running' AND caller.call_depth + 1 <= $7::int -RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth +RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs ` type EnqueueCallTaskParams struct { @@ -112,6 +112,7 @@ func (q *Queries) EnqueueCallTask(ctx context.Context, arg EnqueueCallTaskParams &i.WorkspaceID, &i.CallParentTaskID, &i.CallDepth, + &i.Outputs, ) return i, err } diff --git a/internal/infrastructure/postgres/postgresdb/models.go b/internal/infrastructure/postgres/postgresdb/models.go index 97042f682..11b8f39e7 100644 --- a/internal/infrastructure/postgres/postgresdb/models.go +++ b/internal/infrastructure/postgres/postgresdb/models.go @@ -392,6 +392,7 @@ type Task struct { WorkspaceID string CallParentTaskID int64 CallDepth int32 + Outputs string } type ToolCall struct { diff --git a/internal/infrastructure/postgres/postgresdb/tasks.sql.go b/internal/infrastructure/postgres/postgresdb/tasks.sql.go index e27cd2202..96e7c7441 100644 --- a/internal/infrastructure/postgres/postgresdb/tasks.sql.go +++ b/internal/infrastructure/postgres/postgresdb/tasks.sql.go @@ -46,9 +46,9 @@ func (q *Queries) BeginRemediationTask(ctx context.Context, arg BeginRemediation } const claimByIssue = `-- name: ClaimByIssue :one -UPDATE tasks SET status = 'running', attempt = attempt + 1, updated_at = now() +UPDATE tasks SET status = 'running', attempt = attempt + 1, outputs = '', updated_at = now() WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND status = 'queued' -RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth +RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs ` type ClaimByIssueParams struct { @@ -57,6 +57,8 @@ type ClaimByIssueParams struct { IssueNumber int64 } +// The claim starts the attempt's output set empty, as ClaimNextTask does: +// outputs is attempt-scoped (docs/prds/workflow-call-outputs.md). func (q *Queries) ClaimByIssue(ctx context.Context, arg ClaimByIssueParams) (Task, error) { row := q.db.QueryRow(ctx, claimByIssue, arg.Owner, arg.Repo, arg.IssueNumber) var i Task @@ -98,13 +100,14 @@ func (q *Queries) ClaimByIssue(ctx context.Context, arg ClaimByIssueParams) (Tas &i.WorkspaceID, &i.CallParentTaskID, &i.CallDepth, + &i.Outputs, ) return i, err } const claimNextTask = `-- name: ClaimNextTask :one UPDATE tasks -SET status = 'running', attempt = attempt + 1, updated_at = now() +SET status = 'running', attempt = attempt + 1, outputs = '', updated_at = now() WHERE id = ( SELECT id FROM tasks WHERE status = 'queued' @@ -112,12 +115,15 @@ WHERE id = ( FOR UPDATE SKIP LOCKED LIMIT 1 ) -RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth +RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs ` // FOR UPDATE SKIP LOCKED replaces the SQLite single-writer assumption: two // claimers running at once take different rows instead of blocking, so the // daemon no longer depends on holding the only connection to the file. +// The claim also starts the attempt's output set empty: outputs is +// attempt-scoped, so nothing the previous attempt wrote survives +// (docs/prds/workflow-call-outputs.md). func (q *Queries) ClaimNextTask(ctx context.Context) (Task, error) { row := q.db.QueryRow(ctx, claimNextTask) var i Task @@ -159,6 +165,7 @@ func (q *Queries) ClaimNextTask(ctx context.Context) (Task, error) { &i.WorkspaceID, &i.CallParentTaskID, &i.CallDepth, + &i.Outputs, ) return i, err } @@ -269,7 +276,7 @@ VALUES ( ), $8 ) -RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth +RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs ` type InsertChatTaskParams struct { @@ -340,6 +347,7 @@ func (q *Queries) InsertChatTask(ctx context.Context, arg InsertChatTaskParams) &i.WorkspaceID, &i.CallParentTaskID, &i.CallDepth, + &i.Outputs, ) return i, err } @@ -367,7 +375,7 @@ func (q *Queries) InsertTransition(ctx context.Context, arg InsertTransitionPara } const listOpenPRs = `-- name: ListOpenPRs :many -SELECT id, owner, repo, issue_number, pr_number, status, source, identity, attempt, review_cursor, watch_comment_id +SELECT id, owner, repo, issue_number, pr_number, status, source, identity, attempt, review_cursor, watch_comment_id, outputs FROM tasks WHERE status = 'pr_open' ` @@ -383,6 +391,7 @@ type ListOpenPRsRow struct { Attempt int64 ReviewCursor int64 WatchCommentID int64 + Outputs string } func (q *Queries) ListOpenPRs(ctx context.Context) ([]ListOpenPRsRow, error) { @@ -406,6 +415,7 @@ func (q *Queries) ListOpenPRs(ctx context.Context) ([]ListOpenPRsRow, error) { &i.Attempt, &i.ReviewCursor, &i.WatchCommentID, + &i.Outputs, ); err != nil { return nil, err } @@ -420,7 +430,8 @@ func (q *Queries) ListOpenPRs(ctx context.Context) ([]ListOpenPRsRow, error) { const listTaskSummaries = `-- name: ListTaskSummaries :many SELECT id, owner, repo, issue_number, title, status, workflow, pr_number, tokens_used, iterations, attempt, park_reason, retry_count, - created_at, updated_at, plan, source, identity, binding_id, binding_version + created_at, updated_at, plan, source, identity, binding_id, binding_version, + outputs FROM tasks ORDER BY updated_at DESC LIMIT $1 ` @@ -445,11 +456,14 @@ type ListTaskSummariesRow struct { Identity string BindingID string BindingVersion int64 + Outputs string } // The dashboard's list. This projection is deliberately narrow: Plan gates // the "Decision required" panel and Source labels the row, and widening it -// back would send rows without them. +// back would send rows without them. outputs rides it because the task row's +// own structured result is part of the summary a caller or the dashboard +// reads (docs/prds/workflow-call-outputs.md, "Storage and wire"). func (q *Queries) ListTaskSummaries(ctx context.Context, limit int32) ([]ListTaskSummariesRow, error) { rows, err := q.db.Query(ctx, listTaskSummaries, limit) if err != nil { @@ -480,6 +494,7 @@ func (q *Queries) ListTaskSummaries(ctx context.Context, limit int32) ([]ListTas &i.Identity, &i.BindingID, &i.BindingVersion, + &i.Outputs, ); err != nil { return nil, err } @@ -620,7 +635,7 @@ func (q *Queries) StampTaskBinding(ctx context.Context, arg StampTaskBindingPara } const taskByID = `-- name: TaskByID :one -SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth FROM tasks WHERE id = $1 +SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs FROM tasks WHERE id = $1 ` func (q *Queries) TaskByID(ctx context.Context, id int64) (Task, error) { @@ -664,12 +679,13 @@ func (q *Queries) TaskByID(ctx context.Context, id int64) (Task, error) { &i.WorkspaceID, &i.CallParentTaskID, &i.CallDepth, + &i.Outputs, ) return i, err } const taskByIssue = `-- name: TaskByIssue :one -SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth FROM tasks WHERE owner = $1 AND repo = $2 AND issue_number = $3 +SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs FROM tasks WHERE owner = $1 AND repo = $2 AND issue_number = $3 ` type TaskByIssueParams struct { @@ -719,12 +735,13 @@ func (q *Queries) TaskByIssue(ctx context.Context, arg TaskByIssueParams) (Task, &i.WorkspaceID, &i.CallParentTaskID, &i.CallDepth, + &i.Outputs, ) return i, err } const taskByPR = `-- name: TaskByPR :one -SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth FROM tasks WHERE owner = $1 AND repo = $2 AND pr_number = $3 AND status = $4 +SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs FROM tasks WHERE owner = $1 AND repo = $2 AND pr_number = $3 AND status = $4 ` type TaskByPRParams struct { @@ -783,6 +800,7 @@ func (q *Queries) TaskByPR(ctx context.Context, arg TaskByPRParams) (Task, error &i.WorkspaceID, &i.CallParentTaskID, &i.CallDepth, + &i.Outputs, ) return i, err } @@ -844,6 +862,7 @@ UPDATE tasks SET workflow = $2, branch = $3, plan = $4, notes = $5, watch_comment_id = $10, retry_count = $11, remediation_rounds = $12, review_payload = $13, workflow_definition_version = $14, workflow_definition_digest = $15, workflow_definition_yaml = $16, + outputs = $17, updated_at = now() WHERE id = $1 ` @@ -865,6 +884,7 @@ type UpdateTaskParams struct { WorkflowDefinitionVersion int64 WorkflowDefinitionDigest string WorkflowDefinitionYaml string + Outputs string } func (q *Queries) UpdateTask(ctx context.Context, arg UpdateTaskParams) error { @@ -885,6 +905,7 @@ func (q *Queries) UpdateTask(ctx context.Context, arg UpdateTaskParams) error { arg.WorkflowDefinitionVersion, arg.WorkflowDefinitionDigest, arg.WorkflowDefinitionYaml, + arg.Outputs, ) return err } diff --git a/internal/infrastructure/postgres/queries/tasks.sql b/internal/infrastructure/postgres/queries/tasks.sql index b488cb342..64fa7c1ae 100644 --- a/internal/infrastructure/postgres/queries/tasks.sql +++ b/internal/infrastructure/postgres/queries/tasks.sql @@ -2,8 +2,11 @@ -- FOR UPDATE SKIP LOCKED replaces the SQLite single-writer assumption: two -- claimers running at once take different rows instead of blocking, so the -- daemon no longer depends on holding the only connection to the file. +-- The claim also starts the attempt's output set empty: outputs is +-- attempt-scoped, so nothing the previous attempt wrote survives +-- (docs/prds/workflow-call-outputs.md). UPDATE tasks -SET status = 'running', attempt = attempt + 1, updated_at = now() +SET status = 'running', attempt = attempt + 1, outputs = '', updated_at = now() WHERE id = ( SELECT id FROM tasks WHERE status = 'queued' @@ -28,10 +31,13 @@ SELECT * FROM tasks WHERE owner = $1 AND repo = $2 AND pr_number = $3 AND status -- name: ListTaskSummaries :many -- The dashboard's list. This projection is deliberately narrow: Plan gates -- the "Decision required" panel and Source labels the row, and widening it --- back would send rows without them. +-- back would send rows without them. outputs rides it because the task row's +-- own structured result is part of the summary a caller or the dashboard +-- reads (docs/prds/workflow-call-outputs.md, "Storage and wire"). SELECT id, owner, repo, issue_number, title, status, workflow, pr_number, tokens_used, iterations, attempt, park_reason, retry_count, - created_at, updated_at, plan, source, identity, binding_id, binding_version + created_at, updated_at, plan, source, identity, binding_id, binding_version, + outputs FROM tasks ORDER BY updated_at DESC LIMIT $1; -- name: CountTasksByStatus :many @@ -74,6 +80,7 @@ UPDATE tasks SET workflow = $2, branch = $3, plan = $4, notes = $5, watch_comment_id = $10, retry_count = $11, remediation_rounds = $12, review_payload = $13, workflow_definition_version = $14, workflow_definition_digest = $15, workflow_definition_yaml = $16, + outputs = $17, updated_at = now() WHERE id = $1; @@ -128,7 +135,9 @@ VALUES ( ON CONFLICT (org_id, identity, owner, repo, issue_number) DO NOTHING; -- name: ClaimByIssue :one -UPDATE tasks SET status = 'running', attempt = attempt + 1, updated_at = now() +-- The claim starts the attempt's output set empty, as ClaimNextTask does: +-- outputs is attempt-scoped (docs/prds/workflow-call-outputs.md). +UPDATE tasks SET status = 'running', attempt = attempt + 1, outputs = '', updated_at = now() WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND status = 'queued' RETURNING *; @@ -160,5 +169,5 @@ WHERE id = @id AND status = @from_status; DELETE FROM tasks WHERE id = $1 AND status = $2; -- name: ListOpenPRs :many -SELECT id, owner, repo, issue_number, pr_number, status, source, identity, attempt, review_cursor, watch_comment_id +SELECT id, owner, repo, issue_number, pr_number, status, source, identity, attempt, review_cursor, watch_comment_id, outputs FROM tasks WHERE status = 'pr_open'; diff --git a/internal/infrastructure/postgres/store.go b/internal/infrastructure/postgres/store.go index 21635f6aa..cacfdc15a 100644 --- a/internal/infrastructure/postgres/store.go +++ b/internal/infrastructure/postgres/store.go @@ -74,12 +74,21 @@ func (s *Store) Close() error { return nil } // JSON-safe floor that keeps synthetic issue numbers clear of real ones. const syntheticIssueNumberBase = 1_000_000_000_000_000 +// structuredPayloadBytes is the write-side bound the store applies to a +// structured payload (review_payload today, task outputs now). A value that +// would exceed it is refused for outputs -- clipped JSON does not parse, and a +// caller must receive an object or nothing -- where the legacy review payload +// is still clipped (docs/prds/workflow-call-outputs.md, "Storage and wire"). +const structuredPayloadBytes = 4000 + // taskFromRow maps the generated task row to the workflow.Task the daemon and // webui consume. func taskFromRow(t postgresdb.Task) *workflow.Task { - // The inputs column is only ever written by EncodeInputs, so a decode - // failure cannot arise from stored data this package produced. + // The inputs and outputs columns are only ever written by EncodeInputs and + // EncodeOutputs, so a decode failure cannot arise from stored data this + // package produced. inputs, _ := task.DecodeInputs(t.Inputs) + outputs, _ := task.DecodeOutputs(t.Outputs) return &workflow.Task{ ID: t.ID, Owner: t.Owner, @@ -110,6 +119,7 @@ func taskFromRow(t postgresdb.Task) *workflow.Task { BindingID: t.BindingID, BindingVersion: int(t.BindingVersion), Inputs: inputs, + Outputs: outputs, ReviewPayload: t.ReviewPayload, ParkClass: t.ParkClass, RemediationRounds: int(t.RemediationRounds), @@ -282,6 +292,18 @@ func (s *Store) ParkTask(ctx context.Context, taskID int64, from, detail, class // Update persists mutable task fields written by workflows. func (s *Store) Update(ctx context.Context, t *workflow.Task) error { + // The row write is where a run's outputs land (docs/prds/workflow-call-outputs.md). + // An encoded set past the bound the store applies to a structured payload + // is refused rather than clipped: clipped JSON does not parse, and a caller + // would receive a broken object as a value. The engine turns the refusal + // into a park, so nothing silently loses its structured result. + outputs, err := task.EncodeOutputs(t.Outputs) + if err != nil { + return fmt.Errorf("store: encode task %d outputs: %w", t.ID, err) + } + if len(outputs) > structuredPayloadBytes { + return fmt.Errorf("store: task %d outputs are %d bytes, past the %d-byte structured-payload bound; refuse rather than clip", t.ID, len(outputs), structuredPayloadBytes) + } return s.queries().UpdateTask(ctx, postgresdb.UpdateTaskParams{ ID: t.ID, Workflow: t.Workflow, @@ -299,6 +321,7 @@ func (s *Store) Update(ctx context.Context, t *workflow.Task) error { WorkflowDefinitionVersion: t.WorkflowDefinitionVersion, WorkflowDefinitionDigest: t.WorkflowDefinitionDigest, WorkflowDefinitionYaml: t.WorkflowDefinitionYAML, + Outputs: outputs, }) } @@ -504,11 +527,13 @@ func (s *Store) OpenPRs(ctx context.Context) ([]workflow.Task, error) { } tasks := make([]workflow.Task, 0, len(rows)) for _, r := range rows { + outputs, _ := task.DecodeOutputs(r.Outputs) tasks = append(tasks, workflow.Task{ ID: r.ID, Owner: r.Owner, Repo: r.Repo, IssueNumber: int(r.IssueNumber), PRNumber: int(r.PrNumber), Status: r.Status, Source: r.Source, Identity: r.Identity, Attempt: int(r.Attempt), ReviewCursor: r.ReviewCursor, WatchCommentID: r.WatchCommentID, + Outputs: outputs, }) } return tasks, nil @@ -527,6 +552,7 @@ func (s *Store) Tasks(ctx context.Context, limit int) ([]workflow.Task, error) { } tasks := make([]workflow.Task, 0, len(rows)) for _, r := range rows { + outputs, _ := task.DecodeOutputs(r.Outputs) tasks = append(tasks, workflow.Task{ ID: r.ID, Owner: r.Owner, Repo: r.Repo, IssueNumber: int(r.IssueNumber), Title: r.Title, Status: r.Status, Workflow: r.Workflow, @@ -534,6 +560,7 @@ func (s *Store) Tasks(ctx context.Context, limit int) ([]workflow.Task, error) { Attempt: int(r.Attempt), ParkReason: r.ParkReason, RetryCount: int(r.RetryCount), CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt, Plan: r.Plan, Source: r.Source, Identity: r.Identity, BindingID: r.BindingID, BindingVersion: int(r.BindingVersion), + Outputs: outputs, }) } return tasks, nil @@ -643,28 +670,34 @@ func (s *Store) callRefusal(ctx context.Context, callerTaskID int64) error { return fmt.Errorf("store: workflow.call from task %d at depth %d: %w", callerTaskID, caller.CallDepth, storecontract.ErrCallDepthExceeded) } -// CallStatus reads one call's callee for a waiting caller: its status and -// latest transition detail. The parent check is the row check the wire grant -// cannot do: a caller reads only the tasks it started. -func (s *Store) CallStatus(ctx context.Context, callerTaskID, callTaskID int64) (string, string, error) { +// CallStatus reads one call's callee for a waiting caller: its status, its +// latest transition detail and the outputs its finished run wrote. The parent +// check is the row check the wire grant cannot do: a caller reads only the +// tasks it started. +func (s *Store) CallStatus(ctx context.Context, callerTaskID, callTaskID int64) (string, string, map[string]any, error) { callee, err := s.queries().TaskByID(ctx, callTaskID) if errors.Is(err, pgx.ErrNoRows) { - return "", "", storecontract.ErrCallNotYours + return "", "", nil, storecontract.ErrCallNotYours } if err != nil { - return "", "", err + return "", "", nil, err } if callee.CallParentTaskID != callerTaskID { - return "", "", storecontract.ErrCallNotYours + return "", "", nil, storecontract.ErrCallNotYours } + // The row is the callee's own; its outputs travel beside status and detail + // (docs/prds/workflow-call-outputs.md, "Storage and wire"). The caller + // decides whether to use them -- only a successful terminal state's values + // reach it. + outputs, _ := task.DecodeOutputs(callee.Outputs) // A callee that has not moved yet has no transition row; its queue // status is the whole answer, and the empty detail says so. detail, err := s.queries().CallStatusDetail(ctx, callTaskID) if errors.Is(err, pgx.ErrNoRows) { - return callee.Status, "", nil + return callee.Status, "", outputs, nil } if err != nil { - return "", "", err + return "", "", nil, err } - return callee.Status, detail, nil + return callee.Status, detail, outputs, nil } diff --git a/internal/infrastructure/postgres/tasks_outputs_test.go b/internal/infrastructure/postgres/tasks_outputs_test.go new file mode 100644 index 000000000..07b6d4f14 --- /dev/null +++ b/internal/infrastructure/postgres/tasks_outputs_test.go @@ -0,0 +1,129 @@ +// Declared workflow outputs on the task row: the attempt-scoped structured +// results a run writes (docs/prds/workflow-call-outputs.md, "Storage and +// wire"). +package postgres + +import ( + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/domain/workflow" +) + +// newStoreForOutputs is a migrated pool with the Store built over it. +func newStoreForOutputs(t *testing.T) *Store { + t.Helper() + pool, _ := migrated(t) + return New(pool) +} + +// TestTaskOutputs pins the task row half of the outputs model: +// +// - Update persists the run's written set, decoded exactly on the way out; +// - an encoded set past the structured-payload bound is refused, not +// clipped (clipped JSON does not parse); +// - a claim starts the next attempt with an empty set, so nothing a +// retried attempt does not rewrite survives. +func TestTaskOutputs(t *testing.T) { + t.Run("Update persists the run's outputs and TaskByID reads them back", func(t *testing.T) { + st := newStoreForOutputs(t) + ctx := t.Context() + task, err := st.EnqueueChatTask(ctx, "acme", "widget", "callee", "body", "w", "", nil) + if err != nil { + t.Fatal(err) + } + if _, err := st.ClaimNext(ctx); err != nil { + t.Fatal(err) + } + row, err := st.TaskByID(ctx, task.ID) + if err != nil { + t.Fatal(err) + } + row.Outputs = map[string]any{"contained": true, "n": "3"} + if err := st.Update(ctx, row); err != nil { + t.Fatalf("Update: %v", err) + } + back, err := st.TaskByID(ctx, task.ID) + if err != nil { + t.Fatal(err) + } + if back.Outputs["contained"] != true || back.Outputs["n"] != "3" { + t.Fatalf("row outputs = %+v, want the written set read back", back.Outputs) + } + }) + + t.Run("an encoded set past the structured-payload bound is refused", func(t *testing.T) { + st := newStoreForOutputs(t) + ctx := t.Context() + task, err := st.EnqueueChatTask(ctx, "acme", "widget", "callee", "body", "w", "", nil) + if err != nil { + t.Fatal(err) + } + if _, err := st.ClaimNext(ctx); err != nil { + t.Fatal(err) + } + row, err := st.TaskByID(ctx, task.ID) + if err != nil { + t.Fatal(err) + } + row.Outputs = map[string]any{"summary": strings.Repeat("x", 5000)} + err = st.Update(ctx, row) + if err == nil { + t.Fatal("Update accepted a set past the structured-payload bound; it must be refused, not clipped") + } + if !strings.Contains(err.Error(), "outputs") { + t.Fatalf("refusal = %v, want it to name the outputs", err) + } + }) + + t.Run("a retried attempt starts with an empty set", func(t *testing.T) { + st := newStoreForOutputs(t) + ctx := t.Context() + task, err := st.EnqueueChatTask(ctx, "acme", "widget", "callee", "body", "w", "", nil) + if err != nil { + t.Fatal(err) + } + first, err := st.ClaimNext(ctx) + if err != nil || first == nil { + t.Fatalf("first claim: (%+v, %v)", first, err) + } + if first.Outputs != nil { + t.Fatalf("first claim outputs = %+v, want the empty set the fresh attempt starts with", first.Outputs) + } + row, err := st.TaskByID(ctx, task.ID) + if err != nil { + t.Fatal(err) + } + row.Outputs = map[string]any{"contained": true} + if err := st.Update(ctx, row); err != nil { + t.Fatalf("Update: %v", err) + } + back, err := st.TaskByID(ctx, task.ID) + if err != nil || back.Outputs["contained"] != true { + t.Fatalf("setup: the attempt's outputs did not reach the row: (%+v, %v)", back.Outputs, err) + } + if err := st.Transition(ctx, task.ID, workflow.StatusRunning, workflow.StatusParked, "needs a look"); err != nil { + t.Fatal(err) + } + if err := st.RetryTask(ctx, task.ID, workflow.StatusParked, ""); err != nil { + t.Fatal(err) + } + second, err := st.ClaimNext(ctx) + if err != nil || second == nil { + t.Fatalf("retried claim: (%+v, %v)", second, err) + } + if task.ID != second.ID { + t.Skipf("another queued task claimed first; the idempotent queue handed out %d", second.ID) + } + if second.Outputs != nil { + t.Fatalf("retried attempt outputs = %+v, want the empty set", second.Outputs) + } + final, err := st.TaskByID(ctx, task.ID) + if err != nil { + t.Fatal(err) + } + if _, still := final.Outputs["contained"]; still { + t.Fatalf("row outputs = %+v, want the emptiness the new attempt starts with", final.Outputs) + } + }) +} diff --git a/internal/infrastructure/staterpc/client.go b/internal/infrastructure/staterpc/client.go index afccfbbc1..656637794 100644 --- a/internal/infrastructure/staterpc/client.go +++ b/internal/infrastructure/staterpc/client.go @@ -652,12 +652,12 @@ func (c *Client) StartCall(ctx context.Context, callerTaskID int64, wf string, i return taskValue(r.Task), nil } -func (c *Client) CallStatus(ctx context.Context, callerTaskID, callTaskID int64) (string, string, error) { +func (c *Client) CallStatus(ctx context.Context, callerTaskID, callTaskID int64) (string, string, map[string]any, error) { r, err := c.client.WorkflowCallStatus(ctx, &pb.WorkflowCallStatusRequest{CallerTaskId: callerTaskID, CallTaskId: callTaskID}) if err != nil { - return "", "", unmapError(err) + return "", "", nil, unmapError(err) } - return r.Status, r.Detail, nil + return r.Status, r.Detail, outputsValue(r.OutputsJson), nil } // Task log diff --git a/internal/infrastructure/staterpc/server.go b/internal/infrastructure/staterpc/server.go index 34a46e94e..65eca9909 100644 --- a/internal/infrastructure/staterpc/server.go +++ b/internal/infrastructure/staterpc/server.go @@ -951,11 +951,11 @@ func (s *server) WorkflowCallStatus(ctx context.Context, r *pb.WorkflowCallStatu if s.deps.WorkflowCalls == nil { return nil, errWorkflowCallerUnavailable } - status, detail, err := s.deps.WorkflowCalls.CallStatus(ctx, r.CallerTaskId, r.CallTaskId) + status, detail, outputs, err := s.deps.WorkflowCalls.CallStatus(ctx, r.CallerTaskId, r.CallTaskId) if err != nil { return nil, s.logErr("WorkflowCallStatus", err) } - return &pb.WorkflowCallStatusResponse{Status: status, Detail: detail}, nil + return &pb.WorkflowCallStatusResponse{Status: status, Detail: detail, OutputsJson: outputsJSON(outputs)}, nil } // Task log diff --git a/internal/infrastructure/staterpc/task_grants_call_test.go b/internal/infrastructure/staterpc/task_grants_call_test.go index ed371c47b..3f66843ce 100644 --- a/internal/infrastructure/staterpc/task_grants_call_test.go +++ b/internal/infrastructure/staterpc/task_grants_call_test.go @@ -48,10 +48,10 @@ func TestTaskGrantAuthorizesTheWorkflowCallRPCs(t *testing.T) { if callee.CallParentTaskID != caller.ID || callee.CallDepth != 1 { t.Fatalf("callee = %+v, want the parent link and depth 1", callee) } - if _, _, err := worker.CallStatus(ctx, caller.ID, callee.ID); err != nil { + if _, _, _, err := worker.CallStatus(ctx, caller.ID, callee.ID); err != nil { t.Fatalf("task grant should authorize CallStatus on its own callees: %v", err) } - if _, _, err := worker.CallStatus(ctx, caller.ID, other.ID); !errors.Is(err, storecontract.ErrCallNotYours) { + if _, _, _, err := worker.CallStatus(ctx, caller.ID, other.ID); !errors.Is(err, storecontract.ErrCallNotYours) { t.Fatalf("CallStatus for a task that is not this caller's callee = %v, want ErrCallNotYours", err) } @@ -61,7 +61,7 @@ func TestTaskGrantAuthorizesTheWorkflowCallRPCs(t *testing.T) { if _, err := worker.StartCall(ctx, other.ID, "callee", nil); status.Code(err) != codes.PermissionDenied { t.Fatalf("StartCall for another task = %v, want PermissionDenied", err) } - if _, _, err := worker.CallStatus(ctx, other.ID, callee.ID); status.Code(err) != codes.PermissionDenied { + if _, _, _, err := worker.CallStatus(ctx, other.ID, callee.ID); status.Code(err) != codes.PermissionDenied { t.Fatalf("CallStatus for another task = %v, want PermissionDenied", err) } diff --git a/internal/infrastructure/staterpc/values.go b/internal/infrastructure/staterpc/values.go index 9b133fcfc..739c0f92d 100644 --- a/internal/infrastructure/staterpc/values.go +++ b/internal/infrastructure/staterpc/values.go @@ -73,8 +73,9 @@ func taskProto(t *task.Task) *pb.Task { //nolint:dupl // mirror-image field-by-f Attempt: int64(t.Attempt), ParkReason: t.ParkReason, RetryCount: int64(t.RetryCount), WatchCommentId: t.WatchCommentID, Source: t.Source, Identity: t.Identity, BindingId: t.BindingID, BindingVersion: int64(t.BindingVersion), - InputsJson: inputsJSON(t.Inputs), - CreatedAt: timestamp(t.CreatedAt), UpdatedAt: timestamp(t.UpdatedAt), + InputsJson: inputsJSON(t.Inputs), + OutputsJson: outputsJSON(t.Outputs), + CreatedAt: timestamp(t.CreatedAt), UpdatedAt: timestamp(t.UpdatedAt), ReviewPayload: t.ReviewPayload, ReviewCursor: t.ReviewCursor, ParkClass: t.ParkClass, @@ -101,6 +102,7 @@ func taskValue(t *pb.Task) *task.Task { //nolint:dupl // see taskProto above WatchCommentID: t.WatchCommentId, Source: t.Source, Identity: t.Identity, BindingID: t.BindingId, BindingVersion: int(t.BindingVersion), Inputs: inputsValue(t.InputsJson), + Outputs: outputsValue(t.OutputsJson), CreatedAt: timeValue(t.CreatedAt), UpdatedAt: timeValue(t.UpdatedAt), ReviewPayload: t.ReviewPayload, ReviewCursor: t.ReviewCursor, @@ -128,6 +130,18 @@ func inputsValue(s string) map[string]any { return inputs } +// outputsJSON and outputsValue carry a task's written outputs in +// task.EncodeOutputs form, exactly as inputsJSON/inputsValue carry inputs. +func outputsJSON(outputs map[string]any) string { + s, _ := task.EncodeOutputs(outputs) + return s +} + +func outputsValue(s string) map[string]any { + outputs, _ := task.DecodeOutputs(s) + return outputs +} + // eventDataJSON and eventDataValue convert events.Event.Data (map[string]any) // to/from a JSON object string, mirroring the store's own on-disk column. func eventDataJSON(data map[string]any) string { diff --git a/internal/infrastructure/staterpc/workflow_call_conformance_test.go b/internal/infrastructure/staterpc/workflow_call_conformance_test.go index 204fcdd31..92f4fc76c 100644 --- a/internal/infrastructure/staterpc/workflow_call_conformance_test.go +++ b/internal/infrastructure/staterpc/workflow_call_conformance_test.go @@ -1,6 +1,7 @@ package staterpc import ( + "encoding/json" "errors" "testing" @@ -59,27 +60,53 @@ func TestWorkflowCallerConformance(t *testing.T) { // A callee that has not moved yet answers its queued status with // no detail; after a transition the detail is the latest one. - status, detail, err := c.CallStatus(ctx, caller.ID, callee.ID) + status, detail, outputs, err := c.CallStatus(ctx, caller.ID, callee.ID) if err != nil || status != workflow.StatusQueued || detail != "" { t.Fatalf("CallStatus queued = (%q, %q, %v), want (queued, \"\", nil)", status, detail, err) } + if outputs != nil { + t.Fatalf("CallStatus queued outputs = %+v, want none", outputs) + } if err := local.Transition(ctx, callee.ID, callee.Status, workflow.StatusRunning, "started"); err != nil { t.Fatalf("Transition callee to running: %v", err) } if err := local.Transition(ctx, callee.ID, workflow.StatusRunning, workflow.StatusCompleted, "contained 10.0.0.9"); err != nil { t.Fatalf("Transition callee to completed: %v", err) } - status, detail, err = c.CallStatus(ctx, caller.ID, callee.ID) + status, detail, _, err = c.CallStatus(ctx, caller.ID, callee.ID) if err != nil || status != workflow.StatusCompleted || detail != "contained 10.0.0.9" { t.Fatalf("CallStatus completed = (%q, %q, %v), want the terminal status and detail", status, detail, err) } + // The callee's written outputs ride the same answer + // (docs/prds/workflow-call-outputs.md, "Storage and wire"): + // decoded from the callee's row, carried beside status and detail, + // over both adapters identically. + if row, err := local.TaskByID(ctx, callee.ID); err != nil { + t.Fatalf("TaskByID callee: %v", err) + } else { + row.Outputs = map[string]any{"contained": true, "detail_number": 3} + if err := local.Update(ctx, row); err != nil { + t.Fatalf("Update callee outputs: %v", err) + } + } + _, _, outputs, err = c.CallStatus(ctx, caller.ID, callee.ID) + if err != nil { + t.Fatalf("CallStatus with outputs: %v", err) + } + if outputs["contained"] != true { + t.Fatalf("CallStatus outputs = %+v, want the callee's written set", outputs) + } + if got, ok := outputs["detail_number"].(json.Number); !ok || got != "3" { + t.Fatalf("CallStatus outputs.detail_number = %v/%T, want the exact number", outputs["detail_number"], outputs["detail_number"]) + } + // A caller reads only its own callees, over both adapters. - _, _, err = c.CallStatus(ctx, 99999, callee.ID) + _, _, _, err = c.CallStatus(ctx, 99999, callee.ID) if !errors.Is(err, storecontract.ErrCallNotYours) { t.Fatalf("CallStatus foreign caller = %v, want ErrCallNotYours", err) } - _, _, err = c.CallStatus(ctx, caller.ID, 99999) + _, _, _, err = c.CallStatus(ctx, caller.ID, 99999) if !errors.Is(err, storecontract.ErrCallNotYours) { t.Fatalf("CallStatus missing callee = %v, want ErrCallNotYours", err) } diff --git a/proto/state/v1/state.proto b/proto/state/v1/state.proto index e541046ee..78817c7e2 100644 --- a/proto/state/v1/state.proto +++ b/proto/state/v1/state.proto @@ -335,6 +335,12 @@ message Task { // writer reads as a depth-0 root run. int64 call_parent_task_id = 36; int32 call_depth = 37; + // outputs_json is the JSON object of declared-output values this run wrote + // (task.EncodeOutputs), empty for a run that wrote none. The container's + // Update sends the whole row, so a column the message cannot carry is a + // column that write drops (docs/prds/workflow-call-outputs.md, "Storage and + // wire"). + string outputs_json = 38; } // Event mirrors internal/events.Event. Data is carried as a JSON object @@ -1041,9 +1047,13 @@ message WorkflowCallStatusRequest { } message WorkflowCallStatusResponse { // status is the callee's task status and detail its latest transition - // detail (empty before the callee first moves). + // detail (empty before the callee first moves). outputs_json is the + // callee's written outputs (task.EncodeOutputs), so a wait:true caller can + // publish one as its own (docs/prds/workflow-call-outputs.md, "Storage and + // wire"). string status = 1; string detail = 2; + string outputs_json = 3; } message InstalledPackage {