From 36b25f78cfc88b43bab43390a0601eaeb62dbc42 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 26 Sep 2026 03:16:44 +1000 Subject: [PATCH] feat(taskstate): count completed runs in the per-workflow stats The transition table moved the no-change outcomes to StatusCompleted, so a run that opens no pull request now ends in a status the per-workflow stats did not count at all: it was not merged, not in review, not parked, and the workflows page would have shown it as a run that went nowhere. The count travels the whole State Store path, since every dashboard consumer reads it through the gRPC client rather than the local store. - WorkflowStats filters `completed` beside merged and pr_open; the SQL is the count, so a wrong filter reads as zero rather than failing, and TestWorkflowStatsCountsCompletedRuns holds it against a real store. - storecontract.WorkflowStat gains Completed (json "completed"), the proto message gains field 9, and workflowStatProto/workflowStatValue carry it; TestWorkflowStatProtoCarriesEveryField round-trips every field by name. - The gateway's chat task report names the status. - The dashboard counts a completed run as delivered in GatePulseCard's totals and badge numerator, and the WorkflowStat/WorkflowStats/WorkflowRow interfaces carry the field. - TestOutcomeSitesAreTransitionsFromRunning is the guard that keeps the producer and the table agreeing: it re-reads this package's tc.Outcome assignment sites from source and requires each status to be one internal/taskstate allows from running, so a new or moved site has to be adjudicated against the table instead of discovered on a live run. --- internal/contracts/state/v1/state.pb.go | 13 +- .../domain/storecontract/storecontract.go | 1 + .../domain/workflow/outcome_sites_test.go | 157 ++++++++++++++++++ internal/gateway/gateway.go | 2 + internal/infrastructure/postgres/events.go | 1 + .../postgres/postgresdb/events.sql.go | 3 + .../postgres/queries/events.sql | 1 + .../postgres/workflow_stats_test.go | 48 ++++++ internal/infrastructure/staterpc/values.go | 8 +- .../staterpc/values_parity_test.go | 36 ++++ proto/state/v1/state.proto | 1 + ui/src/dashboard/GatePulseCard.vue | 14 +- ui/src/dashboard/state.ts | 1 + ui/src/workflows/workflow-rows.ts | 3 + 14 files changed, 279 insertions(+), 10 deletions(-) create mode 100644 internal/domain/workflow/outcome_sites_test.go create mode 100644 internal/infrastructure/postgres/workflow_stats_test.go diff --git a/internal/contracts/state/v1/state.pb.go b/internal/contracts/state/v1/state.pb.go index 380375d24..d9d2d5854 100644 --- a/internal/contracts/state/v1/state.pb.go +++ b/internal/contracts/state/v1/state.pb.go @@ -2555,6 +2555,7 @@ type WorkflowStat struct { AvgTokens int64 `protobuf:"varint,6,opt,name=avg_tokens,json=avgTokens,proto3" json:"avg_tokens,omitempty"` AvgSteps float64 `protobuf:"fixed64,7,opt,name=avg_steps,json=avgSteps,proto3" json:"avg_steps,omitempty"` TotalTokens int64 `protobuf:"varint,8,opt,name=total_tokens,json=totalTokens,proto3" json:"total_tokens,omitempty"` + Completed int64 `protobuf:"varint,9,opt,name=completed,proto3" json:"completed,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2645,6 +2646,13 @@ func (x *WorkflowStat) GetTotalTokens() int64 { return 0 } +func (x *WorkflowStat) GetCompleted() int64 { + if x != nil { + return x.Completed + } + return 0 +} + // StageStat mirrors storecontract.StageStat. type StageStat struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -9317,7 +9325,7 @@ const file_state_v1_state_proto_rawDesc = "" + "\n" + "created_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x129\n" + "\n" + - "updated_at\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\tupdatedAt\"\xe6\x01\n" + + "updated_at\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\tupdatedAt\"\x84\x02\n" + "\fWorkflowStat\x12\x1a\n" + "\bworkflow\x18\x01 \x01(\tR\bworkflow\x12\x12\n" + "\x04runs\x18\x02 \x01(\x03R\x04runs\x12\x16\n" + @@ -9327,7 +9335,8 @@ const file_state_v1_state_proto_rawDesc = "" + "\n" + "avg_tokens\x18\x06 \x01(\x03R\tavgTokens\x12\x1b\n" + "\tavg_steps\x18\a \x01(\x01R\bavgSteps\x12!\n" + - "\ftotal_tokens\x18\b \x01(\x03R\vtotalTokens\"\x80\x01\n" + + "\ftotal_tokens\x18\b \x01(\x03R\vtotalTokens\x12\x1c\n" + + "\tcompleted\x18\t \x01(\x03R\tcompleted\"\x80\x01\n" + "\tStageStat\x12\x1a\n" + "\bworkflow\x18\x01 \x01(\tR\bworkflow\x12\x14\n" + "\x05stage\x18\x02 \x01(\tR\x05stage\x12\x12\n" + diff --git a/internal/domain/storecontract/storecontract.go b/internal/domain/storecontract/storecontract.go index b99bc3a12..2e9769318 100644 --- a/internal/domain/storecontract/storecontract.go +++ b/internal/domain/storecontract/storecontract.go @@ -368,6 +368,7 @@ type WorkflowStat struct { Workflow string `json:"workflow"` Runs int `json:"runs"` Merged int `json:"merged"` + Completed int `json:"completed"` PROpen int `json:"pr_open"` Parked int `json:"parked"` AvgTokens int `json:"avg_tokens"` diff --git a/internal/domain/workflow/outcome_sites_test.go b/internal/domain/workflow/outcome_sites_test.go new file mode 100644 index 000000000..e80704a05 --- /dev/null +++ b/internal/domain/workflow/outcome_sites_test.go @@ -0,0 +1,157 @@ +package workflow + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "slices" + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/taskstate" +) + +// outcomeSite is one place in this package that ends a workflow by assigning +// tc.Outcome: the file it lives in, the status identifier it assigns, and what +// the site decides. +type outcomeSite struct { + file string + // ident is the package status identifier the site assigns, spelled the way + // the source spells it -- that is the string this test matches source + // against. + ident string + // why names the decision, so a reader adjudicating a diff here knows which + // site moved without opening the file. + why string +} + +// outcomeSites is every tc.Outcome assignment in this package's non-test +// sources. Workflow.Run persists any of them as a single transition out of +// StatusRunning (finish, workflow.go), so each status below has to be a row +// internal/taskstate's table allows from Running -- the two no-change paths +// that set StatusMerged while running are exactly what that table caught. +// +// The list is checked against the source in both directions by +// TestOutcomeSitesAreTransitionsFromRunning: a site added, removed or moved to +// another status fails there and has to be adjudicated against the table +// rather than discovered on a live run. +var outcomeSites = []outcomeSite{ + {"agent_run.go", "StatusCompleted", "a chat task's agent run produced its answer: nothing to open a PR for"}, + {"diff_rules.go", "StatusParked", "the diff breaks one of the repo's blocking rules"}, + {"feasibility.go", "StatusClosedWontDo", "feasibility decided this work should not be done"}, + {"feasibility.go", "StatusWaitingHuman", "the PRD is delivered and the run waits for a go/no-go"}, + {"remediate.go", "StatusParked", "the remediation round cap is spent"}, + {"remediate.go", "StatusPROpen", "a remediation round pushed the task back to its open PR"}, + {"review.go", "StatusParked", "review is enabled but no reviewer is wired"}, + {"review.go", "StatusParked", "review confirmed an error-level finding"}, + {"steps.go", "StatusCompleted", "a no-change build closed the forge issue"}, + {"steps.go", "StatusCompleted", "a no-change build on a task with no forge issue"}, + {"steps.go", "StatusPROpen", "the PR opened"}, + {"steps.go", "StatusParked", "the diff exceeds diff_cap_lines"}, + {"triage.go", "StatusCompleted", "triage found no code change is needed"}, + {"triage.go", "StatusQueued", "triage routed the task to another workflow"}, +} + +// outcomeStatusIdentifiers resolves the identifiers above to the stored +// strings they alias (task_aliases.go re-exports them from the task package, +// which re-exports them from internal/taskstate), so the table can name a site +// the way the source does and still ask the transition table about it. +var outcomeStatusIdentifiers = map[string]string{ + "StatusQueued": StatusQueued, + "StatusRunning": StatusRunning, + "StatusWaitingHuman": StatusWaitingHuman, + "StatusPROpen": StatusPROpen, + "StatusMerged": StatusMerged, + "StatusParked": StatusParked, + "StatusDead": StatusDead, + "StatusRejected": StatusRejected, + "StatusClosedWontDo": StatusClosedWontDo, + "StatusCompleted": StatusCompleted, +} + +// TestOutcomeSitesAreTransitionsFromRunning holds the producer half of the +// transition contract: every status this package finishes a run with is one +// internal/taskstate lets a running task move to. It fails for an unadjudicated +// site (the table and the source disagree) and for a status the table forbids +// (the run would have produced a transition the State Store must refuse). +func TestOutcomeSitesAreTransitionsFromRunning(t *testing.T) { + t.Parallel() + + want := make([]string, 0, len(outcomeSites)) + for _, site := range outcomeSites { + want = append(want, site.file+" "+site.ident) + } + slices.Sort(want) + + found := outcomeSitesInSource(t) + if !slices.Equal(found, want) { + t.Errorf("the outcome sites in this package are %v, but the table here lists %v: add, remove or correct the entry (and adjudicate the status against internal/taskstate)", found, want) + } + + for _, site := range outcomeSites { + status, known := outcomeStatusIdentifiers[site.ident] + if !known { + t.Errorf("%s names %s, which is not a task status identifier; the table here cannot be asked about it", site.file, site.ident) + continue + } + if !taskstate.CanTransition(taskstate.Running, status) { + t.Errorf("%s finishes the run as %s (%q), which internal/taskstate does not allow from %s: the task would end in a transition the State Store has to refuse", site.file, site.ident, status, taskstate.Running) + } + } +} + +// outcomeSitesInSource returns every tc.Outcome assignment in this package's +// non-test sources as " ", sorted, duplicates kept: a +// file with two distinct outcome decisions has to be two entries here. +func outcomeSitesInSource(t *testing.T) []string { + t.Helper() + + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read the package directory: %v", err) + } + found := make([]string, 0, len(outcomeSites)) + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + file, err := parser.ParseFile(token.NewFileSet(), name, nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", name, err) + } + ast.Inspect(file, func(node ast.Node) bool { + literal, ok := node.(*ast.CompositeLit) + if !ok { + return true + } + kind, ok := literal.Type.(*ast.Ident) + if !ok || kind.Name != "Outcome" { + return true + } + for _, element := range literal.Elts { + field, ok := element.(*ast.KeyValueExpr) + if !ok { + continue + } + key, ok := field.Key.(*ast.Ident) + if !ok || key.Name != "Status" { + continue + } + status, ok := field.Value.(*ast.Ident) + if !ok { + // An expression this test cannot read by name is still a + // site that has to be adjudicated, so it lands in the + // comparison as a mismatch rather than being skipped. + found = append(found, name+" ") + continue + } + found = append(found, name+" "+status.Name) + } + return true + }) + } + slices.Sort(found) + return found +} diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 75cf57c85..f1fc6160a 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -748,6 +748,7 @@ var taskStateDisplays = map[string]taskStateDisplay{ taskstate.Queued: {icon: "⏳", label: "Queued"}, taskstate.PROpen: {icon: "🔀", label: "PR open"}, taskstate.Merged: {icon: "✅", label: "Merged"}, + taskstate.Completed: {icon: "✅", label: "Completed"}, taskstate.Rejected: {icon: "❌", label: "Rejected"}, taskstate.Declined: {icon: "🚫", label: "Declined"}, "declined": {icon: "🚫", label: "Declined"}, @@ -761,6 +762,7 @@ var statusOrder = []string{ taskstate.Queued, taskstate.PROpen, taskstate.Merged, + taskstate.Completed, taskstate.Rejected, taskstate.Declined, "declined", diff --git a/internal/infrastructure/postgres/events.go b/internal/infrastructure/postgres/events.go index d489a7df7..63a48e7cb 100644 --- a/internal/infrastructure/postgres/events.go +++ b/internal/infrastructure/postgres/events.go @@ -108,6 +108,7 @@ func (s *Store) WorkflowStats(ctx context.Context) ([]storecontract.WorkflowStat Workflow: r.Workflow, Runs: int(r.Runs), Merged: int(r.Merged), + Completed: int(r.Completed), PROpen: int(r.PrOpen), Parked: int(r.Parked), AvgTokens: int(r.AvgTokens), diff --git a/internal/infrastructure/postgres/postgresdb/events.sql.go b/internal/infrastructure/postgres/postgresdb/events.sql.go index f9dc54797..03844619c 100644 --- a/internal/infrastructure/postgres/postgresdb/events.sql.go +++ b/internal/infrastructure/postgres/postgresdb/events.sql.go @@ -295,6 +295,7 @@ const workflowStats = `-- name: WorkflowStats :many SELECT workflow, COUNT(*)::int AS runs, COUNT(*) FILTER (WHERE status = 'merged')::int AS merged, + COUNT(*) FILTER (WHERE status = 'completed')::int AS completed, COUNT(*) FILTER (WHERE status = 'pr_open')::int AS pr_open, COUNT(*) FILTER (WHERE status = 'parked')::int AS parked, CAST(AVG(tokens_used) AS bigint) AS avg_tokens, @@ -310,6 +311,7 @@ type WorkflowStatsRow struct { Workflow string Runs int32 Merged int32 + Completed int32 PrOpen int32 Parked int32 AvgTokens int64 @@ -330,6 +332,7 @@ func (q *Queries) WorkflowStats(ctx context.Context) ([]WorkflowStatsRow, error) &i.Workflow, &i.Runs, &i.Merged, + &i.Completed, &i.PrOpen, &i.Parked, &i.AvgTokens, diff --git a/internal/infrastructure/postgres/queries/events.sql b/internal/infrastructure/postgres/queries/events.sql index 936172496..95a473b8e 100644 --- a/internal/infrastructure/postgres/queries/events.sql +++ b/internal/infrastructure/postgres/queries/events.sql @@ -39,6 +39,7 @@ ORDER BY id; SELECT workflow, COUNT(*)::int AS runs, COUNT(*) FILTER (WHERE status = 'merged')::int AS merged, + COUNT(*) FILTER (WHERE status = 'completed')::int AS completed, COUNT(*) FILTER (WHERE status = 'pr_open')::int AS pr_open, COUNT(*) FILTER (WHERE status = 'parked')::int AS parked, CAST(AVG(tokens_used) AS bigint) AS avg_tokens, diff --git a/internal/infrastructure/postgres/workflow_stats_test.go b/internal/infrastructure/postgres/workflow_stats_test.go new file mode 100644 index 000000000..a2d8ad14d --- /dev/null +++ b/internal/infrastructure/postgres/workflow_stats_test.go @@ -0,0 +1,48 @@ +package postgres + +import ( + "testing" + + "github.com/samcharles93/archie-core/internal/taskstate" +) + +// TestWorkflowStatsCountsCompletedRuns pins the per-workflow count the +// transition table made reachable: a run that finishes with no pull request +// (a no-change build, a triage that needed no code) ends in taskstate.Completed, +// so the workflows page has to be able to see those runs at all. The count is a +// status filter in SQL, which fails silently when it is wrong -- a typo'd or +// missing filter reads as zero, not as an error -- so it is asserted against the +// store rather than left to the dashboard to notice. +func TestWorkflowStatsCountsCompletedRuns(t *testing.T) { + pool, q := migrated(t) + store := New(pool) + + finished := insertChatTask(t, q) + merged := insertChatTask(t, q) + finish := func(id int64, to string) { + t.Helper() + if err := store.Transition(t.Context(), id, taskstate.Queued, taskstate.Running, ""); err != nil { + t.Fatalf("queued -> running: %v", err) + } + if err := store.Transition(t.Context(), id, taskstate.Running, to, ""); err != nil { + t.Fatalf("running -> %s: %v", to, err) + } + } + finish(finished.ID, taskstate.Completed) + finish(merged.ID, taskstate.Merged) + + stats, err := store.WorkflowStats(t.Context()) + if err != nil { + t.Fatalf("WorkflowStats: %v", err) + } + if len(stats) != 1 { + t.Fatalf("WorkflowStats = %+v, want one row for %q", stats, finished.Workflow) + } + got := stats[0] + if got.Workflow != merged.Workflow { + t.Fatalf("WorkflowStats row is %q, want %q", got.Workflow, merged.Workflow) + } + if got.Runs != 2 || got.Completed != 1 || got.Merged != 1 { + t.Errorf("WorkflowStats = %+v, want runs 2, completed 1, merged 1: the completed count has to be its own status, not a share of merged", got) + } +} diff --git a/internal/infrastructure/staterpc/values.go b/internal/infrastructure/staterpc/values.go index 8a28dc187..f1b9586e7 100644 --- a/internal/infrastructure/staterpc/values.go +++ b/internal/infrastructure/staterpc/values.go @@ -282,8 +282,8 @@ func bindingInputsValue(s string) map[string]binding.InputSource { func workflowStatProto(w storecontract.WorkflowStat) *pb.WorkflowStat { return &pb.WorkflowStat{ Workflow: w.Workflow, Runs: int64(w.Runs), Merged: int64(w.Merged), - PrOpen: int64(w.PROpen), Parked: int64(w.Parked), AvgTokens: int64(w.AvgTokens), - AvgSteps: w.AvgSteps, TotalTokens: int64(w.TotalToken), + Completed: int64(w.Completed), PrOpen: int64(w.PROpen), Parked: int64(w.Parked), + AvgTokens: int64(w.AvgTokens), AvgSteps: w.AvgSteps, TotalTokens: int64(w.TotalToken), } } @@ -293,8 +293,8 @@ func workflowStatValue(w *pb.WorkflowStat) storecontract.WorkflowStat { } return storecontract.WorkflowStat{ Workflow: w.Workflow, Runs: int(w.Runs), Merged: int(w.Merged), - PROpen: int(w.PrOpen), Parked: int(w.Parked), AvgTokens: int(w.AvgTokens), - AvgSteps: w.AvgSteps, TotalToken: int(w.TotalTokens), + Completed: int(w.Completed), PROpen: int(w.PrOpen), Parked: int(w.Parked), + AvgTokens: int(w.AvgTokens), AvgSteps: w.AvgSteps, TotalToken: int(w.TotalTokens), } } diff --git a/internal/infrastructure/staterpc/values_parity_test.go b/internal/infrastructure/staterpc/values_parity_test.go index 778d071c4..baff23abb 100644 --- a/internal/infrastructure/staterpc/values_parity_test.go +++ b/internal/infrastructure/staterpc/values_parity_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/workflow/task" ) @@ -49,3 +50,38 @@ func TestTaskProtoCarriesEveryDomainField(t *testing.T) { } } } + +// TestWorkflowStatProtoCarriesEveryField is the same contract for the +// per-workflow metric row, and the reason it exists is the same: the dashboard +// reads these counts off the wire, so a count that only the producer knows +// (Completed, the runs that finished with no pull request to open) would read +// as zero everywhere without anything failing. Each field gets a value of its +// own, so a mapping that assigns two of them from the same source -- or drops +// one -- is caught by name. +func TestWorkflowStatProtoCarriesEveryField(t *testing.T) { + want := storecontract.WorkflowStat{} + v := reflect.ValueOf(&want).Elem() + for i := range v.NumField() { + field := v.Field(i) + switch { + case field.Kind() == reflect.String: + field.SetString(v.Type().Field(i).Name + "-value") + case field.Kind() == reflect.Float64: + field.SetFloat(float64(i) + 0.5) + case field.CanInt(): + field.SetInt(int64(i) + 1) + default: + t.Fatalf("field %s has kind %s, which this test does not know how to fill; extend it alongside the field", v.Type().Field(i).Name, field.Kind()) + } + } + + got := reflect.ValueOf(workflowStatValue(workflowStatProto(want))) + for i := range v.NumField() { + name := v.Type().Field(i).Name + gotField := got.Field(i).Interface() + wantField := v.Field(i).Interface() + if !reflect.DeepEqual(gotField, wantField) { + t.Errorf("WorkflowStat.%s did not survive the proto round trip: got %v, want %v", name, gotField, wantField) + } + } +} diff --git a/proto/state/v1/state.proto b/proto/state/v1/state.proto index 192f94081..51cb4913d 100644 --- a/proto/state/v1/state.proto +++ b/proto/state/v1/state.proto @@ -419,6 +419,7 @@ message WorkflowStat { int64 avg_tokens = 6; double avg_steps = 7; int64 total_tokens = 8; + int64 completed = 9; } // StageStat mirrors storecontract.StageStat. diff --git a/ui/src/dashboard/GatePulseCard.vue b/ui/src/dashboard/GatePulseCard.vue index 70c296006..0a357e598 100644 --- a/ui/src/dashboard/GatePulseCard.vue +++ b/ui/src/dashboard/GatePulseCard.vue @@ -4,15 +4,21 @@ import { computed } from "vue"; import { Badge } from "@/components/ui/badge"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Empty, EmptyHeader, EmptyTitle } from "@/components/ui/empty"; -import { workflows } from "./state"; +import { workflows, type WorkflowStat } from "./state"; /** Sharer of runs: how much of the work that finished passed its gates. */ const stats = computed(() => workflows.value?.workflows ?? []); +// A run that finished without opening a pull request (a no-change build, a +// triage that needed no code) is delivered like a merge or a PR in review, so +// both totals count it: the pass rate would otherwise read every `completed` +// row as a failed gate. +const delivered = (w: WorkflowStat) => (w.merged || 0) + (w.completed || 0); + const totals = computed(() => ({ runs: stats.value.reduce((a, w) => a + (w.runs || 0), 0), merged: stats.value.reduce( - (a, w) => a + (w.merged || 0) + (w.pr_open || 0), + (a, w) => a + delivered(w) + (w.pr_open || 0), 0, ), })); @@ -61,14 +67,14 @@ const pct = computed(() => { }} - {{ w.merged || 0 }}/{{ w.runs || 0 }} + {{ delivered(w) }}/{{ w.runs || 0 }} diff --git a/ui/src/dashboard/state.ts b/ui/src/dashboard/state.ts index dd71a849d..2825ee10b 100644 --- a/ui/src/dashboard/state.ts +++ b/ui/src/dashboard/state.ts @@ -22,6 +22,7 @@ export interface WorkflowStat { workflow?: string; runs?: number; merged?: number; + completed?: number; pr_open?: number; parked?: number; } diff --git a/ui/src/workflows/workflow-rows.ts b/ui/src/workflows/workflow-rows.ts index 3c73ba32a..85f312c77 100644 --- a/ui/src/workflows/workflow-rows.ts +++ b/ui/src/workflows/workflow-rows.ts @@ -14,6 +14,7 @@ export interface WorkflowStats { workflow: string; runs?: number; merged?: number; + completed?: number; avg_tokens?: number; avg_steps?: number; origin?: string; @@ -34,6 +35,7 @@ export interface WorkflowRow { workflow: string; runs?: number; merged?: number; + completed?: number; avg_tokens?: number; avg_steps?: number; origin?: string; @@ -59,6 +61,7 @@ export function workflowRows( workflow: definition.id, runs: 0, merged: 0, + completed: 0, }), }); }