diff --git a/go.mod b/go.mod index f26e50114..270501eb8 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,6 @@ require ( github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 // indirect - go.opentelemetry.io/otel v1.47.0 // indirect go.opentelemetry.io/otel/log v1.47.0 // indirect go.opentelemetry.io/otel/metric v1.47.0 // indirect go.uber.org/multierr v1.11.0 // indirect @@ -99,6 +98,7 @@ require ( go.lsp.dev/jsonrpc2 v1.0.1 go.lsp.dev/protocol v1.0.1 go.lsp.dev/uri v1.0.1 + go.opentelemetry.io/otel v1.47.0 go.opentelemetry.io/otel/trace v1.47.0 golang.org/x/crypto v0.58.0 golang.org/x/mod v0.42.0 diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 6b54e5976..ab8e87a6d 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -508,7 +508,7 @@ func (b *boot) setupGatewayClient(ctx context.Context) error { b.addCleanup(closeMessaging) b.messaging = messaging posted := func(taskID int64) { - b.bus.Publish(events.Event{At: time.Now().UTC(), Kind: events.KindChatOutcome, TaskID: taskID}) + b.bus.Publish(ctx, events.Event{At: time.Now().UTC(), Kind: events.KindChatOutcome, TaskID: taskID}) } go postOutcomes(ctx, b.stateStore, deliverOutcome(contract, messaging), posted, b.log) b.setupReadinessProbes() diff --git a/internal/app/archied/kit_launcher.go b/internal/app/archied/kit_launcher.go index 7d9feac28..15dffa0bd 100644 --- a/internal/app/archied/kit_launcher.go +++ b/internal/app/archied/kit_launcher.go @@ -75,9 +75,9 @@ func (b *boot) setupKitLauncher(ctx context.Context) { Resolver: resolver, OAuthStore: oauthStore, TaskTools: gateway, // A test run's recorded deliveries land on the task's timeline through // the same bus every other event uses. - Events: func(event events.Event) { + Events: func(event events.Event) { //nolint:contextcheck // the egress callback carries no request context if b.bus != nil { - b.bus.Publish(event) + b.bus.Publish(context.Background(), event) } }, }) diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index 326a5e6c9..591836746 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -329,7 +329,7 @@ func subscribeAgentEvents(nc *natsio.Conn, bus *events.Bus, log *slog.Logger) (u log.Warn("task event message undecodable", "task", taskID, "err", err) return } - bus.Publish(e) + bus.Publish(context.Background(), e) }) if err != nil { return nil, err diff --git a/internal/app/archied/task_actions.go b/internal/app/archied/task_actions.go index fbb7f78da..731fd74b9 100644 --- a/internal/app/archied/task_actions.go +++ b/internal/app/archied/task_actions.go @@ -80,8 +80,8 @@ func (b *boot) removeTaskWorktree(task *taskactions.Task) error { return b.d.RemoveWorktree(&workflow.Task{Owner: task.Owner, Repo: task.Repo, Identity: task.Identity, IssueNumber: task.IssueNumber, Org: task.Org}) } -func (b *boot) publishEvent(e events.Event) { +func (b *boot) publishEvent(ctx context.Context, e events.Event) { if b.bus != nil { - b.bus.Publish(e) + b.bus.Publish(ctx, e) } } diff --git a/internal/app/archied/update_relay.go b/internal/app/archied/update_relay.go index bf0ae6201..7c0273029 100644 --- a/internal/app/archied/update_relay.go +++ b/internal/app/archied/update_relay.go @@ -31,7 +31,7 @@ func (b *boot) startUpdateRelay(ctx context.Context, path string) { if path == "" || b.bus == nil { return } - b.relayUpdateReport(path) + b.relayUpdateReport(ctx, path) go func() { ticker := time.NewTicker(updateReportPollInterval) defer ticker.Stop() @@ -40,13 +40,13 @@ func (b *boot) startUpdateRelay(ctx context.Context, path string) { case <-ctx.Done(): return case <-ticker.C: - b.relayUpdateReport(path) + b.relayUpdateReport(ctx, path) } } }() } -func (b *boot) relayUpdateReport(path string) { +func (b *boot) relayUpdateReport(ctx context.Context, path string) { report, found, err := releaseupdate.ReadPendingReport(path) if err != nil { b.log.Warn("read pending update report failed; discarding it", "err", err) @@ -58,7 +58,7 @@ func (b *boot) relayUpdateReport(path string) { if !found { return } - b.publishEvent(releaseupdate.ReportEvent(report, daemonRunningVersions(b.agentStatus))) + b.publishEvent(ctx, releaseupdate.ReportEvent(report, daemonRunningVersions(b.agentStatus))) if err := releaseupdate.ClearPendingReport(path); err != nil { b.log.Warn("clear pending update report failed", "err", err) } diff --git a/internal/app/archiegateway/curators.go b/internal/app/archiegateway/curators.go index 0fccc482a..426d3d960 100644 --- a/internal/app/archiegateway/curators.go +++ b/internal/app/archiegateway/curators.go @@ -92,7 +92,7 @@ type curatorEventSink struct { } func (s curatorEventSink) Emit(kind, detail string, data map[string]any) { - s.b.Publish(events.Event{Kind: kind, Detail: detail, Data: data}) + s.b.Publish(context.Background(), events.Event{Kind: kind, Detail: detail, Data: data}) } // curatorLLMRunner adapts the shared ai-sdk runtime to the curator diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 17b21f833..4af0e8800 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -17,6 +17,9 @@ import ( "sync" "time" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + "github.com/samcharles93/archie-core/internal/agentexec" "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/container" @@ -36,6 +39,7 @@ import ( "github.com/samcharles93/archie-core/internal/forge" agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats" "github.com/samcharles93/archie-core/internal/logging" + "github.com/samcharles93/archie-core/internal/natsrpc" "github.com/samcharles93/archie-core/internal/storage" "github.com/samcharles93/archie-core/internal/taskrun" "github.com/samcharles93/archie-core/internal/taskstate" @@ -54,7 +58,7 @@ type TaskBus interface { // authorizes the task's own State Store calls and the push of its branch. // It lapses at the task's time limit plus a margin if never revoked. type RunCredentialIssuer interface { - Issue(task *workflow.Task, limit time.Duration) (token string, revoke func(), err error) + Issue(ctx context.Context, task *workflow.Task, limit time.Duration) (token string, revoke func(), err error) } // NATSEndpoint is the broker address and credential archied connected with at @@ -268,9 +272,9 @@ func (d *Daemon) sourceTag(source IssueSource) string { } // emit publishes an observability event; safe on a nil bus. -func (d *Daemon) emit(e events.Event) { +func (d *Daemon) emit(ctx context.Context, e events.Event) { if d.Bus != nil { - d.Bus.Publish(e) + d.Bus.Publish(ctx, e) } } @@ -1371,7 +1375,7 @@ func (d *Daemon) acknowledge(ctx context.Context, fg forge.Forge, cfg config.Con d.Log.Warn("ack reaction failed", "issue", is.Number, "err", err) } } - d.emit(events.Event{ + d.emit(ctx, events.Event{ Kind: events.KindTaskQueued, Repo: repo.FullName(), Issue: is.Number, Detail: is.Title, }) @@ -1399,6 +1403,8 @@ func (d *Daemon) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, // and runs the workflow. The message is ack'd on terminal (park is a valid // outcome). Nak on transient errors so NATS redelivers. func (d *Daemon) processNATSTask(ctx context.Context, msg eventbus.Message, hold *taskHold) { + ctx, span := natsrpc.StartConsumer(ctx, msg.Subject(), msg.Header()) + defer span.End() tm, err := workintake.DecodeTask(msg.Data()) if err != nil { d.Log.Error("nats decode failed", "err", err) @@ -1504,7 +1510,7 @@ func (d *Daemon) reconcilePRs(ctx context.Context) { "Closed by #%d, merged by archie.", t.PRNumber, )) d.Log.Info("PR merged", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber) - d.emit(events.Event{ + d.emit(ctx, events.Event{ Kind: events.KindPRMerged, TaskID: t.ID, Attempt: attempt, Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber, Data: map[string]any{"pr": t.PRNumber}, @@ -1513,7 +1519,7 @@ func (d *Daemon) reconcilePRs(ctx context.Context) { _ = d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusRejected, "PR closed without merge") _ = trees.Cleanup(t.Owner, t.Repo, t.IssueNumber) d.Log.Info("PR rejected", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber) - d.emit(events.Event{ + d.emit(ctx, events.Event{ Kind: events.KindPRRejected, TaskID: t.ID, Attempt: attempt, Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber, Data: map[string]any{"pr": t.PRNumber}, @@ -1805,7 +1811,7 @@ func (d *Daemon) acquireTaskContainer( return nil, "", nil, false } - credential, revokeCredential, err := d.runCredential(task) + credential, revokeCredential, err := d.runCredential(ctx, task) if err != nil { d.teardownStorage(ctx, task, repo, workDir) park("run credential failed", err) @@ -1903,7 +1909,7 @@ func (d *Daemon) parkBlockedTask(ctx context.Context, task *workflow.Task, reaso // parked would be a lie when another actor moved it out of running first // -- the same race the parked event is withheld for. d.recordPark(ctx, task.ID, reason) - d.emit(events.Event{ + d.emit(ctx, events.Event{ Kind: events.KindParked, TaskID: task.ID, Attempt: task.Attempt, Repo: strings.Trim(task.Owner+"/"+task.Repo, "/"), Issue: task.IssueNumber, Workflow: task.Workflow, Detail: reason, Data: map[string]any{"park_class": class}, @@ -1955,6 +1961,8 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo conf Tools: d.toolAllowlist(ctx, task, profile.Tools), Harness: harness, } + req.Trace = map[string]string{} + otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(req.Trace)) data, err := json.Marshal(req) if err != nil { d.Log.Error("taskrun encode failed", "task", task.ID, "err", err) @@ -2375,14 +2383,14 @@ func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken, stateStoreUR // runCredential issues the task's run credential, or "" when no State Store // is configured. It fails when no issuer is wired. -func (d *Daemon) runCredential(task *workflow.Task) (string, func(), error) { +func (d *Daemon) runCredential(ctx context.Context, task *workflow.Task) (string, func(), error) { if d.ConnectedStateStore.URL == "" { return "", func() {}, nil } if d.RunCredentials == nil { return "", nil, fmt.Errorf("state store is configured but no run credential issuer is wired") } - return d.RunCredentials.Issue(task, d.configFor(task).Budgets.TaskWallClock.Std()) + return d.RunCredentials.Issue(ctx, task, d.configFor(task).Budgets.TaskWallClock.Std()) } func (d *Daemon) configFor(task *workflow.Task) config.Config { @@ -2429,7 +2437,7 @@ func (d *Daemon) captureAttemptConfig(ctx context.Context, task *workflow.Task, // The assigned ID tells the event sink to broadcast without inserting a // duplicate row. event.ID = id - d.emit(event) + d.emit(ctx, event) } // identityMayAct reports whether a task's identity may act, and the park diff --git a/internal/daemon/kit_task.go b/internal/daemon/kit_task.go index 4016968e0..69b19c33d 100644 --- a/internal/daemon/kit_task.go +++ b/internal/daemon/kit_task.go @@ -40,7 +40,7 @@ func (d *Daemon) runKitTask(ctx context.Context, task *workflow.Task, repo confi park("task.json write failed", err) return } - credential, revokeCredential, err := d.runCredential(task) + credential, revokeCredential, err := d.runCredential(ctx, task) if err != nil { park("run credential failed", err) return diff --git a/internal/daemon/reactions.go b/internal/daemon/reactions.go index 8559ef866..7b77fbd59 100644 --- a/internal/daemon/reactions.go +++ b/internal/daemon/reactions.go @@ -12,6 +12,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/workintake" "github.com/samcharles93/archie-core/internal/eventbus" "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/natsrpc" ) // ReactionSource is the pull surface the reaction consumer needs: fetch one @@ -85,6 +86,8 @@ func (c *reactionConsumer) drain(ctx context.Context, src ReactionSource, limit // path (drop, dedup, or work queued), Nak only on a transient store failure // the next delivery can succeed against. func (c *reactionConsumer) handle(ctx context.Context, msg eventbus.Message) { + ctx, span := natsrpc.StartConsumer(ctx, msg.Subject(), msg.Header()) + defer span.End() envelope, err := workintake.DecodeReviewComment(msg.Data()) if err != nil { c.log.Error("review reaction decode failed", "subject", msg.Subject(), "err", err) diff --git a/internal/domain/taskactions/service.go b/internal/domain/taskactions/service.go index 581230ce5..c4c13092c 100644 --- a/internal/domain/taskactions/service.go +++ b/internal/domain/taskactions/service.go @@ -212,7 +212,7 @@ type Service struct { // RemoveWorktree discards the task's clone on archive, the reap path for a // worktree terminal cleanup kept because it may hold uncaptured work. RemoveWorktree func(*Task) error - Publish func(events.Event) + Publish func(context.Context, events.Event) Warn func(string, ...any) } @@ -648,7 +648,7 @@ func (s Service) emit(ctx context.Context, e events.Event) { } } if s.Publish != nil { - s.Publish(e) + s.Publish(ctx, e) } } diff --git a/internal/domain/workflow/agent.go b/internal/domain/workflow/agent.go index 771f02bc4..c5c26f752 100644 --- a/internal/domain/workflow/agent.go +++ b/internal/domain/workflow/agent.go @@ -69,7 +69,7 @@ func (a AgentStage) Stage() Stage { // The agent call is a child of the stage's recorded step: the store // says what the stage's runtime did, not just what the stage did. res, runErr := tc.RunAgentChild(ctx, a.Name, a.Model, modelRef, func() (agentrun.Result, error) { - return tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter(a.Name)) + return tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter(ctx, a.Name)) }) return a.handleResult(ctx, tc, req, res, runErr, modelRef) }} diff --git a/internal/domain/workflow/implement.go b/internal/domain/workflow/implement.go index 0765b8d4a..916bd9745 100644 --- a/internal/domain/workflow/implement.go +++ b/internal/domain/workflow/implement.go @@ -97,7 +97,7 @@ func runBaselineFix(ctx context.Context, tc *TaskContext, argv []string, out []b } res, agentErr := tc.RunAgentChild(ctx, "baseline-fix", "", modelRef, func() (agentrun.Result, error) { - return tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter("baseline-fix")) + return tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter(ctx, "baseline-fix")) }) if agentErr != nil && res.Version == 0 { return baselineFixFailedError(argv, string(out), fmt.Sprintf("the builder could not run: %v", agentErr)) diff --git a/internal/domain/workflow/prreview_stages.go b/internal/domain/workflow/prreview_stages.go index 0dceeab61..373908e79 100644 --- a/internal/domain/workflow/prreview_stages.go +++ b/internal/domain/workflow/prreview_stages.go @@ -579,7 +579,7 @@ func runReviewerAgent( if err != nil { return agentrun.Result{}, err } - res, runErr := tc.Agent.Run(ctx, tc.prReview.snapshotDir, req, tc.toolCallReporter(name)) + res, runErr := tc.Agent.Run(ctx, tc.prReview.snapshotDir, req, tc.toolCallReporter(ctx, name)) tc.prReview.tokensSpent.Add(int64(res.TokensUsed)) to, detail := taskstate.StepSucceeded, res.Summary switch { @@ -805,7 +805,7 @@ func runPRReviewAgentRecorded( CaptureTools: captureTools, } res, err := tc.RunAgentChild(ctx, name, "", modelRef, func() (agentrun.Result, error) { - return tc.Agent.Run(ctx, workspace, req, tc.toolCallReporter(name)) + return tc.Agent.Run(ctx, workspace, req, tc.toolCallReporter(ctx, name)) }) tc.prReview.tokensSpent.Add(int64(res.TokensUsed)) return res, err diff --git a/internal/domain/workflow/step_activity_test.go b/internal/domain/workflow/step_activity_test.go index 894a2fd94..95473d003 100644 --- a/internal/domain/workflow/step_activity_test.go +++ b/internal/domain/workflow/step_activity_test.go @@ -1,6 +1,7 @@ package workflow import ( + "context" "log/slog" "testing" @@ -19,7 +20,7 @@ func TestStepActivityAttribution(t *testing.T) { tc = tc.branch(branch) } data := map[string]any{"tool": "read"} - tc.Emit(events.KindToolCall, "agent.run", "read a file", data) + tc.Emit(context.Background(), events.KindToolCall, "agent.run", "read a file", data) ev := <-sub.C if ev.Data["step_id"] != int64(7) || ev.Data["branch"] != branch { t.Fatalf("activity lost its step or branch: %#v", ev.Data) diff --git a/internal/domain/workflow/test_run.go b/internal/domain/workflow/test_run.go index 06e22c993..ddc8954db 100644 --- a/internal/domain/workflow/test_run.go +++ b/internal/domain/workflow/test_run.go @@ -36,8 +36,8 @@ type recordDeliveries struct { tc *TaskContext } -func (r recordDeliveries) Record(_ context.Context, detail string) { - r.tc.Emit(events.KindDeliveryRecorded, r.tc.Stage, "would "+detail, nil) +func (r recordDeliveries) Record(ctx context.Context, detail string) { + r.tc.Emit(ctx, events.KindDeliveryRecorded, r.tc.Stage, "would "+detail, nil) } type recordedForge struct { diff --git a/internal/domain/workflow/workflow.go b/internal/domain/workflow/workflow.go index 68f716abc..dc56ed164 100644 --- a/internal/domain/workflow/workflow.go +++ b/internal/domain/workflow/workflow.go @@ -185,11 +185,11 @@ type TaskContext struct { // Emit publishes an observability event stamped with the task and its // attempt. Safe on a nil bus. -func (tc *TaskContext) Emit(kind, stage, detail string, data map[string]any) { +func (tc *TaskContext) Emit(ctx context.Context, kind, stage, detail string, data map[string]any) { if tc.Bus == nil { return } - tc.Bus.Publish(events.Event{ + tc.Bus.Publish(ctx, events.Event{ Kind: kind, TaskID: tc.Task.ID, Repo: tc.Task.Owner + "/" + tc.Task.Repo, @@ -218,7 +218,7 @@ func (tc *TaskContext) activityData(kind string, data map[string]any) map[string // without inserting a duplicate row. func (tc *TaskContext) EmitDurable(ctx context.Context, kind, stage, detail string, data map[string]any) error { if tc.Store == nil { - tc.Emit(kind, stage, detail, data) + tc.Emit(ctx, kind, stage, detail, data) return nil } event := events.Event{ @@ -232,7 +232,7 @@ func (tc *TaskContext) EmitDurable(ctx context.Context, kind, stage, detail stri } event.ID = id if tc.Bus != nil { - tc.Bus.Publish(event) + tc.Bus.Publish(ctx, event) } return nil } @@ -240,9 +240,9 @@ func (tc *TaskContext) EmitDurable(ctx context.Context, kind, stage, detail stri // toolCallReporter builds the agentrun.ToolCallReporter an agent stage passes // to tc.Agent.Run, so every completed tool call during that run surfaces as a // tool_call event on the task timeline. Safe to call on a nil Bus: Emit no-ops. -func (tc *TaskContext) toolCallReporter(stage string) agentrun.ToolCallReporter { +func (tc *TaskContext) toolCallReporter(ctx context.Context, stage string) agentrun.ToolCallReporter { return func(report agentrun.ToolCallReport) { - tc.Emit(events.KindToolCall, stage, report.Detail, map[string]any{ + tc.Emit(ctx, events.KindToolCall, stage, report.Detail, map[string]any{ "tool": report.Tool, "failed": report.Failed, }) @@ -347,7 +347,7 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) { park(ctx, tc, fmt.Sprintf("stage %s could not be recorded: %v", stage.Name, err)) return } - publishEvent(tc, startEvent) + publishEvent(ctx, tc, startEvent) // The stage's own step is the parent of every agent or call step the // stage body records; zero it when the body returns, so a caller that // outlives the stage cannot parent to a step that has finished. @@ -374,7 +374,7 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) { if recordErr != nil { stageLog.Warn("interrupted step finish could not be recorded", "stage", stage.Name, "err", recordErr) } else { - publishEvent(tc, finishEvent) + publishEvent(ctx, tc, finishEvent) } stageLog.Info("stage interrupted", "err", err) return @@ -393,7 +393,7 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) { park(ctx, tc, fmt.Sprintf("stage %s could not be recorded: %v", stage.Name, recordErr)) return } - publishEvent(tc, finishEvent) + publishEvent(ctx, tc, finishEvent) if ended := endsRun(ctx, tc, stage, err, log, stageLog); ended { return @@ -475,11 +475,11 @@ func endsRun(ctx context.Context, tc *TaskContext, stage Stage, err error, log, // publishEvent puts an already-persisted event on the run's bus after its // write committed: the assigned ID is what the daemon's event sink reads to // skip the second insert, the same convention EmitDurable established. -func publishEvent(tc *TaskContext, e events.Event) { +func publishEvent(ctx context.Context, tc *TaskContext, e events.Event) { if tc.Bus == nil || e.ID == 0 { return } - tc.Bus.Publish(e) + tc.Bus.Publish(ctx, e) } func finish(ctx context.Context, tc *TaskContext, log *slog.Logger) { @@ -504,7 +504,7 @@ func finish(ctx context.Context, tc *TaskContext, log *slog.Logger) { log.Error("workflow outcome transition failed", "err", err) return } - tc.Emit(events.KindOutcome, tc.Stage, tc.Outcome.Detail, map[string]any{"status": tc.Outcome.Status}) + tc.Emit(ctx, events.KindOutcome, tc.Stage, tc.Outcome.Detail, map[string]any{"status": tc.Outcome.Status}) log.Info("workflow finished", "status", tc.Outcome.Status) } @@ -518,7 +518,7 @@ func park(ctx context.Context, tc *TaskContext, reason string) { tc.Log.Error("task park transition failed", "err", err) return } - tc.Emit(events.KindParked, tc.Stage, reason, nil) + tc.Emit(ctx, events.KindParked, tc.Stage, reason, nil) // The run's own log is the surface an operator downloads to answer "why did // this park?", and the event above lands on the timeline, which the log // never carries. This is the one place a park's reason is final, so it is @@ -638,7 +638,7 @@ func (tc *TaskContext) startChildStep(ctx context.Context, kind, name string) (i if err != nil { return 0, events.Event{}, err } - publishEvent(tc, event) + publishEvent(ctx, tc, event) return stepID, event, nil } @@ -655,7 +655,7 @@ func (tc *TaskContext) finishChildStep(ctx context.Context, stepID int64, to tas if err != nil { return err } - publishEvent(tc, event) + publishEvent(ctx, tc, event) return nil } diff --git a/internal/domain/workflow/workflow_call.go b/internal/domain/workflow/workflow_call.go index 80ac11615..b398c0620 100644 --- a/internal/domain/workflow/workflow_call.go +++ b/internal/domain/workflow/workflow_call.go @@ -214,7 +214,7 @@ func (tc *TaskContext) startCallStep(ctx context.Context, path string, calleeTas if err != nil { return 0, events.Event{}, err } - publishEvent(tc, event) + publishEvent(ctx, tc, event) return stepID, event, nil } diff --git a/internal/eventbus/eventbus.go b/internal/eventbus/eventbus.go index 8e0f5c749..50d9b9601 100644 --- a/internal/eventbus/eventbus.go +++ b/internal/eventbus/eventbus.go @@ -33,6 +33,9 @@ type Message interface { // Subject returns the address the message arrived on. Subject() string + // Header returns the message headers, or nil if it has none. + Header() map[string][]string + // Ack marks the message handled so it is not redelivered. Ack() error diff --git a/internal/events/events.go b/internal/events/events.go index 5e3979aad..b0f1bff16 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -5,6 +5,7 @@ package events import ( + "context" "slices" "sync" "sync/atomic" @@ -225,7 +226,7 @@ func (b *Bus) Subscribe(buffer int) *Sub { // Publish stamps and delivers the event to every subscriber. Full // subscriber buffers drop the event (counted) rather than blocking. -func (b *Bus) Publish(e Event) { +func (b *Bus) Publish(_ context.Context, e Event) { if e.At.IsZero() { e.At = time.Now().UTC() } diff --git a/internal/forgerpc/forgerpc.go b/internal/forgerpc/forgerpc.go index 64442a5f2..8856bd51c 100644 --- a/internal/forgerpc/forgerpc.go +++ b/internal/forgerpc/forgerpc.go @@ -157,13 +157,13 @@ type Server struct { // RegisterFor registers all handlers under the subjects scoped to identity. func (s *Server) RegisterFor(nc *nats.Conn, identity string) (unsubscribe func(), err error) { - as := func(h func(string, *nats.Msg)) nats.MsgHandler { - return func(msg *nats.Msg) { + as := func(h func(context.Context, string, *nats.Msg)) func(context.Context, *nats.Msg) { + return func(ctx context.Context, msg *nats.Msg) { scoped := identity if identity == "*" { scoped = strings.Split(msg.Subject, ".")[2] } - h(scoped, msg) + h(ctx, scoped, msg) } } return natsrpc.RegisterAll(nc, []natsrpc.Registration{ @@ -181,11 +181,11 @@ func (s *Server) RegisterFor(nc *nats.Conn, identity string) (unsubscribe func() // server's identity and that the run's repository is the one targeted. A run // of a package's workflow also needs the forge permission accepted for that // package. -func (s *Server) authorize(identity string, target Target, permission string) error { +func (s *Server) authorize(ctx context.Context, identity string, target Target, permission string) error { if s.Runs == nil || target.Credential == "" { return errors.New("forge request needs a run credential") } - task, err := s.Runs.TaskForCredential(context.Background(), target.Credential) + task, err := s.Runs.TaskForCredential(ctx, target.Credential) if err != nil { return fmt.Errorf("forge request credential is invalid: %w", err) } @@ -203,7 +203,7 @@ func (s *Server) authorize(identity string, target Target, permission string) er if s.Authorities == nil { return fmt.Errorf("package %q: forge permission %q is not accepted", name, permission) } - accepted, err := s.Authorities.AcceptedAuthority(context.Background(), name) + accepted, err := s.Authorities.AcceptedAuthority(ctx, name) if err != nil { return fmt.Errorf("package %q authority: %w", name, err) } @@ -220,87 +220,87 @@ func serverSubject(identity, base string) string { return SubjectFor(identity, base) } -func (s *Server) authorizedForge(identity string, target Target, permission string) (forge.Forge, error) { - if err := s.authorize(identity, target, permission); err != nil { +func (s *Server) authorizedForge(ctx context.Context, identity string, target Target, permission string) (forge.Forge, error) { + if err := s.authorize(ctx, identity, target, permission); err != nil { return nil, err } if s.Resolve == nil { return nil, errors.New("forge connection unavailable") } - task, err := s.Runs.TaskForCredential(context.Background(), target.Credential) + task, err := s.Runs.TaskForCredential(ctx, target.Credential) if err != nil { return nil, errors.New("forge request credential is invalid") } - return s.Resolve(context.Background(), task) + return s.Resolve(ctx, task) } -func (s *Server) handleComment(identity string, msg *nats.Msg) { +func (s *Server) handleComment(ctx context.Context, identity string, msg *nats.Msg) { var req CommentRequest if err := json.Unmarshal(msg.Data, &req); err != nil { s.respond(msg, CommentResponse{Envelope: natsrpc.NewEnvelope(fmt.Errorf("decode comment request: %w", err))}) return } - client, err := s.authorizedForge(identity, req.Target, "comment") + client, err := s.authorizedForge(ctx, identity, req.Target, "comment") if err != nil { s.respond(msg, CommentResponse{Envelope: natsrpc.NewEnvelope(err)}) return } - id, err := client.Comment(context.Background(), req.Owner, req.Repo, req.Number, req.Body) + id, err := client.Comment(ctx, req.Owner, req.Repo, req.Number, req.Body) s.respond(msg, CommentResponse{ID: id, Envelope: natsrpc.NewEnvelope(err)}) } -func (s *Server) handleCloseIssue(identity string, msg *nats.Msg) { +func (s *Server) handleCloseIssue(ctx context.Context, identity string, msg *nats.Msg) { var req CloseIssueRequest if err := json.Unmarshal(msg.Data, &req); err != nil { s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(fmt.Errorf("decode close_issue request: %w", err))}) return } - client, err := s.authorizedForge(identity, req.Target, "comment") + client, err := s.authorizedForge(ctx, identity, req.Target, "comment") if err != nil { s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(err)}) return } - err = client.CloseIssue(context.Background(), req.Owner, req.Repo, req.Number, req.Comment) + err = client.CloseIssue(ctx, req.Owner, req.Repo, req.Number, req.Comment) s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(err)}) } -func (s *Server) handleCreatePR(identity string, msg *nats.Msg) { +func (s *Server) handleCreatePR(ctx context.Context, identity string, msg *nats.Msg) { var req CreatePRRequest if err := json.Unmarshal(msg.Data, &req); err != nil { s.respond(msg, CreatePRResponse{Envelope: natsrpc.NewEnvelope(fmt.Errorf("decode create_pr request: %w", err))}) return } - client, err := s.authorizedForge(identity, req.Target, "open_pr") + client, err := s.authorizedForge(ctx, identity, req.Target, "open_pr") if err != nil { s.respond(msg, CreatePRResponse{Envelope: natsrpc.NewEnvelope(err)}) return } - num, err := client.CreatePR(context.Background(), req.Owner, req.Repo, req.Title, req.Head, req.Base, req.Body) + num, err := client.CreatePR(ctx, req.Owner, req.Repo, req.Title, req.Head, req.Base, req.Body) s.respond(msg, CreatePRResponse{Number: num, Envelope: natsrpc.NewEnvelope(err)}) } -func (s *Server) handleLinkBranch(identity string, msg *nats.Msg) { +func (s *Server) handleLinkBranch(ctx context.Context, identity string, msg *nats.Msg) { var req LinkBranchRequest if err := json.Unmarshal(msg.Data, &req); err != nil { s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(fmt.Errorf("decode link_branch request: %w", err))}) return } - client, err := s.authorizedForge(identity, req.Target, "open_pr") + client, err := s.authorizedForge(ctx, identity, req.Target, "open_pr") if err != nil { s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(err)}) return } - err = client.LinkBranch(context.Background(), req.Owner, req.Repo, req.IssueNumber, req.Branch) + err = client.LinkBranch(ctx, req.Owner, req.Repo, req.IssueNumber, req.Branch) s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(err)}) } -func (s *Server) handleCreateReviewComments(identity string, msg *nats.Msg) { +func (s *Server) handleCreateReviewComments(ctx context.Context, identity string, msg *nats.Msg) { var req CreateReviewCommentsRequest if err := json.Unmarshal(msg.Data, &req); err != nil { s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(fmt.Errorf("decode create_review_comments request: %w", err))}) return } - client, err := s.authorizedForge(identity, req.Target, "review") + client, err := s.authorizedForge(ctx, identity, req.Target, "review") if err != nil { s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(err)}) return @@ -315,17 +315,17 @@ func (s *Server) handleCreateReviewComments(identity string, msg *nats.Msg) { comments = append(comments, forge.InlineReviewComment{Path: cm.Path, Line: cm.Line, Body: cm.Body}) } review := forge.ReviewSubmission{Event: req.Event, Body: req.Body, Comments: comments} - err = writer.CreateReviewComments(context.Background(), req.Owner, req.Repo, req.Number, req.ReviewedHeadSHA, review) + err = writer.CreateReviewComments(ctx, req.Owner, req.Repo, req.Number, req.ReviewedHeadSHA, review) s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(err)}) } -func (s *Server) handleReplyToReview(identity string, msg *nats.Msg) { +func (s *Server) handleReplyToReview(ctx context.Context, identity string, msg *nats.Msg) { var req ReplyToReviewRequest if err := json.Unmarshal(msg.Data, &req); err != nil { s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(fmt.Errorf("decode reply_to_review request: %w", err))}) return } - client, err := s.authorizedForge(identity, req.Target, "review") + client, err := s.authorizedForge(ctx, identity, req.Target, "review") if err != nil { s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(err)}) return @@ -335,22 +335,22 @@ func (s *Server) handleReplyToReview(identity string, msg *nats.Msg) { s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(errors.New("this forge cannot reply to reviews"))}) return } - err = reader.ReplyToReview(context.Background(), req.Owner, req.Repo, req.Number, req.CommentID, req.Body) + err = reader.ReplyToReview(ctx, req.Owner, req.Repo, req.Number, req.CommentID, req.Body) s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(err)}) } -func (s *Server) handleSetStateLabel(identity string, msg *nats.Msg) { +func (s *Server) handleSetStateLabel(ctx context.Context, identity string, msg *nats.Msg) { var req SetStateLabelRequest if err := json.Unmarshal(msg.Data, &req); err != nil { s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(fmt.Errorf("decode set_state_label request: %w", err))}) return } - client, err := s.authorizedForge(identity, req.Target, "comment") + client, err := s.authorizedForge(ctx, identity, req.Target, "comment") if err != nil { s.respond(msg, Response{Envelope: natsrpc.NewEnvelope(err)}) return } - client.SetStateLabel(context.Background(), req.Owner, req.Repo, req.Number, req.Label, req.KnownLabels) + client.SetStateLabel(ctx, req.Owner, req.Repo, req.Number, req.Label, req.KnownLabels) s.respond(msg, Response{}) } diff --git a/internal/forgerpc/forgerpc_test.go b/internal/forgerpc/forgerpc_test.go index 28250418c..8059459c0 100644 --- a/internal/forgerpc/forgerpc_test.go +++ b/internal/forgerpc/forgerpc_test.go @@ -61,7 +61,7 @@ func TestForgeCallStaysInItsRunsAuthority(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := s.authorize(tt.identity, tt.target, tt.permission); (err == nil) != tt.allowed { + if err := s.authorize(context.Background(), tt.identity, tt.target, tt.permission); (err == nil) != tt.allowed { t.Fatalf("err %v, want allowed %v", err, tt.allowed) } }) diff --git a/internal/gateway/turn.go b/internal/gateway/turn.go index 946862957..b994c7d5a 100644 --- a/internal/gateway/turn.go +++ b/internal/gateway/turn.go @@ -65,7 +65,7 @@ type SoulSource interface { // TurnEventPublisher receives completed primary chat-turn events. type TurnEventPublisher interface { - Publish(events.Event) + Publish(context.Context, events.Event) } // TurnReplayStore provides a durable source-ID lookup for redeliveries that @@ -539,7 +539,7 @@ func (r *TurnRunner) generateAndComplete( return "", r.failTurn(ctx, turn, fmt.Errorf("save completed chat turn: %w", err)) } if r.Bus != nil { - r.Bus.Publish(events.Event{ + r.Bus.Publish(ctx, events.Event{ Kind: events.KindTurnCompleted, Detail: sessionID, Data: map[string]any{"session": sessionID, "channel": r.Channel}, diff --git a/internal/infrastructure/agenttransport/nats/transport.go b/internal/infrastructure/agenttransport/nats/transport.go index 711833364..7f1fd0805 100644 --- a/internal/infrastructure/agenttransport/nats/transport.go +++ b/internal/infrastructure/agenttransport/nats/transport.go @@ -12,6 +12,8 @@ import ( "time" natsio "github.com/nats-io/nats.go" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" "github.com/samcharles93/archie-core/internal/agentexec" "github.com/samcharles93/archie-core/internal/domain/storecontract" @@ -21,6 +23,7 @@ import ( "github.com/samcharles93/archie-core/internal/forgerpc" eventbus "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" + "github.com/samcharles93/archie-core/internal/natsrpc" "github.com/samcharles93/archie-core/internal/taskrun" "github.com/samcharles93/archie-core/internal/worktreerpc" ) @@ -312,6 +315,9 @@ func (t *Transport) handleTask( t.respondTask(msg, nil, fmt.Errorf("validate taskrun request: %w", err), log) return } + ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(request.Trace)) + ctx, span := natsrpc.StartServer(ctx, msg) + defer span.End() response, err := handler(ctx, request) t.respondTask(msg, response, err, log) } @@ -345,7 +351,7 @@ func (*Transport) respondTask(msg *natsio.Msg, response *taskrun.Response, runEr log.Error("taskrun encode response failed", "err", err) return } - if err := msg.Respond(data); err != nil { + if err := msg.RespondMsg(&natsio.Msg{Data: data}); err != nil { log.Error("taskrun respond failed", "err", err) } } diff --git a/internal/infrastructure/cronevents/sink.go b/internal/infrastructure/cronevents/sink.go index 7fc4a05c5..f2580ce68 100644 --- a/internal/infrastructure/cronevents/sink.go +++ b/internal/infrastructure/cronevents/sink.go @@ -2,6 +2,8 @@ package cronevents import ( + "context" + "github.com/samcharles93/archie-core/internal/domain/scheduling" "github.com/samcharles93/archie-core/internal/events" ) @@ -19,7 +21,7 @@ type busSink struct { } func (s busSink) Emit(kind, detail string, data map[string]any) { - s.bus.Publish(events.Event{Kind: kind, Detail: detail, Data: data}) + s.bus.Publish(context.Background(), events.Event{Kind: kind, Detail: detail, Data: data}) } var _ scheduling.Sink = busSink{} diff --git a/internal/infrastructure/eventbus/nats/consumer.go b/internal/infrastructure/eventbus/nats/consumer.go index 929e45c46..e451ac7ea 100644 --- a/internal/infrastructure/eventbus/nats/consumer.go +++ b/internal/infrastructure/eventbus/nats/consumer.go @@ -7,6 +7,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "github.com/samcharles93/archie-core/internal/eventbus" + "github.com/samcharles93/archie-core/internal/natsrpc" ) // Fetch returns the next message, or eventbus.ErrNoMessage when the poll @@ -59,7 +60,10 @@ func (c *Client) Subscribe(ctx context.Context, filterSubject string, handle fun } consumeCtx, err := consumer.Consume(func(msg jetstream.Msg) { + _, span := natsrpc.StartConsumer(ctx, msg.Subject(), msg.Headers()) + defer span.End() if err := handle(message{msg: msg}); err != nil { + span.RecordError(err) c.log.Error("message handler failed", "subject", msg.Subject(), "error", err) _ = msg.Nak() diff --git a/internal/infrastructure/eventbus/nats/message.go b/internal/infrastructure/eventbus/nats/message.go index d865ed7aa..90ffe0f4c 100644 --- a/internal/infrastructure/eventbus/nats/message.go +++ b/internal/infrastructure/eventbus/nats/message.go @@ -25,6 +25,9 @@ func (m message) Data() []byte { return m.msg.Data() } // Subject returns the subject the message arrived on. func (m message) Subject() string { return m.msg.Subject() } +// Header returns the message headers, which carry the producer's trace context. +func (m message) Header() map[string][]string { return m.msg.Headers() } + // Ack marks the message handled so JetStream will not redeliver it. func (m message) Ack() error { if err := m.msg.Ack(); err != nil { diff --git a/internal/infrastructure/eventbus/nats/publisher.go b/internal/infrastructure/eventbus/nats/publisher.go index 5d5989989..0d193453e 100644 --- a/internal/infrastructure/eventbus/nats/publisher.go +++ b/internal/infrastructure/eventbus/nats/publisher.go @@ -8,19 +8,20 @@ import ( "github.com/nats-io/nats.go" "github.com/samcharles93/archie-core/internal/eventbus" + "github.com/samcharles93/archie-core/internal/natsrpc" ) // Publish sends a payload to a subject with no reply routing. func (c *Client) Publish(ctx context.Context, subject string, payload []byte) error { - return c.publish(ctx, &nats.Msg{Subject: subject, Data: payload}) + return c.publish(ctx, natsrpc.NewMsg(ctx, subject, payload)) } // PublishUnique sends a payload carrying an idempotency key, so republishing // the same key inside Config.DedupWindow is suppressed by the server. func (c *Client) PublishUnique(ctx context.Context, subject, idempotencyKey string, payload []byte) error { - header := nats.Header{} - header.Set(idempotencyHeader, idempotencyKey) - return c.publish(ctx, &nats.Msg{Subject: subject, Data: payload, Header: header}) + msg := natsrpc.NewMsg(ctx, subject, payload) + msg.Header.Set(idempotencyHeader, idempotencyKey) + return c.publish(ctx, msg) } // publish is the single JetStream publish path, so every caller gets the same @@ -42,8 +43,11 @@ func (c *Client) Request(ctx context.Context, subject string, payload []byte) ([ if err != nil { return nil, fmt.Errorf("request %s: %w", subject, err) } - msg, err := conn.RequestWithContext(ctx, subject, payload) + ctx, span := natsrpc.StartClient(ctx, subject) + defer span.End() + msg, err := conn.RequestMsgWithContext(ctx, natsrpc.NewMsg(ctx, subject, payload)) if err != nil { + span.RecordError(err) // Translate the SDK's no-responder error into the contract's, so a // caller can retry a not-yet-subscribed worker without importing // the NATS SDK to name the condition. diff --git a/internal/infrastructure/logrpc/nats.go b/internal/infrastructure/logrpc/nats.go index 5cf539727..6a3ca048f 100644 --- a/internal/infrastructure/logrpc/nats.go +++ b/internal/infrastructure/logrpc/nats.go @@ -15,7 +15,7 @@ import ( const daemonSubject = "archie.logs.archied" func RegisterDaemon(nc *nats.Conn, feed *logging.Feed, log *slog.Logger) (func(), error) { - return natsrpc.RegisterAll(nc, []natsrpc.Registration{{Subject: daemonSubject, Handler: func(msg *nats.Msg) { + return natsrpc.RegisterAll(nc, []natsrpc.Registration{{Subject: daemonSubject, Handler: func(_ context.Context, msg *nats.Msg) { var q logging.Query if err := json.Unmarshal(msg.Data, &q); err != nil { natsrpc.Respond(msg, log, "logs", reply{Error: "invalid log query"}) @@ -40,8 +40,11 @@ func ReadDaemon(ctx context.Context, nc *nats.Conn, q logging.Query) (logging.Re if err != nil { return logging.Result{}, err } - msg, err := nc.RequestWithContext(ctx, daemonSubject, raw) + ctx, span := natsrpc.StartClient(ctx, daemonSubject) + defer span.End() + msg, err := nc.RequestMsgWithContext(ctx, natsrpc.NewMsg(ctx, daemonSubject, raw)) if err != nil { + span.RecordError(err) return logging.Result{}, err } var out reply diff --git a/internal/infrastructure/staterpc/task_grants.go b/internal/infrastructure/staterpc/task_grants.go index c7ebf956c..b3164bce0 100644 --- a/internal/infrastructure/staterpc/task_grants.go +++ b/internal/infrastructure/staterpc/task_grants.go @@ -315,8 +315,8 @@ type GrantIssuer struct { Client *Client } -func (g *GrantIssuer) Issue(task *task.Task, limit time.Duration) (string, func(), error) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) +func (g *GrantIssuer) Issue(parent context.Context, task *task.Task, limit time.Duration) (string, func(), error) { + ctx, cancel := context.WithTimeout(parent, 10*time.Second) defer cancel() lifetime := min(limit+grantMargin, maxGrantLifetime) if limit <= 0 { @@ -327,7 +327,7 @@ func (g *GrantIssuer) Issue(task *task.Task, limit time.Duration) (string, func( return "", nil, err } return token, func() { - revokeCtx, revokeCancel := context.WithTimeout(context.Background(), 10*time.Second) + revokeCtx, revokeCancel := context.WithTimeout(context.WithoutCancel(parent), 10*time.Second) defer revokeCancel() _ = g.Client.RevokeTaskGrant(revokeCtx, token) }, nil diff --git a/internal/infrastructure/taskactions/rpc.go b/internal/infrastructure/taskactions/rpc.go index e32d6ca0c..fb67f545a 100644 --- a/internal/infrastructure/taskactions/rpc.go +++ b/internal/infrastructure/taskactions/rpc.go @@ -108,18 +108,17 @@ func actionErrorFor(kind string, err error) error { return err } -// Register serves identity-scoped task actions over NATS. Handlers use -// context.Background(). +// Register serves identity-scoped task actions over NATS. func Register(nc *nats.Conn, service taskactions.Service, log *slog.Logger) (func(), error) { return natsrpc.RegisterAll(nc, []natsrpc.Registration{{ Subject: actionSubject, - Handler: func(msg *nats.Msg) { + Handler: func(ctx context.Context, msg *nats.Msg) { var req actionRequest if err := json.Unmarshal(msg.Data, &req); err != nil { natsrpc.Respond(msg, log, "taskactions", actionResponse{Envelope: natsrpc.NewEnvelope(err)}) return } - err := service.Apply(context.Background(), req.Identity, req.Actor.actor(), req.TaskID, req.Action, taskactions.ActionPayload{ + err := service.Apply(ctx, req.Identity, req.Actor.actor(), req.TaskID, req.Action, taskactions.ActionPayload{ Instructions: req.Instructions, Findings: req.Findings, RetryMode: req.RetryMode, diff --git a/internal/infrastructure/taskactions/service.go b/internal/infrastructure/taskactions/service.go index ca5ca2a4e..b5c6f081e 100644 --- a/internal/infrastructure/taskactions/service.go +++ b/internal/infrastructure/taskactions/service.go @@ -20,7 +20,7 @@ func NewService( mergePR func(context.Context, string, string, int) error, removeLogs func(int64) error, removeWorktree func(*taskactions.Task) error, - publish func(events.Event), + publish func(context.Context, events.Event), warn func(string, ...any), ) taskactions.Service { return taskactions.Service{ diff --git a/internal/natsrpc/natsrpc.go b/internal/natsrpc/natsrpc.go index 69cd28063..baaa671ae 100644 --- a/internal/natsrpc/natsrpc.go +++ b/internal/natsrpc/natsrpc.go @@ -11,6 +11,7 @@ import ( "time" "github.com/nats-io/nats.go" + "go.opentelemetry.io/otel/codes" ) // Envelope is the error half of an RPC response. Embed it in a @@ -51,8 +52,12 @@ func (c *Client) Request(ctx context.Context, subject string, data []byte) ([]by ctx, cancel = context.WithTimeout(ctx, c.Timeout) defer cancel() } - reply, err := c.Conn.RequestWithContext(ctx, subject, data) + ctx, span := StartClient(ctx, subject) + defer span.End() + reply, err := c.Conn.RequestMsgWithContext(ctx, NewMsg(ctx, subject, data)) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) return nil, fmt.Errorf("%s: %w", subject, err) } return reply.Data, nil @@ -82,7 +87,9 @@ func Call[Resp any](ctx context.Context, c *Client, subject string, req any) (Re // Registration pairs a subject with its handler for RegisterAll. type Registration struct { Subject string - Handler nats.MsgHandler + // Handler receives a context carrying the caller's trace and a SERVER + // span that ends when it returns. + Handler func(ctx context.Context, msg *nats.Msg) } // RegisterAll subscribes every registration and flushes, unsubscribing all @@ -96,7 +103,7 @@ func RegisterAll(nc *nats.Conn, regs []Registration) (unsubscribe func(), err er } for _, r := range regs { - sub, subErr := nc.Subscribe(r.Subject, r.Handler) + sub, subErr := nc.Subscribe(r.Subject, serve(r.Handler)) if subErr != nil { unsubscribeAll() return nil, fmt.Errorf("subscribe %s: %w", r.Subject, subErr) @@ -112,6 +119,14 @@ func RegisterAll(nc *nats.Conn, regs []Registration) (unsubscribe func(), err er return unsubscribeAll, nil } +func serve(h func(context.Context, *nats.Msg)) nats.MsgHandler { + return func(msg *nats.Msg) { + ctx, span := StartServer(context.Background(), msg) + defer span.End() + h(ctx, msg) + } +} + // Respond marshals v and replies to msg, logging any failure. func Respond(msg *nats.Msg, log *slog.Logger, pkg string, v any) { data, err := json.Marshal(v) @@ -121,7 +136,7 @@ func Respond(msg *nats.Msg, log *slog.Logger, pkg string, v any) { } return } - if err := msg.Respond(data); err != nil && log != nil { + if err := msg.RespondMsg(&nats.Msg{Data: data}); err != nil && log != nil { log.Error(pkg+": respond failed", "err", err) } } diff --git a/internal/natsrpc/trace.go b/internal/natsrpc/trace.go new file mode 100644 index 000000000..c943a4447 --- /dev/null +++ b/internal/natsrpc/trace.go @@ -0,0 +1,73 @@ +package natsrpc + +import ( + "context" + "maps" + "slices" + + "github.com/nats-io/nats.go" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +const tracerName = "github.com/samcharles93/archie-core/internal/natsrpc" + +// carrier adapts nats.Header to a propagation.TextMapCarrier. nats.Header is +// case-sensitive and propagation.HeaderCarrier canonicalises keys, which would +// miss lowercase senders. +type carrier nats.Header + +func (c carrier) Get(k string) string { return nats.Header(c).Get(k) } +func (c carrier) Set(k, v string) { nats.Header(c).Set(k, v) } +func (c carrier) Keys() []string { return slices.Collect(maps.Keys(c)) } + +// NewMsg builds a message for subject carrying ctx's trace context. +func NewMsg(ctx context.Context, subject string, data []byte) *nats.Msg { + msg := nats.NewMsg(subject) + msg.Data = data + otel.GetTextMapPropagator().Inject(ctx, carrier(msg.Header)) + return msg +} + +// Extract returns ctx carrying the trace context found in header. +func Extract(ctx context.Context, header map[string][]string) context.Context { + if header == nil { + return ctx + } + return otel.GetTextMapPropagator().Extract(ctx, carrier(header)) +} + +func messagingAttrs(subject, op string) trace.SpanStartEventOption { + return trace.WithAttributes( + attribute.String("messaging.system", "nats"), + attribute.String("messaging.destination.name", subject), + attribute.String("messaging.operation.type", op), + ) +} + +// StartClient starts the CLIENT span of a request/reply call. +func StartClient(ctx context.Context, subject string) (context.Context, trace.Span) { + return otel.Tracer(tracerName).Start(ctx, subject+" request", + trace.WithSpanKind(trace.SpanKindClient), messagingAttrs(subject, "send")) +} + +// StartServer starts the SERVER span for a received request, parented to the +// caller's span. +func StartServer(ctx context.Context, msg *nats.Msg) (context.Context, trace.Span) { + ctx = Extract(ctx, msg.Header) + return otel.Tracer(tracerName).Start(ctx, msg.Subject+" process", + trace.WithSpanKind(trace.SpanKindServer), messagingAttrs(msg.Subject, "process")) +} + +// StartConsumer starts a CONSUMER span for a delivered stream message. The +// producer is linked rather than made the parent because its span has usually +// ended by delivery time. +func StartConsumer(ctx context.Context, subject string, header map[string][]string) (context.Context, trace.Span) { + producer := trace.SpanContextFromContext(Extract(context.Background(), header)) + opts := []trace.SpanStartOption{trace.WithSpanKind(trace.SpanKindConsumer), messagingAttrs(subject, "process")} + if producer.IsValid() { + opts = append(opts, trace.WithLinks(trace.Link{SpanContext: producer})) + } + return otel.Tracer(tracerName).Start(ctx, subject+" process", opts...) +} diff --git a/internal/taskrun/taskrun.go b/internal/taskrun/taskrun.go index 1fee900c7..c48519fb3 100644 --- a/internal/taskrun/taskrun.go +++ b/internal/taskrun/taskrun.go @@ -36,6 +36,9 @@ type Request struct { // Instructions are the instance and organisation layers every agent // stage follows. Instructions workflow.Instructions `json:"instructions"` + // Trace is the daemon's W3C trace context, so the agent's spans join the + // task's trace. + Trace map[string]string `json:"trace,omitempty"` } // Validate rejects a full-task request that cannot be correlated to a real diff --git a/internal/webui/api_config.go b/internal/webui/api_config.go index 495a83eae..0266e9096 100644 --- a/internal/webui/api_config.go +++ b/internal/webui/api_config.go @@ -35,8 +35,8 @@ type ChannelView struct { // the fields this derives from. func (s *Server) handleChannels(w http.ResponseWriter, r *http.Request) { if s.Channels != nil { - views := make([]ChannelView, 0, len(s.Channels.Snapshot())) - for _, status := range s.Channels.Snapshot() { + views := make([]ChannelView, 0, len(s.Channels.Snapshot(r.Context()))) + for _, status := range s.Channels.Snapshot(r.Context()) { views = append(views, channelViewFromStatus(status)) } writeJSON(w, map[string]any{"channels": views}) @@ -55,7 +55,7 @@ func (s *Server) handleChannelReload(w http.ResponseWriter, r *http.Request) { return } var status *status.Status - for _, candidate := range s.Channels.Snapshot() { + for _, candidate := range s.Channels.Snapshot(r.Context()) { if candidate.ID == id { status = &candidate break @@ -244,7 +244,7 @@ func chatChannelConfigured(chat config.ChatConfig) bool { // told so rather than offering a control that would 503. // ChannelStatusSource reports the live state of each configured channel. type ChannelStatusSource interface { - Snapshot() []status.Status + Snapshot(ctx context.Context) []status.Status } // RemoteChannelStatus reads channel state from the State Store; unreachable @@ -257,11 +257,11 @@ type remoteChannelStatus struct { channels storecontract.ChannelStatusStore } -func (r remoteChannelStatus) Snapshot() []status.Status { +func (r remoteChannelStatus) Snapshot(ctx context.Context) []status.Status { // The handler has no error to return, so a failed read is an empty report. // That is the honest answer: the dashboard must not show a stale "running" // for a channel it cannot currently ask about. - rows, err := r.channels.ChannelStatus(context.Background()) + rows, err := r.channels.ChannelStatus(ctx) if err != nil { return nil } diff --git a/internal/worktreerpc/worktreerpc.go b/internal/worktreerpc/worktreerpc.go index ed30dd246..89c561e6e 100644 --- a/internal/worktreerpc/worktreerpc.go +++ b/internal/worktreerpc/worktreerpc.go @@ -83,12 +83,12 @@ type Server struct { } // handlerContext returns a context bounded by s.Timeout. -func (s *Server) handlerContext() (context.Context, context.CancelFunc) { +func (s *Server) handlerContext(parent context.Context) (context.Context, context.CancelFunc) { timeout := s.Timeout if timeout <= 0 { timeout = defaultHandlerTimeout } - return context.WithTimeout(context.Background(), timeout) + return context.WithTimeout(parent, timeout) } // RegisterFor registers publication under the subject scoped to identity. @@ -98,23 +98,23 @@ func (s *Server) RegisterFor(nc *nats.Conn, identity string) (unsubscribe func() subject = "archie.worktree.*.push" } return natsrpc.RegisterAll(nc, []natsrpc.Registration{ - {Subject: subject, Handler: func(msg *nats.Msg) { + {Subject: subject, Handler: func(ctx context.Context, msg *nats.Msg) { scoped := identity if identity == "*" { scoped = strings.Split(msg.Subject, ".")[2] } - s.handlePush(scoped, msg) + s.handlePush(ctx, scoped, msg) }}, }) } -func (s *Server) handlePush(identity string, msg *nats.Msg) { +func (s *Server) handlePush(parent context.Context, identity string, msg *nats.Msg) { var req PushRequest if err := json.Unmarshal(msg.Data, &req); err != nil { s.respond(msg, err) return } - ctx, cancel := s.handlerContext() + ctx, cancel := s.handlerContext(parent) defer cancel() task, err := publishable(ctx, s.Runs, req.Grant, identity) if err != nil {