diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 58d4cd09c..8c810b3d5 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -32,8 +32,6 @@ import ( "github.com/samcharles93/archie-core/internal/domain/alert" "github.com/samcharles93/archie-core/internal/domain/applystatus" "github.com/samcharles93/archie-core/internal/domain/health" - "github.com/samcharles93/archie-core/internal/domain/identity" - "github.com/samcharles93/archie-core/internal/domain/org" "github.com/samcharles93/archie-core/internal/domain/presence" "github.com/samcharles93/archie-core/internal/domain/scheduling" "github.com/samcharles93/archie-core/internal/domain/storecontract" @@ -94,7 +92,10 @@ type boot struct { // dialed to the standalone archie-state-store gRPC service at // [services.state].target. It is set by openStateStoreAdapter, which // requires [services.state].target to be set. - stateStore storecontract.TaskStore + stateStore storecontract.TaskStore + // stateClient is stateStore's concrete type, which the daemon's typed store + // fields are assigned from directly. + stateClient *staterpc.Client taskModels daemon.TaskModels taskInstructions daemon.TaskInstructions // accessChain is the daemon's policy engine, built once from the stored @@ -288,6 +289,7 @@ func (b *boot) openStateStoreAdapter(ctx context.Context) error { return err } b.stateStore = client + b.stateClient = client if alerts, ok := b.stateStore.(alert.Store); ok { b.credentialWatch = &credentialWatch{reporter: alert.Reporter{Raiser: alerts, Log: b.log, By: "archied"}} } @@ -596,6 +598,16 @@ func (b *boot) buildDaemon(ctx context.Context) { ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken, Scoped: b.cfg.NATS.Mode != config.NATSModeExternal}, ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), CA: b.stateStoreCA(ctx)}, Store: b.stateStore, + Mappings: b.stateClient, + Bindings: b.stateClient, + BindingDispatcher: b.stateClient, + MappingMatches: b.stateClient, + BindingTaskCreator: b.stateClient, + IdentityRepository: b.stateClient, + AgentAssignments: b.stateClient, + Principals: b.stateClient, + Denials: b.stateClient, + PlaybookLedger: b.stateClient, Bus: b.bus, Forge: b.forgeClient, Trees: b.trees, @@ -618,50 +630,8 @@ func (b *boot) buildDaemon(ctx context.Context) { Playbooks: b.livePlaybooks(), } - if identities, ok := b.stateStore.(identity.Repository); ok { - b.d.IdentityRepository = identities - } - if assignments, ok := b.stateStore.(org.AgentAssignments); ok { - b.d.AgentAssignments = assignments - } - - if ms, ok := b.stateStore.(storecontract.MappingStore); ok { - b.d.Mappings = ms - } - - if bs, ok := b.stateStore.(storecontract.BindingStore); ok { - b.d.Bindings = bs - } - - if bd, ok := b.stateStore.(storecontract.BindingDispatcher); ok { - b.d.BindingDispatcher = bd - } - - if mm, ok := b.stateStore.(storecontract.MappingMatchRecorder); ok { - b.d.MappingMatches = mm - } - - if btc, ok := b.stateStore.(storecontract.BindingTaskCreator); ok { - b.d.BindingTaskCreator = btc - } b.d.WithdrawTask = b.taskActions().Withdraw - b.d.Access = b.accessChain - if principals, ok := b.stateStore.(access.PrincipalSource); ok { - b.d.Principals = principals - } - if denials, ok := b.stateStore.(access.DenialRecorder); ok { - b.d.Denials = denials - } - b.d.PlaybookLedger = playbookLedger(ctx, b.stateStore, b.log) -} - -func playbookLedger(ctx context.Context, source any, log *slog.Logger) storecontract.PlaybookDispatcher { - if pd, ok := source.(storecontract.PlaybookDispatcher); ok { - return pd - } - log.WarnContext(ctx, "eda action playbooks will not run: no dispatch ledger is wired") - return nil } func (b *boot) publishConfig(ctx context.Context, cfg config.Config) { diff --git a/internal/app/archied/repository_check.go b/internal/app/archied/repository_check.go index 0419710cd..719e6c16a 100644 --- a/internal/app/archied/repository_check.go +++ b/internal/app/archied/repository_check.go @@ -34,7 +34,7 @@ func (s *repositoryServer) CheckRepository(ctx context.Context, r *pb.CheckRepos if !ok || caller.IdentityID == "" { return nil, status.Error(codes.Unauthenticated, "Sign in before checking repository access.") } - if b.d == nil || b.d.Access == nil || b.d.Principals == nil { + if b.d == nil { return nil, status.Error(codes.Unavailable, "Repository checks are temporarily unavailable.") } decision := b.d.Access.Authorize(ctx, caller, access.ActionUpdate, access.Resource{Kind: access.KindPolicy, ID: "repository-policies", Org: caller.Org}, access.Context{}) diff --git a/internal/daemon/agent_status_test.go b/internal/daemon/agent_status_test.go index 2db1cb82a..8a519778d 100644 --- a/internal/daemon/agent_status_test.go +++ b/internal/daemon/agent_status_test.go @@ -64,12 +64,12 @@ func TestRunViaAgentRequiresTerminalStatus(t *testing.T) { t.Fatalf("marshal response: %v", err) } store := &statusStore{} - d := &Daemon{ + d := wired(&Daemon{ Cfg: config.NewHolder(config.Config{}), Store: store, Tasks: statusBus{reply: reply}, Log: slog.New(slog.DiscardHandler), - } + }) d.runViaAgent(t.Context(), &workflow.Task{ID: 7, Attempt: 1, Owner: "owner", Repo: "repo", Workflow: "implement"}, config.Repo{}, config.AgentProfile{}, "") diff --git a/internal/daemon/binding_dispatch.go b/internal/daemon/binding_dispatch.go new file mode 100644 index 000000000..1734f0714 --- /dev/null +++ b/internal/daemon/binding_dispatch.go @@ -0,0 +1,358 @@ +package daemon + +import ( + "context" + "errors" + "fmt" + "maps" + "strings" + + "github.com/samcharles93/archie-core/internal/domain/access" + "github.com/samcharles93/archie-core/internal/domain/binding" + "github.com/samcharles93/archie-core/internal/domain/identity" + "github.com/samcharles93/archie-core/internal/domain/mapping" + "github.com/samcharles93/archie-core/internal/domain/org" + "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/domain/workflow" + workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task" + "github.com/samcharles93/archie-core/internal/events" +) + +// dispatchBindings offers each identified capture to every armed binding on +// its source. +func (d *Daemon) dispatchBindings(ctx context.Context) { + principals := d.OrgPrincipals(ctx) + seen := make(map[org.OrgID]bool) + for _, principal := range principals { + if seen[principal.Org] { + continue + } + seen[principal.Org] = true + d.dispatchOrgBindings(org.WithOrg(access.WithPrincipal(ctx, principal), principal.Org)) + } +} + +func (d *Daemon) dispatchOrgBindings(ctx context.Context) { + // Gather the armed bindings whose workflow is enabled first, so the + // oldest-first captures query is not held by captures that only a binding + // on a disabled workflow would take. + bindings, err := d.Bindings.ListBindings(ctx) + if err != nil { + d.Log.WarnContext(ctx, "list bindings failed", "error", err) + return + } + enablement, err := d.workflowEnablement(ctx) + if err != nil { + d.Log.WarnContext(ctx, "binding dispatch: workflow enablement unavailable", "error", err) + return + } + enabled := make([]string, 0, len(bindings)) + armed := make([]binding.Binding, 0, len(bindings)) + for _, b := range bindings { + if b.OrgID == org.OrgFromContext(ctx) && b.Status == binding.StatusArmed && b.Matcher.Source != "" && enablement.Enabled(b.OrgID, b.Workflow) { + enabled = append(enabled, b.ID) + armed = append(armed, b) + } + } + if len(enabled) == 0 { + return + } + + captures, err := d.BindingDispatcher.ListUndispatchedCaptures(ctx, enabled, bindingDispatchBatchLimit) + if err != nil { + d.Log.WarnContext(ctx, "list undispatched captures failed", "error", err) + return + } + workflows, err := d.activeWorkflows(ctx) + if err != nil { + d.Log.WarnContext(ctx, "binding dispatch: workflow definitions unavailable", "error", err) + return + } + for _, c := range captures { + d.dispatchCapture(ctx, c, armed, workflows) + } +} + +// dispatchCapture offers one capture to every armed binding that matches it. +// armed holds only the org's armed bindings on enabled workflows, so a +// disabled workflow's capture waits rather than recording a failure every pass. +func (d *Daemon) dispatchCapture(ctx context.Context, c storecontract.CapturedEvent, armed []binding.Binding, workflows workflow.WorkflowDefinitionCollection) { + for _, b := range armed { + if b.Matcher.Matches(c.Source, c.Dispatchable()) { + d.dispatchOneBinding(ctx, b, c, workflows) + } + } +} + +// bindingMapping returns b's mapping, or nil after logging why it is +// unavailable. A mapping that no longer exists is a terminal outcome. +func (d *Daemon) bindingMapping(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent) *mapping.Mapping { + m, err := d.Mappings.GetMapping(ctx, b.MappingID) + if err != nil { + d.Log.WarnContext(ctx, "binding dispatch: get mapping", "binding", b.ID, "mapping", b.MappingID, "error", err) + return nil + } + if m == nil { + d.recordDispatchFailure(ctx, b, c, fmt.Sprintf("mapping %q does not exist", b.MappingID), nil) + } + return m +} + +// dispatchOneBinding offers one capture to one binding: the mapping must +// match the capture's event type and resolve required fields, the filter +// must admit the values, and the workflow must accept the inputs. The +// ledger claim and the task commit together, so each binding fires exactly +// once per delivery. +func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, workflows workflow.WorkflowDefinitionCollection) { + m := d.bindingMapping(ctx, b, c) + if m == nil || m.EventTypeID != c.EventType { + return + } + values, failures := mapping.Resolve(m.Fields, []byte(c.Body)) + if hasBlockingFailure(m.Fields, failures) { + d.recordDispatchFailure(ctx, b, c, "required field failed", map[string]any{"failures": failures}) + return + } + if err := d.MappingMatches.RecordMappingMatch(ctx, m.ID, c.ID); err != nil { + d.Log.WarnContext(ctx, "binding dispatch: record mapping match", "mapping", m.ID, "capture", c.ID, "error", err) + } + filter, err := binding.CompileFilter(b.Filter, m.Fields) + if err != nil { + d.recordDispatchFailure(ctx, b, c, "filter does not compile", map[string]any{"error": err.Error()}) + return + } + if admitted, _ := filter.Admits(values); !admitted { + d.recordEvaluated(ctx, b, c, "filtered") + return + } + + target, reason, ok := d.resolveBindingTarget(b, values, workflows) + if !ok { + d.recordDispatchFailure(ctx, b, c, reason, nil) + return + } + // The chain decides dispatch: the binding's principal may `run` this + // workflow in its workspace, and the event's signature result and address + // travel with the request. A denial is terminal; an unavailable principal + // or policy chain leaves the capture listed for the next cycle. + if !d.authorizeDispatch(ctx, b, c, target) { + return + } + d.claimAndEnqueue(ctx, b, c, target, values) +} + +// claimAndEnqueue asks the State Store to claim the dispatch and enqueue its +// task as one unit. A failure leaves the delivery undispatched, so the next +// cycle offers it again; a delivery already dispatched is refused. +func (d *Daemon) claimAndEnqueue(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget, values map[string]any) { + title := fmt.Sprintf("binding %s/%d from %s", b.Name, b.Version, c.Source) + body := renderBindingBody(values, c) + if target.declared { + body = fmt.Sprintf("Started by binding %q from capture %q; the workflow's inputs travel with the task.", b.Name, c.ID) + } + taskIdentity := string(identity.SystemID) + if principal, ok := access.PrincipalFromContext(ctx); ok { + taskIdentity = string(principal.IdentityID) + } + task, err := d.BindingTaskCreator.EnqueueBindingTask(ctx, target.owner, target.repo, title, body, b.Workflow, taskIdentity, b.ID, b.Version, c.ID, target.inputs) + if err != nil { + if !errors.Is(err, storecontract.ErrAlreadyDispatched) { + d.Log.WarnContext(ctx, "binding dispatch: enqueue", "binding", b.ID, "capture", c.ID, "error", err) + } + return + } + d.markBindingStart(ctx, task.ID, b, c) +} + +// authorizeDispatch checks the binding's principal, its org's dispatching +// identity, may start the workflow. A denial is recorded and reported as a +// binding_dispatch_failure. The run itself acts as the workflow's declared +// identity, which adoptWorkflowIdentity checks when the task is processed. +func (d *Daemon) authorizeDispatch(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget) bool { + principal, ok := access.PrincipalFromContext(ctx) + if !ok { + d.Log.WarnContext(ctx, "binding dispatch: no principal; dispatch left for the next cycle", "binding", b.ID, "capture", c.ID) + return false + } + resource := access.Resource{ + Kind: access.KindWorkflow, ID: b.Workflow, Org: principal.Org, Workspace: b.WorkspaceID, + } + contextValue := access.Context{Signature: signatureOf(c), Addr: c.RemoteAddr} + decision := d.Access.Authorize(ctx, principal, access.ActionRun, resource, contextValue) + if decision.Allowed { + return true + } + if errors.Is(decision.Err, access.ErrChainUnavailable) { + // Nothing was evaluated. Recording it would mark the pair evaluated for + // good, so a State Store blip would drop the capture instead of leaving + // it for the next cycle. + d.Log.WarnContext(ctx, "binding dispatch: access chain unavailable; dispatch left for the next cycle", "binding", b.ID, "capture", c.ID, "err", decision.Err) + return false + } + denial := access.Denial{ + Principal: principal.IdentityID, Org: principal.Org, Action: access.ActionRun, + Kind: resource.Kind, ResourceID: resource.ID, Level: decision.Level, + Policies: decision.Policies, + } + if err := d.Denials.RecordDenial(ctx, denial); err != nil { + d.Log.WarnContext(ctx, "binding dispatch: record denial", "binding", b.ID, "error", err) + } + d.recordDispatchFailure(ctx, b, c, "access denied", map[string]any{ + "level": string(decision.Level), "policies": decision.Policies, + }) + return false +} + +// markBindingStart records on a task's timeline the binding and captured +// event that started it. +func (d *Daemon) markBindingStart(ctx context.Context, taskID int64, b binding.Binding, c storecontract.CapturedEvent) { + if _, err := d.Store.InsertEvent(ctx, events.Event{ + TaskID: taskID, + Kind: events.KindBindingStarted, + Detail: fmt.Sprintf("started by binding %q from source %q", b.Name, c.Source), + Data: map[string]any{ + "binding_id": b.ID, "binding_name": b.Name, "binding_version": b.Version, + "capture_id": c.ID, "source": c.Source, "unsigned": c.Unsigned, + }, + }); err != nil { + d.Log.WarnContext(ctx, "binding dispatch: start marker", "task", taskID, "error", err) + } +} + +// recordDispatchFailure records why a binding did not dispatch a capture and +// marks the pair evaluated, so the capture is not offered to it again. +func (d *Daemon) recordDispatchFailure(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, reason string, extra map[string]any) { + d.recordEvaluated(ctx, b, c, reason) + data := map[string]any{ + "binding_id": b.ID, + "binding_version": b.Version, + "capture_id": c.ID, + } + maps.Copy(data, extra) + _, _ = d.Store.InsertEvent(ctx, events.Event{ + Kind: "binding_dispatch_failure", + Detail: fmt.Sprintf("binding %q (capture %q): %s", b.ID, c.ID, reason), + Data: data, + }) +} + +// recordEvaluated writes the ledger row for an evaluation that started no +// task. Only terminal outcomes call it; transient ones leave the capture +// listed for the next cycle. +func (d *Daemon) recordEvaluated(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, reason string) { + err := d.BindingDispatcher.RecordDispatch(ctx, b.ID, int64(b.Version), c.ID, 0, reason) + if err != nil && !errors.Is(err, storecontract.ErrAlreadyDispatched) { + d.Log.WarnContext(ctx, "binding dispatch: record evaluation", "binding", b.ID, "capture", c.ID, "error", err) + } +} + +// resolveBindingRepo returns the binding's pinned owner/repo, or the only +// configured repo. With zero or several repos and no pin it refuses. +func (d *Daemon) resolveBindingRepo(b binding.Binding) (owner, repo, reason string) { + if b.Owner != "" && b.Repo != "" { + return b.Owner, b.Repo, "" + } + repos := d.Cfg.Get().Repos + if len(repos) == 1 { + return repos[0].Owner, repos[0].Name, "" + } + return "", "", fmt.Sprintf("cannot pick a repository: %d are configured; pin the binding to one", len(repos)) +} + +// activeWorkflows returns the definitions a dispatch cycle checks bindings +// against, the same set a task is later pinned from. +func (d *Daemon) activeWorkflows(ctx context.Context) (workflow.WorkflowDefinitionCollection, error) { + collection, _, err := d.WorkflowDefinitions.WorkflowDefinitions(ctx) + return collection, err +} + +// workflowEnablement returns which workflows each org has disabled; with no +// source wired every workflow is enabled. +func (d *Daemon) workflowEnablement(ctx context.Context) (workflowtask.WorkflowEnablement, error) { + return d.WorkflowEnablement.WorkflowEnablement(ctx) +} + +// disabledRoutedWorkflow reports the workflow an issue's labels route to and +// whether the identity's org has that workflow disabled, matching the +// enablement filter binding dispatch applies. A read failure (org, +// definitions, enablement) is an error so the caller falls through to +// publishing rather than dropping the issue; no routable definition is not an +// error, it is reported as not disabled and the pin path parks the task. +func (d *Daemon) disabledRoutedWorkflow(ctx context.Context, labels, identityID string) (routed string, disabled bool, err error) { + orgID, err := d.identityOrg(ctx, identity.IdentityID(identityID)) + if err != nil { + return "", false, err + } + definitions, err := d.activeWorkflows(ctx) + if err != nil { + return "", false, err + } + available := make(map[string]struct{}, len(definitions.Definitions)) + for _, definition := range definitions.Definitions { + available[definition.ID] = struct{}{} + } + routed, resolveErr := workflow.ResolveWorkflowID(&workflow.Task{Labels: labels}, available) + if resolveErr == nil { + enablement, err := d.workflowEnablement(ctx) + if err != nil { + return "", false, err + } + return routed, !enablement.Enabled(orgID, routed), nil + } + // No routable definition is a normal outcome here, not a read failure: + // keep the publish behaviour and let the pin path park the task. + return "", false, nil +} + +// bindingDispatchBatchLimit caps how many undispatched captures a single +// dispatchBindings pass will fetch from the store. The cap is defensive +// (the captures table is also pruned by retention/maxEvents) and bounds a +// single cycle's worst-case latency on a backlog. +const bindingDispatchBatchLimit = 100 + +// signatureOf names the event's signature result as the dispatch context +// carries it: a signed event and an approved unsigned event are the two +// dispatchable shapes (storecontract.CapturedEvent.Dispatchable). +func signatureOf(c storecontract.CapturedEvent) string { + switch { + case c.Authenticated: + return "valid" + case c.Unsigned: + return "unsigned" + } + return "unverified" +} + +// hasBlockingFailure reports whether any failure is on a required field. +func hasBlockingFailure(fields []mapping.Field, failures []mapping.Failure) bool { + if len(failures) == 0 { + return false + } + required := make(map[string]bool, len(fields)) + for _, f := range fields { + if f.Required { + required[f.Name] = true + } + } + for _, fail := range failures { + if required[fail.FieldName] { + return true + } + } + return false +} + +// renderBindingBody builds the task body from the values a mapping +// resolved, one "key=value" pair per line. Empty maps still produce +// a body that records the source capture id so the workflow has +// provenance to hand back to the operator. +func renderBindingBody(values map[string]any, c storecontract.CapturedEvent) string { + if len(values) == 0 { + return fmt.Sprintf("(no fields resolved from capture %q)", c.ID) + } + parts := make([]string, 0, len(values)) + for k, v := range values { + parts = append(parts, fmt.Sprintf("%s=%v", k, v)) + } + return strings.Join(parts, "\n") +} diff --git a/internal/daemon/binding_dispatch_test.go b/internal/daemon/binding_dispatch_test.go index dae94ab0d..73c3d9426 100644 --- a/internal/daemon/binding_dispatch_test.go +++ b/internal/daemon/binding_dispatch_test.go @@ -106,11 +106,11 @@ func TestBindingDispatchCreatesOneTask(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { store := &ledgerStore{claimed: map[string]bool{}, failures: tc.failures} - d := &Daemon{ + d := wired(&Daemon{ Cfg: config.NewHolder(config.Config{}), Store: store, Log: slog.New(slog.DiscardHandler), Bindings: store, BindingDispatcher: store, BindingTaskCreator: store, Mappings: store, WorkflowDefinitions: orgDefinitions{org.DefaultOrgID: "id: deploy\nrepository: none\nsteps:\n - type: workflow.finish\n"}, - } + }) for range tc.cycles { var listed, done sync.WaitGroup listed.Add(tc.concurrent) diff --git a/internal/daemon/binding_target.go b/internal/daemon/binding_target.go index 7d26bfc3b..c45069eeb 100644 --- a/internal/daemon/binding_target.go +++ b/internal/daemon/binding_target.go @@ -1,11 +1,9 @@ package daemon import ( - "context" "fmt" "github.com/samcharles93/archie-core/internal/domain/binding" - "github.com/samcharles93/archie-core/internal/domain/identity" "github.com/samcharles93/archie-core/internal/domain/workflow" workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task" ) @@ -20,57 +18,6 @@ type bindingTarget struct { declared bool } -// activeWorkflows returns the definitions a dispatch cycle checks bindings -// against, the same set a task is later pinned from. -func (d *Daemon) activeWorkflows(ctx context.Context) (workflow.WorkflowDefinitionCollection, error) { - if d.WorkflowDefinitions == nil { - return workflow.ShippedDefinitions(), nil - } - collection, _, err := d.WorkflowDefinitions.WorkflowDefinitions(ctx) - return collection, err -} - -// workflowEnablement returns which workflows each org has disabled; with no -// source wired every workflow is enabled. -func (d *Daemon) workflowEnablement(ctx context.Context) (workflowtask.WorkflowEnablement, error) { - if d.WorkflowEnablement == nil { - return workflowtask.WorkflowEnablement{}, nil - } - return d.WorkflowEnablement.WorkflowEnablement(ctx) -} - -// disabledRoutedWorkflow reports the workflow an issue's labels route to and -// whether the identity's org has that workflow disabled, matching the -// enablement filter binding dispatch applies. A read failure (org, -// definitions, enablement) is an error so the caller falls through to -// publishing rather than dropping the issue; no routable definition is not an -// error, it is reported as not disabled and the pin path parks the task. -func (d *Daemon) disabledRoutedWorkflow(ctx context.Context, labels, identityID string) (routed string, disabled bool, err error) { - orgID, err := d.identityOrg(ctx, identity.IdentityID(identityID)) - if err != nil { - return "", false, err - } - definitions, err := d.activeWorkflows(ctx) - if err != nil { - return "", false, err - } - available := make(map[string]struct{}, len(definitions.Definitions)) - for _, definition := range definitions.Definitions { - available[definition.ID] = struct{}{} - } - routed, resolveErr := workflow.ResolveWorkflowID(&workflow.Task{Labels: labels}, available) - if resolveErr == nil { - enablement, err := d.workflowEnablement(ctx) - if err != nil { - return "", false, err - } - return routed, !enablement.Enabled(orgID, routed), nil - } - // No routable definition is a normal outcome here, not a read failure: - // keep the publish behaviour and let the pin path park the task. - return "", false, nil -} - // resolveBindingTarget checks a matched binding against the workflow it // targets and resolves the task's repository and inputs. A non-empty reason // means the binding does not dispatch and the reason is recorded on it. diff --git a/internal/daemon/call_waits.go b/internal/daemon/call_waits.go index e689f6af6..80f81c277 100644 --- a/internal/daemon/call_waits.go +++ b/internal/daemon/call_waits.go @@ -21,7 +21,7 @@ const callWaitBuffer = 1024 // carry the caller's task id. func (d *Daemon) watchCallWaits(ctx context.Context) { reader, ok := d.Store.(storecontract.StepReader) - if d.Bus == nil || !ok { + if !ok { return } sub := d.Bus.Subscribe(callWaitBuffer) diff --git a/internal/daemon/container_run.go b/internal/daemon/container_run.go new file mode 100644 index 000000000..fa2b6376d --- /dev/null +++ b/internal/daemon/container_run.go @@ -0,0 +1,397 @@ +package daemon + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + + "github.com/samcharles93/archie-core/internal/agentexec" + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/container" + "github.com/samcharles93/archie-core/internal/domain/workflow" + "github.com/samcharles93/archie-core/internal/eventbus" + "github.com/samcharles93/archie-core/internal/events" + agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats" + "github.com/samcharles93/archie-core/internal/storage" + "github.com/samcharles93/archie-core/internal/taskrun" + "github.com/samcharles93/archie-core/internal/taskstate" +) + +// acquireTaskContainer prepares and acquires the sandbox container for a +// task, reporting false when it could not -- having already parked the +// task and logged why. The caller releases the container it returns. +func (d *Daemon) acquireTaskContainer( + ctx context.Context, + task *workflow.Task, + repo config.Repo, + workDir string, + image string, +) (*container.Container, string, func(), bool) { + park := func(reason string, err error) { + d.Log.ErrorContext(ctx, reason, "err", err) + d.parkBlockedTask(ctx, task, reason+": "+err.Error(), taskstate.ParkTransient, parkBlocker(err)) + } + + // Resolve the PR number from inputs before the brief is written. + if task.Workflow == "pr-review" { + task.PRNumber = task.EffectivePRNumber() + } + + if err := writeTaskBrief(workDir, task); err != nil { + park("task.json write failed", err) + return nil, "", nil, false + } + + // Fetch the pull request with the daemon's forge credential into workDir, + // so the container reads it without credentials. + if task.Workflow == "pr-review" { + if err := prefetchPRReview(ctx, d.forgeFor(ctx, task), task.Owner, task.Repo, task.PRNumber, workDir); err != nil { + park("pr-review prefetch failed", err) + return nil, "", nil, false + } + } + + // Guard: Storage may be nil if the daemon was wired incorrectly. In + // normal operation, Storage is always set when ContainerPool is set. + if d.Storage == nil { + d.Log.ErrorContext(ctx, "storage backend is nil -- cannot acquire container") + d.parkRunningTask(ctx, task, "storage backend not configured", taskstate.ParkTransient) + return nil, "", nil, false + } + + mounts, err := d.Storage.Setup(ctx, storage.TaskRef{ + WorktreeDir: workDir, + Ecosystem: repo.Ecosystem, + PersistentStorage: repo.PersistentStorage, + Owner: task.Owner, + Repo: task.Repo, + }) + if err != nil { + park("storage setup failed", err) + return nil, "", nil, false + } + + credential, revokeCredential, err := d.runCredential(ctx, task) + if err != nil { + d.teardownStorage(ctx, task, repo, workDir) + park("run credential failed", err) + return nil, "", nil, false + } + + model, closeModel, err := d.openModelEgress(ctx, task, credential) + if err != nil { + revokeCredential() + d.teardownStorage(ctx, task, repo, workDir) + park("model egress failed", err) + return nil, "", nil, false + } + revokeRun := revokeCredential + revokeCredential = func() { closeModel(); revokeRun() } + + // The container lives in its own network namespace, so a loopback State + // Store target names the container itself. Give this task an address on + // the host bridge that forwards to it, closed with the task. + stateStoreURL, relay, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential) + if err != nil { + revokeCredential() + d.teardownStorage(ctx, task, repo, workDir) + park("state store relay failed", err) + return nil, "", nil, false + } + revokeCredential = release + + ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(d.containerEnv(task, credential, stateStoreURL), model.Env...)) + if err != nil { + revokeCredential() + // Roll back the storage we just set up: the mounts were created + // for this task but no container will use them. Leaking them until + // a later TTL sweep is not acceptable on a backend that allocates + // real resources (volumes, NFS leases). + d.teardownStorage(ctx, task, repo, workDir) + park("container acquire failed", err) + return nil, "", nil, false + } + d.admitToRelay(ctx, relay, task, ctr) + return ctr, credential, revokeCredential, true +} + +// admitToRelay lets ctr, and only ctr, through relay. A container whose +// addresses cannot be read is refused rather than let anyone through. +func (d *Daemon) admitToRelay(ctx context.Context, relay *stateStoreRelay, task *workflow.Task, ctr *container.Container) { + if relay == nil { + return + } + addrs, err := d.ContainerPool.ContainerAddrs(ctx, ctr.ID) + if err != nil { + d.Log.WarnContext(ctx, "state store relay: container addresses unknown, refusing its connections", "task", task.ID, "err", err) + } + relay.Admit(addrs) +} + +// taskStateStoreEndpoint is the State Store address one task container is +// handed, the relay carrying it (nil when none is needed) and the release that +// ends that relay. The container lives +// in its own network namespace, so a loopback target names the container +// itself; behind a relay, the address is the host bridge's. +func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, *stateStoreRelay, func(), error) { + relay, err := d.openTaskStateStoreRelay(ctx) + if err != nil { + return "", nil, release, err + } + if relay == nil { + return d.ConnectedStateStore.URL, nil, release, nil + } + return relay.Addr(), relay, func() { relay.Close(); release() }, nil +} + +// openTaskStateStoreRelay starts the relay that carries one task container's +// State Store traffic to the daemon's target. The target stays bound where the +// operator put it; the container is handed an address on the host bridge, the +// same way it is handed NATS's. Nil when the target already reaches the +// containers, or when no bridge address is known. +func (d *Daemon) openTaskStateStoreRelay(ctx context.Context) (*stateStoreRelay, error) { + addr, loopback := loopbackStateStoreTarget(d.ConnectedStateStore.URL) + if !loopback { + return nil, nil + } + gateway := "" + if d.ContainerPool != nil { + gateway = d.ContainerPool.HostGateway() + } + if gateway == "" { + d.Log.WarnContext(ctx, "state store target is loopback and the container pool has no host bridge address; task containers cannot reach it", + "target", d.ConnectedStateStore.URL) + return nil, nil + } + return openStateStoreRelay(ctx, gateway, addr) +} + +// containerEnv is the environment a task container starts with. stateStoreURL +// is the address the container can reach the State Store by, which is the +// relay's rather than the daemon's own target. +func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken, stateStoreURL string) []string { + var env []string + // The endpoint the daemon's own client connected with at startup, not + // the live config: a reloaded [nats] section must not point new + // containers at a server the daemon is not publishing on. + env = append(env, "NATS_URL="+d.ConnectedNATS.URL) + if token := d.ConnectedNATS.ContainerToken(); token != "" { + env = append(env, "NATS_TOKEN="+token) + } + // Pass the container-reachable endpoint with this task's scoped token, + // never the daemon's own. + if stateStoreURL != "" { + env = append(env, "STATE_STORE_URL="+stateStoreURL) + if len(d.ConnectedStateStore.CA) > 0 { + env = append(env, "STATE_STORE_CA="+string(d.ConnectedStateStore.CA)) + } + if stateStoreToken != "" { + env = append(env, "STATE_STORE_TOKEN="+stateStoreToken) + } + } + // The agent runs as root in the container, so its commits land owned by + // UID 0 on the host. The daemon's UID/GID let the agent hand the worktree + // back before the daemon reads it to push. + env = append(env, fmt.Sprintf("WORKTREE_UID=%d", os.Getuid())) + env = append(env, fmt.Sprintf("WORKTREE_GID=%d", os.Getgid())) + return env +} + +// runCredential issues the task's run credential, or "" when no State Store +// is configured. +func (d *Daemon) runCredential(ctx context.Context, task *workflow.Task) (string, func(), error) { + if d.ConnectedStateStore.URL == "" { + return "", func() {}, nil + } + return d.RunCredentials.Issue(ctx, task, d.configFor(task).Budgets.TaskWallClock.Std()) +} + +// runViaAgent hands a task to archie-agent and waits for its result. It parks +// the task only when the agent never answered or failed before recording an +// outcome. +// +// credential is the task's run credential; the agent presents it to push the +// task's branch. +func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo config.Repo, profile config.AgentProfile, credential string) { + cfg, err := d.taskConfig(ctx, task) + if err != nil { + d.parkRunningTask(ctx, task, err.Error(), taskstate.ParkTransient) + return + } + taskCfg := cfg.ForTask() + d.captureAttemptConfig(ctx, task, taskCfg) + var instructions workflow.Instructions + instructions.Instance, instructions.Organisation, err = d.TaskInstructions.TaskInstructions(ctx, task.ID) + if err != nil { + d.parkRunningTask(ctx, task, "resolve the task's instructions: "+err.Error(), taskstate.ParkTransient) + return + } + req := taskrun.Request{ + Instructions: instructions, + Task: task, + Repo: repo, + Cfg: taskCfg, + Providers: agentexec.ProvidersFromConfig(cfg.Providers), + MCPServers: cfg.Tools.MCPServers, + RunCredential: credential, + WorkflowDefinition: task.WorkflowDefinitionYAML, + Tools: d.toolAllowlist(ctx, task, profile.Tools), + } + req.Trace = map[string]string{} + otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(req.Trace)) + data, err := json.Marshal(req) + if err != nil { + d.Log.ErrorContext(ctx, "taskrun encode failed", "task", task.ID, "err", err) + d.parkRunningTask(ctx, task, "taskrun encode failed: "+err.Error(), taskstate.ParkTransient) + return + } + + reply, err := d.requestTaskRun(ctx, task.ID, data) + if err != nil { + if stoppingOnly(ctx) { + d.Log.InfoContext(ctx, "daemon stopping; task left running for the next daemon to re-queue", "task", task.ID) + return + } + err = runFailure(ctx, err) + d.Log.ErrorContext(ctx, "taskrun request failed", "task", task.ID, "err", err) + d.parkRunningTask(ctx, task, "taskrun request failed: "+err.Error(), taskstate.ParkTransient) + return + } + + var resp taskrun.Response + if err := json.Unmarshal(reply, &resp); err != nil { + d.Log.ErrorContext(ctx, "taskrun decode response failed", "task", task.ID, "err", err) + d.parkRunningTask(ctx, task, "taskrun decode response failed: "+err.Error(), taskstate.ParkTransient) + return + } + if resp.Error != "" { + d.Log.ErrorContext(ctx, "taskrun run failed", "task", task.ID, "err", resp.Error) + d.parkRunningTask(ctx, task, "taskrun run failed: "+resp.Error, taskstate.ParkTransient) + return + } + // The agent applies the task's own outcome transition, so a status the + // task cannot reach from running means nothing landed: empty, unknown, or + // still running. Transient, because the cause is environmental -- a State + // Store that refused the transition -- and a retry can record one. + if !taskstate.CanTransition(taskstate.Running, resp.Status) { + reason := noOutcomeReason(resp.Status) + d.Log.ErrorContext(ctx, "taskrun run recorded no outcome", "task", task.ID, "status", resp.Status) + d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient) + return + } + + // Reached only past every failure return above, so this is always a + // fully decoded, error-free response -- a version is never recorded off + // a request that never reached an agent, or a response describing a + // parked/errored task. + if resp.AgentVersion != "" { + d.AgentStatus.Observe(resp.AgentVersion, resp.AgentInstallType) + } + + d.Log.InfoContext(ctx, "taskrun complete", "task", task.ID, "status", resp.Status) +} + +// requestTaskRun publishes the taskrun request, retrying on +// nats.ErrNoResponders until the container subscribes. Other errors return +// at once. +func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte) ([]byte, error) { + subject := agentnats.SubjectForTask(taskID) + deadline := time.Now().Add(taskRunReadyTimeout) + backoff := taskRunRetryBackoff + for { + reply, err := d.Tasks.Request(ctx, subject, data) + if err == nil { + return reply, nil + } + if !errors.Is(err, eventbus.ErrNoResponders) || !time.Now().Before(deadline) { + return nil, err + } + wait := min(backoff, time.Until(deadline)) + if wait <= 0 { + return nil, err + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(wait): + } + } +} + +// captureAttemptConfig records the attempt's effective configuration as an +// event. Failures are logged. +func (d *Daemon) captureAttemptConfig(ctx context.Context, task *workflow.Task, cfg config.TaskConfig) { + document, err := json.Marshal(cfg) + if err != nil { + d.Log.WarnContext(ctx, "attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err) + return + } + event := events.Event{ + At: time.Now().UTC(), + Kind: events.KindConfigCaptured, + TaskID: task.ID, + Repo: task.Owner + "/" + task.Repo, + Issue: task.IssueNumber, + Workflow: task.Workflow, + Attempt: task.Attempt, + Data: map[string]any{ + "schema": events.ConfigCapturedSchema, + // Raw so the document is embedded once, as it was encoded, rather + // than as a string a reader would have to decode again. + "document": json.RawMessage(document), + }, + } + id, err := d.Store.InsertEvent(ctx, event) + if err != nil { + d.Log.WarnContext(ctx, "attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err) + return + } + // The assigned ID tells the event sink to broadcast without inserting a + // duplicate row. + event.ID = id + d.emit(ctx, event) +} + +// writeTaskBrief writes task.json, the container's boot-time brief. +func writeTaskBrief(workDir string, task *workflow.Task) error { + return container.WriteTaskJSON(workDir, container.TaskPayload{ + ID: task.ID, Owner: task.Owner, Repo: task.Repo, + Number: task.IssueNumber, Title: task.Title, Body: task.Body, + Labels: strings.Split(task.Labels, ","), + Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan, + Inputs: task.Inputs, + }) +} + +// noOutcomeReason names why a taskrun reply left the task without an outcome, +// quoting the status so the log line points at what the agent actually said. +func noOutcomeReason(status string) string { + if status == "" { + return "agent returned no status: the run could not record an outcome" + } + return fmt.Sprintf("agent returned unrecognised status %q: the run could not record an outcome", status) +} + +const ( + // taskRunReadyTimeout is how long requestTaskRun retries a + // nats.ErrNoResponders while a new container has not subscribed yet. + taskRunReadyTimeout = 20 * time.Second + // taskRunRetryBackoff is the delay between those retries. + taskRunRetryBackoff = 250 * time.Millisecond +) + +// runFailure names why a taskrun request failed: the container's exit or the +// time limit when either cancelled it, else the request's own error. +func runFailure(ctx context.Context, err error) error { + if cause := context.Cause(ctx); errors.Is(cause, errContainerExited) || errors.Is(cause, errTaskTimeLimit) { + return cause + } + return err +} diff --git a/internal/daemon/cycle_test.go b/internal/daemon/cycle_test.go index c5dbe8030..9caac0084 100644 --- a/internal/daemon/cycle_test.go +++ b/internal/daemon/cycle_test.go @@ -8,7 +8,7 @@ import ( // The systemd watchdog beats only while StalestLoop is fresh, so a cycle that // doesn't stamp it gets archied killed shortly after every start. func TestCycleMarksPass(t *testing.T) { - d := &Daemon{} + d := wired(&Daemon{Store: &retryStore{}}) d.Cycle(context.Background()) if loop, at := d.StalestLoop(); loop != "cycle" || at.IsZero() { t.Fatalf("StalestLoop() = %q, %v; want a cycle pass", loop, at) diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 33542bc4e..1f7c30334 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -5,42 +5,27 @@ package daemon import ( "context" - "encoding/json" "errors" "fmt" - "io/fs" "log/slog" - "maps" - "os" - "path/filepath" "strings" "sync" "time" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/propagation" - - "github.com/samcharles93/archie-core/internal/agentexec" "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/container" "github.com/samcharles93/archie-core/internal/domain/access" - "github.com/samcharles93/archie-core/internal/domain/binding" "github.com/samcharles93/archie-core/internal/domain/eda/playbook" "github.com/samcharles93/archie-core/internal/domain/identity" - "github.com/samcharles93/archie-core/internal/domain/mapping" "github.com/samcharles93/archie-core/internal/domain/org" "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/workflow" workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task" - "github.com/samcharles93/archie-core/internal/domain/workintake" "github.com/samcharles93/archie-core/internal/eventbus" "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/forge" - agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats" "github.com/samcharles93/archie-core/internal/logging" - "github.com/samcharles93/archie-core/internal/natsrpc" "github.com/samcharles93/archie-core/internal/storage" - "github.com/samcharles93/archie-core/internal/taskrun" "github.com/samcharles93/archie-core/internal/taskstate" "github.com/samcharles93/archie-core/internal/worktree" ) @@ -100,43 +85,37 @@ type Daemon struct { // startup, passed to containers. ConnectedStateStore StateStoreEndpoint Store storecontract.TaskStore - // Mappings persists payload field mappings. - // Used by the binding dispatch loop to resolve capture bodies against - // the mapping a binding names. Optional: nil disables the binding - // dispatch loop (legacy behaviour). + // Mappings persists payload field mappings, which the binding dispatch + // loop uses to resolve capture bodies against the mapping a binding names. Mappings storecontract.MappingStore // Bindings persists playbook bindings. - // Optional: nil disables the binding dispatch loop (legacy behaviour). Bindings storecontract.BindingStore // BindingDispatcher lists undispatched captures, finds armed bindings and - // records dispatches. Nil disables binding dispatch. + // records dispatches. BindingDispatcher storecontract.BindingDispatcher // MappingMatches counts each event a mapping resolved at dispatch. - // Optional: nil leaves match counts unrecorded. MappingMatches storecontract.MappingMatchRecorder // BindingTaskCreator enqueues a task triggered by a binding. Split // off TaskLifecycle so the lifecycle surface stays narrow and the // dispatch loop does not acquire the full task-creation contract. - // Optional: nil disables the dispatch loop. BindingTaskCreator storecontract.BindingTaskCreator // Access evaluates the policy chain at dispatch, the second of the two // Authorizer call sites: the workflow's identity may `run` the workflow in - // its workspace. Optional: nil dispatches without the chain, which is the - // behaviour of an install that has not built it. + // its workspace. Access access.Authorizer // Principals assembles the dispatch principal for the workflow's - // identity. Wired with Access or not at all. + // identity. Principals access.PrincipalSource - // Denials records dispatch refusals. Optional: nil skips the record. + // Denials records dispatch refusals. Denials access.DenialRecorder Forge forge.Forge Trees *worktree.Manager Bus *events.Bus Log *slog.Logger - // Tasks is the NATS task bus. Always set in production. + // Tasks is the NATS task bus. Tasks TaskBus - // reactionConsumer is built lazily by drainReactions and kept for its - // lifetime drop counter. + // reactionConsumer is built by Startup and kept for its lifetime drop + // counter. reactionConsumer *reactionConsumer // lastPolled remembers each repo's eligible issues from its previous @@ -144,11 +123,11 @@ type Daemon struct { // issue that was eligible and dropped out reads as withdrawn. lastPolledMu sync.Mutex lastPolled map[string]polledIssues - // reactionPublisher is built lazily by scanPRReviews; an indirection - // only so tests can swap the delivery without the bus. + // reactionPublisher delivers review reactions. Startup sets it to + // PublishReaction; tests swap it to deliver without the bus. reactionPublisher ReactionPublisher // Reactions is the review-reaction stream, drained into remediate runs each - // cycle. Nil disables it. + // cycle. Reactions ReactionSource // RunCredentials issues each task's run credential (see runCredential). // Required whenever ConnectedStateStore.URL is set: the task is parked @@ -158,20 +137,11 @@ type Daemon struct { // keys. Nil parks a task whose providers need a key. ModelEgress ModelEgress // TaskModels answers the model aliases a task's org resolves against. - // Nil runs every task on the instance aliases. TaskModels TaskModels // TaskInstructions answers the instance and organisation instructions a - // task's agent runs follow. Nil runs without them. + // task's agent runs follow. TaskInstructions TaskInstructions - // TaskRunReadyTimeout bounds how long runViaAgent retries a taskrun request - // while the new container has not subscribed yet. Zero uses - // defaultTaskRunReadyTimeout. - TaskRunReadyTimeout time.Duration - // TaskRunRetryBackoff is the delay between retry attempts within - // TaskRunReadyTimeout. Zero uses defaultTaskRunRetryBackoff. - TaskRunRetryBackoff time.Duration - // AgentStatus records agent versions for update verification. Nil disables - // it. + // AgentStatus records agent versions for update verification. AgentStatus *AgentStatus // ContainerPool manages Docker container lifecycle. Nil means autonomous @@ -188,10 +158,10 @@ type Daemon struct { // A runnable task requires it; acquireTaskContainer parks on nil. Storage storage.Backend // IssueSources resolves, each poll cycle, every organisation connection - // to poll and the identity whose permit reaches it. Nil polls nothing. + // to poll and the identity whose permit reaches it. IssueSources func(context.Context) []IssueSource // OrgPrincipals is one acting principal per organisation, system org - // first. Nil dispatches the root identity's org alone. + // first. OrgPrincipals func(context.Context) []access.Principal // IdentityRepository is the durable lifecycle authority. Polling checks it // every cycle so suspend/reactivate commands do not require a restart. @@ -202,13 +172,12 @@ type Daemon struct { PackageAuthorities PackageAuthorities // Playbooks are the loaded EDA playbooks, consulted before kind/label - // bindings when pinning a workflow. Nil means none. + // bindings when pinning a workflow. Playbooks interface { Dispatch(context.Context, playbook.DispatchInput) (playbook.Decision, bool) Run(context.Context, storecontract.PlaybookDispatcher, *slog.Logger, playbook.DispatchInput) error } // PlaybookLedger is the at-most-once gate action playbooks run through. - // Nil means action playbooks do not run: they never fire unguarded. PlaybookLedger storecontract.PlaybookDispatcher // WorkflowDefinitions supplies the active database definitions. A task is @@ -217,12 +186,11 @@ type Daemon struct { WorkflowDefinitions(context.Context) (workflow.WorkflowDefinitionCollection, int64, error) } // WithdrawTask declines a task whose issue a complete poll no longer - // returns. Nil leaves withdrawn issues' work running. + // returns. WithdrawTask func(ctx context.Context, taskID int64, reason string) error // WorkflowEnablement supplies which workflows each org has disabled. A - // binding targeting a workflow its org disabled does not dispatch. Nil - // means every workflow is enabled. + // binding targeting a workflow its org disabled does not dispatch. WorkflowEnablement interface { WorkflowEnablement(context.Context) (workflowtask.WorkflowEnablement, error) } @@ -256,778 +224,6 @@ type IssueSource struct { Repos []config.Repo } -func (d *Daemon) issueSources(ctx context.Context) []IssueSource { - if d.IssueSources == nil { - return nil - } - return d.IssueSources(ctx) -} - -// emit publishes an observability event; safe on a nil bus. -func (d *Daemon) emit(ctx context.Context, e events.Event) { - if d.Bus != nil { - d.Bus.Publish(ctx, e) - } -} - -// Startup runs crash recovery and access verification once. -func (d *Daemon) Startup(ctx context.Context) error { - if err := d.cleanupExpiredStorage(ctx); err != nil { - d.Log.WarnContext(ctx, "startup volume cleanup failed", "err", err) - } - n, err := d.Store.RecoverStale(ctx) - if err != nil { - return err - } - if n > 0 { - d.Log.InfoContext(ctx, "re-queued tasks left running by a previous daemon", "count", n) - } - - d.sweepAccess(ctx) - return nil -} - -// sweepTimeout bounds one forge's boot sweep. Startup gates the gateways and -// the run loop, and a forge client carries no request deadline of its own, so -// an unreachable host would otherwise hold boot open indefinitely. -const sweepTimeout = 2 * time.Minute - -// A clone waits, holding its slot, until the work volume has minCloneSpace -// free, and parks as transient if it does not within diskWaitLimit. Clones -// share objects with the repository's mirror, so the floor covers a checkout -// and its build output, not a full clone. -const ( - minCloneSpace = 1 << 30 - diskCheckEvery = 15 * time.Second - diskWaitLimit = 10 * time.Minute - // cloneRetention is how long an untouched clone, such as a parked task's, - // is kept before a full work volume may reclaim it. - cloneRetention = 72 * time.Hour -) - -// sweepAccess accepts pending invitations and verifies push access through -// every organisation connection, each in its own goroutine. -func (d *Daemon) sweepAccess(ctx context.Context) { - sweep := func(source IssueSource) { - log := d.Log.With("identity", source.Identity) - ctx, cancel := context.WithTimeout(ctx, sweepTimeout) - defer cancel() - if err := source.Forge.AcceptInvitations(ctx, repoRefs(source.Repos)); err != nil { - log.WarnContext(ctx, "invitation sweep failed", "err", err) - } - for _, r := range source.Repos { - if err := source.Forge.VerifyPush(ctx, r.Owner, r.Name); err != nil { - // A sweep cut short by its own context is not a repo failure: name - // the budget when it expired, stay quiet on a shutdown. - switch { - case errors.Is(ctx.Err(), context.DeadlineExceeded): - // Every later call would return the same error, so say it once. - log.WarnContext(ctx, "push check abandoned: sweep budget exhausted", "repo", r.FullName(), "err", err) - return - case ctx.Err() != nil: - return - } - log.WarnContext(ctx, "repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err) - } - } - } - - var wg sync.WaitGroup - for _, source := range d.issueSources(ctx) { - wg.Go(func() { sweep(source) }) - } - wg.Wait() -} - -// repoRefs is a source's configured repositories as the allowlist the forge -// accepts invitations against. -func repoRefs(repos []config.Repo) []forge.RepoRef { - out := make([]forge.RepoRef, 0, len(repos)) - for _, r := range repos { - out = append(out, forge.RepoRef{Owner: r.Owner, Repo: r.Name}) - } - return out -} - -// Run polls until the context ends. -func (d *Daemon) Run(ctx context.Context) error { - // A pass no longer waits for the tasks it submitted, so the run does, once - // the ctx is cancelled. - defer d.WaitForTasks() - d.watchCallWaits(ctx) - if d.Tasks != nil { - drained := make(chan struct{}) - defer func() { <-drained }() - go func() { - defer close(drained) - d.drainLoop(ctx) - }() - } - interval := d.Cfg.Get().PollInterval.Std() - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - if d.Tasks == nil { - d.dispatchBindings(ctx) - } - d.Cycle(ctx) - select { - case <-ctx.Done(): - return ctx.Err() - case <-ticker.C: - // PollInterval may have been reloaded; reset the ticker so a - // change takes effect on the next cycle. - if iv := d.Cfg.Get().PollInterval.Std(); iv != interval { - interval = iv - ticker.Reset(iv) - } - } - } -} - -// drainLoop is the only task drainer and capture dispatcher while Run is -// active. It runs apart from the poll tick so a captured event becomes a -// running task within about a second regardless of poll_interval or forge -// latency. Submit never blocks, so a full dispatcher -// just leaves tasks waiting on slots and the loop stays paced by the ticker. -func (d *Daemon) drainLoop(ctx context.Context) { - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - for { - d.markPass("drain") - d.dispatchBindings(ctx) - d.drainNATS(ctx) - select { - case <-ctx.Done(): - return - case <-ticker.C: - } - } -} - -// pollIssuesWithConfig returns the repo's issues the dispatch rule makes -// work, using the given forge client and dispatch config. complete is false when any part of the poll failed or was skipped, so -// the result must not be read as the whole eligible set. -func (d *Daemon) pollIssuesWithConfig(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo) (issues []forge.Issue, complete bool) { - labelled := func() ([]forge.Issue, bool) { - // A label trigger needs a non-empty label or it would match every open issue. - if cfg.Label == "" { - d.Log.ErrorContext(ctx, "label trigger configured with an empty label; refusing to poll (an empty label matches every open issue)", "repo", repo.FullName()) - return nil, false - } - issues, err := fg.IssuesWithLabel(ctx, repo.Owner, repo.Name, cfg.Label) - if err != nil { - d.Log.ErrorContext(ctx, "label poll failed", "repo", repo.FullName(), "err", err) - return nil, false - } - return issues, true - } - assigned := func() ([]forge.Issue, bool) { - if cfg.BotUser == "" { - d.Log.ErrorContext(ctx, "assignee trigger needs the connection's account; refusing to poll", "repo", repo.FullName()) - return nil, false - } - issues, err := fg.AssignedIssues(ctx, repo.Owner, repo.Name, cfg.BotUser) - if err != nil { - d.Log.ErrorContext(ctx, "assigned poll failed", "repo", repo.FullName(), "err", err) - return nil, false - } - return issues, true - } - switch cfg.Dispatch.Trigger { - case "label": - return labelled() - case "either": - byAssignee, assignedOK := assigned() - byLabel, labelledOK := labelled() - seen := map[int]bool{} - for _, is := range byAssignee { - seen[is.Number] = true - } - issues = byAssignee - for _, is := range byLabel { - if !seen[is.Number] { - issues = append(issues, is) - } - } - return issues, assignedOK && labelledOK - default: // "assignee" (default) - return assigned() - } -} - -// RunOnce runs one cycle and drains until no task is in flight. A run waiting -// on a callee stays in flight while the callee sits queued, so the queue is -// drained again until every task, callees included, has returned. -func (d *Daemon) RunOnce(ctx context.Context) { - defer d.WaitForTasks() - d.watchCallWaits(ctx) - // One pass has no scheduler, so it runs the maintenance jobs itself. - for _, job := range d.BuiltinJobs() { - if err := job.Run(ctx); err != nil { - d.Log.WarnContext(ctx, "built-in job failed", "job", job.ID, "err", err) - } - } - d.dispatchBindings(ctx) - d.Cycle(ctx) - if d.Tasks == nil { - return - } - d.drainNATS(ctx) - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - for d.taskDispatcher().inFlight() > 0 { - select { - case <-ctx.Done(): - return - case <-ticker.C: - d.drainNATS(ctx) - } - } -} - -// Cycle is one poll pass: turn review reactions into tasks; maintenance runs -// as built-in schedules. Captured events are dispatched by the caller -// (drainLoop in Run, once in RunOnce) so they start within about a second. -func (d *Daemon) Cycle(ctx context.Context) { - d.markPass("cycle") - d.drainReactions(ctx) - d.retryTransientParks(ctx) -} - -// retryTransientParks requeues tasks parked for an environmental cause once -// their backoff has passed. The store parks a task needs_decision, which -// raises the alert, when its retries run out, so this never loops on one. -func (d *Daemon) retryTransientParks(ctx context.Context) { - if d.Store == nil { - return - } - now := time.Now() - var after storecontract.TaskCursor - for { - page, err := d.Store.TasksPage(ctx, storecontract.TaskPage{Statuses: []string{workflow.StatusParked}, After: after, Limit: 200}) - if err != nil { - d.Log.WarnContext(ctx, "list parked tasks for retry failed", "err", err) - return - } - for _, t := range page { - if taskstate.TransientRetryDue(t.ParkClass, t.RetryCount, t.UpdatedAt, now) { - d.retryTransientPark(ctx, t.ID) - } - } - if len(page) < 200 { - return - } - last := page[len(page)-1] - after = storecontract.TaskCursor{UpdatedAt: last.UpdatedAt, ID: last.ID} - } -} - -// retryTransientPark requeues one task on its stored retry mode, which the -// list projection omits: a remediation must keep continuing its branch. -func (d *Daemon) retryTransientPark(ctx context.Context, id int64) { - t, err := d.Store.TaskByID(ctx, id) - if err != nil { - d.Log.WarnContext(ctx, "load task for transient retry failed", "task", id, "err", err) - return - } - if err := d.Store.RetryTask(ctx, id, workflow.StatusParked, "", t.RetryMode, workflowtask.Resume{}, false); err != nil { - if !errors.Is(err, storecontract.ErrStaleTransition) { - d.Log.WarnContext(ctx, "transient park retry failed", "task", id, "err", err) - } - return - } - d.emit(ctx, events.Event{ - Kind: events.KindTaskRetried, TaskID: id, Workflow: t.Workflow, - Repo: strings.Trim(t.Owner+"/"+t.Repo, "/"), Issue: t.IssueNumber, - Detail: "retried automatically after a transient park", - Data: map[string]any{"retry_count": t.RetryCount + 1, "previous_reason": t.ParkReason}, - }) -} - -func (d *Daemon) cleanupExpiredStorage(ctx context.Context) error { - ttl := d.Cfg.Get().Containers.VolumeTTL.Std() - if ttl <= 0 || d.Storage == nil { - return nil - } - n, err := d.Storage.CleanupExpired(ctx, ttl) - if n > 0 { - d.Log.InfoContext(ctx, "expired persistent volumes removed", "count", n) - } - if err != nil { - return fmt.Errorf("persistent volume cleanup: %w", err) - } - return nil -} - -// bindingDispatchBatchLimit caps how many undispatched captures a single -// dispatchBindings pass will fetch from the store. The cap is defensive -// (the captures table is also pruned by retention/maxEvents) and bounds a -// single cycle's worst-case latency on a backlog. -const bindingDispatchBatchLimit = 100 - -// dispatchBindings offers each identified capture to every armed binding on -// its source. Nil Bindings, BindingDispatcher or BindingTaskCreator disables -// it. -func (d *Daemon) dispatchBindings(ctx context.Context) { - if d.Bindings == nil || d.BindingDispatcher == nil || d.BindingTaskCreator == nil { - return - } - if d.Principals == nil { - d.dispatchOrgBindings(ctx) - return - } - var principals []access.Principal - if d.OrgPrincipals != nil { - principals = d.OrgPrincipals(ctx) - } else { - principal, err := d.Principals.PrincipalFor(ctx, identity.SystemID) - if err != nil { - d.Log.WarnContext(ctx, "binding dispatch: principal unavailable", "identity", identity.SystemID, "err", err) - return - } - principals = []access.Principal{principal} - } - seen := make(map[org.OrgID]bool) - for _, principal := range principals { - if seen[principal.Org] { - continue - } - seen[principal.Org] = true - d.dispatchOrgBindings(org.WithOrg(access.WithPrincipal(ctx, principal), principal.Org)) - } -} - -func (d *Daemon) dispatchOrgBindings(ctx context.Context) { - // Gather the armed bindings whose workflow is enabled first, so the - // oldest-first captures query is not held by captures that only a binding - // on a disabled workflow would take. - bindings, err := d.Bindings.ListBindings(ctx) - if err != nil { - d.Log.WarnContext(ctx, "list bindings failed", "error", err) - return - } - enablement, err := d.workflowEnablement(ctx) - if err != nil { - d.Log.WarnContext(ctx, "binding dispatch: workflow enablement unavailable", "error", err) - return - } - enabled := make([]string, 0, len(bindings)) - armed := make([]binding.Binding, 0, len(bindings)) - for _, b := range bindings { - if b.OrgID == org.OrgFromContext(ctx) && b.Status == binding.StatusArmed && b.Matcher.Source != "" && enablement.Enabled(b.OrgID, b.Workflow) { - enabled = append(enabled, b.ID) - armed = append(armed, b) - } - } - if len(enabled) == 0 { - return - } - - captures, err := d.BindingDispatcher.ListUndispatchedCaptures(ctx, enabled, bindingDispatchBatchLimit) - if err != nil { - d.Log.WarnContext(ctx, "list undispatched captures failed", "error", err) - return - } - workflows, err := d.activeWorkflows(ctx) - if err != nil { - d.Log.WarnContext(ctx, "binding dispatch: workflow definitions unavailable", "error", err) - return - } - for _, c := range captures { - d.dispatchCapture(ctx, c, armed, workflows) - } -} - -// dispatchCapture offers one capture to every armed binding that matches it. -// armed holds only the org's armed bindings on enabled workflows, so a -// disabled workflow's capture waits rather than recording a failure every pass. -func (d *Daemon) dispatchCapture(ctx context.Context, c storecontract.CapturedEvent, armed []binding.Binding, workflows workflow.WorkflowDefinitionCollection) { - for _, b := range armed { - if b.Matcher.Matches(c.Source, c.Dispatchable()) { - d.dispatchOneBinding(ctx, b, c, workflows) - } - } -} - -// bindingMapping returns b's mapping, or nil after logging why it is -// unavailable. A mapping that no longer exists is a terminal outcome. -func (d *Daemon) bindingMapping(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent) *mapping.Mapping { - if d.Mappings == nil { - d.Log.WarnContext(ctx, "binding dispatch: mapping store unavailable", "binding", b.ID) - return nil - } - m, err := d.Mappings.GetMapping(ctx, b.MappingID) - if err != nil { - d.Log.WarnContext(ctx, "binding dispatch: get mapping", "binding", b.ID, "mapping", b.MappingID, "error", err) - return nil - } - if m == nil { - d.recordDispatchFailure(ctx, b, c, fmt.Sprintf("mapping %q does not exist", b.MappingID), nil) - } - return m -} - -// dispatchOneBinding offers one capture to one binding: the mapping must -// match the capture's event type and resolve required fields, the filter -// must admit the values, and the workflow must accept the inputs. The -// ledger claim and the task commit together, so each binding fires exactly -// once per delivery. -func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, workflows workflow.WorkflowDefinitionCollection) { - m := d.bindingMapping(ctx, b, c) - if m == nil || m.EventTypeID != c.EventType { - return - } - values, failures := mapping.Resolve(m.Fields, []byte(c.Body)) - if hasBlockingFailure(m.Fields, failures) { - d.recordDispatchFailure(ctx, b, c, "required field failed", map[string]any{"failures": failures}) - return - } - if d.MappingMatches != nil { - if err := d.MappingMatches.RecordMappingMatch(ctx, m.ID, c.ID); err != nil { - d.Log.WarnContext(ctx, "binding dispatch: record mapping match", "mapping", m.ID, "capture", c.ID, "error", err) - } - } - filter, err := binding.CompileFilter(b.Filter, m.Fields) - if err != nil { - d.recordDispatchFailure(ctx, b, c, "filter does not compile", map[string]any{"error": err.Error()}) - return - } - if admitted, _ := filter.Admits(values); !admitted { - d.recordEvaluated(ctx, b, c, "filtered") - return - } - - target, reason, ok := d.resolveBindingTarget(b, values, workflows) - if !ok { - d.recordDispatchFailure(ctx, b, c, reason, nil) - return - } - // The chain decides dispatch: the binding's principal may `run` this - // workflow in its workspace, and the event's signature result and address - // travel with the request. A denial is terminal; an unavailable principal - // or policy chain leaves the capture listed for the next cycle. - if !d.authorizeDispatch(ctx, b, c, target) { - return - } - d.claimAndEnqueue(ctx, b, c, target, values) -} - -// claimAndEnqueue asks the State Store to claim the dispatch and enqueue its -// task as one unit. A failure leaves the delivery undispatched, so the next -// cycle offers it again; a delivery already dispatched is refused. -func (d *Daemon) claimAndEnqueue(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget, values map[string]any) { - title := fmt.Sprintf("binding %s/%d from %s", b.Name, b.Version, c.Source) - body := renderBindingBody(values, c) - if target.declared { - body = fmt.Sprintf("Started by binding %q from capture %q; the workflow's inputs travel with the task.", b.Name, c.ID) - } - taskIdentity := string(identity.SystemID) - if principal, ok := access.PrincipalFromContext(ctx); ok { - taskIdentity = string(principal.IdentityID) - } - task, err := d.BindingTaskCreator.EnqueueBindingTask(ctx, target.owner, target.repo, title, body, b.Workflow, taskIdentity, b.ID, b.Version, c.ID, target.inputs) - if err != nil { - if !errors.Is(err, storecontract.ErrAlreadyDispatched) { - d.Log.WarnContext(ctx, "binding dispatch: enqueue", "binding", b.ID, "capture", c.ID, "error", err) - } - return - } - d.markBindingStart(ctx, task.ID, b, c) -} - -// authorizeDispatch checks the binding's principal, its org's dispatching -// identity, may start the workflow. A denial is recorded and reported as a -// binding_dispatch_failure. The run itself acts as the workflow's declared -// identity, which adoptWorkflowIdentity checks when the task is processed. -func (d *Daemon) authorizeDispatch(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget) bool { - if d.Access == nil || d.Principals == nil { - return true - } - principal, ok := access.PrincipalFromContext(ctx) - if !ok { - d.Log.WarnContext(ctx, "binding dispatch: no principal; dispatch left for the next cycle", "binding", b.ID, "capture", c.ID) - return false - } - resource := access.Resource{ - Kind: access.KindWorkflow, ID: b.Workflow, Org: principal.Org, Workspace: b.WorkspaceID, - } - contextValue := access.Context{Signature: signatureOf(c), Addr: c.RemoteAddr} - decision := d.Access.Authorize(ctx, principal, access.ActionRun, resource, contextValue) - if decision.Allowed { - return true - } - if errors.Is(decision.Err, access.ErrChainUnavailable) { - // Nothing was evaluated. Recording it would mark the pair evaluated for - // good, so a State Store blip would drop the capture instead of leaving - // it for the next cycle. - d.Log.WarnContext(ctx, "binding dispatch: access chain unavailable; dispatch left for the next cycle", "binding", b.ID, "capture", c.ID, "err", decision.Err) - return false - } - if d.Denials != nil { - denial := access.Denial{ - Principal: principal.IdentityID, Org: principal.Org, Action: access.ActionRun, - Kind: resource.Kind, ResourceID: resource.ID, Level: decision.Level, - Policies: decision.Policies, - } - if err := d.Denials.RecordDenial(ctx, denial); err != nil { - d.Log.WarnContext(ctx, "binding dispatch: record denial", "binding", b.ID, "error", err) - } - } - d.recordDispatchFailure(ctx, b, c, "access denied", map[string]any{ - "level": string(decision.Level), "policies": decision.Policies, - }) - return false -} - -// signatureOf names the event's signature result as the dispatch context -// carries it: a signed event and an approved unsigned event are the two -// dispatchable shapes (storecontract.CapturedEvent.Dispatchable). -func signatureOf(c storecontract.CapturedEvent) string { - switch { - case c.Authenticated: - return "valid" - case c.Unsigned: - return "unsigned" - } - return "unverified" -} - -// markBindingStart records on a task's timeline the binding and captured -// event that started it. -func (d *Daemon) markBindingStart(ctx context.Context, taskID int64, b binding.Binding, c storecontract.CapturedEvent) { - if _, err := d.Store.InsertEvent(ctx, events.Event{ - TaskID: taskID, - Kind: events.KindBindingStarted, - Detail: fmt.Sprintf("started by binding %q from source %q", b.Name, c.Source), - Data: map[string]any{ - "binding_id": b.ID, "binding_name": b.Name, "binding_version": b.Version, - "capture_id": c.ID, "source": c.Source, "unsigned": c.Unsigned, - }, - }); err != nil { - d.Log.WarnContext(ctx, "binding dispatch: start marker", "task", taskID, "error", err) - } -} - -// recordDispatchFailure records why a binding did not dispatch a capture and -// marks the pair evaluated, so the capture is not offered to it again. -func (d *Daemon) recordDispatchFailure(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, reason string, extra map[string]any) { - d.recordEvaluated(ctx, b, c, reason) - data := map[string]any{ - "binding_id": b.ID, - "binding_version": b.Version, - "capture_id": c.ID, - } - maps.Copy(data, extra) - _, _ = d.Store.InsertEvent(ctx, events.Event{ - Kind: "binding_dispatch_failure", - Detail: fmt.Sprintf("binding %q (capture %q): %s", b.ID, c.ID, reason), - Data: data, - }) -} - -// recordEvaluated writes the ledger row for an evaluation that started no -// task. Only terminal outcomes call it; transient ones leave the capture -// listed for the next cycle. -func (d *Daemon) recordEvaluated(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, reason string) { - err := d.BindingDispatcher.RecordDispatch(ctx, b.ID, int64(b.Version), c.ID, 0, reason) - if err != nil && !errors.Is(err, storecontract.ErrAlreadyDispatched) { - d.Log.WarnContext(ctx, "binding dispatch: record evaluation", "binding", b.ID, "capture", c.ID, "error", err) - } -} - -// hasBlockingFailure reports whether any failure is on a required field. -func hasBlockingFailure(fields []mapping.Field, failures []mapping.Failure) bool { - if len(failures) == 0 { - return false - } - required := make(map[string]bool, len(fields)) - for _, f := range fields { - if f.Required { - required[f.Name] = true - } - } - for _, fail := range failures { - if required[fail.FieldName] { - return true - } - } - return false -} - -// renderBindingBody builds the task body from the values a mapping -// resolved, one "key=value" pair per line. Empty maps still produce -// a body that records the source capture id so the workflow has -// provenance to hand back to the operator. -func renderBindingBody(values map[string]any, c storecontract.CapturedEvent) string { - if len(values) == 0 { - return fmt.Sprintf("(no fields resolved from capture %q)", c.ID) - } - parts := make([]string, 0, len(values)) - for k, v := range values { - parts = append(parts, fmt.Sprintf("%s=%v", k, v)) - } - return strings.Join(parts, "\n") -} - -// resolveBindingRepo returns the binding's pinned owner/repo, or the only -// configured repo. With zero or several repos and no pin it refuses. -func (d *Daemon) resolveBindingRepo(b binding.Binding) (owner, repo, reason string) { - if b.Owner != "" && b.Repo != "" { - return b.Owner, b.Repo, "" - } - repos := d.Cfg.Get().Repos - if len(repos) == 1 { - return repos[0].Owner, repos[0].Name, "" - } - return "", "", fmt.Sprintf("cannot pick a repository: %d are configured; pin the binding to one", len(repos)) -} - -// drainReactions turns available review reactions into queued remediate -// runs. It runs before the task drain so a reaction queued in this pass can -// be claimed by it; the consumer is built lazily once, so its drop counter -// spans the process lifetime. -func (d *Daemon) drainReactions(ctx context.Context) { - if d.Reactions == nil || d.Store == nil { - return - } - if d.reactionConsumer == nil { - d.reactionConsumer = newReactionConsumer( - // The store's owned-PR lookup is the authorization boundary: a - // reaction only ever causes work against a PR archie opened and - // still owns (pr-review-remediation.md decision 3). - d.Store, - d.Store, - d.Store, - d.botUserForTask, - d.Log, - ) - } - if _, err := d.reactionConsumer.drain(ctx, d.Reactions, maxReactionsPerCycle); err != nil { - d.Log.ErrorContext(ctx, "reaction drain failed", "err", err) - } -} - -// botUserForTask resolves the bot login that owns task, so the reaction -// consumer can exclude Archie's own comments: the global forge bot user for -// root tasks. -func (d *Daemon) botUserForTask(task *workflowtask.Task) string { - if !d.rootTask(task) { - return "" - } - return d.Cfg.Get().BotUser -} - -// drainNATS processes tasks from NATS, falling back to the task store's ClaimNext for -// requeued tasks (waiting_approval approval, retry-parked) that didn't come -// through a NATS publish. It returns once the stream and the claim fallback -// are both empty: in-flight tasks keep running, so a task waiting on a callee -// does not hold up the pass that would claim it. Shutdown waits for them -// through WaitForTasks. -func (d *Daemon) drainNATS(ctx context.Context) { - dispatcher := d.taskDispatcher() - // A full dispatcher leaves work in the stream and the store, not parked - // on slots where it already reads as running. - for ctx.Err() == nil && dispatcher.hasRoom() { - msg, err := d.Tasks.Fetch(ctx) - if err != nil && !errors.Is(err, eventbus.ErrNoMessage) { - d.Log.ErrorContext(ctx, "nats fetch failed", "err", err) - break - } - if err == nil { - d.submitNATSTask(ctx, dispatcher, msg) - d.markPass("drain") - continue - } - - // ErrNoMessage means the stream is drained. That is precisely when - // the SQLite fallback must run: requeued tasks (waiting_approval - // approval, retry-parked) never went through a NATS publish, so - // nothing will ever arrive on the stream to trigger them. - d.claimRequeued(ctx, dispatcher) - return - } -} - -// claimRequeued submits tasks from the store's claim fallback while the -// dispatcher has room; the rest stay unclaimed for a later pass. It does -// not go back to the stream between claims: each empty fetch waits a full poll -// timeout, so a backlog left by a restart would outlast the systemd watchdog. -func (d *Daemon) claimRequeued(ctx context.Context, dispatcher *taskDispatcher) { - for ctx.Err() == nil && dispatcher.hasRoom() { - task, err := d.Store.ClaimNext(ctx) - if err != nil { - d.Log.ErrorContext(ctx, "sqlite claim failed", "err", err) - return - } - if task == nil { - return - } - dispatcher.Submit(ctx, task, d.process) - d.markPass("drain") - } -} - -// taskDispatcher returns the daemon's long-lived global dispatcher, building -// it from the running config on first use. It outlives a single drain pass so -// ResizeTaskDispatcher can resize the running dispatcher. -func (d *Daemon) taskDispatcher() *taskDispatcher { - d.dispatcherOnce.Do(func() { - d.dispatcher = newTaskDispatcher(d.Cfg.Get().Containers.MaxConcurrency, d.allowConcurrentForTask) - }) - return d.dispatcher -} - -// WaitForTasks blocks until every submitted task's process has returned. A -// drain pass no longer waits for its own tasks, so shutdown is where in-flight -// work is waited for. -func (d *Daemon) WaitForTasks() { - d.taskDispatcher().Wait() -} - -// ResizeTaskDispatcher applies a new containers.max_concurrency to the -// running global task dispatcher without a restart. Running tasks keep their -// slots; a lowered limit is honoured as they finish, and a raised limit -// admits queued work immediately. -func (d *Daemon) ResizeTaskDispatcher(maxConcurrency int) { - d.taskDispatcher().SetMaxConcurrency(maxConcurrency) -} - -// submitNATSTask decodes a fetched message and queues it for processing. A -// message that cannot be decoded is acked rather than redelivered: it will -// never decode on a retry, so leaving it unacked would block the queue. -func (d *Daemon) submitNATSTask(ctx context.Context, dispatcher *taskDispatcher, msg eventbus.Message) { - envelope, err := workintake.DecodeTask(msg.Data()) - if err != nil { - d.Log.ErrorContext(ctx, "nats decode failed", "subject", msg.Subject(), "err", err) - _ = msg.Ack() - return - } - task := &workflow.Task{Owner: envelope.Owner, Repo: envelope.Repo, Identity: envelope.Identity} - dispatcher.Submit(ctx, task, func(ctx context.Context, _ *workflow.Task, hold *taskHold) { - d.processNATSTask(ctx, msg, hold) - }) -} - -// taskDispatcher limits concurrent tasks globally and runs one task per repo -// at a time, in submit order, unless the repo allows concurrency. -type taskDispatcher struct { - allowConcurrent func(task *workflow.Task) bool - - mu sync.Mutex - limit int - active int - slotsFree *sync.Cond - repoTail map[string]chan struct{} - pending int - // waiting counts submitted tasks that hold no slot yet, so a drain pass - // stops claiming once queued work alone fills the limit. - waiting int - // holds maps a task id to the capacity its run holds, so a run waiting on - // a callee can give that capacity back and take it again by task id. - holds map[int64]*taskHold -} - // taskHold is one submitted task's claim on execution capacity: its slot under // the global limit, and its place at the tail of its repo's chain, which the // next task for that repo is blocked on. A run that waits on a callee gives @@ -1068,248 +264,6 @@ func (h *taskHold) name(id int64, attempt int) { h.dispatcher.mu.Unlock() } -// SetMaxConcurrency accepts a new global limit without a restart. Running -// tasks keep their slots; a lowered limit is enforced as they finish, and a -// raised limit admits queued work immediately. -func (d *taskDispatcher) SetMaxConcurrency(maxConcurrency int) { - d.mu.Lock() - d.limit = maxConcurrency - d.mu.Unlock() - d.slotsFree.Broadcast() -} - -func newTaskDispatcher(maxConcurrency int, allowConcurrent func(task *workflow.Task) bool) *taskDispatcher { - if allowConcurrent == nil { - allowConcurrent = func(*workflow.Task) bool { return false } - } - d := &taskDispatcher{ - limit: maxConcurrency, - allowConcurrent: allowConcurrent, - repoTail: make(map[string]chan struct{}), - holds: make(map[int64]*taskHold), - } - d.slotsFree = sync.NewCond(&d.mu) - return d -} - -// joinRepoLocked appends baton to the tail of repo's chain and returns the -// baton ahead of it, which the joining task is blocked on. Callers hold d.mu. -func (d *taskDispatcher) joinRepoLocked(repo string, baton chan struct{}) chan struct{} { - previous := d.repoTail[repo] - d.repoTail[repo] = baton - return previous -} - -// takeSlotLocked blocks until the global limit has room, then counts the run -// in. limit <= 0 means unlimited. The limit is re-read on every wake, so -// SetMaxConcurrency reaches waiting tasks without recreating the dispatcher. -// Callers hold d.mu. -func (d *taskDispatcher) takeSlotLocked() { - for d.limit > 0 && d.active >= d.limit { - d.slotsFree.Wait() - } - d.active++ -} - -// giveBackLocked gives back everything the run holds: its slot under the -// global limit, and its place in its repo's chain, which releases the task -// queued behind it. Callers hold d.mu. -func (d *taskDispatcher) giveBackLocked(hold *taskHold) { - if hold.slot { - d.active-- - hold.slot = false - } - if hold.baton == nil { - return - } - close(hold.baton) - if d.repoTail[hold.repo] == hold.baton { - delete(d.repoTail, hold.repo) - } - hold.baton = nil -} - -// finishRun retires a returned run: it holds no capacity any more, and no -// Resume may take any for it. Whatever an in-flight resume takes later is -// given back by reacquire, which sees finished. -func (d *taskDispatcher) finishRun(hold *taskHold) { - d.mu.Lock() - hold.finished = true - if hold.id != 0 && d.holds[hold.id] == hold { - delete(d.holds, hold.id) - } - d.giveBackLocked(hold) - d.pending-- - d.mu.Unlock() - d.slotsFree.Broadcast() -} - -// setCallWaits applies a durable snapshot without accumulating duplicate events. -func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) { - d.mu.Lock() - if d.holds[hold.id] != hold || hold.finished { - d.mu.Unlock() - return - } - previous := hold.waits - hold.waits = waits - if waits > 0 { - d.giveBackLocked(hold) - } - d.mu.Unlock() - if waits > 0 { - d.slotsFree.Broadcast() - } else if previous > 0 { - // Reacquiring may block behind another task; the caller keeps running. - go d.reacquire(hold) - } -} - -// reacquire returns a resumed run's capacity: its place in its repo's chain -// and a slot under the global limit. The run keeps executing meanwhile -- a -// wait on a callee is not a pause. A hold records one slot and one place, so a -// re-acquire queued while another is still waiting gives back whichever of the -// two it finds already recorded instead of taking a second. -func (d *taskDispatcher) reacquire(hold *taskHold) { - d.mu.Lock() - if hold.finished || hold.waits > 0 { - // The run returned, or started waiting again, after the reconciliation that - // queued this: it needs no capacity. - d.mu.Unlock() - return - } - var previous chan struct{} - // A place another re-acquire already queued is the run's one place; joining - // behind it would queue a second, which the task behind it would wait on - // forever. - if hold.chain && hold.baton == nil { - hold.baton = make(chan struct{}) - previous = d.joinRepoLocked(hold.repo, hold.baton) - } - d.mu.Unlock() - - if previous != nil { - <-previous - } - - d.mu.Lock() - d.takeSlotLocked() - if hold.slot { - // Another re-acquire took the run's one slot while this waited: give - // back the slot this one just took. The place is the other's, so it is - // left alone. - d.active-- - d.mu.Unlock() - d.slotsFree.Broadcast() - return - } - hold.slot = true - if hold.finished || hold.waits > 0 { - // The run ended, or started waiting again, while this was queued for - // capacity: give back what it took, so no slot or place in the repo - // chain leaks. - d.giveBackLocked(hold) - d.mu.Unlock() - d.slotsFree.Broadcast() - return - } - d.mu.Unlock() -} - -// Submit queues a task's process behind the global limit and, unless the repo -// allows concurrency, behind the task already holding that repo. process is -// handed the run's capacity so it can name it once it knows the task's id and -// give it back while it waits on a callee. -func (d *taskDispatcher) Submit( - ctx context.Context, - task *workflow.Task, - process func(context.Context, *workflow.Task, *taskHold), -) { - repo := task.Owner + "/" + task.Repo - hold := &taskHold{dispatcher: d, repo: repo} - - var previous chan struct{} - if !d.allowConcurrent(task) { - hold.chain = true - hold.baton = make(chan struct{}) - d.mu.Lock() - previous = d.joinRepoLocked(repo, hold.baton) - d.mu.Unlock() - } - - d.mu.Lock() - d.pending++ - d.waiting++ - d.mu.Unlock() - go func() { - defer d.finishRun(hold) - if previous != nil { - <-previous - } - d.mu.Lock() - d.takeSlotLocked() - d.waiting-- - hold.slot = true - d.mu.Unlock() - process(ctx, task, hold) - }() -} - -// hasRoom reports whether a newly submitted task could take a slot without -// queueing behind work already submitted. A run waiting on a callee holds no -// slot, so its callee still finds room. -func (d *taskDispatcher) hasRoom() bool { - d.mu.Lock() - defer d.mu.Unlock() - return d.limit <= 0 || d.active+d.waiting < d.limit -} - -// inFlight is how many submitted tasks have not returned yet. -func (d *taskDispatcher) inFlight() int { - d.mu.Lock() - defer d.mu.Unlock() - return d.pending -} - -func (d *taskDispatcher) Wait() { - d.mu.Lock() - defer d.mu.Unlock() - for d.pending > 0 { - d.slotsFree.Wait() - } -} - -// poll discovers work through every organisation connection, resolved -// afresh each cycle so a rotated credential or a new connection applies to -// the next poll. -func (d *Daemon) poll(ctx context.Context) error { - var failed []string - room := d.Cfg.Get().IntakePerPass - for _, source := range d.issueSources(ctx) { - if !d.identityActive(ctx, source.Identity) { - continue - } - cfg := d.Cfg.Get() - cfg.BotUser = source.BotUser - if source.Dispatch.Trigger != "" { - cfg.Dispatch, cfg.Label = source.Dispatch, source.Label - } - tag := string(source.Identity) - for _, repo := range source.Repos { - issues, complete := d.pollIssuesWithConfig(ctx, source.Forge, cfg, repo) - if complete { - d.withdrawUnpolled(ctx, repo, tag, dispatchRule(cfg), issues) - } else { - failed = append(failed, repo.FullName()) - } - if d.Tasks != nil { - d.publishDiscovered(ctx, source.Forge, cfg, repo, issues, tag, &room) - } - } - } - return passFailed("issue poll", failed) -} - // passFailed reports a maintenance pass whose items failed; each item's cause // is already logged where it happened. func passFailed(pass string, failed []string) error { @@ -1325,966 +279,6 @@ type polledIssues struct { eligible map[int]bool } -// rememberPoll records a repo's latest eligible set and returns the previous -// one, reporting whether it was taken under the same rule. -func (d *Daemon) rememberPoll(key string, current polledIssues) (polledIssues, bool) { - d.lastPolledMu.Lock() - defer d.lastPolledMu.Unlock() - previous, seen := d.lastPolled[key] - if d.lastPolled == nil { - d.lastPolled = map[string]polledIssues{} - } - d.lastPolled[key] = current - return previous, seen && previous.rule == current.rule -} - -// dispatchRule identifies the rule a poll selects issues by. -func dispatchRule(cfg config.Config) string { - return cfg.Dispatch.Trigger + "\x00" + cfg.Label + "\x00" + cfg.BotUser -} - -// withdrawUnpolled declines the queued or running work of an identity's -// issues in repo that a complete poll no longer returns: the issue was -// closed, unlabelled or unassigned. -func (d *Daemon) withdrawUnpolled(ctx context.Context, repo config.Repo, identity, rule string, polled []forge.Issue) { - if d.WithdrawTask == nil { - return - } - eligible := make(map[int]bool, len(polled)) - for _, is := range polled { - eligible[is.Number] = true - } - // Only an issue the same rule found eligible last time and no longer - // finds was withdrawn. A changed label, trigger or bot user, or the first - // poll after a restart, only records the new set: tasks picked up under - // the old rule were not withdrawn by anyone. - previous, sameRule := d.rememberPoll(identity+"\x00"+repo.FullName(), polledIssues{rule: rule, eligible: eligible}) - if !sameRule { - return - } - active, err := d.Store.TasksPage(ctx, storecontract.TaskPage{ - Statuses: []string{taskstate.Queued, taskstate.Running}, Limit: 500, - }) - if err != nil { - d.Log.WarnContext(ctx, "withdrawal check: active tasks unavailable", "repo", repo.FullName(), "err", err) - return - } - for _, t := range active { - if t.Owner != repo.Owner || t.Repo != repo.Name || t.Identity != identity || - !t.IsForgeBacked() || t.PRNumber > 0 || eligible[t.IssueNumber] || !previous.eligible[t.IssueNumber] { - continue - } - if err := d.WithdrawTask(ctx, t.ID, "the issue is no longer assigned, labelled or open"); err != nil { - d.Log.WarnContext(ctx, "withdraw task", "task", t.ID, "err", err) - } - } -} - -// publishDiscovered publishes a repo's discovered issues until the pass has -// created its cap of tasks. room is what is left of the cap; the issues left -// over are found again by the next poll, because an issue with a task is -// skipped and one without is not. A cap of zero or less is no cap. -func (d *Daemon) publishDiscovered(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, issues []forge.Issue, identity string, room *int) { - capped := *room > 0 - for _, is := range issues { - if capped && *room == 0 { - d.Log.InfoContext(ctx, "intake cap reached; remaining issues wait for the next poll", "repo", repo.FullName()) - return - } - if d.pollNATS(ctx, fg, cfg, repo, is, strings.Join(is.Labels, ","), identity) && capped { - *room-- - } - } -} - -// pollNATS publishes a discovered issue to NATS and reports whether it created -// a task. -func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) bool { - // Read-only existence check prevents rediscovering an existing task. - existing, err := d.Store.TaskByIssue(ctx, repo.Owner, repo.Name, is.Number) - if err != nil { - d.Log.ErrorContext(ctx, "task lookup failed", "repo", repo.FullName(), "issue", is.Number, "err", err) - return false - } - if existing != nil { - return false - } - // An issue whose label-routed workflow the identity's org disabled must - // not queue a task at all. Skip without acknowledging, so a later poll - // picks the issue up again if the workflow is re-enabled. - if routed, disabled, err := d.disabledRoutedWorkflow(ctx, labels, identity); err != nil { - d.Log.WarnContext(ctx, "routed workflow check failed; publishing anyway", "repo", repo.FullName(), "issue", is.Number, "err", err) - } else if disabled { - d.Log.InfoContext(ctx, "issue skipped: its routed workflow is disabled for this org", "repo", repo.FullName(), "issue", is.Number, "workflow", routed) - return false - } - // Classify and encode here: the envelope, its routing kind and its - // subject belong to the work-intake domain, not to the transport. - parsed := workintake.SplitLabels(labels) - task := workintake.TaskEnvelope{ - Owner: repo.Owner, - Repo: repo.Name, - Number: is.Number, - Title: is.Title, - Body: is.Body, - Labels: parsed, - Identity: identity, - Kind: workintake.KindForLabels(parsed), - } - if err := d.PublishTask(ctx, task); err != nil { - d.Log.ErrorContext(ctx, "task publish failed", "repo", repo.FullName(), "issue", is.Number, "err", err) - return false - } - d.acknowledge(ctx, fg, cfg, repo, is) - return true -} - -// PublishTask validates a task envelope, stamps the producing identity's -// org and publishes it. The idempotency key dedups rediscovered issues. -func (d *Daemon) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error { - if d.Tasks == nil { - return fmt.Errorf("publish task %s: no task bus configured", task.Ref()) - } - if err := task.Kind.Validate(); err != nil { - return fmt.Errorf("publish task %s: %w", task.Ref(), err) - } - if task.Identity == "" { - return fmt.Errorf("publish task %s: no identity", task.Ref()) - } - resolved, err := d.identityOrg(ctx, identity.IdentityID(task.Identity)) - if err != nil { - return fmt.Errorf("publish task %s: resolve org: %w", task.Ref(), err) - } - task.Org = resolved - payload, err := task.Encode() - if err != nil { - return err - } - return d.Tasks.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload) -} - -// PublishReaction stamps the org on a review reaction, defaulting to the -// root identity's, and publishes it. -func (d *Daemon) PublishReaction(ctx context.Context, reaction workintake.ReviewCommentEnvelope) error { - if d.Tasks == nil { - return fmt.Errorf("publish reaction %s: no task bus configured", reaction.Ref()) - } - if reaction.Org == "" { - resolved, err := d.identityOrg(ctx, identity.SystemID) - if err != nil { - return fmt.Errorf("publish reaction %s: resolve org: %w", reaction.Ref(), err) - } - reaction.Org = resolved - } - payload, err := reaction.Encode() - if err != nil { - return err - } - return d.Tasks.PublishUnique(ctx, reaction.Subject(), reaction.IdempotencyKey(), payload) -} - -// identityOrg resolves an identity's org. No PrincipalSource means the -// default org; a store error is returned. -func (d *Daemon) identityOrg(ctx context.Context, id identity.IdentityID) (org.OrgID, error) { - if d.Principals == nil { - return org.DefaultOrgID, nil - } - principal, err := d.Principals.PrincipalFor(ctx, id) - if err != nil { - return "", err - } - return principal.Org, nil -} - -// acknowledge posts the pickup reaction -// using fg -- the forge client that owns repo (identity-scoped or root) -- -// and the ack reaction from cfg, the dispatch config that owns this poll. -func (d *Daemon) acknowledge(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue) { - d.Log.InfoContext(ctx, "issue queued", "repo", repo.FullName(), "issue", is.Number, "title", is.Title) - if ack := cfg.Dispatch.AckReaction; ack != "" { - if err := fg.React(ctx, repo.Owner, repo.Name, is.Number, ack); err != nil { - d.Log.WarnContext(ctx, "ack reaction failed", "issue", is.Number, "err", err) - } - } -} - -// emitTaskQueued records the queued task under the identity that polled it, -// which is both the actor and the authority the task was created with. -func (d *Daemon) emitTaskQueued(ctx context.Context, task *workflow.Task) { - e := events.Event{ - Kind: events.KindTaskQueued, TaskID: task.ID, Repo: task.Owner + "/" + task.Repo, - Issue: task.IssueNumber, Detail: task.Title, Attempt: task.Attempt, - ActorID: task.Identity, PrincipalID: task.Identity, - } - if d.IdentityRepository != nil { - if value, err := d.IdentityRepository.Get(ctx, identity.IdentityID(task.Identity)); err == nil { - e.ActorKind = string(value.Kind) - } - } - d.emit(ctx, e) -} - -func (d *Daemon) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, name string) bool { - if ok, reason, class := d.identityMayAct(ctx, name); !ok { - d.Log.ErrorContext(ctx, "nats intake rejected: "+reason, "subject", msg.Subject()) - if class == taskstate.ParkTransient { - if err := msg.Nak(); err != nil { - d.Log.WarnContext(ctx, "nak failed", "err", err) - } - return false - } - if err := msg.Ack(); err != nil { - d.Log.WarnContext(ctx, "ack failed", "err", err) - } - return false - } - - return true -} - -// processNATSTask decodes a NATS message, writes it to SQLite, claims it, -// and runs the workflow. The message is ack'd on terminal (park is a valid -// outcome). Nak on transient errors so NATS redelivers. -func (d *Daemon) processNATSTask(ctx context.Context, msg eventbus.Message, hold *taskHold) { - ctx, span := natsrpc.StartConsumer(ctx, msg.Subject(), msg.Header()) - defer span.End() - tm, err := workintake.DecodeTask(msg.Data()) - if err != nil { - d.Log.ErrorContext(ctx, "nats decode failed", "err", err) - if err := msg.Ack(); err != nil { - d.Log.WarnContext(ctx, "ack failed", "err", err) - } - // bad message, don't retry - return - } - - if !d.acceptTaskPublisher(ctx, msg, tm.Identity) { - return - } - - inserted, err := d.Store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity) - if err != nil { - d.Log.ErrorContext(ctx, "nats enqueue failed", "err", err) - if err := msg.Nak(); err != nil { - d.Log.WarnContext(ctx, "nats nak failed", "err", err) - } - return - } - if !inserted { - // Already tracked -- dedup. Ack and move on. - if err := msg.Ack(); err != nil { - d.Log.WarnContext(ctx, "ack failed", "err", err) - } - return - } - - task, err := d.Store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number) - if err != nil { - d.Log.ErrorContext(ctx, "nats claim failed", "err", err) - if err := msg.Nak(); err != nil { - d.Log.WarnContext(ctx, "nats nak failed", "err", err) - } - return - } - if task == nil { - // Claimed by another consumer, or task is not queued. Ack. - if err := msg.Ack(); err != nil { - d.Log.WarnContext(ctx, "ack failed", "err", err) - } - return - } - - d.emitTaskQueued(ctx, task) - d.process(ctx, task, hold) - if err := msg.Ack(); err != nil { - d.Log.WarnContext(ctx, "ack failed", "err", err) - } -} - -// closeResolvedIssue closes a finished task's forge issue, skipping chat -// tasks. Failures are logged. -func (d *Daemon) closeResolvedIssue(ctx context.Context, fg forge.Forge, task *workflow.Task, comment string) { - if !task.IsForgeBacked() { - return - } - if d.ResolveRepository != nil { - resolved, err := d.Repository(ctx, task, config.OperationPR) - if err != nil { - d.Log.WarnContext(ctx, "issue closure not permitted", "task", task.ID) - return - } - fg = resolved.Forge - } - if err := fg.CloseIssue(ctx, task.Owner, task.Repo, task.IssueNumber, comment); err != nil { - d.Log.WarnContext(ctx, "closing resolved issue failed; it stays open and will be re-polled", - "repo", task.Owner+"/"+task.Repo, "issue", task.IssueNumber, "err", err) - } -} - -// reconcilePRs moves pr_open tasks to merged/rejected from GitHub state -// and cleans up their worktrees. -func (d *Daemon) reconcilePRs(ctx context.Context) error { - tasks, err := d.Store.OpenPRs(ctx) - if err != nil { - return fmt.Errorf("reconcile query: %w", err) - } - var failed []string - for _, t := range tasks { - fg := d.forgeFor(ctx, &t) - if _, unavailable := fg.(*forge.NoopForge); unavailable { - continue - } - trees := d.treesFor(&t) - // PRState/PR reconciliation is valid regardless of Source -- a - // chat-sourced task can still open a real PR against a real - // branch. Only the issue-label call (tied to the synthetic - // issue number) is forge-only. - state, err := fg.PRState(ctx, t.Owner, t.Repo, t.PRNumber) - if err != nil { - d.Log.WarnContext(ctx, "PR state check failed", "pr", t.PRNumber, "err", err) - failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber)) - continue - } - attempt := t.Attempt - switch state { - case "merged": - if err := d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusMerged, ""); err != nil { - d.Log.ErrorContext(ctx, "PR merged but task transition failed", "task", t.ID, "pr", t.PRNumber, "err", err) - failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber)) - continue - } - if err := trees.Cleanup(t.Owner, t.Repo, t.IssueNumber); err != nil { - d.Log.WarnContext(ctx, "worktree cleanup failed", "task", t.ID, "err", err) - } - // Close the issue explicitly; nothing else does. - d.closeResolvedIssue(ctx, fg, &t, fmt.Sprintf( - "Closed by #%d, merged by archie.", t.PRNumber, - )) - d.Log.InfoContext(ctx, "PR merged", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber) - d.emit(ctx, events.Event{ - Kind: events.KindPRMerged, TaskID: t.ID, Attempt: attempt, - Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber, - Data: map[string]any{"pr": t.PRNumber}, - }) - case "closed": - if err := d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusRejected, "PR closed without merge"); err != nil { - d.Log.ErrorContext(ctx, "PR rejected but task transition failed", "task", t.ID, "pr", t.PRNumber, "err", err) - failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber)) - continue - } - if err := trees.Cleanup(t.Owner, t.Repo, t.IssueNumber); err != nil { - d.Log.WarnContext(ctx, "worktree cleanup failed", "task", t.ID, "err", err) - } - d.Log.InfoContext(ctx, "PR rejected", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber) - d.emit(ctx, events.Event{ - Kind: events.KindPRRejected, TaskID: t.ID, Attempt: attempt, - Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber, - Data: map[string]any{"pr": t.PRNumber}, - }) - } - } - return passFailed("PR reconcile", failed) -} - -func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHold) { - // A task off the NATS stream only has an id once the store has claimed its - // row, inside this call, so its capacity is named here. - hold.name(task.ID, task.Attempt) - // The log opens before any check that can park, so every park reason - // reaches the attempt log. - defer d.openTaskLog(task)() - // The workflow may name the identity the run acts as, which everything - // below is chosen by, so it is settled before anything else. - if !d.adoptWorkflowIdentity(ctx, task) { - return - } - // Register first so the task can be stopped during setup. - ctx, finished := d.running.begin(ctx, task.ID, task.Identity) - defer finished() - - if d.parkIfIdentityUnresolvable(ctx, task) { - return - } - - trees, repo, err := d.taskWorkspace(ctx, task) - if err != nil { - d.parkBlockedTask(ctx, task, err.Error(), pinParkClass(err), parkBlocker(err)) - return - } - - if d.ContainerPool == nil || d.Tasks == nil { - d.parkCapabilityUnavailable(ctx, task) - return - } - - d.Log.InfoContext(ctx, "processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "attempt", task.Attempt) - workDir, ok := d.prepareWorkspace(ctx, task, trees, repo) - if !ok { - return - } - if !task.HasRepository() { - defer func() { - if err := trees.RemoveScratch(task.ID); err != nil { - d.Log.WarnContext(ctx, "scratch workspace cleanup failed", "task", task.ID, "err", err) - } - }() - } - - // The workflow is pinned before the container starts, because the - // profile it names decides the container's image. - profile, ok := d.pinTaskProfile(ctx, task) - if !ok { - return - } - ctr, credential, revokeCredential, ok := d.acquireTaskContainer(ctx, task, repo, workDir, profile.Image) - if !ok { - return - } - defer d.ContainerPool.Release(ctx, ctr) - defer revokeCredential() - - // Run the whole task in archie-agent, which calls back to the daemon for - // store, forge and push operations. - limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std()) - runCtx, stopWatch := withContainerExit(limitCtx, ctr) - d.runViaAgent(runCtx, task, repo, profile, credential) - stopWatch() - stopLimit() - - d.teardownStorage(ctx, task, repo, workDir) - d.cleanupTerminalTaskWorktree(ctx, task, trees) -} - -// teardownStorage runs after the workflow completes. The Docker backend is -// a no-op; future backends (temp volumes, NFS leases) use this hook. -func (d *Daemon) teardownStorage(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string) { - if d.Storage == nil { - return - } - if err := d.Storage.Teardown(ctx, storage.TaskRef{ - WorktreeDir: workDir, - Ecosystem: repo.Ecosystem, - PersistentStorage: repo.PersistentStorage, - Owner: task.Owner, - Repo: task.Repo, - }); err != nil { - d.Log.WarnContext(ctx, "storage teardown failed", "task", task.ID, "err", err) - } -} - -func (d *Daemon) taskWorkspace(ctx context.Context, task *workflow.Task) (*worktree.Manager, config.Repo, error) { - if task.HasRepository() && d.ResolveRepository != nil { - resolved, err := d.Repository(ctx, task, config.OperationClone) - if err == nil && !task.IsTestRun() && workflow.DefinitionPushes(task.WorkflowDefinitionYAML) { - // Checked before the run rather than at its commit step, so a - // missing permit parks in seconds instead of after the agent's work. - _, err = d.Repository(ctx, task, config.OperationPush) - } - return resolved.Trees, resolved.Repository, err - } - trees := d.treesFor(task) - repo, ok := d.repoFor(task) - if !ok && task.HasRepository() { - return nil, repo, pinFailure{taskstate.ParkTerminal, errors.New("repository is no longer configured")} - } - return trees, repo, nil -} - -// prepareWorkspace makes the directory the container binds as its -// workspace, which must exist before Acquire: a fresh clone of the task's -// repository, or an empty scratch directory for a task with none. It parks -// the task and reports false when it cannot. -func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, trees *worktree.Manager, repo config.Repo) (string, bool) { - if !task.HasRepository() { - dir, err := trees.PrepareScratch(task.ID) - if err != nil { - d.Log.ErrorContext(ctx, "scratch workspace prepare failed", "err", err) - d.parkRunningTask(ctx, task, "scratch workspace prepare failed: "+err.Error(), taskstate.ParkTransient) - return "", false - } - return dir, true - } - // The retry mode on the task decides whether to resume the pushed branch or - // reset onto base. Prepare clones a missing worktree either way. - target := worktree.Fresh - if taskstate.NormalizeRetryMode(task.RetryMode) == taskstate.RetryKeepWorktree { - target = worktree.Keep - // Consumed: a later plain requeue must not skip the reset. - task.RetryMode = "" - } - if taskstate.NormalizeRetryMode(task.RetryMode) == taskstate.RetryContinuePushedWork { - if task.Branch == "" { - // The action refuses this combination, so an empty branch here is a - // dispatch bug upstream and no retry can fix it. - d.Log.ErrorContext(ctx, "task has no persisted branch to continue", "task", task.ID) - d.parkRunningTask(ctx, task, "worktree resume failed: task has no branch to continue", taskstate.ParkNeedsDecision) - return "", false - } - target = worktree.Target(task.Branch) - } - d.reclaimStaleClones(ctx, task, trees) - if err := worktree.AwaitSpace(ctx, trees.FreeBytes, minCloneSpace, diskCheckEvery, diskWaitLimit); err != nil { - d.Log.WarnContext(ctx, "worktree prepare delayed", "task", task.ID, "err", err) - d.parkRunningTask(ctx, task, "worktree prepare failed: "+err.Error(), taskstate.ParkTransient) - return "", false - } - // Every task gets an independent full clone. - prepare := func() (string, string, error) { - return trees.Prepare(ctx, task.Owner, task.Repo, repo.BaseBranch(), task.IssueNumber, task.Title, task.Body, task.Labels, target) - } - dir, branch, err := prepare() - if d.reownDenied(ctx, trees.Dir(task.Owner, task.Repo, task.IssueNumber), err) { - dir, branch, err = prepare() - } - if err != nil { - reason := "worktree prepare failed: " + err.Error() - if target != worktree.Fresh { - reason = "worktree resume failed: " + err.Error() - } - d.Log.ErrorContext(ctx, "worktree prepare failed", "task", task.ID, "err", err) - d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient) - return "", false - } - task.Branch = branch - if err := d.Store.Update(ctx, task); err != nil { - d.Log.WarnContext(ctx, "task branch not persisted", "task", task.ID, "err", err) - } - return dir, true -} - -// reclaimStaleClones frees the work volume before a clone waits on it, by -// removing the oldest clones nothing has touched for cloneRetention. A clone -// with uncommitted or unpushed work is kept. -func (d *Daemon) reclaimStaleClones(ctx context.Context, task *workflow.Task, trees *worktree.Manager) { - if have, err := trees.FreeBytes(); err != nil || have >= minCloneSpace { - return - } - n, err := worktree.ReclaimStale(d.Cfg.Get().WorkDir, trees.Dir(task.Owner, task.Repo, task.IssueNumber), cloneRetention, trees.FreeBytes, minCloneSpace, func(dir string) bool { return trees.HoldsUnsharedWork(ctx, dir) }) - if err != nil { - d.Log.WarnContext(ctx, "stale clone reclaim failed", "task", task.ID, "err", err) - } - if n > 0 { - d.Log.InfoContext(ctx, "reclaimed stale clones on a full work volume", "task", task.ID, "clones", n) - } -} - -// cleanupTerminalTaskWorktree removes a settled task's worktree, unless it -// holds uncommitted work or its state cannot be read. -func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow.Task, trees *worktree.Manager) { - if d.Store == nil || trees == nil || task == nil || !task.HasRepository() { - return - } - cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - defer cancel() - - latest, err := d.Store.TaskByID(cleanupCtx, task.ID) - if err != nil { - d.Log.WarnContext(ctx, "terminal worktree cleanup task lookup failed", "task", task.ID, "err", err) - return - } - // A run that handed off shares its work item's worktree with its - // successor, which owns it from here. - if latest == nil || latest.HandoffToTaskID != 0 { - return - } - // Settled statuses. Keep the worktree while a PR exists. - switch latest.Status { - case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted, workflow.StatusDead: - if latest.PRNumber != 0 { - return - } - if d.worktreeHoldsUncapturedWork(cleanupCtx, trees, task) { - return - } - if err := d.cleanupWorktree(cleanupCtx, trees, task.Owner, task.Repo, task.IssueNumber); err != nil { - d.Log.WarnContext(ctx, "terminal worktree cleanup failed", "task", task.ID, "err", err) - } - } -} - -// worktreeHoldsUncapturedWork reports whether a terminal task's worktree has -// to survive cleanup because it may hold work no commit captured. A worktree -// that is not on disk at all has nothing to keep, so a task whose clone was -// already removed still cleans up whatever leftovers share its path. -func (d *Daemon) worktreeHoldsUncapturedWork(ctx context.Context, trees *worktree.Manager, task *workflow.Task) bool { - dir := trees.Dir(task.Owner, task.Repo, task.IssueNumber) - if _, err := os.Stat(dir); err != nil { - if errors.Is(err, fs.ErrNotExist) { - return false - } - d.Log.WarnContext(ctx, "terminal worktree kept: it could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err) - return true - } - uncommitted, err := trees.HasUncommittedChanges(ctx, dir) - if err != nil { - d.Log.WarnContext(ctx, "terminal worktree kept: uncommitted work could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err) - return true - } - if uncommitted { - d.Log.InfoContext(ctx, "terminal worktree kept: it holds uncommitted work; archive the task to remove it", "task", task.ID, "dir", dir) - return true - } - return false -} - -// openTaskLog opens the task's log sink for one process() call and returns -// its closer. Failure to open is logged, not fatal. -func (d *Daemon) openTaskLog(task *workflow.Task) func() { - if err := d.TaskLogs.Open(task.ID, task.Attempt); err != nil { - d.Log.Warn("task log sink unavailable", "task", task.ID, "attempt", task.Attempt, "err", err) - } - return func() { - if err := d.TaskLogs.Close(task.ID); err != nil { - d.Log.Warn("task log sink close failed", "task", task.ID, "err", err) - } - } -} - -// acquireTaskContainer prepares and acquires the sandbox container for a -// task, reporting false when it could not -- having already parked the -// task and logged why. The caller releases the container it returns. -func (d *Daemon) acquireTaskContainer( - ctx context.Context, - task *workflow.Task, - repo config.Repo, - workDir string, - image string, -) (*container.Container, string, func(), bool) { - park := func(reason string, err error) { - d.Log.ErrorContext(ctx, reason, "err", err) - d.parkBlockedTask(ctx, task, reason+": "+err.Error(), taskstate.ParkTransient, parkBlocker(err)) - } - - // Resolve the PR number from inputs before the brief is written. - if task.Workflow == "pr-review" { - task.PRNumber = task.EffectivePRNumber() - } - - if err := writeTaskBrief(workDir, task); err != nil { - park("task.json write failed", err) - return nil, "", nil, false - } - - // Fetch the pull request with the daemon's forge credential into workDir, - // so the container reads it without credentials. - if task.Workflow == "pr-review" { - if err := prefetchPRReview(ctx, d.forgeFor(ctx, task), task.Owner, task.Repo, task.PRNumber, workDir); err != nil { - park("pr-review prefetch failed", err) - return nil, "", nil, false - } - } - - // Guard: Storage may be nil if the daemon was wired incorrectly. In - // normal operation, Storage is always set when ContainerPool is set. - if d.Storage == nil { - d.Log.ErrorContext(ctx, "storage backend is nil -- cannot acquire container") - d.parkRunningTask(ctx, task, "storage backend not configured", taskstate.ParkTransient) - return nil, "", nil, false - } - - mounts, err := d.Storage.Setup(ctx, storage.TaskRef{ - WorktreeDir: workDir, - Ecosystem: repo.Ecosystem, - PersistentStorage: repo.PersistentStorage, - Owner: task.Owner, - Repo: task.Repo, - }) - if err != nil { - park("storage setup failed", err) - return nil, "", nil, false - } - - credential, revokeCredential, err := d.runCredential(ctx, task) - if err != nil { - d.teardownStorage(ctx, task, repo, workDir) - park("run credential failed", err) - return nil, "", nil, false - } - - model, closeModel, err := d.openModelEgress(ctx, task, credential) - if err != nil { - revokeCredential() - d.teardownStorage(ctx, task, repo, workDir) - park("model egress failed", err) - return nil, "", nil, false - } - revokeRun := revokeCredential - revokeCredential = func() { closeModel(); revokeRun() } - - // The container lives in its own network namespace, so a loopback State - // Store target names the container itself. Give this task an address on - // the host bridge that forwards to it, closed with the task. - stateStoreURL, relay, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential) - if err != nil { - revokeCredential() - d.teardownStorage(ctx, task, repo, workDir) - park("state store relay failed", err) - return nil, "", nil, false - } - revokeCredential = release - - ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(d.containerEnv(task, credential, stateStoreURL), model.Env...)) - if err != nil { - revokeCredential() - // Roll back the storage we just set up: the mounts were created - // for this task but no container will use them. Leaking them until - // a later TTL sweep is not acceptable on a backend that allocates - // real resources (volumes, NFS leases). - d.teardownStorage(ctx, task, repo, workDir) - park("container acquire failed", err) - return nil, "", nil, false - } - d.admitToRelay(ctx, relay, task, ctr) - return ctr, credential, revokeCredential, true -} - -// writeTaskBrief writes task.json, the container's boot-time brief. -func writeTaskBrief(workDir string, task *workflow.Task) error { - return container.WriteTaskJSON(workDir, container.TaskPayload{ - ID: task.ID, Owner: task.Owner, Repo: task.Repo, - Number: task.IssueNumber, Title: task.Title, Body: task.Body, - Labels: strings.Split(task.Labels, ","), - Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan, - Inputs: task.Inputs, - }) -} - -// parkCapabilityUnavailable parks a task whose daemon-side capability is -// missing (container pool or task transport): both are environmental, so -// the park class is transient and the KindParked event carries the class -// for the dashboard. The reason distinguishes the two in logs and events. -func (d *Daemon) parkCapabilityUnavailable(ctx context.Context, task *workflow.Task) { - var reason string - if d.ContainerPool == nil { - reason = "managed agent container pool is unavailable; refusing to run this task on the host" - d.Log.ErrorContext(ctx, "task parked: managed worker unavailable", "task", task.ID, - "hint", "enable containers and make the archie-agent image available, then retry the task") - } else { - reason = "agent task transport is unavailable; refusing to run this task on the host" - d.Log.ErrorContext(ctx, "task parked: agent task transport unavailable", "task", task.ID) - } - d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient) -} - -// parkRunningTask moves a running task to parked with class, using a -// bounded context that ignores cancellation. ErrStaleTransition is ignored; -// other errors are logged. -func (d *Daemon) parkRunningTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass) { - d.parkBlockedTask(ctx, task, reason, class, "") -} - -// parkBlockedTask is parkRunningTask recording the blocker the park site -// knows, so the dashboard can link to its fix. -func (d *Daemon) parkBlockedTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass, blocker taskstate.Blocker) { - if d.Store == nil { - return - } - writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - defer cancel() - - if err := d.Store.ParkTask(writeCtx, task.ID, workflow.StatusRunning, reason, class, blocker); err != nil { - if !errors.Is(err, storecontract.ErrStaleTransition) { - d.Log.WarnContext(ctx, "terminal park transition failed", "task", task.ID, "reason", reason, "err", err) - } - return - } - // Recorded only once the transition landed. An entry claiming the task - // parked would be a lie when another actor moved it out of running first - // -- the same race the parked event is withheld for. - d.recordPark(ctx, task.ID, reason) - d.emit(ctx, events.Event{ - Kind: events.KindParked, TaskID: task.ID, Attempt: task.Attempt, - Repo: strings.Trim(task.Owner+"/"+task.Repo, "/"), Issue: task.IssueNumber, Workflow: task.Workflow, - Detail: reason, Data: map[string]any{"park_class": class}, - }) -} - -// recordPark writes a park reason to the attempt's log. Dropped when no sink -// is open. -func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) { - d.TaskLogs.Write(ctx, taskID, logging.Entry{ - Time: time.Now(), - Level: slog.LevelError.String(), - Message: "task parked: " + reason, - Fields: map[string]any{"component": "daemon", "task": taskID}, - }) -} - -// runViaAgent hands a task to archie-agent and waits for its result. It parks -// the task only when the agent never answered or failed before recording an -// outcome. -// -// credential is the task's run credential; the agent presents it to push the -// task's branch. -func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo config.Repo, profile config.AgentProfile, credential string) { - cfg, err := d.taskConfig(ctx, task) - if err != nil { - d.parkRunningTask(ctx, task, err.Error(), taskstate.ParkTransient) - return - } - taskCfg := cfg.ForTask() - d.captureAttemptConfig(ctx, task, taskCfg) - var instructions workflow.Instructions - if d.TaskInstructions != nil { - instructions.Instance, instructions.Organisation, err = d.TaskInstructions.TaskInstructions(ctx, task.ID) - if err != nil { - d.parkRunningTask(ctx, task, "resolve the task's instructions: "+err.Error(), taskstate.ParkTransient) - return - } - } - req := taskrun.Request{ - Instructions: instructions, - Task: task, - Repo: repo, - Cfg: taskCfg, - Providers: agentexec.ProvidersFromConfig(cfg.Providers), - MCPServers: cfg.Tools.MCPServers, - RunCredential: credential, - WorkflowDefinition: task.WorkflowDefinitionYAML, - Tools: d.toolAllowlist(ctx, task, profile.Tools), - } - req.Trace = map[string]string{} - otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(req.Trace)) - data, err := json.Marshal(req) - if err != nil { - d.Log.ErrorContext(ctx, "taskrun encode failed", "task", task.ID, "err", err) - d.parkRunningTask(ctx, task, "taskrun encode failed: "+err.Error(), taskstate.ParkTransient) - return - } - - reply, err := d.requestTaskRun(ctx, task.ID, data) - if err != nil { - if stoppingOnly(ctx) { - d.Log.InfoContext(ctx, "daemon stopping; task left running for the next daemon to re-queue", "task", task.ID) - return - } - err = runFailure(ctx, err) - d.Log.ErrorContext(ctx, "taskrun request failed", "task", task.ID, "err", err) - d.parkRunningTask(ctx, task, "taskrun request failed: "+err.Error(), taskstate.ParkTransient) - return - } - - var resp taskrun.Response - if err := json.Unmarshal(reply, &resp); err != nil { - d.Log.ErrorContext(ctx, "taskrun decode response failed", "task", task.ID, "err", err) - d.parkRunningTask(ctx, task, "taskrun decode response failed: "+err.Error(), taskstate.ParkTransient) - return - } - if resp.Error != "" { - d.Log.ErrorContext(ctx, "taskrun run failed", "task", task.ID, "err", resp.Error) - d.parkRunningTask(ctx, task, "taskrun run failed: "+resp.Error, taskstate.ParkTransient) - return - } - // The agent applies the task's own outcome transition, so a status the - // task cannot reach from running means nothing landed: empty, unknown, or - // still running. Transient, because the cause is environmental -- a State - // Store that refused the transition -- and a retry can record one. - if !taskstate.CanTransition(taskstate.Running, resp.Status) { - reason := noOutcomeReason(resp.Status) - d.Log.ErrorContext(ctx, "taskrun run recorded no outcome", "task", task.ID, "status", resp.Status) - d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient) - return - } - - // Reached only past every failure return above, so this is always a - // fully decoded, error-free response -- a version is never recorded off - // a request that never reached an agent, or a response describing a - // parked/errored task. - if d.AgentStatus != nil && resp.AgentVersion != "" { - d.AgentStatus.Observe(resp.AgentVersion, resp.AgentInstallType) - } - - d.Log.InfoContext(ctx, "taskrun complete", "task", task.ID, "status", resp.Status) -} - -// noOutcomeReason names why a taskrun reply left the task without an outcome, -// quoting the status so the log line points at what the agent actually said. -func noOutcomeReason(status string) string { - if status == "" { - return "agent returned no status: the run could not record an outcome" - } - return fmt.Sprintf("agent returned unrecognised status %q: the run could not record an outcome", status) -} - -// pinTaskProfile pins the task's workflow definition and resolves its agent -// profile, parking the task on failure. -func (d *Daemon) pinTaskProfile(ctx context.Context, task *workflow.Task) (config.AgentProfile, bool) { - if err := d.pinWorkflowDefinition(ctx, task); err != nil { - d.Log.ErrorContext(ctx, "pin workflow definition failed", "task", task.ID, "err", err) - d.parkBlockedTask(ctx, task, "pin workflow definition: "+err.Error(), pinParkClass(err), parkBlocker(err)) - return config.AgentProfile{}, false - } - iface, err := workflowtask.ParseWorkflowInterface(task.WorkflowDefinitionYAML) - if err == nil { - // Every producer meets the declared interface here, including the - // ones that cannot read definitions when they enqueue (chat and - // dashboard spawns go through the Gateway, which never does). - err = iface.CheckInputs(task.Inputs) - } - if err == nil && iface.RepositoryMode() == workflowtask.RepositoryRequired && !task.HasRepository() { - // An optional workflow may hand off to one that requires a - // repository, because its task may carry one; a task that does not - // is parked here rather than run the target's steps in a scratch - // workspace. - err = fmt.Errorf("requires a repository and this task has none") - } - if err == nil { - var profile config.AgentProfile - if profile, err = d.configFor(task).Containers.Profile(iface.Profile); err == nil { - return profile, true - } - } - d.Log.ErrorContext(ctx, "agent profile unavailable", "task", task.ID, "err", err) - d.parkRunningTask(ctx, task, "workflow "+task.Workflow+": "+err.Error(), taskstate.ParkNeedsDecision) - return config.AgentProfile{}, false -} - -// pinWorkflowDefinition ensures the task carries a valid definition pin for -// the workflow it names and that its org has the workflow enabled. Every run -// passes through here, whatever started it, so this is the one enablement -// check for issue labels, handoffs, approvals, calls and playbooks alike. -func (d *Daemon) pinWorkflowDefinition(ctx context.Context, task *workflow.Task) error { - ctx, err := d.workflowContext(ctx, task) - if err != nil { - return err - } - if err := d.pinDefinition(ctx, task); err != nil { - return err - } - enablement, err := d.workflowEnablement(ctx) - if err != nil { - return pinFailure{taskstate.ParkTransient, fmt.Errorf("workflow enablement: %w", err)} - } - if !enablement.Enabled(task.Org, task.Workflow) { - return blockedPark{taskstate.BlockerWorkflowDisabled, pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("workflow %s is disabled", task.Workflow)}} - } - if err := d.checkPackageTrigger(ctx, task); err != nil { - return pinFailure{taskstate.ParkNeedsDecision, err} - } - return nil -} - -// pinDefinition keeps a valid pin and resolves a missing or mismatched one -// again from the active definitions. -func (d *Daemon) pinDefinition(ctx context.Context, task *workflow.Task) error { - pinned, ok, err := workflow.PinnedDefinitionID(task) - if err != nil { - return pinFailure{taskstate.ParkNeedsDecision, err} - } - if ok && pinned == task.Workflow { - return nil - } - d.runActionPlaybooks(ctx, task) - if d.WorkflowDefinitions == nil { - return d.pinWorkflowFromCollection(ctx, task, workflow.ShippedDefinitions(), 0) - } - collection, version, err := d.WorkflowDefinitions.WorkflowDefinitions(ctx) - if err != nil { - return pinFailure{taskstate.ParkTransient, err} - } - return d.pinWorkflowFromCollection(ctx, task, collection, version) -} - // pinFailure carries the park class for a pin failure: operator-actionable // for a bad pin or missing workflow, transient for a store error. type pinFailure struct { @@ -2339,505 +333,3 @@ func accessBlocker(code string) taskstate.Blocker { } return "" } - -// runActionPlaybooks runs the action playbooks matching a task before it is -// pinned. Failures never fail the task; tasks already naming a workflow are -// skipped. -func (d *Daemon) runActionPlaybooks(ctx context.Context, task *workflow.Task) { - if d.Playbooks == nil || d.PlaybookLedger == nil || task.Workflow != "" { - return - } - if err := d.Playbooks.Run(ctx, d.PlaybookLedger, d.Log, playbookInput(task)); err != nil { - d.Log.WarnContext(ctx, "action playbook run failed", "task", task.ID, "err", err) - } -} - -// resolveWorkflowID picks the definition to pin: a matching playbook, then -// the kind defaults. A task already naming a workflow keeps it. -func (d *Daemon) resolveWorkflowID(ctx context.Context, task *workflow.Task, available map[string]struct{}) (string, error) { - if d.Playbooks == nil || task.Workflow != "" { - return workflow.ResolveWorkflowID(task, available) - } - decision, matched := d.Playbooks.Dispatch(ctx, playbookInput(task)) - if !matched { - return workflow.ResolveWorkflowID(task, available) - } - if _, defined := available[decision.Workflow]; !defined { - // Reported, not silently downgraded to the binding's choice: the - // operator bound this trigger to a named workflow, and running a - // different one under their rule is worse than parking the task. - return "", fmt.Errorf("playbook %q binds this trigger to workflow %q, which is not defined", decision.PlaybookID, decision.Workflow) - } - d.Log.InfoContext(ctx, "playbook selected workflow", - "task", task.ID, "playbook", decision.PlaybookID, - "playbook_version", decision.Version, "workflow", decision.Workflow) - return decision.Workflow, nil -} - -// playbookInput builds the coordinator's dispatch input from a task row. The -// event surface is what the intake point actually knows about the originating -// issue -- the fields a `when` condition can read; it is deliberately not -// padded with values the daemon would have to invent. -func playbookInput(task *workflow.Task) playbook.DispatchInput { - labels := workintake.SplitLabels(task.Labels) - kind := string(workintake.KindForLabels(labels)) - return playbook.DispatchInput{ - Labels: labels, - Kind: kind, - TaskID: workintake.TaskEnvelope{Org: task.Org, Identity: task.Identity, Owner: task.Owner, Repo: task.Repo, Number: task.IssueNumber}.IdempotencyKey(), - Event: map[string]any{ - "kind": kind, - "labels": labels, - "owner": task.Owner, - "repo": task.Repo, - "number": task.IssueNumber, - "title": task.Title, - "body": task.Body, - "source": task.Source, - }, - } -} - -func (d *Daemon) pinWorkflowFromCollection(ctx context.Context, task *workflow.Task, collection workflow.WorkflowDefinitionCollection, version int64) error { - available := make(map[string]struct{}, len(collection.Definitions)) - for _, definition := range collection.Definitions { - available[definition.ID] = struct{}{} - } - id, err := d.resolveWorkflowID(ctx, task, available) - if err != nil { - return pinFailure{taskstate.ParkNeedsDecision, err} - } - definition, ok := collection.DefinitionByID(id) - if !ok { - return pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("workflow definition %q disappeared", id)} - } - task.Workflow = id - task.WorkflowDefinitionVersion = version - task.WorkflowDefinitionYAML = definition.YAML - task.WorkflowDefinitionDigest = workflow.DigestDefinition(definition.YAML) - if err := d.Store.Update(ctx, task); err != nil { - return pinFailure{taskstate.ParkTransient, fmt.Errorf("persist workflow definition pin: %w", err)} - } - return nil -} - -const ( - // defaultTaskRunReadyTimeout is how long requestTaskRun retries a - // nats.ErrNoResponders before giving up, when Daemon.TaskRunReadyTimeout - // is unset. - defaultTaskRunReadyTimeout = 20 * time.Second - // defaultTaskRunRetryBackoff is the delay between retry attempts, when - // Daemon.TaskRunRetryBackoff is unset. - defaultTaskRunRetryBackoff = 250 * time.Millisecond -) - -func (d *Daemon) taskRunReadyTimeout() time.Duration { - if d.TaskRunReadyTimeout > 0 { - return d.TaskRunReadyTimeout - } - return defaultTaskRunReadyTimeout -} - -func (d *Daemon) taskRunRetryBackoff() time.Duration { - if d.TaskRunRetryBackoff > 0 { - return d.TaskRunRetryBackoff - } - return defaultTaskRunRetryBackoff -} - -// requestTaskRun publishes the taskrun request, retrying on -// nats.ErrNoResponders until the container subscribes. Other errors return -// at once. -func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte) ([]byte, error) { - subject := agentnats.SubjectForTask(taskID) - deadline := time.Now().Add(d.taskRunReadyTimeout()) - backoff := d.taskRunRetryBackoff() - for { - reply, err := d.Tasks.Request(ctx, subject, data) - if err == nil { - return reply, nil - } - if !errors.Is(err, eventbus.ErrNoResponders) || !time.Now().Before(deadline) { - return nil, err - } - wait := min(backoff, time.Until(deadline)) - if wait <= 0 { - return nil, err - } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(wait): - } - } -} - -// taskStateStoreEndpoint is the State Store address one task container is -// handed, the relay carrying it (nil when none is needed) and the release that -// ends that relay. The container lives -// in its own network namespace, so a loopback target names the container -// itself; behind a relay, the address is the host bridge's. -func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, *stateStoreRelay, func(), error) { - relay, err := d.openTaskStateStoreRelay(ctx) - if err != nil { - return "", nil, release, err - } - if relay == nil { - return d.ConnectedStateStore.URL, nil, release, nil - } - return relay.Addr(), relay, func() { relay.Close(); release() }, nil -} - -// admitToRelay lets ctr, and only ctr, through relay. A container whose -// addresses cannot be read is refused rather than let anyone through. -func (d *Daemon) admitToRelay(ctx context.Context, relay *stateStoreRelay, task *workflow.Task, ctr *container.Container) { - if relay == nil { - return - } - addrs, err := d.ContainerPool.ContainerAddrs(ctx, ctr.ID) - if err != nil { - d.Log.WarnContext(ctx, "state store relay: container addresses unknown, refusing its connections", "task", task.ID, "err", err) - } - relay.Admit(addrs) -} - -// openTaskStateStoreRelay starts the relay that carries one task container's -// State Store traffic to the daemon's target. The target stays bound where the -// operator put it; the container is handed an address on the host bridge, the -// same way it is handed NATS's. Nil when the target already reaches the -// containers, or when no bridge address is known. -func (d *Daemon) openTaskStateStoreRelay(ctx context.Context) (*stateStoreRelay, error) { - addr, loopback := loopbackStateStoreTarget(d.ConnectedStateStore.URL) - if !loopback { - return nil, nil - } - gateway := "" - if d.ContainerPool != nil { - gateway = d.ContainerPool.HostGateway() - } - if gateway == "" { - d.Log.WarnContext(ctx, "state store target is loopback and the container pool has no host bridge address; task containers cannot reach it", - "target", d.ConnectedStateStore.URL) - return nil, nil - } - return openStateStoreRelay(ctx, gateway, addr) -} - -// containerEnv is the environment a task container starts with. stateStoreURL -// is the address the container can reach the State Store by, which is the -// relay's rather than the daemon's own target. -func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken, stateStoreURL string) []string { - var env []string - // The endpoint the daemon's own client connected with at startup, not - // the live config: a reloaded [nats] section must not point new - // containers at a server the daemon is not publishing on. - env = append(env, "NATS_URL="+d.ConnectedNATS.URL) - if token := d.ConnectedNATS.ContainerToken(); token != "" { - env = append(env, "NATS_TOKEN="+token) - } - // Pass the container-reachable endpoint with this task's scoped token, - // never the daemon's own. - if stateStoreURL != "" { - env = append(env, "STATE_STORE_URL="+stateStoreURL) - if len(d.ConnectedStateStore.CA) > 0 { - env = append(env, "STATE_STORE_CA="+string(d.ConnectedStateStore.CA)) - } - if stateStoreToken != "" { - env = append(env, "STATE_STORE_TOKEN="+stateStoreToken) - } - } - // The agent runs as root in the container, so its commits land owned by - // UID 0 on the host. The daemon's UID/GID let the agent hand the worktree - // back before the daemon reads it to push. - env = append(env, fmt.Sprintf("WORKTREE_UID=%d", os.Getuid())) - env = append(env, fmt.Sprintf("WORKTREE_GID=%d", os.Getgid())) - return env -} - -// runCredential issues the task's run credential, or "" when no State Store -// is configured. It fails when no issuer is wired. -func (d *Daemon) runCredential(ctx context.Context, task *workflow.Task) (string, func(), error) { - if d.ConnectedStateStore.URL == "" { - return "", func() {}, nil - } - if d.RunCredentials == nil { - return "", nil, fmt.Errorf("state store is configured but no run credential issuer is wired") - } - return d.RunCredentials.Issue(ctx, task, d.configFor(task).Budgets.TaskWallClock.Std()) -} - -func (d *Daemon) configFor(task *workflow.Task) config.Config { - cfg := d.Cfg.Get() - if !d.rootTask(task) { - cfg.BotUser, cfg.BotEmail, cfg.Forge = "", "", config.Forge{} - cfg.Org, cfg.GrantedCredentials, cfg.Identities, cfg.IdentityGrants = task.OrgOrDefault(), nil, nil, nil - cfg.Repos, cfg.Models, cfg.Providers = nil, nil, nil - } - return cfg -} - -// captureAttemptConfig records the attempt's effective configuration as an -// event. Failures are logged. -func (d *Daemon) captureAttemptConfig(ctx context.Context, task *workflow.Task, cfg config.TaskConfig) { - if d.Store == nil { - return - } - document, err := json.Marshal(cfg) - if err != nil { - d.Log.WarnContext(ctx, "attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err) - return - } - event := events.Event{ - At: time.Now().UTC(), - Kind: events.KindConfigCaptured, - TaskID: task.ID, - Repo: task.Owner + "/" + task.Repo, - Issue: task.IssueNumber, - Workflow: task.Workflow, - Attempt: task.Attempt, - Data: map[string]any{ - "schema": events.ConfigCapturedSchema, - // Raw so the document is embedded once, as it was encoded, rather - // than as a string a reader would have to decode again. - "document": json.RawMessage(document), - }, - } - id, err := d.Store.InsertEvent(ctx, event) - if err != nil { - d.Log.WarnContext(ctx, "attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err) - return - } - // The assigned ID tells the event sink to broadcast without inserting a - // duplicate row. - event.ID = id - d.emit(ctx, event) -} - -// identityMayAct reports whether a task's identity may act, and the park -// class if not. An empty, unknown or inactive identity never falls back to -// the System forge. -func (d *Daemon) identityMayAct(ctx context.Context, name string) (bool, string, taskstate.ParkClass) { - if name == "" { - return false, "task has no identity", taskstate.ParkNeedsDecision - } - id := identity.IdentityID(name) - if d.IdentityRepository != nil { - value, err := d.IdentityRepository.Get(ctx, id) - if err != nil { - return false, "execution identity unavailable: " + err.Error(), taskstate.ParkTransient - } - if value.Lifecycle != identity.LifecycleActive { - return false, "identity may not act", taskstate.ParkNeedsDecision - } - return true, "", "" - } - if name == string(identity.SystemID) { - return true, "", "" - } - return false, "identity " + name + " no longer resolves", taskstate.ParkTransient -} - -// parkIfIdentityUnresolvable parks task when its identity cannot be acted on, -// reporting whether it did. -func (d *Daemon) parkIfIdentityUnresolvable(ctx context.Context, task *workflow.Task) bool { - ok, reason, class := d.identityMayAct(ctx, task.Identity) - if ok { - return false - } - d.parkRunningTask(ctx, task, reason, class) - return true -} - -func (d *Daemon) identityActive(ctx context.Context, id identity.IdentityID) bool { - if d.IdentityRepository == nil { - return true - } - if id == "" { - d.Log.ErrorContext(ctx, "identity lifecycle check failed", "err", "identity ID is empty") - return false - } - value, err := d.IdentityRepository.Get(ctx, id) - if err != nil { - d.Log.ErrorContext(ctx, "identity lifecycle check failed", "identity", id, "err", err) - return false - } - return value.Lifecycle == identity.LifecycleActive -} - -// forgeFor returns the forge client that owns task: its organisation -// connection, else the root d.Forge for the root alone. This is the safety -// boundary that keeps one identity's forge token from being used against -// another identity's repos. -func (d *Daemon) forgeFor(ctx context.Context, task *workflow.Task) forge.Forge { - if d.ResolveRepository != nil && task.HasRepository() { - resolved, err := d.Repository(ctx, task, config.OperationClone) - if err != nil { - return forge.NewNoop(d.Log) - } - return resolved.Forge - } - if !d.rootTask(task) { - return forge.NewNoop(d.Log) - } - return d.Forge -} - -// treesFor returns the worktree manager that owns task, mirroring forgeFor. -func (d *Daemon) treesFor(task *workflow.Task) *worktree.Manager { - if d.ResolveRepository != nil && task.HasRepository() { - // Cleanup needs only the org's path, not a still-valid external credential. - return &worktree.Manager{WorkDir: worktree.OrgWorkDir(d.Cfg.Get().WorkDir, task.OrgOrDefault())} - } - if !d.rootTask(task) { - return &worktree.Manager{WorkDir: filepath.Join(d.Cfg.Get().WorkDir, "identity-"+string(identity.StableID(task.Identity)))} - } - return d.Trees -} - -func (d *Daemon) rootTask(task *workflow.Task) bool { - return identity.IdentityID(task.Identity) == identity.SystemID -} - -// repoFor resolves a root task's repo config from Cfg.Repos. -func (d *Daemon) repoFor(t *workflow.Task) (config.Repo, bool) { - if !d.rootTask(t) { - return config.Repo{}, false - } - for _, r := range d.Cfg.Get().Repos { - if r.Owner == t.Owner && r.Name == t.Repo { - return r, true - } - } - return config.Repo{}, false -} - -// allowConcurrentForTask reports whether the task's repo allows concurrent -// tasks. Unknown repos do not. -func (d *Daemon) allowConcurrentForTask(task *workflow.Task) bool { - if !task.HasRepository() { - return true - } - repo, ok := d.repoFor(task) - return ok && repo.AllowConcurrent -} - -// StalestLoop reports the run loop that least recently began a pass and when, -// or the zero time when no loop has started yet. The systemd watchdog beats -// only while this is fresh, so every loop Run starts must mark its passes. -func (d *Daemon) StalestLoop() (string, time.Time) { - var loop string - var oldest int64 - d.loopPasses.Range(func(k, v any) bool { - name, okName := k.(string) - if ns, ok := v.(int64); ok && okName && (oldest == 0 || ns < oldest) { - loop, oldest = name, ns - } - return true - }) - if oldest == 0 { - return "", time.Time{} - } - return loop, time.Unix(0, oldest).UTC() -} - -// markPass stamps the start of one loop's pass, so a hung pass shows as stale. -func (d *Daemon) markPass(loop string) { - d.loopPasses.Store(loop, time.Now().UnixNano()) -} - -// errContainerExited is the cancellation cause when a task's agent container -// stops before answering its taskrun request. -var errContainerExited = errors.New("agent container exited before the task finished") - -// runFailure names why a taskrun request failed: the container's exit or the -// time limit when either cancelled it, else the request's own error. -func runFailure(ctx context.Context, err error) error { - if cause := context.Cause(ctx); errors.Is(cause, errContainerExited) || errors.Is(cause, errTaskTimeLimit) { - return cause - } - return err -} - -// exitWatcher is a running agent container, seen only for how it ends. -type exitWatcher interface { - Exited() <-chan struct{} - ExitDetail() string -} - -// withContainerExit derives a context cancelled when exited closes. A dead -// container never answers its taskrun request, so without this the request -// blocks forever and the task row stays running. -func withContainerExit(ctx context.Context, ctr exitWatcher) (context.Context, func()) { - runCtx, cancel := context.WithCancelCause(ctx) - go func() { - select { - case <-ctr.Exited(): - if detail := ctr.ExitDetail(); detail != "" { - cancel(fmt.Errorf("%w: %s", errContainerExited, detail)) - return - } - cancel(errContainerExited) - case <-runCtx.Done(): - } - }() - return runCtx, func() { cancel(nil) } -} - -// errTaskTimeLimit is the cancellation cause when a task outruns its overall -// time limit (the execution settings' max task runtime). -var errTaskTimeLimit = errors.New("task exceeded its time limit") - -// withTaskTimeLimit bounds a whole task run. A limit of zero or less applies -// no bound. -func withTaskTimeLimit(ctx context.Context, limit time.Duration) (context.Context, func()) { - if limit <= 0 { - return ctx, func() {} - } - return context.WithTimeoutCause(ctx, limit, fmt.Errorf("%w (%s)", errTaskTimeLimit, limit)) -} - -// RemoveWorktree deletes a task's clone from the trees of the identity that -// owns it. Archiving calls it: terminal cleanup keeps a worktree that may hold -// uncaptured work, and the operator's archive is the decision to discard it. -func (d *Daemon) RemoveWorktree(task *workflow.Task) error { - trees := d.treesFor(task) - if trees == nil { - return nil - } - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - return d.cleanupWorktree(ctx, trees, task.Owner, task.Repo, task.IssueNumber) -} - -// cleanupWorktree removes a task's clone, first handing back any root-owned -// files a killed container left in it. -func (d *Daemon) cleanupWorktree(ctx context.Context, trees *worktree.Manager, owner, repo string, issue int) error { - err := trees.Cleanup(owner, repo, issue) - if d.reownDenied(ctx, trees.Dir(owner, repo, issue), err) { - return trees.Cleanup(owner, repo, issue) - } - return err -} - -// reownDenied repairs dir when err says the daemon was refused access to it, -// and reports whether the caller should retry. A SIGKILLed task container -// leaves root-owned files that only a container can chown back. -func (d *Daemon) reownDenied(ctx context.Context, dir string, err error) bool { - if err == nil || !errors.Is(err, fs.ErrPermission) || d.ContainerPool == nil { - return false - } - if rerr := d.ContainerPool.Reown(ctx, dir, os.Getuid(), os.Getgid()); rerr != nil { - d.Log.WarnContext(ctx, "worktree ownership repair failed", "dir", dir, "err", rerr) - return false - } - return true -} - -// InFlight counts the same submitted work that graceful shutdown waits for, -// including tasks waiting for a repository or concurrency slot. -func (d *Daemon) InFlight() int { - dispatcher := d.taskDispatcher() - dispatcher.mu.Lock() - defer dispatcher.mu.Unlock() - return dispatcher.pending -} diff --git a/internal/daemon/dispatcher_test.go b/internal/daemon/dispatcher_test.go index 6f59251d1..f2755ebd2 100644 --- a/internal/daemon/dispatcher_test.go +++ b/internal/daemon/dispatcher_test.go @@ -413,13 +413,6 @@ func TestResumeAfterTheRunReturnedLeavesNoCapacity(t *testing.T) { d.Wait() } -// A daemon with no bus still starts and stops: the watcher is optional. -func TestCallWaitsWithoutABus(t *testing.T) { - d := &Daemon{Cfg: config.NewHolder(config.Config{}), Log: slog.New(slog.DiscardHandler)} - d.watchCallWaits(t.Context()) - d.WaitForTasks() -} - type callWaitStore struct { storecontract.TaskStore mu sync.Mutex @@ -442,7 +435,7 @@ func TestCallWaitsRecoverDroppedStart(t *testing.T) { }} cfg := config.Config{} cfg.Containers.MaxConcurrency = 1 - d := &Daemon{Cfg: config.NewHolder(cfg), Store: store, Bus: events.NewBus(), Log: slog.New(slog.DiscardHandler)} + d := wired(&Daemon{Cfg: config.NewHolder(cfg), Store: store, Bus: events.NewBus(), Log: slog.New(slog.DiscardHandler)}) dispatcher := d.taskDispatcher() started := make(chan int64, 2) a, b := make(chan struct{}), make(chan struct{}) @@ -532,12 +525,12 @@ func TestDrainPassDoesNotWaitForItsTasks(t *testing.T) { <-release }, } - d := &Daemon{ + d := wired(&Daemon{ Cfg: config.NewHolder(config.Config{}), Log: slog.New(slog.DiscardHandler), Store: store, Tasks: stubTasks{}, - } + }) drained := make(chan struct{}) go func() { @@ -611,12 +604,12 @@ func TestDrainPassClaimsABacklogWithoutWaitingPerTask(t *testing.T) { store.queue = append(store.queue, &workflow.Task{ID: id + 1, Owner: "acme", Repo: "app"}) } tasks := &countingTasks{} - d := &Daemon{ + d := wired(&Daemon{ Cfg: config.NewHolder(config.Config{}), Log: slog.New(slog.DiscardHandler), Store: store, Tasks: tasks, - } + }) d.drainNATS(context.Background()) d.WaitForTasks() if len(store.queue) != 0 { @@ -639,12 +632,12 @@ func TestDrainPassClaimsOnlyWhatTheDispatcherCanRun(t *testing.T) { } cfg := config.Config{} cfg.Containers.MaxConcurrency = 2 - d := &Daemon{ + d := wired(&Daemon{ Cfg: config.NewHolder(cfg), Log: slog.New(slog.DiscardHandler), Store: store, Tasks: &countingTasks{}, - } + }) d.drainNATS(context.Background()) if got := len(store.queue); got != 8 { t.Fatalf("pass left %d tasks unclaimed, want 8", got) diff --git a/internal/daemon/identity.go b/internal/daemon/identity.go new file mode 100644 index 000000000..c4e0801d5 --- /dev/null +++ b/internal/daemon/identity.go @@ -0,0 +1,58 @@ +package daemon + +import ( + "context" + + "github.com/samcharles93/archie-core/internal/domain/identity" + "github.com/samcharles93/archie-core/internal/domain/org" + "github.com/samcharles93/archie-core/internal/taskstate" +) + +// identityOrg resolves an identity's org. No PrincipalSource means the +// default org; a store error is returned. +// orgResolver is the identity-to-org surface the identity checks read through. +type orgResolver interface { + identityOrg(ctx context.Context, id identity.IdentityID) (org.OrgID, error) + identityMayAct(ctx context.Context, name string) (bool, string, taskstate.ParkClass) + identityActive(ctx context.Context, id identity.IdentityID) bool +} + +var _ orgResolver = (*Daemon)(nil) + +func (d *Daemon) identityOrg(ctx context.Context, id identity.IdentityID) (org.OrgID, error) { + principal, err := d.Principals.PrincipalFor(ctx, id) + if err != nil { + return "", err + } + return principal.Org, nil +} + +func (d *Daemon) identityActive(ctx context.Context, id identity.IdentityID) bool { + if id == "" { + d.Log.ErrorContext(ctx, "identity lifecycle check failed", "err", "identity ID is empty") + return false + } + value, err := d.IdentityRepository.Get(ctx, id) + if err != nil { + d.Log.ErrorContext(ctx, "identity lifecycle check failed", "identity", id, "err", err) + return false + } + return value.Lifecycle == identity.LifecycleActive +} + +// identityMayAct reports whether a task's identity may act, and the park +// class if not. An empty, unknown or inactive identity never falls back to +// the System forge. +func (d *Daemon) identityMayAct(ctx context.Context, name string) (bool, string, taskstate.ParkClass) { + if name == "" { + return false, "task has no identity", taskstate.ParkNeedsDecision + } + value, err := d.IdentityRepository.Get(ctx, identity.IdentityID(name)) + if err != nil { + return false, "execution identity unavailable: " + err.Error(), taskstate.ParkTransient + } + if value.Lifecycle != identity.LifecycleActive { + return false, "identity may not act", taskstate.ParkNeedsDecision + } + return true, "", "" +} diff --git a/internal/daemon/intake_cap_test.go b/internal/daemon/intake_cap_test.go index 83e1086df..cd4604876 100644 --- a/internal/daemon/intake_cap_test.go +++ b/internal/daemon/intake_cap_test.go @@ -55,7 +55,7 @@ func TestPublishDiscoveredCap(t *testing.T) { for _, n := range tt.existing { have[n] = true } - d := &Daemon{Log: slog.New(slog.DiscardHandler), Store: taskLookup{have: have}, Tasks: bus} + d := wired(&Daemon{Log: slog.New(slog.DiscardHandler), Store: taskLookup{have: have}, Tasks: bus}) var issues []forge.Issue for n := 1; n <= tt.issues; n++ { issues = append(issues, forge.Issue{Number: n, Labels: []string{"archie"}}) diff --git a/internal/daemon/intake_disabled_test.go b/internal/daemon/intake_disabled_test.go index 72f739c78..875e77303 100644 --- a/internal/daemon/intake_disabled_test.go +++ b/internal/daemon/intake_disabled_test.go @@ -90,11 +90,11 @@ func TestPollNATSSkipsDisabledRoutedWorkflow(t *testing.T) { bus := &publishCounter{} fg := &reactCounter{} var logs bytes.Buffer - d := &Daemon{ + d := wired(&Daemon{ Log: slog.New(slog.NewTextHandler(&logs, nil)), Store: taskLookup{}, Tasks: bus, WorkflowDefinitions: tt.definitions, WorkflowEnablement: tt.enablement, Principals: tt.principals, - } + }) is := forge.Issue{Number: 42, Title: "crash on startup"} if created := d.pollNATS(t.Context(), fg, cfg, repo, is, tt.labels, "acme-bot"); created != tt.wantCreated { t.Errorf("pollNATS created %v, want %v", created, tt.wantCreated) diff --git a/internal/daemon/issue_poll.go b/internal/daemon/issue_poll.go new file mode 100644 index 000000000..eb344a2ee --- /dev/null +++ b/internal/daemon/issue_poll.go @@ -0,0 +1,219 @@ +package daemon + +import ( + "context" + "strings" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/domain/workintake" + "github.com/samcharles93/archie-core/internal/forge" + "github.com/samcharles93/archie-core/internal/taskstate" +) + +func (d *Daemon) issueSources(ctx context.Context) []IssueSource { + return d.IssueSources(ctx) +} + +// pollIssuesWithConfig returns the repo's issues the dispatch rule makes +// work, using the given forge client and dispatch config. complete is false when any part of the poll failed or was skipped, so +// the result must not be read as the whole eligible set. +func (d *Daemon) pollIssuesWithConfig(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo) (issues []forge.Issue, complete bool) { + labelled := func() ([]forge.Issue, bool) { + // A label trigger needs a non-empty label or it would match every open issue. + if cfg.Label == "" { + d.Log.ErrorContext(ctx, "label trigger configured with an empty label; refusing to poll (an empty label matches every open issue)", "repo", repo.FullName()) + return nil, false + } + issues, err := fg.IssuesWithLabel(ctx, repo.Owner, repo.Name, cfg.Label) + if err != nil { + d.Log.ErrorContext(ctx, "label poll failed", "repo", repo.FullName(), "err", err) + return nil, false + } + return issues, true + } + assigned := func() ([]forge.Issue, bool) { + if cfg.BotUser == "" { + d.Log.ErrorContext(ctx, "assignee trigger needs the connection's account; refusing to poll", "repo", repo.FullName()) + return nil, false + } + issues, err := fg.AssignedIssues(ctx, repo.Owner, repo.Name, cfg.BotUser) + if err != nil { + d.Log.ErrorContext(ctx, "assigned poll failed", "repo", repo.FullName(), "err", err) + return nil, false + } + return issues, true + } + switch cfg.Dispatch.Trigger { + case "label": + return labelled() + case "either": + byAssignee, assignedOK := assigned() + byLabel, labelledOK := labelled() + seen := map[int]bool{} + for _, is := range byAssignee { + seen[is.Number] = true + } + issues = byAssignee + for _, is := range byLabel { + if !seen[is.Number] { + issues = append(issues, is) + } + } + return issues, assignedOK && labelledOK + default: // "assignee" (default) + return assigned() + } +} + +// poll discovers work through every organisation connection, resolved +// afresh each cycle so a rotated credential or a new connection applies to +// the next poll. +func (d *Daemon) poll(ctx context.Context) error { + var failed []string + room := d.Cfg.Get().IntakePerPass + for _, source := range d.issueSources(ctx) { + if !d.identityActive(ctx, source.Identity) { + continue + } + cfg := d.Cfg.Get() + cfg.BotUser = source.BotUser + if source.Dispatch.Trigger != "" { + cfg.Dispatch, cfg.Label = source.Dispatch, source.Label + } + tag := string(source.Identity) + for _, repo := range source.Repos { + issues, complete := d.pollIssuesWithConfig(ctx, source.Forge, cfg, repo) + if complete { + d.withdrawUnpolled(ctx, repo, tag, dispatchRule(cfg), issues) + } else { + failed = append(failed, repo.FullName()) + } + d.publishDiscovered(ctx, source.Forge, cfg, repo, issues, tag, &room) + } + } + return passFailed("issue poll", failed) +} + +// rememberPoll records a repo's latest eligible set and returns the previous +// one, reporting whether it was taken under the same rule. +func (d *Daemon) rememberPoll(key string, current polledIssues) (polledIssues, bool) { + d.lastPolledMu.Lock() + defer d.lastPolledMu.Unlock() + previous, seen := d.lastPolled[key] + if d.lastPolled == nil { + d.lastPolled = map[string]polledIssues{} + } + d.lastPolled[key] = current + return previous, seen && previous.rule == current.rule +} + +// withdrawUnpolled declines the queued or running work of an identity's +// issues in repo that a complete poll no longer returns: the issue was +// closed, unlabelled or unassigned. +func (d *Daemon) withdrawUnpolled(ctx context.Context, repo config.Repo, identity, rule string, polled []forge.Issue) { + eligible := make(map[int]bool, len(polled)) + for _, is := range polled { + eligible[is.Number] = true + } + // Only an issue the same rule found eligible last time and no longer + // finds was withdrawn. A changed label, trigger or bot user, or the first + // poll after a restart, only records the new set: tasks picked up under + // the old rule were not withdrawn by anyone. + previous, sameRule := d.rememberPoll(identity+"\x00"+repo.FullName(), polledIssues{rule: rule, eligible: eligible}) + if !sameRule { + return + } + active, err := d.Store.TasksPage(ctx, storecontract.TaskPage{ + Statuses: []string{taskstate.Queued, taskstate.Running}, Limit: 500, + }) + if err != nil { + d.Log.WarnContext(ctx, "withdrawal check: active tasks unavailable", "repo", repo.FullName(), "err", err) + return + } + for _, t := range active { + if t.Owner != repo.Owner || t.Repo != repo.Name || t.Identity != identity || + !t.IsForgeBacked() || t.PRNumber > 0 || eligible[t.IssueNumber] || !previous.eligible[t.IssueNumber] { + continue + } + if err := d.WithdrawTask(ctx, t.ID, "the issue is no longer assigned, labelled or open"); err != nil { + d.Log.WarnContext(ctx, "withdraw task", "task", t.ID, "err", err) + } + } +} + +// publishDiscovered publishes a repo's discovered issues until the pass has +// created its cap of tasks. room is what is left of the cap; the issues left +// over are found again by the next poll, because an issue with a task is +// skipped and one without is not. A cap of zero or less is no cap. +func (d *Daemon) publishDiscovered(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, issues []forge.Issue, identity string, room *int) { + capped := *room > 0 + for _, is := range issues { + if capped && *room == 0 { + d.Log.InfoContext(ctx, "intake cap reached; remaining issues wait for the next poll", "repo", repo.FullName()) + return + } + if d.pollNATS(ctx, fg, cfg, repo, is, strings.Join(is.Labels, ","), identity) && capped { + *room-- + } + } +} + +// pollNATS publishes a discovered issue to NATS and reports whether it created +// a task. +func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) bool { + // Read-only existence check prevents rediscovering an existing task. + existing, err := d.Store.TaskByIssue(ctx, repo.Owner, repo.Name, is.Number) + if err != nil { + d.Log.ErrorContext(ctx, "task lookup failed", "repo", repo.FullName(), "issue", is.Number, "err", err) + return false + } + if existing != nil { + return false + } + // An issue whose label-routed workflow the identity's org disabled must + // not queue a task at all. Skip without acknowledging, so a later poll + // picks the issue up again if the workflow is re-enabled. + if routed, disabled, err := d.disabledRoutedWorkflow(ctx, labels, identity); err != nil { + d.Log.WarnContext(ctx, "routed workflow check failed; publishing anyway", "repo", repo.FullName(), "issue", is.Number, "err", err) + } else if disabled { + d.Log.InfoContext(ctx, "issue skipped: its routed workflow is disabled for this org", "repo", repo.FullName(), "issue", is.Number, "workflow", routed) + return false + } + // Classify and encode here: the envelope, its routing kind and its + // subject belong to the work-intake domain, not to the transport. + parsed := workintake.SplitLabels(labels) + task := workintake.TaskEnvelope{ + Owner: repo.Owner, + Repo: repo.Name, + Number: is.Number, + Title: is.Title, + Body: is.Body, + Labels: parsed, + Identity: identity, + Kind: workintake.KindForLabels(parsed), + } + if err := d.PublishTask(ctx, task); err != nil { + d.Log.ErrorContext(ctx, "task publish failed", "repo", repo.FullName(), "issue", is.Number, "err", err) + return false + } + d.acknowledge(ctx, fg, cfg, repo, is) + return true +} + +// acknowledge posts the pickup reaction +// using fg -- the forge client that owns repo (identity-scoped or root) -- +// and the ack reaction from cfg, the dispatch config that owns this poll. +func (d *Daemon) acknowledge(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue) { + d.Log.InfoContext(ctx, "issue queued", "repo", repo.FullName(), "issue", is.Number, "title", is.Title) + if ack := cfg.Dispatch.AckReaction; ack != "" { + if err := fg.React(ctx, repo.Owner, repo.Name, is.Number, ack); err != nil { + d.Log.WarnContext(ctx, "ack reaction failed", "issue", is.Number, "err", err) + } + } +} + +// dispatchRule identifies the rule a poll selects issues by. +func dispatchRule(cfg config.Config) string { + return cfg.Dispatch.Trigger + "\x00" + cfg.Label + "\x00" + cfg.BotUser +} diff --git a/internal/daemon/lifecycle.go b/internal/daemon/lifecycle.go new file mode 100644 index 000000000..56075d384 --- /dev/null +++ b/internal/daemon/lifecycle.go @@ -0,0 +1,183 @@ +package daemon + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/forge" +) + +// Startup runs crash recovery and access verification once. +func (d *Daemon) Startup(ctx context.Context) error { + d.wireReactions() + if err := d.cleanupExpiredStorage(ctx); err != nil { + d.Log.WarnContext(ctx, "startup volume cleanup failed", "err", err) + } + n, err := d.Store.RecoverStale(ctx) + if err != nil { + return err + } + if n > 0 { + d.Log.InfoContext(ctx, "re-queued tasks left running by a previous daemon", "count", n) + } + + d.sweepAccess(ctx) + return nil +} + +// wireReactions builds the review-reaction consumer and publisher. Startup +// calls it before the run loop and the scheduler start, so nothing builds them +// concurrently. +func (d *Daemon) wireReactions() { + d.reactionConsumer = newReactionConsumer(d.Store, d.Store, d.Store, d.botUserForTask, d.Log) + d.reactionPublisher = d.PublishReaction +} + +// sweepAccess accepts pending invitations and verifies push access through +// every organisation connection, each in its own goroutine. +func (d *Daemon) sweepAccess(ctx context.Context) { + sweep := func(source IssueSource) { + log := d.Log.With("identity", source.Identity) + ctx, cancel := context.WithTimeout(ctx, sweepTimeout) + defer cancel() + if err := source.Forge.AcceptInvitations(ctx, repoRefs(source.Repos)); err != nil { + log.WarnContext(ctx, "invitation sweep failed", "err", err) + } + for _, r := range source.Repos { + if err := source.Forge.VerifyPush(ctx, r.Owner, r.Name); err != nil { + // A sweep cut short by its own context is not a repo failure: name + // the budget when it expired, stay quiet on a shutdown. + switch { + case errors.Is(ctx.Err(), context.DeadlineExceeded): + // Every later call would return the same error, so say it once. + log.WarnContext(ctx, "push check abandoned: sweep budget exhausted", "repo", r.FullName(), "err", err) + return + case ctx.Err() != nil: + return + } + log.WarnContext(ctx, "repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err) + } + } + } + + var wg sync.WaitGroup + for _, source := range d.issueSources(ctx) { + wg.Go(func() { sweep(source) }) + } + wg.Wait() +} + +// repoRefs is a source's configured repositories as the allowlist the forge +// accepts invitations against. +func repoRefs(repos []config.Repo) []forge.RepoRef { + out := make([]forge.RepoRef, 0, len(repos)) + for _, r := range repos { + out = append(out, forge.RepoRef{Owner: r.Owner, Repo: r.Name}) + } + return out +} + +// sweepTimeout bounds one forge's boot sweep. Startup gates the gateways and +// the run loop, and a forge client carries no request deadline of its own, so +// an unreachable host would otherwise hold boot open indefinitely. +const sweepTimeout = 2 * time.Minute + +// Run polls until the context ends. +func (d *Daemon) Run(ctx context.Context) error { + // A pass no longer waits for the tasks it submitted, so the run does, once + // the ctx is cancelled. + defer d.WaitForTasks() + d.watchCallWaits(ctx) + drained := make(chan struct{}) + defer func() { <-drained }() + go func() { + defer close(drained) + d.drainLoop(ctx) + }() + interval := d.Cfg.Get().PollInterval.Std() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + d.Cycle(ctx) + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + // PollInterval may have been reloaded; reset the ticker so a + // change takes effect on the next cycle. + if iv := d.Cfg.Get().PollInterval.Std(); iv != interval { + interval = iv + ticker.Reset(iv) + } + } + } +} + +// RunOnce runs one cycle and drains until no task is in flight. A run waiting +// on a callee stays in flight while the callee sits queued, so the queue is +// drained again until every task, callees included, has returned. +func (d *Daemon) RunOnce(ctx context.Context) { + defer d.WaitForTasks() + d.watchCallWaits(ctx) + // One pass has no scheduler, so it runs the maintenance jobs itself. + for _, job := range d.BuiltinJobs() { + if err := job.Run(ctx); err != nil { + d.Log.WarnContext(ctx, "built-in job failed", "job", job.ID, "err", err) + } + } + d.dispatchBindings(ctx) + d.Cycle(ctx) + d.drainNATS(ctx) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for d.taskDispatcher().inFlight() > 0 { + select { + case <-ctx.Done(): + return + case <-ticker.C: + d.drainNATS(ctx) + } + } +} + +// Cycle is one poll pass: turn review reactions into tasks; maintenance runs +// as built-in schedules. Captured events are dispatched by the caller +// (drainLoop in Run, once in RunOnce) so they start within about a second. +func (d *Daemon) Cycle(ctx context.Context) { + d.markPass("cycle") + d.drainReactions(ctx) + d.retryTransientParks(ctx) +} + +// markPass stamps the start of one loop's pass, so a hung pass shows as stale. +func (d *Daemon) markPass(loop string) { + d.loopPasses.Store(loop, time.Now().UnixNano()) +} + +// StalestLoop reports the run loop that least recently began a pass and when, +// or the zero time when no loop has started yet. The systemd watchdog beats +// only while this is fresh, so every loop Run starts must mark its passes. +func (d *Daemon) StalestLoop() (string, time.Time) { + var loop string + var oldest int64 + d.loopPasses.Range(func(k, v any) bool { + name, okName := k.(string) + if ns, ok := v.(int64); ok && okName && (oldest == 0 || ns < oldest) { + loop, oldest = name, ns + } + return true + }) + if oldest == 0 { + return "", time.Time{} + } + return loop, time.Unix(0, oldest).UTC() +} + +// emit publishes an observability event. +func (d *Daemon) emit(ctx context.Context, e events.Event) { + d.Bus.Publish(ctx, e) +} diff --git a/internal/daemon/model_egress.go b/internal/daemon/model_egress.go index f8f17e5f0..b85c284e8 100644 --- a/internal/daemon/model_egress.go +++ b/internal/daemon/model_egress.go @@ -36,13 +36,11 @@ type TaskModels interface { // never touch it. func (d *Daemon) taskConfig(ctx context.Context, task *workflow.Task) (config.Config, error) { cfg := d.configFor(task) - if d.TaskModels != nil { - models, err := d.TaskModels.TaskModels(ctx, task.ID) - if err != nil { - return config.Config{}, fmt.Errorf("resolve the task's models: %w", err) - } - cfg = cfg.WithOrgModels(models) + models, err := d.TaskModels.TaskModels(ctx, task.ID) + if err != nil { + return config.Config{}, fmt.Errorf("resolve the task's models: %w", err) } + cfg = cfg.WithOrgModels(models) cfg.Providers = cfg.ModelProviders() return cfg, nil } diff --git a/internal/daemon/model_egress_test.go b/internal/daemon/model_egress_test.go index 0609a7c99..9d13d57b9 100644 --- a/internal/daemon/model_egress_test.go +++ b/internal/daemon/model_egress_test.go @@ -36,7 +36,7 @@ func TestOpenModelEgressCarriesOnlyReferencedProviders(t *testing.T) { t.Parallel() egressProxy := &recordingModelEgress{} - d := &Daemon{ + d := wired(&Daemon{ Cfg: config.NewHolder(config.Config{ Models: map[string]string{"default": "used/model-one"}, Providers: map[string]config.Provider{ @@ -45,7 +45,7 @@ func TestOpenModelEgressCarriesOnlyReferencedProviders(t *testing.T) { }, }), ModelEgress: egressProxy, - } + }) task := &workflow.Task{ID: 7, Identity: string(sys), WorkflowDefinitionYAML: "id: w\nsteps:\n - type: agent.run\n"} if _, _, err := d.openModelEgress(t.Context(), task, "run-credential"); err != nil { @@ -89,7 +89,7 @@ func TestRunViaAgentCarriesProviderCredentialVariable(t *testing.T) { t.Fatalf("marshal response: %v", err) } bus := &capturingBus{reply: reply} - d := &Daemon{ + d := wired(&Daemon{ Cfg: config.NewHolder(config.Config{ Models: map[string]string{"default": "used/model-one"}, Providers: map[string]config.Provider{ @@ -98,7 +98,7 @@ func TestRunViaAgentCarriesProviderCredentialVariable(t *testing.T) { }), Tasks: bus, Log: slog.New(slog.DiscardHandler), - } + }) task := &workflow.Task{ID: 7, Identity: string(sys), Attempt: 1, Workflow: "implement", WorkflowDefinitionYAML: "id: w\nsteps:\n - type: agent.run\n"} d.runViaAgent(t.Context(), task, config.Repo{}, config.AgentProfile{}, "run-credential") diff --git a/internal/daemon/package_tools.go b/internal/daemon/package_tools.go index 729446435..8f07f690c 100644 --- a/internal/daemon/package_tools.go +++ b/internal/daemon/package_tools.go @@ -23,13 +23,11 @@ func (d *Daemon) toolAllowlist(ctx context.Context, task *workflow.Task, profile return storepkg.AllowedTools(profileTools, false, nil) } var accepted []string - if d.PackageAuthorities != nil { - authority, err := d.PackageAuthorities.AcceptedAuthority(ctx, name) - if err != nil { - d.Log.WarnContext(ctx, "package authority unavailable; offering no tools", "task", task.ID, "package", name, "err", err) - } else { - accepted = authority.Tools - } + authority, err := d.PackageAuthorities.AcceptedAuthority(ctx, name) + if err != nil { + d.Log.WarnContext(ctx, "package authority unavailable; offering no tools", "task", task.ID, "package", name, "err", err) + } else { + accepted = authority.Tools } return storepkg.AllowedTools(profileTools, true, accepted) } diff --git a/internal/daemon/package_tools_test.go b/internal/daemon/package_tools_test.go index 05af693e1..d7abe935e 100644 --- a/internal/daemon/package_tools_test.go +++ b/internal/daemon/package_tools_test.go @@ -24,10 +24,10 @@ func (a acceptedTools) AcceptedAuthority(_ context.Context, name string) (storep // offered only the tools accepted for its package, and none when the package // is missing or unaccepted, while an operator workflow keeps its profile's list. func TestToolAllowlistFollowsPackageAuthority(t *testing.T) { - d := &Daemon{Log: slog.Default(), PackageAuthorities: acceptedTools{ + d := wired(&Daemon{Log: slog.Default(), PackageAuthorities: acceptedTools{ "triage": {Tools: []string{"search", "fetch"}}, "unsigned": {}, - }} + }}) tests := []struct { name string yaml string diff --git a/internal/daemon/package_trigger.go b/internal/daemon/package_trigger.go index bf265ae08..f93fd4e98 100644 --- a/internal/daemon/package_trigger.go +++ b/internal/daemon/package_trigger.go @@ -42,9 +42,6 @@ func (d *Daemon) checkPackageTrigger(ctx context.Context, task *workflow.Task) e if name == "" { return nil } - if d.PackageAuthorities == nil { - return fmt.Errorf("package %s: no accepted authority is available", name) - } authority, err := d.PackageAuthorities.AcceptedAuthority(ctx, name) if err != nil { return fmt.Errorf("package %s: accepted authority unavailable: %w", name, err) diff --git a/internal/daemon/package_trigger_test.go b/internal/daemon/package_trigger_test.go index 258594321..0d137f425 100644 --- a/internal/daemon/package_trigger_test.go +++ b/internal/daemon/package_trigger_test.go @@ -12,11 +12,11 @@ import ( // trigger accepted for its package, failing closed when none is accepted or // the package is unknown, while operator workflows are unbounded. func TestCheckPackageTrigger(t *testing.T) { - d := &Daemon{PackageAuthorities: acceptedTools{ + d := wired(&Daemon{PackageAuthorities: acceptedTools{ "triage": storepkg.Authority{Triggers: []string{"label:triage", "binding:b1", "call"}}, "chatty": storepkg.Authority{Triggers: []string{"source:chat"}}, "unsigned": storepkg.Authority{}, - }} + }}) tests := []struct { name string task workflow.Task diff --git a/internal/daemon/park.go b/internal/daemon/park.go new file mode 100644 index 000000000..ed8d4c54a --- /dev/null +++ b/internal/daemon/park.go @@ -0,0 +1,157 @@ +package daemon + +import ( + "context" + "errors" + "log/slog" + "strings" + "time" + + "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/domain/workflow" + workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task" + "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/logging" + "github.com/samcharles93/archie-core/internal/taskstate" +) + +// parkCapabilityUnavailable parks a task whose daemon-side capability is +// missing (container pool or task transport): both are environmental, so +// the park class is transient and the KindParked event carries the class +// for the dashboard. The reason distinguishes the two in logs and events. +func (d *Daemon) parkCapabilityUnavailable(ctx context.Context, task *workflow.Task) { + var reason string + if d.ContainerPool == nil { + reason = "managed agent container pool is unavailable; refusing to run this task on the host" + d.Log.ErrorContext(ctx, "task parked: managed worker unavailable", "task", task.ID, + "hint", "enable containers and make the archie-agent image available, then retry the task") + } else { + reason = "agent task transport is unavailable; refusing to run this task on the host" + d.Log.ErrorContext(ctx, "task parked: agent task transport unavailable", "task", task.ID) + } + d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient) +} + +// parkRunningTask moves a running task to parked with class, using a +// bounded context that ignores cancellation. ErrStaleTransition is ignored; +// other errors are logged. +func (d *Daemon) parkRunningTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass) { + d.parkBlockedTask(ctx, task, reason, class, "") +} + +// parkBlockedTask is parkRunningTask recording the blocker the park site +// knows, so the dashboard can link to its fix. +func (d *Daemon) parkBlockedTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass, blocker taskstate.Blocker) { + writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + + if err := d.Store.ParkTask(writeCtx, task.ID, workflow.StatusRunning, reason, class, blocker); err != nil { + if !errors.Is(err, storecontract.ErrStaleTransition) { + d.Log.WarnContext(ctx, "terminal park transition failed", "task", task.ID, "reason", reason, "err", err) + } + return + } + // Recorded only once the transition landed. An entry claiming the task + // parked would be a lie when another actor moved it out of running first + // -- the same race the parked event is withheld for. + d.recordPark(ctx, task.ID, reason) + d.emit(ctx, events.Event{ + Kind: events.KindParked, TaskID: task.ID, Attempt: task.Attempt, + Repo: strings.Trim(task.Owner+"/"+task.Repo, "/"), Issue: task.IssueNumber, Workflow: task.Workflow, + Detail: reason, Data: map[string]any{"park_class": class}, + }) +} + +// recordPark writes a park reason to the attempt's log. Dropped when no sink +// is open. +func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) { + d.TaskLogs.Write(ctx, taskID, logging.Entry{ + Time: time.Now(), + Level: slog.LevelError.String(), + Message: "task parked: " + reason, + Fields: map[string]any{"component": "daemon", "task": taskID}, + }) +} + +// parkIfIdentityUnresolvable parks task when its identity cannot be acted on, +// reporting whether it did. +func (d *Daemon) parkIfIdentityUnresolvable(ctx context.Context, task *workflow.Task) bool { + ok, reason, class := d.identityMayAct(ctx, task.Identity) + if ok { + return false + } + d.parkRunningTask(ctx, task, reason, class) + return true +} + +// retryTransientParks requeues tasks parked for an environmental cause once +// their backoff has passed. The store parks a task needs_decision, which +// raises the alert, when its retries run out, so this never loops on one. +func (d *Daemon) retryTransientParks(ctx context.Context) { + now := time.Now() + var after storecontract.TaskCursor + for { + page, err := d.Store.TasksPage(ctx, storecontract.TaskPage{Statuses: []string{workflow.StatusParked}, After: after, Limit: 200}) + if err != nil { + d.Log.WarnContext(ctx, "list parked tasks for retry failed", "err", err) + return + } + for _, t := range page { + if taskstate.TransientRetryDue(t.ParkClass, t.RetryCount, t.UpdatedAt, now) { + d.retryTransientPark(ctx, t.ID) + } + } + if len(page) < 200 { + return + } + last := page[len(page)-1] + after = storecontract.TaskCursor{UpdatedAt: last.UpdatedAt, ID: last.ID} + } +} + +// retryTransientPark requeues one task on its stored retry mode, which the +// list projection omits: a remediation must keep continuing its branch. +func (d *Daemon) retryTransientPark(ctx context.Context, id int64) { + t, err := d.Store.TaskByID(ctx, id) + if err != nil { + d.Log.WarnContext(ctx, "load task for transient retry failed", "task", id, "err", err) + return + } + if err := d.Store.RetryTask(ctx, id, workflow.StatusParked, "", t.RetryMode, workflowtask.Resume{}, false); err != nil { + if !errors.Is(err, storecontract.ErrStaleTransition) { + d.Log.WarnContext(ctx, "transient park retry failed", "task", id, "err", err) + } + return + } + d.emit(ctx, events.Event{ + Kind: events.KindTaskRetried, TaskID: id, Workflow: t.Workflow, + Repo: strings.Trim(t.Owner+"/"+t.Repo, "/"), Issue: t.IssueNumber, + Detail: "retried automatically after a transient park", + Data: map[string]any{"retry_count": t.RetryCount + 1, "previous_reason": t.ParkReason}, + }) +} + +// parkRefusedRun parks a task whose workflow identity check did not permit it. +// A denial is the operator's to fix, but a chain that could not be read is a +// store that will answer again, so it parks transient rather than demanding an +// operator for a blip. +func (d *Daemon) parkRefusedRun(ctx context.Context, task *workflow.Task, reason string, err error) { + class, blocker := taskstate.ParkNeedsDecision, taskstate.BlockerPolicyDenied + if err != nil { + class, blocker = pinParkClass(err), parkBlocker(err) + } + d.parkBlockedTask(ctx, task, reason, class, blocker) +} + +// openTaskLog opens the task's log sink for one process() call and returns +// its closer. Failure to open is logged, not fatal. +func (d *Daemon) openTaskLog(task *workflow.Task) func() { + if err := d.TaskLogs.Open(task.ID, task.Attempt); err != nil { + d.Log.Warn("task log sink unavailable", "task", task.ID, "attempt", task.Attempt, "err", err) + } + return func() { + if err := d.TaskLogs.Close(task.ID); err != nil { + d.Log.Warn("task log sink close failed", "task", task.ID, "err", err) + } + } +} diff --git a/internal/daemon/park_test.go b/internal/daemon/park_test.go index 327923c18..5d66748eb 100644 --- a/internal/daemon/park_test.go +++ b/internal/daemon/park_test.go @@ -31,7 +31,7 @@ func TestEarlyParkRecordsAttempt(t *testing.T) { defer bus.Close() sub := bus.Subscribe(1) defer sub.Close() - d := &Daemon{Store: parkStore{err: err}, Bus: bus, Log: slog.New(slog.DiscardHandler)} + d := wired(&Daemon{Store: parkStore{err: err}, Bus: bus, Log: slog.New(slog.DiscardHandler)}) task := &workflow.Task{ID: 7, Attempt: 3, Owner: "owner", Repo: "repo", Workflow: "implement"} d.parkRunningTask(t.Context(), task, "clone authentication failed", taskstate.ParkNeedsDecision) select { diff --git a/internal/daemon/pinning.go b/internal/daemon/pinning.go new file mode 100644 index 000000000..04a54a8f6 --- /dev/null +++ b/internal/daemon/pinning.go @@ -0,0 +1,170 @@ +package daemon + +import ( + "context" + "fmt" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/eda/playbook" + "github.com/samcharles93/archie-core/internal/domain/workflow" + workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task" + "github.com/samcharles93/archie-core/internal/domain/workintake" + "github.com/samcharles93/archie-core/internal/taskstate" +) + +// pinTaskProfile pins the task's workflow definition and resolves its agent +// profile, parking the task on failure. +func (d *Daemon) pinTaskProfile(ctx context.Context, task *workflow.Task) (config.AgentProfile, bool) { + if err := d.pinWorkflowDefinition(ctx, task); err != nil { + d.Log.ErrorContext(ctx, "pin workflow definition failed", "task", task.ID, "err", err) + d.parkBlockedTask(ctx, task, "pin workflow definition: "+err.Error(), pinParkClass(err), parkBlocker(err)) + return config.AgentProfile{}, false + } + iface, err := workflowtask.ParseWorkflowInterface(task.WorkflowDefinitionYAML) + if err == nil { + // Every producer meets the declared interface here, including the + // ones that cannot read definitions when they enqueue (chat and + // dashboard spawns go through the Gateway, which never does). + err = iface.CheckInputs(task.Inputs) + } + if err == nil && iface.RepositoryMode() == workflowtask.RepositoryRequired && !task.HasRepository() { + // An optional workflow may hand off to one that requires a + // repository, because its task may carry one; a task that does not + // is parked here rather than run the target's steps in a scratch + // workspace. + err = fmt.Errorf("requires a repository and this task has none") + } + if err == nil { + var profile config.AgentProfile + if profile, err = d.configFor(task).Containers.Profile(iface.Profile); err == nil { + return profile, true + } + } + d.Log.ErrorContext(ctx, "agent profile unavailable", "task", task.ID, "err", err) + d.parkRunningTask(ctx, task, "workflow "+task.Workflow+": "+err.Error(), taskstate.ParkNeedsDecision) + return config.AgentProfile{}, false +} + +// pinWorkflowDefinition ensures the task carries a valid definition pin for +// the workflow it names and that its org has the workflow enabled. Every run +// passes through here, whatever started it, so this is the one enablement +// check for issue labels, handoffs, approvals, calls and playbooks alike. +func (d *Daemon) pinWorkflowDefinition(ctx context.Context, task *workflow.Task) error { + ctx, err := d.workflowContext(ctx, task) + if err != nil { + return err + } + if err := d.pinDefinition(ctx, task); err != nil { + return err + } + enablement, err := d.workflowEnablement(ctx) + if err != nil { + return pinFailure{taskstate.ParkTransient, fmt.Errorf("workflow enablement: %w", err)} + } + if !enablement.Enabled(task.Org, task.Workflow) { + return blockedPark{taskstate.BlockerWorkflowDisabled, pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("workflow %s is disabled", task.Workflow)}} + } + if err := d.checkPackageTrigger(ctx, task); err != nil { + return pinFailure{taskstate.ParkNeedsDecision, err} + } + return nil +} + +// pinDefinition keeps a valid pin and resolves a missing or mismatched one +// again from the active definitions. +func (d *Daemon) pinDefinition(ctx context.Context, task *workflow.Task) error { + pinned, ok, err := workflow.PinnedDefinitionID(task) + if err != nil { + return pinFailure{taskstate.ParkNeedsDecision, err} + } + if ok && pinned == task.Workflow { + return nil + } + d.runActionPlaybooks(ctx, task) + collection, version, err := d.WorkflowDefinitions.WorkflowDefinitions(ctx) + if err != nil { + return pinFailure{taskstate.ParkTransient, err} + } + return d.pinWorkflowFromCollection(ctx, task, collection, version) +} + +func (d *Daemon) pinWorkflowFromCollection(ctx context.Context, task *workflow.Task, collection workflow.WorkflowDefinitionCollection, version int64) error { + available := make(map[string]struct{}, len(collection.Definitions)) + for _, definition := range collection.Definitions { + available[definition.ID] = struct{}{} + } + id, err := d.resolveWorkflowID(ctx, task, available) + if err != nil { + return pinFailure{taskstate.ParkNeedsDecision, err} + } + definition, ok := collection.DefinitionByID(id) + if !ok { + return pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("workflow definition %q disappeared", id)} + } + task.Workflow = id + task.WorkflowDefinitionVersion = version + task.WorkflowDefinitionYAML = definition.YAML + task.WorkflowDefinitionDigest = workflow.DigestDefinition(definition.YAML) + if err := d.Store.Update(ctx, task); err != nil { + return pinFailure{taskstate.ParkTransient, fmt.Errorf("persist workflow definition pin: %w", err)} + } + return nil +} + +// resolveWorkflowID picks the definition to pin: a matching playbook, then +// the kind defaults. A task already naming a workflow keeps it. +func (d *Daemon) resolveWorkflowID(ctx context.Context, task *workflow.Task, available map[string]struct{}) (string, error) { + if task.Workflow != "" { + return workflow.ResolveWorkflowID(task, available) + } + decision, matched := d.Playbooks.Dispatch(ctx, playbookInput(task)) + if !matched { + return workflow.ResolveWorkflowID(task, available) + } + if _, defined := available[decision.Workflow]; !defined { + // Reported, not silently downgraded to the binding's choice: the + // operator bound this trigger to a named workflow, and running a + // different one under their rule is worse than parking the task. + return "", fmt.Errorf("playbook %q binds this trigger to workflow %q, which is not defined", decision.PlaybookID, decision.Workflow) + } + d.Log.InfoContext(ctx, "playbook selected workflow", + "task", task.ID, "playbook", decision.PlaybookID, + "playbook_version", decision.Version, "workflow", decision.Workflow) + return decision.Workflow, nil +} + +// runActionPlaybooks runs the action playbooks matching a task before it is +// pinned. Failures never fail the task; tasks already naming a workflow are +// skipped. +func (d *Daemon) runActionPlaybooks(ctx context.Context, task *workflow.Task) { + if task.Workflow != "" { + return + } + if err := d.Playbooks.Run(ctx, d.PlaybookLedger, d.Log, playbookInput(task)); err != nil { + d.Log.WarnContext(ctx, "action playbook run failed", "task", task.ID, "err", err) + } +} + +// playbookInput builds the coordinator's dispatch input from a task row. The +// event surface is what the intake point actually knows about the originating +// issue -- the fields a `when` condition can read; it is deliberately not +// padded with values the daemon would have to invent. +func playbookInput(task *workflow.Task) playbook.DispatchInput { + labels := workintake.SplitLabels(task.Labels) + kind := string(workintake.KindForLabels(labels)) + return playbook.DispatchInput{ + Labels: labels, + Kind: kind, + TaskID: workintake.TaskEnvelope{Org: task.Org, Identity: task.Identity, Owner: task.Owner, Repo: task.Repo, Number: task.IssueNumber}.IdempotencyKey(), + Event: map[string]any{ + "kind": kind, + "labels": labels, + "owner": task.Owner, + "repo": task.Repo, + "number": task.IssueNumber, + "title": task.Title, + "body": task.Body, + "source": task.Source, + }, + } +} diff --git a/internal/daemon/pipeline.go b/internal/daemon/pipeline.go new file mode 100644 index 000000000..4f4cb9254 --- /dev/null +++ b/internal/daemon/pipeline.go @@ -0,0 +1,480 @@ +package daemon + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "time" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/identity" + "github.com/samcharles93/archie-core/internal/domain/workflow" + workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task" + "github.com/samcharles93/archie-core/internal/forge" + "github.com/samcharles93/archie-core/internal/storage" + "github.com/samcharles93/archie-core/internal/taskstate" + "github.com/samcharles93/archie-core/internal/worktree" +) + +// repoHub is the per-task repository surface the pipeline resolves through. +type repoHub interface { + configFor(task *workflow.Task) config.Config + forgeFor(ctx context.Context, task *workflow.Task) forge.Forge + treesFor(task *workflow.Task) *worktree.Manager + Repository(ctx context.Context, task *workflow.Task, operation string) (RepositoryAccess, error) +} + +var _ repoHub = (*Daemon)(nil) + +func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHold) { + // A task off the NATS stream only has an id once the store has claimed its + // row, inside this call, so its capacity is named here. + hold.name(task.ID, task.Attempt) + // The log opens before any check that can park, so every park reason + // reaches the attempt log. + defer d.openTaskLog(task)() + // The workflow may name the identity the run acts as, which everything + // below is chosen by, so it is settled before anything else. + if !d.adoptWorkflowIdentity(ctx, task) { + return + } + // Register first so the task can be stopped during setup. + ctx, finished := d.running.begin(ctx, task.ID, task.Identity) + defer finished() + + if d.parkIfIdentityUnresolvable(ctx, task) { + return + } + + trees, repo, err := d.taskWorkspace(ctx, task) + if err != nil { + d.parkBlockedTask(ctx, task, err.Error(), pinParkClass(err), parkBlocker(err)) + return + } + + if d.ContainerPool == nil { + d.parkCapabilityUnavailable(ctx, task) + return + } + + d.Log.InfoContext(ctx, "processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "attempt", task.Attempt) + workDir, ok := d.prepareWorkspace(ctx, task, trees, repo) + if !ok { + return + } + if !task.HasRepository() { + defer func() { + if err := trees.RemoveScratch(task.ID); err != nil { + d.Log.WarnContext(ctx, "scratch workspace cleanup failed", "task", task.ID, "err", err) + } + }() + } + + // The workflow is pinned before the container starts, because the + // profile it names decides the container's image. + profile, ok := d.pinTaskProfile(ctx, task) + if !ok { + return + } + ctr, credential, revokeCredential, ok := d.acquireTaskContainer(ctx, task, repo, workDir, profile.Image) + if !ok { + return + } + defer d.ContainerPool.Release(ctx, ctr) + defer revokeCredential() + + // Run the whole task in archie-agent, which calls back to the daemon for + // store, forge and push operations. + limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std()) + runCtx, stopWatch := withContainerExit(limitCtx, ctr) + d.runViaAgent(runCtx, task, repo, profile, credential) + stopWatch() + stopLimit() + + d.teardownStorage(ctx, task, repo, workDir) + d.cleanupTerminalTaskWorktree(ctx, task, trees) +} + +func (d *Daemon) taskWorkspace(ctx context.Context, task *workflow.Task) (*worktree.Manager, config.Repo, error) { + if task.HasRepository() { + resolved, err := d.Repository(ctx, task, config.OperationClone) + if err == nil && !task.IsTestRun() && workflow.DefinitionPushes(task.WorkflowDefinitionYAML) { + // Checked before the run rather than at its commit step, so a + // missing permit parks in seconds instead of after the agent's work. + _, err = d.Repository(ctx, task, config.OperationPush) + } + return resolved.Trees, resolved.Repository, err + } + trees := d.treesFor(task) + repo, ok := d.repoFor(task) + if !ok && task.HasRepository() { + return nil, repo, pinFailure{taskstate.ParkTerminal, errors.New("repository is no longer configured")} + } + return trees, repo, nil +} + +// prepareWorkspace makes the directory the container binds as its +// workspace, which must exist before Acquire: a fresh clone of the task's +// repository, or an empty scratch directory for a task with none. It parks +// the task and reports false when it cannot. +func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, trees *worktree.Manager, repo config.Repo) (string, bool) { + if !task.HasRepository() { + dir, err := trees.PrepareScratch(task.ID) + if err != nil { + d.Log.ErrorContext(ctx, "scratch workspace prepare failed", "err", err) + d.parkRunningTask(ctx, task, "scratch workspace prepare failed: "+err.Error(), taskstate.ParkTransient) + return "", false + } + return dir, true + } + // The retry mode on the task decides whether to resume the pushed branch or + // reset onto base. Prepare clones a missing worktree either way. + target := worktree.Fresh + if taskstate.NormalizeRetryMode(task.RetryMode) == taskstate.RetryKeepWorktree { + target = worktree.Keep + // Consumed: a later plain requeue must not skip the reset. + task.RetryMode = "" + } + if taskstate.NormalizeRetryMode(task.RetryMode) == taskstate.RetryContinuePushedWork { + if task.Branch == "" { + // The action refuses this combination, so an empty branch here is a + // dispatch bug upstream and no retry can fix it. + d.Log.ErrorContext(ctx, "task has no persisted branch to continue", "task", task.ID) + d.parkRunningTask(ctx, task, "worktree resume failed: task has no branch to continue", taskstate.ParkNeedsDecision) + return "", false + } + target = worktree.Target(task.Branch) + } + d.reclaimStaleClones(ctx, task, trees) + if err := worktree.AwaitSpace(ctx, trees.FreeBytes, minCloneSpace, diskCheckEvery, diskWaitLimit); err != nil { + d.Log.WarnContext(ctx, "worktree prepare delayed", "task", task.ID, "err", err) + d.parkRunningTask(ctx, task, "worktree prepare failed: "+err.Error(), taskstate.ParkTransient) + return "", false + } + // Every task gets an independent full clone. + prepare := func() (string, string, error) { + return trees.Prepare(ctx, task.Owner, task.Repo, repo.BaseBranch(), task.IssueNumber, task.Title, task.Body, task.Labels, target) + } + dir, branch, err := prepare() + if d.reownDenied(ctx, trees.Dir(task.Owner, task.Repo, task.IssueNumber), err) { + dir, branch, err = prepare() + } + if err != nil { + reason := "worktree prepare failed: " + err.Error() + if target != worktree.Fresh { + reason = "worktree resume failed: " + err.Error() + } + d.Log.ErrorContext(ctx, "worktree prepare failed", "task", task.ID, "err", err) + d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient) + return "", false + } + task.Branch = branch + if err := d.Store.Update(ctx, task); err != nil { + d.Log.WarnContext(ctx, "task branch not persisted", "task", task.ID, "err", err) + } + return dir, true +} + +// reclaimStaleClones frees the work volume before a clone waits on it, by +// removing the oldest clones nothing has touched for cloneRetention. A clone +// with uncommitted or unpushed work is kept. +func (d *Daemon) reclaimStaleClones(ctx context.Context, task *workflow.Task, trees *worktree.Manager) { + if have, err := trees.FreeBytes(); err != nil || have >= minCloneSpace { + return + } + n, err := worktree.ReclaimStale(d.Cfg.Get().WorkDir, trees.Dir(task.Owner, task.Repo, task.IssueNumber), cloneRetention, trees.FreeBytes, minCloneSpace, func(dir string) bool { return trees.HoldsUnsharedWork(ctx, dir) }) + if err != nil { + d.Log.WarnContext(ctx, "stale clone reclaim failed", "task", task.ID, "err", err) + } + if n > 0 { + d.Log.InfoContext(ctx, "reclaimed stale clones on a full work volume", "task", task.ID, "clones", n) + } +} + +// cleanupTerminalTaskWorktree removes a settled task's worktree, unless it +// holds uncommitted work or its state cannot be read. +func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow.Task, trees *worktree.Manager) { + if trees == nil || task == nil || !task.HasRepository() { + return + } + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + + latest, err := d.Store.TaskByID(cleanupCtx, task.ID) + if err != nil { + d.Log.WarnContext(ctx, "terminal worktree cleanup task lookup failed", "task", task.ID, "err", err) + return + } + // A run that handed off shares its work item's worktree with its + // successor, which owns it from here. + if latest == nil || latest.HandoffToTaskID != 0 { + return + } + // Settled statuses. Keep the worktree while a PR exists. + switch latest.Status { + case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted, workflow.StatusDead: + if latest.PRNumber != 0 { + return + } + if d.worktreeHoldsUncapturedWork(cleanupCtx, trees, task) { + return + } + if err := d.cleanupWorktree(cleanupCtx, trees, task.Owner, task.Repo, task.IssueNumber); err != nil { + d.Log.WarnContext(ctx, "terminal worktree cleanup failed", "task", task.ID, "err", err) + } + } +} + +// worktreeHoldsUncapturedWork reports whether a terminal task's worktree has +// to survive cleanup because it may hold work no commit captured. A worktree +// that is not on disk at all has nothing to keep, so a task whose clone was +// already removed still cleans up whatever leftovers share its path. +func (d *Daemon) worktreeHoldsUncapturedWork(ctx context.Context, trees *worktree.Manager, task *workflow.Task) bool { + dir := trees.Dir(task.Owner, task.Repo, task.IssueNumber) + if _, err := os.Stat(dir); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false + } + d.Log.WarnContext(ctx, "terminal worktree kept: it could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err) + return true + } + uncommitted, err := trees.HasUncommittedChanges(ctx, dir) + if err != nil { + d.Log.WarnContext(ctx, "terminal worktree kept: uncommitted work could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err) + return true + } + if uncommitted { + d.Log.InfoContext(ctx, "terminal worktree kept: it holds uncommitted work; archive the task to remove it", "task", task.ID, "dir", dir) + return true + } + return false +} + +// teardownStorage runs after the workflow completes. The Docker backend is +// a no-op; future backends (temp volumes, NFS leases) use this hook. +func (d *Daemon) teardownStorage(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string) { + if d.Storage == nil { + return + } + if err := d.Storage.Teardown(ctx, storage.TaskRef{ + WorktreeDir: workDir, + Ecosystem: repo.Ecosystem, + PersistentStorage: repo.PersistentStorage, + Owner: task.Owner, + Repo: task.Repo, + }); err != nil { + d.Log.WarnContext(ctx, "storage teardown failed", "task", task.ID, "err", err) + } +} + +func (d *Daemon) cleanupExpiredStorage(ctx context.Context) error { + ttl := d.Cfg.Get().Containers.VolumeTTL.Std() + if ttl <= 0 || d.Storage == nil { + return nil + } + n, err := d.Storage.CleanupExpired(ctx, ttl) + if n > 0 { + d.Log.InfoContext(ctx, "expired persistent volumes removed", "count", n) + } + if err != nil { + return fmt.Errorf("persistent volume cleanup: %w", err) + } + return nil +} + +// forgeFor returns the forge client that owns task: its organisation +// connection, else the root d.Forge for the root alone. This is the safety +// boundary that keeps one identity's forge token from being used against +// another identity's repos. +func (d *Daemon) forgeFor(ctx context.Context, task *workflow.Task) forge.Forge { + if task.HasRepository() { + resolved, err := d.Repository(ctx, task, config.OperationClone) + if err != nil { + return forge.NewNoop(d.Log) + } + return resolved.Forge + } + if !d.rootTask(task) { + return forge.NewNoop(d.Log) + } + return d.Forge +} + +// treesFor returns the worktree manager that owns task, mirroring forgeFor. +func (d *Daemon) treesFor(task *workflow.Task) *worktree.Manager { + if task.HasRepository() { + // Cleanup needs only the org's path, not a still-valid external credential. + return &worktree.Manager{WorkDir: worktree.OrgWorkDir(d.Cfg.Get().WorkDir, task.OrgOrDefault())} + } + if !d.rootTask(task) { + return &worktree.Manager{WorkDir: filepath.Join(d.Cfg.Get().WorkDir, "identity-"+string(identity.StableID(task.Identity)))} + } + return d.Trees +} + +// repoFor resolves a root task's repo config from Cfg.Repos. +func (d *Daemon) repoFor(t *workflow.Task) (config.Repo, bool) { + if !d.rootTask(t) { + return config.Repo{}, false + } + for _, r := range d.Cfg.Get().Repos { + if r.Owner == t.Owner && r.Name == t.Repo { + return r, true + } + } + return config.Repo{}, false +} + +func (d *Daemon) rootTask(task *workflow.Task) bool { + return identity.IdentityID(task.Identity) == identity.SystemID +} + +func (d *Daemon) configFor(task *workflow.Task) config.Config { + cfg := d.Cfg.Get() + if !d.rootTask(task) { + cfg.BotUser, cfg.BotEmail, cfg.Forge = "", "", config.Forge{} + cfg.Org, cfg.GrantedCredentials, cfg.Identities, cfg.IdentityGrants = task.OrgOrDefault(), nil, nil, nil + cfg.Repos, cfg.Models, cfg.Providers = nil, nil, nil + } + return cfg +} + +// allowConcurrentForTask reports whether the task's repo allows concurrent +// tasks. Unknown repos do not. +func (d *Daemon) allowConcurrentForTask(task *workflow.Task) bool { + if !task.HasRepository() { + return true + } + repo, ok := d.repoFor(task) + return ok && repo.AllowConcurrent +} + +// Repository resolves current credentials and permissions for the run's +// execution identity, never the identity that originally requested the task. +func (d *Daemon) Repository(ctx context.Context, task *workflow.Task, operation string) (RepositoryAccess, error) { + run := *task + if run.WorkflowDefinitionYAML != "" { + iface, err := workflowtask.ParseWorkflowInterface(run.WorkflowDefinitionYAML) + if err != nil || iface.Identity == "" { + return RepositoryAccess{}, fmt.Errorf("workflow execution identity unavailable") + } + _, name, err := d.declaredIdentity(ctx, &run, iface.Identity) + if err != nil { + return RepositoryAccess{}, err + } + run.Identity = name + } + ctx, err := d.workflowContext(ctx, &run) + if err != nil { + return RepositoryAccess{}, err + } + if _, err := d.databaseExecutionIdentity(ctx, &run, identity.IdentityID(run.Identity)); err != nil { + return RepositoryAccess{}, err + } + return d.ResolveRepository(ctx, &run, operation) +} + +// botUserForTask resolves the bot login that owns task, so the reaction +// consumer can exclude Archie's own comments: the global forge bot user for +// root tasks. +func (d *Daemon) botUserForTask(task *workflowtask.Task) string { + if !d.rootTask(task) { + return "" + } + return d.Cfg.Get().BotUser +} + +// RemoveWorktree deletes a task's clone from the trees of the identity that +// owns it. Archiving calls it: terminal cleanup keeps a worktree that may hold +// uncaptured work, and the operator's archive is the decision to discard it. +func (d *Daemon) RemoveWorktree(task *workflow.Task) error { + trees := d.treesFor(task) + if trees == nil { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + return d.cleanupWorktree(ctx, trees, task.Owner, task.Repo, task.IssueNumber) +} + +// cleanupWorktree removes a task's clone, first handing back any root-owned +// files a killed container left in it. +func (d *Daemon) cleanupWorktree(ctx context.Context, trees *worktree.Manager, owner, repo string, issue int) error { + err := trees.Cleanup(owner, repo, issue) + if d.reownDenied(ctx, trees.Dir(owner, repo, issue), err) { + return trees.Cleanup(owner, repo, issue) + } + return err +} + +// reownDenied repairs dir when err says the daemon was refused access to it, +// and reports whether the caller should retry. A SIGKILLed task container +// leaves root-owned files that only a container can chown back. +func (d *Daemon) reownDenied(ctx context.Context, dir string, err error) bool { + if err == nil || !errors.Is(err, fs.ErrPermission) || d.ContainerPool == nil { + return false + } + if rerr := d.ContainerPool.Reown(ctx, dir, os.Getuid(), os.Getgid()); rerr != nil { + d.Log.WarnContext(ctx, "worktree ownership repair failed", "dir", dir, "err", rerr) + return false + } + return true +} + +// A clone waits, holding its slot, until the work volume has minCloneSpace +// free, and parks as transient if it does not within diskWaitLimit. Clones +// share objects with the repository's mirror, so the floor covers a checkout +// and its build output, not a full clone. +const ( + minCloneSpace = 1 << 30 + diskCheckEvery = 15 * time.Second + diskWaitLimit = 10 * time.Minute + // cloneRetention is how long an untouched clone, such as a parked task's, + // is kept before a full work volume may reclaim it. + cloneRetention = 72 * time.Hour +) + +// withContainerExit derives a context cancelled when exited closes. A dead +// container never answers its taskrun request, so without this the request +// blocks forever and the task row stays running. +func withContainerExit(ctx context.Context, ctr exitWatcher) (context.Context, func()) { + runCtx, cancel := context.WithCancelCause(ctx) + go func() { + select { + case <-ctr.Exited(): + if detail := ctr.ExitDetail(); detail != "" { + cancel(fmt.Errorf("%w: %s", errContainerExited, detail)) + return + } + cancel(errContainerExited) + case <-runCtx.Done(): + } + }() + return runCtx, func() { cancel(nil) } +} + +// exitWatcher is a running agent container, seen only for how it ends. +type exitWatcher interface { + Exited() <-chan struct{} + ExitDetail() string +} + +// errContainerExited is the cancellation cause when a task's agent container +// stops before answering its taskrun request. +var errContainerExited = errors.New("agent container exited before the task finished") + +// withTaskTimeLimit bounds a whole task run. A limit of zero or less applies +// no bound. +func withTaskTimeLimit(ctx context.Context, limit time.Duration) (context.Context, func()) { + if limit <= 0 { + return ctx, func() {} + } + return context.WithTimeoutCause(ctx, limit, fmt.Errorf("%w (%s)", errTaskTimeLimit, limit)) +} + +// errTaskTimeLimit is the cancellation cause when a task outruns its overall +// time limit (the execution settings' max task runtime). +var errTaskTimeLimit = errors.New("task exceeded its time limit") diff --git a/internal/daemon/poll_identity_test.go b/internal/daemon/poll_identity_test.go index 8198ffd66..68cb64ec4 100644 --- a/internal/daemon/poll_identity_test.go +++ b/internal/daemon/poll_identity_test.go @@ -54,7 +54,7 @@ func TestPolledTaskKeepsItsCreatingIdentity(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { bus := &publishedKeys{} - d := &Daemon{ + d := wired(&Daemon{ Log: slog.New(slog.DiscardHandler), Store: taskLookup{}, Tasks: bus, Cfg: config.NewHolder(config.Config{}), IssueSources: func(context.Context) []IssueSource { @@ -65,7 +65,7 @@ func TestPolledTaskKeepsItsCreatingIdentity(t *testing.T) { Repos: []config.Repo{{Owner: "acme", Name: "widget"}}, }} }, - } + }) if err := d.poll(t.Context()); err != nil { t.Fatal(err) } diff --git a/internal/daemon/repository_access.go b/internal/daemon/repository_access.go index f16ef62e6..fc5924cb7 100644 --- a/internal/daemon/repository_access.go +++ b/internal/daemon/repository_access.go @@ -1,13 +1,7 @@ package daemon import ( - "context" - "fmt" - "github.com/samcharles93/archie-core/internal/config" - "github.com/samcharles93/archie-core/internal/domain/identity" - "github.com/samcharles93/archie-core/internal/domain/workflow" - workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task" "github.com/samcharles93/archie-core/internal/forge" "github.com/samcharles93/archie-core/internal/worktree" ) @@ -20,31 +14,3 @@ type RepositoryAccess struct { Trees *worktree.Manager Forge forge.Forge } - -// Repository resolves current credentials and permissions for the run's -// execution identity, never the identity that originally requested the task. -func (d *Daemon) Repository(ctx context.Context, task *workflow.Task, operation string) (RepositoryAccess, error) { - if d.ResolveRepository == nil { - return RepositoryAccess{}, fmt.Errorf("repository connections unavailable") - } - run := *task - if run.WorkflowDefinitionYAML != "" { - iface, err := workflowtask.ParseWorkflowInterface(run.WorkflowDefinitionYAML) - if err != nil || iface.Identity == "" { - return RepositoryAccess{}, fmt.Errorf("workflow execution identity unavailable") - } - _, name, err := d.declaredIdentity(ctx, &run, iface.Identity) - if err != nil { - return RepositoryAccess{}, err - } - run.Identity = name - } - ctx, err := d.workflowContext(ctx, &run) - if err != nil { - return RepositoryAccess{}, err - } - if _, err := d.databaseExecutionIdentity(ctx, &run, identity.IdentityID(run.Identity)); err != nil { - return RepositoryAccess{}, err - } - return d.ResolveRepository(ctx, &run, operation) -} diff --git a/internal/daemon/review_scan.go b/internal/daemon/review_scan.go index a208a8476..da093f562 100644 --- a/internal/daemon/review_scan.go +++ b/internal/daemon/review_scan.go @@ -5,9 +5,12 @@ import ( "fmt" "log/slog" + "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/domain/identity" + "github.com/samcharles93/archie-core/internal/domain/workflow" workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task" "github.com/samcharles93/archie-core/internal/domain/workintake" + "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/forge" ) @@ -122,17 +125,11 @@ func scanPRReviewReactions( // scanPRReviews scans every open PR task for review activity with its // identity's forge. Forges without review reading are skipped. func (d *Daemon) scanPRReviews(ctx context.Context) error { - if d.Store == nil || d.Tasks == nil { - return nil - } tasks, err := d.Store.OpenPRs(ctx) if err != nil { return fmt.Errorf("review scan query: %w", err) } var failed []string - if d.reactionPublisher == nil { - d.reactionPublisher = d.PublishReaction - } for i := range tasks { t := &tasks[i] reader, ok := d.forgeFor(ctx, t).(forge.PullRequestReviewReader) @@ -184,3 +181,112 @@ func (d *Daemon) scanOnePR(ctx context.Context, reader forge.PullRequestReviewRe } return true } + +// drainReactions turns available review reactions into queued remediate +// runs. It runs before the task drain so a reaction queued in this pass can +// be claimed by it. The consumer is built by Startup, so its drop counter +// spans the process lifetime. +func (d *Daemon) drainReactions(ctx context.Context) { + if _, err := d.reactionConsumer.drain(ctx, d.Reactions, maxReactionsPerCycle); err != nil { + d.Log.ErrorContext(ctx, "reaction drain failed", "err", err) + } +} + +// PublishReaction stamps the org on a review reaction, defaulting to the +// root identity's, and publishes it. +func (d *Daemon) PublishReaction(ctx context.Context, reaction workintake.ReviewCommentEnvelope) error { + if reaction.Org == "" { + resolved, err := d.identityOrg(ctx, identity.SystemID) + if err != nil { + return fmt.Errorf("publish reaction %s: resolve org: %w", reaction.Ref(), err) + } + reaction.Org = resolved + } + payload, err := reaction.Encode() + if err != nil { + return err + } + return d.Tasks.PublishUnique(ctx, reaction.Subject(), reaction.IdempotencyKey(), payload) +} + +// reconcilePRs moves pr_open tasks to merged/rejected from GitHub state +// and cleans up their worktrees. +func (d *Daemon) reconcilePRs(ctx context.Context) error { + tasks, err := d.Store.OpenPRs(ctx) + if err != nil { + return fmt.Errorf("reconcile query: %w", err) + } + var failed []string + for _, t := range tasks { + fg := d.forgeFor(ctx, &t) + if _, unavailable := fg.(*forge.NoopForge); unavailable { + continue + } + trees := d.treesFor(&t) + // PRState/PR reconciliation is valid regardless of Source -- a + // chat-sourced task can still open a real PR against a real + // branch. Only the issue-label call (tied to the synthetic + // issue number) is forge-only. + state, err := fg.PRState(ctx, t.Owner, t.Repo, t.PRNumber) + if err != nil { + d.Log.WarnContext(ctx, "PR state check failed", "pr", t.PRNumber, "err", err) + failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber)) + continue + } + attempt := t.Attempt + switch state { + case "merged": + if err := d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusMerged, ""); err != nil { + d.Log.ErrorContext(ctx, "PR merged but task transition failed", "task", t.ID, "pr", t.PRNumber, "err", err) + failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber)) + continue + } + if err := trees.Cleanup(t.Owner, t.Repo, t.IssueNumber); err != nil { + d.Log.WarnContext(ctx, "worktree cleanup failed", "task", t.ID, "err", err) + } + // Close the issue explicitly; nothing else does. + d.closeResolvedIssue(ctx, &t, fmt.Sprintf( + "Closed by #%d, merged by archie.", t.PRNumber, + )) + d.Log.InfoContext(ctx, "PR merged", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber) + d.emit(ctx, events.Event{ + Kind: events.KindPRMerged, TaskID: t.ID, Attempt: attempt, + Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber, + Data: map[string]any{"pr": t.PRNumber}, + }) + case "closed": + if err := d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusRejected, "PR closed without merge"); err != nil { + d.Log.ErrorContext(ctx, "PR rejected but task transition failed", "task", t.ID, "pr", t.PRNumber, "err", err) + failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber)) + continue + } + if err := trees.Cleanup(t.Owner, t.Repo, t.IssueNumber); err != nil { + d.Log.WarnContext(ctx, "worktree cleanup failed", "task", t.ID, "err", err) + } + d.Log.InfoContext(ctx, "PR rejected", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber) + d.emit(ctx, events.Event{ + Kind: events.KindPRRejected, TaskID: t.ID, Attempt: attempt, + Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber, + Data: map[string]any{"pr": t.PRNumber}, + }) + } + } + return passFailed("PR reconcile", failed) +} + +// closeResolvedIssue closes a finished task's forge issue, skipping chat +// tasks. Failures are logged. +func (d *Daemon) closeResolvedIssue(ctx context.Context, task *workflow.Task, comment string) { + if !task.IsForgeBacked() { + return + } + resolved, err := d.Repository(ctx, task, config.OperationPR) + if err != nil { + d.Log.WarnContext(ctx, "issue closure not permitted", "task", task.ID) + return + } + if err := resolved.Forge.CloseIssue(ctx, task.Owner, task.Repo, task.IssueNumber, comment); err != nil { + d.Log.WarnContext(ctx, "closing resolved issue failed; it stays open and will be re-polled", + "repo", task.Owner+"/"+task.Repo, "issue", task.IssueNumber, "err", err) + } +} diff --git a/internal/daemon/statestore_relay_test.go b/internal/daemon/statestore_relay_test.go index e06548d12..2bf1bda63 100644 --- a/internal/daemon/statestore_relay_test.go +++ b/internal/daemon/statestore_relay_test.go @@ -85,10 +85,10 @@ func TestContainerEnvStateStore(t *testing.T) { } t.Cleanup(relay.Close) - d := &Daemon{ + d := wired(&Daemon{ ConnectedNATS: NATSEndpoint{URL: "nats://10.0.0.5:4222"}, ConnectedStateStore: StateStoreEndpoint{URL: upstream.Addr().String(), CA: []byte("ca-pem")}, - } + }) env := envValues(d.containerEnv(&workflow.Task{ID: 7}, "task-token", relay.Addr())) if got := env["STATE_STORE_URL"]; got != relay.Addr() { diff --git a/internal/daemon/task_bus.go b/internal/daemon/task_bus.go new file mode 100644 index 000000000..9eb5128f3 --- /dev/null +++ b/internal/daemon/task_bus.go @@ -0,0 +1,482 @@ +package daemon + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/samcharles93/archie-core/internal/domain/identity" + "github.com/samcharles93/archie-core/internal/domain/workflow" + "github.com/samcharles93/archie-core/internal/domain/workintake" + "github.com/samcharles93/archie-core/internal/eventbus" + "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/natsrpc" + "github.com/samcharles93/archie-core/internal/taskstate" +) + +// drainLoop is the only task drainer and capture dispatcher while Run is +// active. It runs apart from the poll tick so a captured event becomes a +// running task within about a second regardless of poll_interval or forge +// latency. Submit never blocks, so a full dispatcher +// just leaves tasks waiting on slots and the loop stays paced by the ticker. +func (d *Daemon) drainLoop(ctx context.Context) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + d.markPass("drain") + d.dispatchBindings(ctx) + d.drainNATS(ctx) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +// drainNATS processes tasks from NATS, falling back to the task store's ClaimNext for +// requeued tasks (waiting_approval approval, retry-parked) that didn't come +// through a NATS publish. It returns once the stream and the claim fallback +// are both empty: in-flight tasks keep running, so a task waiting on a callee +// does not hold up the pass that would claim it. Shutdown waits for them +// through WaitForTasks. +func (d *Daemon) drainNATS(ctx context.Context) { + dispatcher := d.taskDispatcher() + // A full dispatcher leaves work in the stream and the store, not parked + // on slots where it already reads as running. + for ctx.Err() == nil && dispatcher.hasRoom() { + msg, err := d.Tasks.Fetch(ctx) + if err != nil && !errors.Is(err, eventbus.ErrNoMessage) { + d.Log.ErrorContext(ctx, "nats fetch failed", "err", err) + break + } + if err == nil { + d.submitNATSTask(ctx, dispatcher, msg) + d.markPass("drain") + continue + } + + // ErrNoMessage means the stream is drained. That is precisely when + // the SQLite fallback must run: requeued tasks (waiting_approval + // approval, retry-parked) never went through a NATS publish, so + // nothing will ever arrive on the stream to trigger them. + d.claimRequeued(ctx, dispatcher) + return + } +} + +// claimRequeued submits tasks from the store's claim fallback while the +// dispatcher has room; the rest stay unclaimed for a later pass. It does +// not go back to the stream between claims: each empty fetch waits a full poll +// timeout, so a backlog left by a restart would outlast the systemd watchdog. +func (d *Daemon) claimRequeued(ctx context.Context, dispatcher *taskDispatcher) { + for ctx.Err() == nil && dispatcher.hasRoom() { + task, err := d.Store.ClaimNext(ctx) + if err != nil { + d.Log.ErrorContext(ctx, "sqlite claim failed", "err", err) + return + } + if task == nil { + return + } + dispatcher.Submit(ctx, task, d.process) + d.markPass("drain") + } +} + +// taskDispatcher returns the daemon's long-lived global dispatcher, building +// it from the running config on first use. It outlives a single drain pass so +// ResizeTaskDispatcher can resize the running dispatcher. +func (d *Daemon) taskDispatcher() *taskDispatcher { + d.dispatcherOnce.Do(func() { + d.dispatcher = newTaskDispatcher(d.Cfg.Get().Containers.MaxConcurrency, d.allowConcurrentForTask) + }) + return d.dispatcher +} + +// WaitForTasks blocks until every submitted task's process has returned. A +// drain pass no longer waits for its own tasks, so shutdown is where in-flight +// work is waited for. +func (d *Daemon) WaitForTasks() { + d.taskDispatcher().Wait() +} + +// ResizeTaskDispatcher applies a new containers.max_concurrency to the +// running global task dispatcher without a restart. Running tasks keep their +// slots; a lowered limit is honoured as they finish, and a raised limit +// admits queued work immediately. +func (d *Daemon) ResizeTaskDispatcher(maxConcurrency int) { + d.taskDispatcher().SetMaxConcurrency(maxConcurrency) +} + +// submitNATSTask decodes a fetched message and queues it for processing. A +// message that cannot be decoded is acked rather than redelivered: it will +// never decode on a retry, so leaving it unacked would block the queue. +func (d *Daemon) submitNATSTask(ctx context.Context, dispatcher *taskDispatcher, msg eventbus.Message) { + envelope, err := workintake.DecodeTask(msg.Data()) + if err != nil { + d.Log.ErrorContext(ctx, "nats decode failed", "subject", msg.Subject(), "err", err) + if err := msg.Ack(); err != nil { + d.Log.WarnContext(ctx, "nats ack failed", "subject", msg.Subject(), "err", err) + } + return + } + task := &workflow.Task{Owner: envelope.Owner, Repo: envelope.Repo, Identity: envelope.Identity} + dispatcher.Submit(ctx, task, func(ctx context.Context, _ *workflow.Task, hold *taskHold) { + d.processNATSTask(ctx, msg, hold) + }) +} + +// processNATSTask decodes a NATS message, writes it to SQLite, claims it, +// and runs the workflow. The message is ack'd on terminal (park is a valid +// outcome). Nak on transient errors so NATS redelivers. +func (d *Daemon) processNATSTask(ctx context.Context, msg eventbus.Message, hold *taskHold) { + ctx, span := natsrpc.StartConsumer(ctx, msg.Subject(), msg.Header()) + defer span.End() + tm, err := workintake.DecodeTask(msg.Data()) + if err != nil { + d.Log.ErrorContext(ctx, "nats decode failed", "err", err) + if err := msg.Ack(); err != nil { + d.Log.WarnContext(ctx, "ack failed", "err", err) + } + // bad message, don't retry + return + } + + if !d.acceptTaskPublisher(ctx, msg, tm.Identity) { + return + } + + inserted, err := d.Store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity) + if err != nil { + d.Log.ErrorContext(ctx, "nats enqueue failed", "err", err) + if err := msg.Nak(); err != nil { + d.Log.WarnContext(ctx, "nats nak failed", "err", err) + } + return + } + if !inserted { + // Already tracked -- dedup. Ack and move on. + if err := msg.Ack(); err != nil { + d.Log.WarnContext(ctx, "ack failed", "err", err) + } + return + } + + task, err := d.Store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number) + if err != nil { + d.Log.ErrorContext(ctx, "nats claim failed", "err", err) + if err := msg.Nak(); err != nil { + d.Log.WarnContext(ctx, "nats nak failed", "err", err) + } + return + } + if task == nil { + // Claimed by another consumer, or task is not queued. Ack. + if err := msg.Ack(); err != nil { + d.Log.WarnContext(ctx, "ack failed", "err", err) + } + return + } + + d.emitTaskQueued(ctx, task) + d.process(ctx, task, hold) + if err := msg.Ack(); err != nil { + d.Log.WarnContext(ctx, "ack failed", "err", err) + } +} + +func (d *Daemon) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, name string) bool { + if ok, reason, class := d.identityMayAct(ctx, name); !ok { + d.Log.ErrorContext(ctx, "nats intake rejected: "+reason, "subject", msg.Subject()) + if class == taskstate.ParkTransient { + if err := msg.Nak(); err != nil { + d.Log.WarnContext(ctx, "nak failed", "err", err) + } + return false + } + if err := msg.Ack(); err != nil { + d.Log.WarnContext(ctx, "ack failed", "err", err) + } + return false + } + + return true +} + +// emitTaskQueued records the queued task under the identity that polled it, +// which is both the actor and the authority the task was created with. +func (d *Daemon) emitTaskQueued(ctx context.Context, task *workflow.Task) { + e := events.Event{ + Kind: events.KindTaskQueued, TaskID: task.ID, Repo: task.Owner + "/" + task.Repo, + Issue: task.IssueNumber, Detail: task.Title, Attempt: task.Attempt, + ActorID: task.Identity, PrincipalID: task.Identity, + } + if value, err := d.IdentityRepository.Get(ctx, identity.IdentityID(task.Identity)); err == nil { + e.ActorKind = string(value.Kind) + } + d.emit(ctx, e) +} + +// PublishTask validates a task envelope, stamps the producing identity's +// org and publishes it. The idempotency key dedups rediscovered issues. +func (d *Daemon) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error { + if err := task.Kind.Validate(); err != nil { + return fmt.Errorf("publish task %s: %w", task.Ref(), err) + } + if task.Identity == "" { + return fmt.Errorf("publish task %s: no identity", task.Ref()) + } + resolved, err := d.identityOrg(ctx, identity.IdentityID(task.Identity)) + if err != nil { + return fmt.Errorf("publish task %s: resolve org: %w", task.Ref(), err) + } + task.Org = resolved + payload, err := task.Encode() + if err != nil { + return err + } + return d.Tasks.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload) +} + +// InFlight counts the same submitted work that graceful shutdown waits for, +// including tasks waiting for a repository or concurrency slot. +func (d *Daemon) InFlight() int { + dispatcher := d.taskDispatcher() + dispatcher.mu.Lock() + defer dispatcher.mu.Unlock() + return dispatcher.pending +} + +// taskDispatcher limits concurrent tasks globally and runs one task per repo +// at a time, in submit order, unless the repo allows concurrency. +type taskDispatcher struct { + allowConcurrent func(task *workflow.Task) bool + + mu sync.Mutex + limit int + active int + slotsFree *sync.Cond + repoTail map[string]chan struct{} + pending int + // waiting counts submitted tasks that hold no slot yet, so a drain pass + // stops claiming once queued work alone fills the limit. + waiting int + // holds maps a task id to the capacity its run holds, so a run waiting on + // a callee can give that capacity back and take it again by task id. + holds map[int64]*taskHold +} + +// SetMaxConcurrency accepts a new global limit without a restart. Running +// tasks keep their slots; a lowered limit is enforced as they finish, and a +// raised limit admits queued work immediately. +func (d *taskDispatcher) SetMaxConcurrency(maxConcurrency int) { + d.mu.Lock() + d.limit = maxConcurrency + d.mu.Unlock() + d.slotsFree.Broadcast() +} + +func newTaskDispatcher(maxConcurrency int, allowConcurrent func(task *workflow.Task) bool) *taskDispatcher { + if allowConcurrent == nil { + allowConcurrent = func(*workflow.Task) bool { return false } + } + d := &taskDispatcher{ + limit: maxConcurrency, + allowConcurrent: allowConcurrent, + repoTail: make(map[string]chan struct{}), + holds: make(map[int64]*taskHold), + } + d.slotsFree = sync.NewCond(&d.mu) + return d +} + +// joinRepoLocked appends baton to the tail of repo's chain and returns the +// baton ahead of it, which the joining task is blocked on. Callers hold d.mu. +func (d *taskDispatcher) joinRepoLocked(repo string, baton chan struct{}) chan struct{} { + previous := d.repoTail[repo] + d.repoTail[repo] = baton + return previous +} + +// takeSlotLocked blocks until the global limit has room, then counts the run +// in. limit <= 0 means unlimited. The limit is re-read on every wake, so +// SetMaxConcurrency reaches waiting tasks without recreating the dispatcher. +// Callers hold d.mu. +func (d *taskDispatcher) takeSlotLocked() { + for d.limit > 0 && d.active >= d.limit { + d.slotsFree.Wait() + } + d.active++ +} + +// giveBackLocked gives back everything the run holds: its slot under the +// global limit, and its place in its repo's chain, which releases the task +// queued behind it. Callers hold d.mu. +func (d *taskDispatcher) giveBackLocked(hold *taskHold) { + if hold.slot { + d.active-- + hold.slot = false + } + if hold.baton == nil { + return + } + close(hold.baton) + if d.repoTail[hold.repo] == hold.baton { + delete(d.repoTail, hold.repo) + } + hold.baton = nil +} + +// finishRun retires a returned run: it holds no capacity any more, and no +// Resume may take any for it. Whatever an in-flight resume takes later is +// given back by reacquire, which sees finished. +func (d *taskDispatcher) finishRun(hold *taskHold) { + d.mu.Lock() + hold.finished = true + if hold.id != 0 && d.holds[hold.id] == hold { + delete(d.holds, hold.id) + } + d.giveBackLocked(hold) + d.pending-- + d.mu.Unlock() + d.slotsFree.Broadcast() +} + +// setCallWaits applies a durable snapshot without accumulating duplicate events. +func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) { + d.mu.Lock() + if d.holds[hold.id] != hold || hold.finished { + d.mu.Unlock() + return + } + previous := hold.waits + hold.waits = waits + if waits > 0 { + d.giveBackLocked(hold) + } + d.mu.Unlock() + if waits > 0 { + d.slotsFree.Broadcast() + } else if previous > 0 { + // Reacquiring may block behind another task; the caller keeps running. + go d.reacquire(hold) + } +} + +// reacquire returns a resumed run's capacity: its place in its repo's chain +// and a slot under the global limit. The run keeps executing meanwhile -- a +// wait on a callee is not a pause. A hold records one slot and one place, so a +// re-acquire queued while another is still waiting gives back whichever of the +// two it finds already recorded instead of taking a second. +func (d *taskDispatcher) reacquire(hold *taskHold) { + d.mu.Lock() + if hold.finished || hold.waits > 0 { + // The run returned, or started waiting again, after the reconciliation that + // queued this: it needs no capacity. + d.mu.Unlock() + return + } + var previous chan struct{} + // A place another re-acquire already queued is the run's one place; joining + // behind it would queue a second, which the task behind it would wait on + // forever. + if hold.chain && hold.baton == nil { + hold.baton = make(chan struct{}) + previous = d.joinRepoLocked(hold.repo, hold.baton) + } + d.mu.Unlock() + + if previous != nil { + <-previous + } + + d.mu.Lock() + d.takeSlotLocked() + if hold.slot { + // Another re-acquire took the run's one slot while this waited: give + // back the slot this one just took. The place is the other's, so it is + // left alone. + d.active-- + d.mu.Unlock() + d.slotsFree.Broadcast() + return + } + hold.slot = true + if hold.finished || hold.waits > 0 { + // The run ended, or started waiting again, while this was queued for + // capacity: give back what it took, so no slot or place in the repo + // chain leaks. + d.giveBackLocked(hold) + d.mu.Unlock() + d.slotsFree.Broadcast() + return + } + d.mu.Unlock() +} + +// Submit queues a task's process behind the global limit and, unless the repo +// allows concurrency, behind the task already holding that repo. process is +// handed the run's capacity so it can name it once it knows the task's id and +// give it back while it waits on a callee. +func (d *taskDispatcher) Submit( + ctx context.Context, + task *workflow.Task, + process func(context.Context, *workflow.Task, *taskHold), +) { + repo := task.Owner + "/" + task.Repo + hold := &taskHold{dispatcher: d, repo: repo} + + var previous chan struct{} + if !d.allowConcurrent(task) { + hold.chain = true + hold.baton = make(chan struct{}) + d.mu.Lock() + previous = d.joinRepoLocked(repo, hold.baton) + d.mu.Unlock() + } + + d.mu.Lock() + d.pending++ + d.waiting++ + d.mu.Unlock() + go func() { + defer d.finishRun(hold) + if previous != nil { + <-previous + } + d.mu.Lock() + d.takeSlotLocked() + d.waiting-- + hold.slot = true + d.mu.Unlock() + process(ctx, task, hold) + }() +} + +// hasRoom reports whether a newly submitted task could take a slot without +// queueing behind work already submitted. A run waiting on a callee holds no +// slot, so its callee still finds room. +func (d *taskDispatcher) hasRoom() bool { + d.mu.Lock() + defer d.mu.Unlock() + return d.limit <= 0 || d.active+d.waiting < d.limit +} + +// inFlight is how many submitted tasks have not returned yet. +func (d *taskDispatcher) inFlight() int { + d.mu.Lock() + defer d.mu.Unlock() + return d.pending +} + +func (d *taskDispatcher) Wait() { + d.mu.Lock() + defer d.mu.Unlock() + for d.pending > 0 { + d.slotsFree.Wait() + } +} diff --git a/internal/daemon/transient_retry_test.go b/internal/daemon/transient_retry_test.go index a6db2ded8..c40e5dc23 100644 --- a/internal/daemon/transient_retry_test.go +++ b/internal/daemon/transient_retry_test.go @@ -39,7 +39,7 @@ func TestRetryTransientParks(t *testing.T) { {ID: 3, ParkClass: taskstate.ParkNeedsDecision, UpdatedAt: old}, {ID: 4, ParkClass: taskstate.ParkTransient, RetryCount: taskstate.MaxTransientRetries, UpdatedAt: old}, }} - d := &Daemon{Store: store, Log: slog.New(slog.DiscardHandler)} + d := wired(&Daemon{Store: store, Log: slog.New(slog.DiscardHandler)}) d.retryTransientParks(t.Context()) if len(store.retried) != 1 || store.retried[0] != 1 { t.Fatalf("retried = %v, want only task 1 (due, transient, retries left)", store.retried) diff --git a/internal/daemon/wiring_test.go b/internal/daemon/wiring_test.go new file mode 100644 index 000000000..6ad5b8ed1 --- /dev/null +++ b/internal/daemon/wiring_test.go @@ -0,0 +1,220 @@ +package daemon + +import ( + "context" + "log/slog" + "time" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/access" + "github.com/samcharles93/archie-core/internal/domain/eda/playbook" + "github.com/samcharles93/archie-core/internal/domain/identity" + "github.com/samcharles93/archie-core/internal/domain/org" + "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/domain/storepkg" + "github.com/samcharles93/archie-core/internal/domain/workflow" + workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task" + "github.com/samcharles93/archie-core/internal/eventbus" + "github.com/samcharles93/archie-core/internal/events" +) + +// wired gives d the dependencies production always sets, with neutral +// stand-ins for any the test left nil. A test that does not exercise one of +// them then never reaches a nil branch in the code under test. A test that +// does exercise one sets its own fake. +func wired(d *Daemon) *Daemon { + if d.Tasks == nil { + d.Tasks = neutralTaskBus{} + } + if d.Store == nil { + d.Store = neutralStore{} + } + if d.Reactions == nil { + d.Reactions = neutralReactions{} + } + if d.Bus == nil { + d.Bus = events.NewBus() + } + if d.RunCredentials == nil { + d.RunCredentials = neutralCredentials{} + } + if d.TaskModels == nil { + d.TaskModels = neutralTaskConfig{} + } + if d.TaskInstructions == nil { + d.TaskInstructions = neutralTaskConfig{} + } + if d.Access == nil { + d.Access = allowAll{} + } + if d.Principals == nil { + d.Principals = neutralPrincipals{} + } + if d.Denials == nil { + d.Denials = neutralDenials{} + } + if d.IdentityRepository == nil { + d.IdentityRepository = neutralIdentities{} + } + if d.AgentAssignments == nil { + d.AgentAssignments = neutralAssignments{} + } + if d.PackageAuthorities == nil { + d.PackageAuthorities = neutralAuthorities{} + } + if d.Playbooks == nil { + d.Playbooks = neutralPlaybooks{} + } + if d.WorkflowDefinitions == nil { + d.WorkflowDefinitions = neutralDefinitions{} + } + if d.WorkflowEnablement == nil { + d.WorkflowEnablement = neutralEnablement{} + } + if d.IssueSources == nil { + d.IssueSources = func(context.Context) []IssueSource { return nil } + } + if d.OrgPrincipals == nil { + d.OrgPrincipals = func(context.Context) []access.Principal { + return []access.Principal{{Org: org.DefaultOrgID}} + } + } + if d.MappingMatches == nil { + d.MappingMatches = neutralMatches{} + } + if d.WithdrawTask == nil { + d.WithdrawTask = func(context.Context, int64, string) error { return nil } + } + if d.AgentStatus == nil { + d.AgentStatus = &AgentStatus{} + } + if d.Log == nil { + d.Log = slog.New(slog.DiscardHandler) + } + d.wireReactions() + return d +} + +// neutralTaskBus takes no work and accepts every publish. +type neutralTaskBus struct{} + +func (neutralTaskBus) PublishUnique(context.Context, string, string, []byte) error { return nil } + +func (neutralTaskBus) Fetch(context.Context) (eventbus.Message, error) { + return nil, eventbus.ErrNoMessage +} + +func (neutralTaskBus) Request(context.Context, string, []byte) ([]byte, error) { return nil, nil } + +// neutralStore records no events. Any other store call panics, so a test that +// reaches one sets its own store. +type neutralStore struct{ storecontract.TaskStore } + +func (neutralStore) InsertEvent(context.Context, events.Event) (int64, error) { return 0, nil } + +// neutralReactions has no review reactions waiting. +type neutralReactions struct{} + +func (neutralReactions) Fetch(context.Context) (eventbus.Message, error) { + return nil, eventbus.ErrNoMessage +} + +// neutralCredentials issues an empty run credential. +type neutralCredentials struct{} + +func (neutralCredentials) Issue(context.Context, *workflow.Task, time.Duration) (string, func(), error) { + return "", func() {}, nil +} + +// neutralTaskConfig has no org model aliases and no instructions. +type neutralTaskConfig struct{} + +func (neutralTaskConfig) TaskModels(context.Context, int64) (config.OrgModels, error) { + return config.OrgModels{}, nil +} + +func (neutralTaskConfig) TaskInstructions(context.Context, int64) (string, string, error) { + return "", "", nil +} + +// allowAll permits every policy request. +type allowAll struct{} + +func (allowAll) Authorize(context.Context, access.Principal, access.Action, access.Resource, access.Context) access.Decision { + return access.Decision{Allowed: true} +} + +// neutralPrincipals resolves every identity to the default org with no role. +type neutralPrincipals struct{} + +func (neutralPrincipals) PrincipalFor(_ context.Context, id identity.IdentityID) (access.Principal, error) { + return access.Principal{IdentityID: id, Org: org.DefaultOrgID}, nil +} + +type neutralDenials struct{} + +func (neutralDenials) RecordDenial(context.Context, access.Denial) error { return nil } + +// neutralIdentities resolves every identity as an active agent. +type neutralIdentities struct{} + +func (neutralIdentities) List(context.Context) ([]identity.Identity, error) { return nil, nil } + +func (neutralIdentities) Get(_ context.Context, id identity.IdentityID) (identity.Identity, error) { + return identity.Identity{ID: id, Kind: identity.KindBot, Lifecycle: identity.LifecycleActive}, nil +} + +func (neutralIdentities) ResolveLegacyName(context.Context, string) (identity.Identity, error) { + return identity.Identity{}, identity.ErrNotFound +} + +func (neutralIdentities) Create(_ context.Context, v identity.Identity, _ identity.Audit) (identity.Identity, error) { + return v, nil +} + +func (neutralIdentities) Apply(_ context.Context, _ identity.IdentityID, _ int64, _ identity.Command, _ identity.Audit) (identity.Identity, error) { + return identity.Identity{}, nil +} + +type neutralAssignments struct{} + +func (neutralAssignments) ListOrgAgents(context.Context, org.OrgID) ([]org.AgentAssignment, error) { + return nil, nil +} + +// neutralAuthorities accepts nothing for any package. +type neutralAuthorities struct{} + +func (neutralAuthorities) AcceptedAuthority(context.Context, string) (storepkg.Authority, error) { + return storepkg.Authority{}, nil +} + +// neutralPlaybooks matches no playbook. +type neutralPlaybooks struct{} + +func (neutralPlaybooks) Dispatch(context.Context, playbook.DispatchInput) (playbook.Decision, bool) { + return playbook.Decision{}, false +} + +func (neutralPlaybooks) Run(context.Context, storecontract.PlaybookDispatcher, *slog.Logger, playbook.DispatchInput) error { + return nil +} + +// neutralMatches records no mapping match. +type neutralMatches struct{} + +func (neutralMatches) RecordMappingMatch(context.Context, string, string) error { return nil } + +// neutralDefinitions serves the shipped workflow definitions at version 0. +type neutralDefinitions struct{} + +func (neutralDefinitions) WorkflowDefinitions(context.Context) (workflow.WorkflowDefinitionCollection, int64, error) { + return workflow.ShippedDefinitions(), 0, nil +} + +// neutralEnablement disables no workflow. +type neutralEnablement struct{} + +func (neutralEnablement) WorkflowEnablement(context.Context) (workflowtask.WorkflowEnablement, error) { + return workflowtask.WorkflowEnablement{}, nil +} diff --git a/internal/daemon/withdraw_test.go b/internal/daemon/withdraw_test.go index 8351eee44..0992a9e04 100644 --- a/internal/daemon/withdraw_test.go +++ b/internal/daemon/withdraw_test.go @@ -37,7 +37,7 @@ func TestWithdrawUnpolled(t *testing.T) { {"eligible again under the new rule", "label:bot", []int{3}, false}, {"dropped under the same rule: withdrawn", "label:bot", nil, true}, } - d := &Daemon{Log: slog.New(slog.DiscardHandler), Store: activeTasks{tasks: []workflow.Task{task}}} + d := wired(&Daemon{Log: slog.New(slog.DiscardHandler), Store: activeTasks{tasks: []workflow.Task{task}}}) for _, step := range steps { withdrawn := false d.WithdrawTask = func(context.Context, int64, string) error { withdrawn = true; return nil } diff --git a/internal/daemon/workflow_identity.go b/internal/daemon/workflow_identity.go index 1b6b3e37c..becc1d971 100644 --- a/internal/daemon/workflow_identity.go +++ b/internal/daemon/workflow_identity.go @@ -60,9 +60,6 @@ func (d *Daemon) adoptWorkflowIdentity(ctx context.Context, task *workflow.Task) // declaredIdentity resolves a workflow's declared identity to the ID it is // authorized as and the name the task carries while it runs. func (d *Daemon) declaredIdentity(ctx context.Context, task *workflow.Task, ref string) (identity.IdentityID, string, error) { - if d.IdentityRepository == nil { - return "", "", pinFailure{taskstate.ParkTransient, fmt.Errorf("identity authority unavailable for %s", ref)} - } value, err := d.databaseExecutionIdentity(ctx, task, identity.IdentityID(ref)) if err != nil { return "", "", err @@ -83,13 +80,7 @@ func (d *Daemon) databaseExecutionIdentity(ctx context.Context, task *workflow.T if !value.Kind.Agent() || value.Lifecycle != identity.LifecycleActive { return identity.Identity{}, blockedPark{taskstate.BlockerAgentAssignment, pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("execution identity %s must be an active bot or service account", id)}} } - if d.AgentAssignments == nil { - return identity.Identity{}, pinFailure{taskstate.ParkTransient, fmt.Errorf("agent assignment authority unavailable")} - } owner := org.OrgID(task.OrgOrDefault()) - if d.Principals == nil { - return identity.Identity{}, pinFailure{taskstate.ParkTransient, fmt.Errorf("principal authority unavailable")} - } principal, err := d.Principals.PrincipalFor(ctx, value.ID) if err != nil { return identity.Identity{}, pinFailure{taskstate.ParkTransient, fmt.Errorf("resolve execution principal: %w", err)} @@ -112,9 +103,6 @@ func (d *Daemon) databaseExecutionIdentity(ctx context.Context, task *workflow.T // an install that has not built one. The error reports a chain that could not // be consulted, which is not a denial: nothing was evaluated. func (d *Daemon) mayRunWorkflow(ctx context.Context, task *workflow.Task, id identity.IdentityID) (bool, error) { - if d.Access == nil || d.Principals == nil { - return true, nil - } principal, err := d.principalIn(ctx, id, task.Org) if err != nil { d.Log.WarnContext(ctx, "workflow identity: principal unavailable", "task", task.ID, "identity", id, "err", err) @@ -133,26 +121,12 @@ func (d *Daemon) mayRunWorkflow(ctx context.Context, task *workflow.Task, id ide // operator to involve: the chain will answer once the store does. return false, pinFailure{taskstate.ParkTransient, decision.Err} } - if d.Denials != nil { - denial := access.Denial{ - Principal: principal.IdentityID, Org: principal.Org, Action: access.ActionRun, - Kind: resource.Kind, ResourceID: resource.ID, Level: decision.Level, Policies: decision.Policies, - } - if err := d.Denials.RecordDenial(ctx, denial); err != nil { - d.Log.WarnContext(ctx, "workflow identity: record denial", "task", task.ID, "err", err) - } + denial := access.Denial{ + Principal: principal.IdentityID, Org: principal.Org, Action: access.ActionRun, + Kind: resource.Kind, ResourceID: resource.ID, Level: decision.Level, Policies: decision.Policies, + } + if err := d.Denials.RecordDenial(ctx, denial); err != nil { + d.Log.WarnContext(ctx, "workflow identity: record denial", "task", task.ID, "err", err) } return false, nil } - -// parkRefusedRun parks a task whose workflow identity check did not permit it. -// A denial is the operator's to fix, but a chain that could not be read is a -// store that will answer again, so it parks transient rather than demanding an -// operator for a blip. -func (d *Daemon) parkRefusedRun(ctx context.Context, task *workflow.Task, reason string, err error) { - class, blocker := taskstate.ParkNeedsDecision, taskstate.BlockerPolicyDenied - if err != nil { - class, blocker = pinParkClass(err), parkBlocker(err) - } - d.parkBlockedTask(ctx, task, reason, class, blocker) -} diff --git a/internal/daemon/workflow_identity_test.go b/internal/daemon/workflow_identity_test.go index 616f43552..33722f3b5 100644 --- a/internal/daemon/workflow_identity_test.go +++ b/internal/daemon/workflow_identity_test.go @@ -129,11 +129,11 @@ func TestWorkflowIdentity(t *testing.T) { t.Run(tt.name, func(t *testing.T) { store := &identityStore{} var denied denials - d := &Daemon{ + d := wired(&Daemon{ Store: store, Log: slog.New(slog.DiscardHandler), Access: tt.allowed, Principals: principalOf{}, Denials: &denied, IdentityRepository: agentDirectory{}, AgentAssignments: agentDirectory{}, - } + }) if tt.disabled { d.WorkflowEnablement = enablementOf(workflowtask.WorkflowEnablement{}.SetEnabled("", "deploy", false)) } @@ -177,11 +177,11 @@ func TestWorkflowRunCarriesEventContext(t *testing.T) { want bool }{{"valid", true}, {"unsigned", false}, {"", false}} { t.Run(tt.signature, func(t *testing.T) { - d := &Daemon{ + d := wired(&Daemon{ Store: &identityStore{}, Log: slog.New(slog.DiscardHandler), Access: signedOnly{runAllowed{sys: true, "id-deployer": true}}, Principals: principalOf{}, IdentityRepository: agentDirectory{}, AgentAssignments: agentDirectory{}, - } + }) task := &workflow.Task{ ID: 1, Identity: string(sys), Workflow: "deploy", BindingSignature: tt.signature, WorkflowDefinitionYAML: deploy, WorkflowDefinitionDigest: workflow.DigestDefinition(deploy), @@ -236,11 +236,11 @@ func TestDashboardExecutionIdentity(t *testing.T) { } store := &identityStore{} cfg := config.Config{BotUser: "root-bot", GrantedCredentials: []string{"root-secret"}, Forge: config.Forge{Token: config.SecretRef{Engine: "env", Key: "ROOT"}}, Repos: []config.Repo{{Owner: "acme", Name: "app"}}} - d := &Daemon{ + d := wired(&Daemon{ Store: store, Log: slog.New(slog.DiscardHandler), Cfg: config.NewHolder(cfg), IdentityRepository: repository, AgentAssignments: assignedAgents{{IdentityID: value.ID, OrgID: tt.assigned}}, Principals: workflowPrincipals{sys: "org-sys", "dashboard-agent": "org-sys"}, Access: runAllowed{sys: true, "dashboard-agent": true}, - } + }) definition := "id: deploy\nidentity: dashboard-agent\nrepository: none\n" newTask := func() *workflow.Task { return &workflow.Task{ID: 1, Identity: string(sys), Workflow: "deploy", WorkflowDefinitionYAML: definition, WorkflowDefinitionDigest: workflow.DigestDefinition(definition)} diff --git a/internal/daemon/workflow_org.go b/internal/daemon/workflow_org.go index d23b882d4..ac057b223 100644 --- a/internal/daemon/workflow_org.go +++ b/internal/daemon/workflow_org.go @@ -18,13 +18,6 @@ func (d *Daemon) workflowContext(ctx context.Context, task *workflow.Task) (cont if owner == "" { owner = org.DefaultOrgID } - if d.Principals == nil { - principal, forwarded := access.PrincipalFromContext(ctx) - if owner != org.DefaultOrgID || org.OrgFromContext(ctx) != owner || (forwarded && principal.Org != "" && principal.Org != owner) { - return nil, pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("workflow org %s: principal unavailable", owner)} - } - return ctx, nil - } principal, err := d.principalIn(ctx, identity.IdentityID(task.Identity), owner) if err != nil { return nil, pinFailure{taskstate.ParkTransient, fmt.Errorf("workflow principal: %w", err)} diff --git a/internal/daemon/workflow_org_test.go b/internal/daemon/workflow_org_test.go index c1a1902a2..354b37437 100644 --- a/internal/daemon/workflow_org_test.go +++ b/internal/daemon/workflow_org_test.go @@ -88,12 +88,12 @@ func TestDispatchOrgWorkflows(t *testing.T) { {ID: "root", MappingID: "root", Matcher: binding.Matcher{Source: "root"}, OrgID: org.DefaultOrgID, Workflow: "deploy", Status: binding.StatusArmed, Owner: "owner", Repo: "repo"}, {ID: "acme", MappingID: "acme", Matcher: binding.Matcher{Source: "acme"}, OrgID: "acme", Workflow: "deploy", Status: binding.StatusArmed}, }} - d := &Daemon{Cfg: config.NewHolder(config.Config{}), Store: store, Log: slog.New(slog.DiscardHandler), Bindings: store, BindingDispatcher: store, BindingTaskCreator: store, Mappings: store, OrgPrincipals: func(context.Context) []access.Principal { + d := wired(&Daemon{Cfg: config.NewHolder(config.Config{}), Store: store, Log: slog.New(slog.DiscardHandler), Bindings: store, BindingDispatcher: store, BindingTaskCreator: store, Mappings: store, OrgPrincipals: func(context.Context) []access.Principal { return []access.Principal{{IdentityID: identity.SystemID, Org: org.DefaultOrgID}, {IdentityID: "acme-bot", Org: "acme"}} }, Principals: workflowPrincipals{sys: org.DefaultOrgID, "acme-bot": "acme"}, WorkflowDefinitions: orgDefinitions{ org.DefaultOrgID: "id: deploy\nrepository: required\nsteps:\n - type: workflow.finish\n", "acme": "id: deploy\nrepository: none\nsteps:\n - type: workflow.finish\n", - }} + }}) if disabled { d.WorkflowEnablement = enablementOf(workflowtask.WorkflowEnablement{}.SetEnabled("acme", "deploy", false)) } @@ -135,27 +135,16 @@ func TestPinWorkflowInTaskOrg(t *testing.T) { principals access.PrincipalSource want string refused bool - contextOrg org.OrgID }{ - {"legacy default org", "", nil, shared, false, ""}, - {"non-default org definition", "acme", workflowPrincipals{"bot": "acme"}, custom, false, ""}, - {"another org cannot supply the definition", "acme", workflowPrincipals{"bot": org.DefaultOrgID}, "", true, ""}, - {"a member of the task's org may start it from another", "acme", memberOf{home: org.DefaultOrgID, other: "acme"}, custom, false, ""}, - {"a non-member cannot start a task in another org", "acme", memberOf{home: org.DefaultOrgID, other: "beta"}, "", true, ""}, - {"unresolved org is refused", "acme", nil, "", true, ""}, - {"unresolved context cannot select another org", "", nil, "", true, "acme"}, + {"non-default org definition", "acme", workflowPrincipals{"bot": "acme"}, custom, false}, + {"another org cannot supply the definition", "acme", workflowPrincipals{"bot": org.DefaultOrgID}, "", true}, + {"a member of the task's org may start it from another", "acme", memberOf{home: org.DefaultOrgID, other: "acme"}, custom, false}, + {"a non-member cannot start a task in another org", "acme", memberOf{home: org.DefaultOrgID, other: "beta"}, "", true}, } { t.Run(tc.name, func(t *testing.T) { - d := &Daemon{Store: listsWorkspaces{&identityStore{}}, WorkflowDefinitions: orgDefinitions{org.DefaultOrgID: shared, "acme": custom}} - if tc.principals != nil { - d.Principals = tc.principals - } + d := wired(&Daemon{Store: listsWorkspaces{&identityStore{}}, Principals: tc.principals, WorkflowDefinitions: orgDefinitions{org.DefaultOrgID: shared, "acme": custom}}) task := &workflow.Task{ID: 1, Identity: "bot", Org: tc.owner, Workflow: "deploy"} - ctx := t.Context() - if tc.contextOrg != "" { - ctx = org.WithOrg(access.WithPrincipal(ctx, access.Principal{IdentityID: "foreign", Org: tc.contextOrg}), tc.contextOrg) - } - err := d.pinWorkflowDefinition(ctx, task) + err := d.pinWorkflowDefinition(t.Context(), task) if tc.refused { if err == nil || pinParkClass(err) != taskstate.ParkNeedsDecision || task.WorkflowDefinitionYAML != "" { t.Fatalf("cross-org pin accepted: task %#v err %v", task, err)