refactor(daemon): drop nil paths production never takes; one file per cluster (#1929)
This commit is contained in:
38 files changed
+2931
-2784
No files matched your search
@@ -32,8 +32,6 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/domain/alert"
|
||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||
"github.com/samcharles93/archie-core/internal/domain/health"
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/org"
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
"github.com/samcharles93/archie-core/internal/domain/scheduling"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
@@ -94,7 +92,10 @@ type boot struct {
|
||||
// dialed to the standalone archie-state-store gRPC service at
|
||||
// [services.state].target. It is set by openStateStoreAdapter, which
|
||||
// requires [services.state].target to be set.
|
||||
stateStore storecontract.TaskStore
|
||||
stateStore storecontract.TaskStore
|
||||
// stateClient is stateStore's concrete type, which the daemon's typed store
|
||||
// fields are assigned from directly.
|
||||
stateClient *staterpc.Client
|
||||
taskModels daemon.TaskModels
|
||||
taskInstructions daemon.TaskInstructions
|
||||
// accessChain is the daemon's policy engine, built once from the stored
|
||||
@@ -288,6 +289,7 @@ func (b *boot) openStateStoreAdapter(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
b.stateStore = client
|
||||
b.stateClient = client
|
||||
if alerts, ok := b.stateStore.(alert.Store); ok {
|
||||
b.credentialWatch = &credentialWatch{reporter: alert.Reporter{Raiser: alerts, Log: b.log, By: "archied"}}
|
||||
}
|
||||
@@ -596,6 +598,16 @@ func (b *boot) buildDaemon(ctx context.Context) {
|
||||
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken, Scoped: b.cfg.NATS.Mode != config.NATSModeExternal},
|
||||
ConnectedStateStore: daemon.StateStoreEndpoint{URL: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target), CA: b.stateStoreCA(ctx)},
|
||||
Store: b.stateStore,
|
||||
Mappings: b.stateClient,
|
||||
Bindings: b.stateClient,
|
||||
BindingDispatcher: b.stateClient,
|
||||
MappingMatches: b.stateClient,
|
||||
BindingTaskCreator: b.stateClient,
|
||||
IdentityRepository: b.stateClient,
|
||||
AgentAssignments: b.stateClient,
|
||||
Principals: b.stateClient,
|
||||
Denials: b.stateClient,
|
||||
PlaybookLedger: b.stateClient,
|
||||
Bus: b.bus,
|
||||
Forge: b.forgeClient,
|
||||
Trees: b.trees,
|
||||
@@ -618,50 +630,8 @@ func (b *boot) buildDaemon(ctx context.Context) {
|
||||
Playbooks: b.livePlaybooks(),
|
||||
}
|
||||
|
||||
if identities, ok := b.stateStore.(identity.Repository); ok {
|
||||
b.d.IdentityRepository = identities
|
||||
}
|
||||
if assignments, ok := b.stateStore.(org.AgentAssignments); ok {
|
||||
b.d.AgentAssignments = assignments
|
||||
}
|
||||
|
||||
if ms, ok := b.stateStore.(storecontract.MappingStore); ok {
|
||||
b.d.Mappings = ms
|
||||
}
|
||||
|
||||
if bs, ok := b.stateStore.(storecontract.BindingStore); ok {
|
||||
b.d.Bindings = bs
|
||||
}
|
||||
|
||||
if bd, ok := b.stateStore.(storecontract.BindingDispatcher); ok {
|
||||
b.d.BindingDispatcher = bd
|
||||
}
|
||||
|
||||
if mm, ok := b.stateStore.(storecontract.MappingMatchRecorder); ok {
|
||||
b.d.MappingMatches = mm
|
||||
}
|
||||
|
||||
if btc, ok := b.stateStore.(storecontract.BindingTaskCreator); ok {
|
||||
b.d.BindingTaskCreator = btc
|
||||
}
|
||||
b.d.WithdrawTask = b.taskActions().Withdraw
|
||||
|
||||
b.d.Access = b.accessChain
|
||||
if principals, ok := b.stateStore.(access.PrincipalSource); ok {
|
||||
b.d.Principals = principals
|
||||
}
|
||||
if denials, ok := b.stateStore.(access.DenialRecorder); ok {
|
||||
b.d.Denials = denials
|
||||
}
|
||||
b.d.PlaybookLedger = playbookLedger(ctx, b.stateStore, b.log)
|
||||
}
|
||||
|
||||
func playbookLedger(ctx context.Context, source any, log *slog.Logger) storecontract.PlaybookDispatcher {
|
||||
if pd, ok := source.(storecontract.PlaybookDispatcher); ok {
|
||||
return pd
|
||||
}
|
||||
log.WarnContext(ctx, "eda action playbooks will not run: no dispatch ledger is wired")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *boot) publishConfig(ctx context.Context, cfg config.Config) {
|
||||
|
||||
@@ -34,7 +34,7 @@ func (s *repositoryServer) CheckRepository(ctx context.Context, r *pb.CheckRepos
|
||||
if !ok || caller.IdentityID == "" {
|
||||
return nil, status.Error(codes.Unauthenticated, "Sign in before checking repository access.")
|
||||
}
|
||||
if b.d == nil || b.d.Access == nil || b.d.Principals == nil {
|
||||
if b.d == nil {
|
||||
return nil, status.Error(codes.Unavailable, "Repository checks are temporarily unavailable.")
|
||||
}
|
||||
decision := b.d.Access.Authorize(ctx, caller, access.ActionUpdate, access.Resource{Kind: access.KindPolicy, ID: "repository-policies", Org: caller.Org}, access.Context{})
|
||||
|
||||
@@ -64,12 +64,12 @@ func TestRunViaAgentRequiresTerminalStatus(t *testing.T) {
|
||||
t.Fatalf("marshal response: %v", err)
|
||||
}
|
||||
store := &statusStore{}
|
||||
d := &Daemon{
|
||||
d := wired(&Daemon{
|
||||
Cfg: config.NewHolder(config.Config{}),
|
||||
Store: store,
|
||||
Tasks: statusBus{reply: reply},
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
}
|
||||
})
|
||||
d.runViaAgent(t.Context(), &workflow.Task{ID: 7, Attempt: 1, Owner: "owner", Repo: "repo", Workflow: "implement"},
|
||||
config.Repo{}, config.AgentProfile{}, "")
|
||||
|
||||
|
||||
@@ -0,0 +1,358 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/access"
|
||||
"github.com/samcharles93/archie-core/internal/domain/binding"
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/mapping"
|
||||
"github.com/samcharles93/archie-core/internal/domain/org"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
)
|
||||
|
||||
// dispatchBindings offers each identified capture to every armed binding on
|
||||
// its source.
|
||||
func (d *Daemon) dispatchBindings(ctx context.Context) {
|
||||
principals := d.OrgPrincipals(ctx)
|
||||
seen := make(map[org.OrgID]bool)
|
||||
for _, principal := range principals {
|
||||
if seen[principal.Org] {
|
||||
continue
|
||||
}
|
||||
seen[principal.Org] = true
|
||||
d.dispatchOrgBindings(org.WithOrg(access.WithPrincipal(ctx, principal), principal.Org))
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Daemon) dispatchOrgBindings(ctx context.Context) {
|
||||
// Gather the armed bindings whose workflow is enabled first, so the
|
||||
// oldest-first captures query is not held by captures that only a binding
|
||||
// on a disabled workflow would take.
|
||||
bindings, err := d.Bindings.ListBindings(ctx)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "list bindings failed", "error", err)
|
||||
return
|
||||
}
|
||||
enablement, err := d.workflowEnablement(ctx)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "binding dispatch: workflow enablement unavailable", "error", err)
|
||||
return
|
||||
}
|
||||
enabled := make([]string, 0, len(bindings))
|
||||
armed := make([]binding.Binding, 0, len(bindings))
|
||||
for _, b := range bindings {
|
||||
if b.OrgID == org.OrgFromContext(ctx) && b.Status == binding.StatusArmed && b.Matcher.Source != "" && enablement.Enabled(b.OrgID, b.Workflow) {
|
||||
enabled = append(enabled, b.ID)
|
||||
armed = append(armed, b)
|
||||
}
|
||||
}
|
||||
if len(enabled) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
captures, err := d.BindingDispatcher.ListUndispatchedCaptures(ctx, enabled, bindingDispatchBatchLimit)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "list undispatched captures failed", "error", err)
|
||||
return
|
||||
}
|
||||
workflows, err := d.activeWorkflows(ctx)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "binding dispatch: workflow definitions unavailable", "error", err)
|
||||
return
|
||||
}
|
||||
for _, c := range captures {
|
||||
d.dispatchCapture(ctx, c, armed, workflows)
|
||||
}
|
||||
}
|
||||
|
||||
// dispatchCapture offers one capture to every armed binding that matches it.
|
||||
// armed holds only the org's armed bindings on enabled workflows, so a
|
||||
// disabled workflow's capture waits rather than recording a failure every pass.
|
||||
func (d *Daemon) dispatchCapture(ctx context.Context, c storecontract.CapturedEvent, armed []binding.Binding, workflows workflow.WorkflowDefinitionCollection) {
|
||||
for _, b := range armed {
|
||||
if b.Matcher.Matches(c.Source, c.Dispatchable()) {
|
||||
d.dispatchOneBinding(ctx, b, c, workflows)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// bindingMapping returns b's mapping, or nil after logging why it is
|
||||
// unavailable. A mapping that no longer exists is a terminal outcome.
|
||||
func (d *Daemon) bindingMapping(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent) *mapping.Mapping {
|
||||
m, err := d.Mappings.GetMapping(ctx, b.MappingID)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "binding dispatch: get mapping", "binding", b.ID, "mapping", b.MappingID, "error", err)
|
||||
return nil
|
||||
}
|
||||
if m == nil {
|
||||
d.recordDispatchFailure(ctx, b, c, fmt.Sprintf("mapping %q does not exist", b.MappingID), nil)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// dispatchOneBinding offers one capture to one binding: the mapping must
|
||||
// match the capture's event type and resolve required fields, the filter
|
||||
// must admit the values, and the workflow must accept the inputs. The
|
||||
// ledger claim and the task commit together, so each binding fires exactly
|
||||
// once per delivery.
|
||||
func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, workflows workflow.WorkflowDefinitionCollection) {
|
||||
m := d.bindingMapping(ctx, b, c)
|
||||
if m == nil || m.EventTypeID != c.EventType {
|
||||
return
|
||||
}
|
||||
values, failures := mapping.Resolve(m.Fields, []byte(c.Body))
|
||||
if hasBlockingFailure(m.Fields, failures) {
|
||||
d.recordDispatchFailure(ctx, b, c, "required field failed", map[string]any{"failures": failures})
|
||||
return
|
||||
}
|
||||
if err := d.MappingMatches.RecordMappingMatch(ctx, m.ID, c.ID); err != nil {
|
||||
d.Log.WarnContext(ctx, "binding dispatch: record mapping match", "mapping", m.ID, "capture", c.ID, "error", err)
|
||||
}
|
||||
filter, err := binding.CompileFilter(b.Filter, m.Fields)
|
||||
if err != nil {
|
||||
d.recordDispatchFailure(ctx, b, c, "filter does not compile", map[string]any{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
if admitted, _ := filter.Admits(values); !admitted {
|
||||
d.recordEvaluated(ctx, b, c, "filtered")
|
||||
return
|
||||
}
|
||||
|
||||
target, reason, ok := d.resolveBindingTarget(b, values, workflows)
|
||||
if !ok {
|
||||
d.recordDispatchFailure(ctx, b, c, reason, nil)
|
||||
return
|
||||
}
|
||||
// The chain decides dispatch: the binding's principal may `run` this
|
||||
// workflow in its workspace, and the event's signature result and address
|
||||
// travel with the request. A denial is terminal; an unavailable principal
|
||||
// or policy chain leaves the capture listed for the next cycle.
|
||||
if !d.authorizeDispatch(ctx, b, c, target) {
|
||||
return
|
||||
}
|
||||
d.claimAndEnqueue(ctx, b, c, target, values)
|
||||
}
|
||||
|
||||
// claimAndEnqueue asks the State Store to claim the dispatch and enqueue its
|
||||
// task as one unit. A failure leaves the delivery undispatched, so the next
|
||||
// cycle offers it again; a delivery already dispatched is refused.
|
||||
func (d *Daemon) claimAndEnqueue(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget, values map[string]any) {
|
||||
title := fmt.Sprintf("binding %s/%d from %s", b.Name, b.Version, c.Source)
|
||||
body := renderBindingBody(values, c)
|
||||
if target.declared {
|
||||
body = fmt.Sprintf("Started by binding %q from capture %q; the workflow's inputs travel with the task.", b.Name, c.ID)
|
||||
}
|
||||
taskIdentity := string(identity.SystemID)
|
||||
if principal, ok := access.PrincipalFromContext(ctx); ok {
|
||||
taskIdentity = string(principal.IdentityID)
|
||||
}
|
||||
task, err := d.BindingTaskCreator.EnqueueBindingTask(ctx, target.owner, target.repo, title, body, b.Workflow, taskIdentity, b.ID, b.Version, c.ID, target.inputs)
|
||||
if err != nil {
|
||||
if !errors.Is(err, storecontract.ErrAlreadyDispatched) {
|
||||
d.Log.WarnContext(ctx, "binding dispatch: enqueue", "binding", b.ID, "capture", c.ID, "error", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
d.markBindingStart(ctx, task.ID, b, c)
|
||||
}
|
||||
|
||||
// authorizeDispatch checks the binding's principal, its org's dispatching
|
||||
// identity, may start the workflow. A denial is recorded and reported as a
|
||||
// binding_dispatch_failure. The run itself acts as the workflow's declared
|
||||
// identity, which adoptWorkflowIdentity checks when the task is processed.
|
||||
func (d *Daemon) authorizeDispatch(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget) bool {
|
||||
principal, ok := access.PrincipalFromContext(ctx)
|
||||
if !ok {
|
||||
d.Log.WarnContext(ctx, "binding dispatch: no principal; dispatch left for the next cycle", "binding", b.ID, "capture", c.ID)
|
||||
return false
|
||||
}
|
||||
resource := access.Resource{
|
||||
Kind: access.KindWorkflow, ID: b.Workflow, Org: principal.Org, Workspace: b.WorkspaceID,
|
||||
}
|
||||
contextValue := access.Context{Signature: signatureOf(c), Addr: c.RemoteAddr}
|
||||
decision := d.Access.Authorize(ctx, principal, access.ActionRun, resource, contextValue)
|
||||
if decision.Allowed {
|
||||
return true
|
||||
}
|
||||
if errors.Is(decision.Err, access.ErrChainUnavailable) {
|
||||
// Nothing was evaluated. Recording it would mark the pair evaluated for
|
||||
// good, so a State Store blip would drop the capture instead of leaving
|
||||
// it for the next cycle.
|
||||
d.Log.WarnContext(ctx, "binding dispatch: access chain unavailable; dispatch left for the next cycle", "binding", b.ID, "capture", c.ID, "err", decision.Err)
|
||||
return false
|
||||
}
|
||||
denial := access.Denial{
|
||||
Principal: principal.IdentityID, Org: principal.Org, Action: access.ActionRun,
|
||||
Kind: resource.Kind, ResourceID: resource.ID, Level: decision.Level,
|
||||
Policies: decision.Policies,
|
||||
}
|
||||
if err := d.Denials.RecordDenial(ctx, denial); err != nil {
|
||||
d.Log.WarnContext(ctx, "binding dispatch: record denial", "binding", b.ID, "error", err)
|
||||
}
|
||||
d.recordDispatchFailure(ctx, b, c, "access denied", map[string]any{
|
||||
"level": string(decision.Level), "policies": decision.Policies,
|
||||
})
|
||||
return false
|
||||
}
|
||||
|
||||
// markBindingStart records on a task's timeline the binding and captured
|
||||
// event that started it.
|
||||
func (d *Daemon) markBindingStart(ctx context.Context, taskID int64, b binding.Binding, c storecontract.CapturedEvent) {
|
||||
if _, err := d.Store.InsertEvent(ctx, events.Event{
|
||||
TaskID: taskID,
|
||||
Kind: events.KindBindingStarted,
|
||||
Detail: fmt.Sprintf("started by binding %q from source %q", b.Name, c.Source),
|
||||
Data: map[string]any{
|
||||
"binding_id": b.ID, "binding_name": b.Name, "binding_version": b.Version,
|
||||
"capture_id": c.ID, "source": c.Source, "unsigned": c.Unsigned,
|
||||
},
|
||||
}); err != nil {
|
||||
d.Log.WarnContext(ctx, "binding dispatch: start marker", "task", taskID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// recordDispatchFailure records why a binding did not dispatch a capture and
|
||||
// marks the pair evaluated, so the capture is not offered to it again.
|
||||
func (d *Daemon) recordDispatchFailure(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, reason string, extra map[string]any) {
|
||||
d.recordEvaluated(ctx, b, c, reason)
|
||||
data := map[string]any{
|
||||
"binding_id": b.ID,
|
||||
"binding_version": b.Version,
|
||||
"capture_id": c.ID,
|
||||
}
|
||||
maps.Copy(data, extra)
|
||||
_, _ = d.Store.InsertEvent(ctx, events.Event{
|
||||
Kind: "binding_dispatch_failure",
|
||||
Detail: fmt.Sprintf("binding %q (capture %q): %s", b.ID, c.ID, reason),
|
||||
Data: data,
|
||||
})
|
||||
}
|
||||
|
||||
// recordEvaluated writes the ledger row for an evaluation that started no
|
||||
// task. Only terminal outcomes call it; transient ones leave the capture
|
||||
// listed for the next cycle.
|
||||
func (d *Daemon) recordEvaluated(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, reason string) {
|
||||
err := d.BindingDispatcher.RecordDispatch(ctx, b.ID, int64(b.Version), c.ID, 0, reason)
|
||||
if err != nil && !errors.Is(err, storecontract.ErrAlreadyDispatched) {
|
||||
d.Log.WarnContext(ctx, "binding dispatch: record evaluation", "binding", b.ID, "capture", c.ID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// resolveBindingRepo returns the binding's pinned owner/repo, or the only
|
||||
// configured repo. With zero or several repos and no pin it refuses.
|
||||
func (d *Daemon) resolveBindingRepo(b binding.Binding) (owner, repo, reason string) {
|
||||
if b.Owner != "" && b.Repo != "" {
|
||||
return b.Owner, b.Repo, ""
|
||||
}
|
||||
repos := d.Cfg.Get().Repos
|
||||
if len(repos) == 1 {
|
||||
return repos[0].Owner, repos[0].Name, ""
|
||||
}
|
||||
return "", "", fmt.Sprintf("cannot pick a repository: %d are configured; pin the binding to one", len(repos))
|
||||
}
|
||||
|
||||
// activeWorkflows returns the definitions a dispatch cycle checks bindings
|
||||
// against, the same set a task is later pinned from.
|
||||
func (d *Daemon) activeWorkflows(ctx context.Context) (workflow.WorkflowDefinitionCollection, error) {
|
||||
collection, _, err := d.WorkflowDefinitions.WorkflowDefinitions(ctx)
|
||||
return collection, err
|
||||
}
|
||||
|
||||
// workflowEnablement returns which workflows each org has disabled; with no
|
||||
// source wired every workflow is enabled.
|
||||
func (d *Daemon) workflowEnablement(ctx context.Context) (workflowtask.WorkflowEnablement, error) {
|
||||
return d.WorkflowEnablement.WorkflowEnablement(ctx)
|
||||
}
|
||||
|
||||
// disabledRoutedWorkflow reports the workflow an issue's labels route to and
|
||||
// whether the identity's org has that workflow disabled, matching the
|
||||
// enablement filter binding dispatch applies. A read failure (org,
|
||||
// definitions, enablement) is an error so the caller falls through to
|
||||
// publishing rather than dropping the issue; no routable definition is not an
|
||||
// error, it is reported as not disabled and the pin path parks the task.
|
||||
func (d *Daemon) disabledRoutedWorkflow(ctx context.Context, labels, identityID string) (routed string, disabled bool, err error) {
|
||||
orgID, err := d.identityOrg(ctx, identity.IdentityID(identityID))
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
definitions, err := d.activeWorkflows(ctx)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
available := make(map[string]struct{}, len(definitions.Definitions))
|
||||
for _, definition := range definitions.Definitions {
|
||||
available[definition.ID] = struct{}{}
|
||||
}
|
||||
routed, resolveErr := workflow.ResolveWorkflowID(&workflow.Task{Labels: labels}, available)
|
||||
if resolveErr == nil {
|
||||
enablement, err := d.workflowEnablement(ctx)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return routed, !enablement.Enabled(orgID, routed), nil
|
||||
}
|
||||
// No routable definition is a normal outcome here, not a read failure:
|
||||
// keep the publish behaviour and let the pin path park the task.
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// bindingDispatchBatchLimit caps how many undispatched captures a single
|
||||
// dispatchBindings pass will fetch from the store. The cap is defensive
|
||||
// (the captures table is also pruned by retention/maxEvents) and bounds a
|
||||
// single cycle's worst-case latency on a backlog.
|
||||
const bindingDispatchBatchLimit = 100
|
||||
|
||||
// signatureOf names the event's signature result as the dispatch context
|
||||
// carries it: a signed event and an approved unsigned event are the two
|
||||
// dispatchable shapes (storecontract.CapturedEvent.Dispatchable).
|
||||
func signatureOf(c storecontract.CapturedEvent) string {
|
||||
switch {
|
||||
case c.Authenticated:
|
||||
return "valid"
|
||||
case c.Unsigned:
|
||||
return "unsigned"
|
||||
}
|
||||
return "unverified"
|
||||
}
|
||||
|
||||
// hasBlockingFailure reports whether any failure is on a required field.
|
||||
func hasBlockingFailure(fields []mapping.Field, failures []mapping.Failure) bool {
|
||||
if len(failures) == 0 {
|
||||
return false
|
||||
}
|
||||
required := make(map[string]bool, len(fields))
|
||||
for _, f := range fields {
|
||||
if f.Required {
|
||||
required[f.Name] = true
|
||||
}
|
||||
}
|
||||
for _, fail := range failures {
|
||||
if required[fail.FieldName] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// renderBindingBody builds the task body from the values a mapping
|
||||
// resolved, one "key=value" pair per line. Empty maps still produce
|
||||
// a body that records the source capture id so the workflow has
|
||||
// provenance to hand back to the operator.
|
||||
func renderBindingBody(values map[string]any, c storecontract.CapturedEvent) string {
|
||||
if len(values) == 0 {
|
||||
return fmt.Sprintf("(no fields resolved from capture %q)", c.ID)
|
||||
}
|
||||
parts := make([]string, 0, len(values))
|
||||
for k, v := range values {
|
||||
parts = append(parts, fmt.Sprintf("%s=%v", k, v))
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
@@ -106,11 +106,11 @@ func TestBindingDispatchCreatesOneTask(t *testing.T) {
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
store := &ledgerStore{claimed: map[string]bool{}, failures: tc.failures}
|
||||
d := &Daemon{
|
||||
d := wired(&Daemon{
|
||||
Cfg: config.NewHolder(config.Config{}), Store: store, Log: slog.New(slog.DiscardHandler),
|
||||
Bindings: store, BindingDispatcher: store, BindingTaskCreator: store, Mappings: store,
|
||||
WorkflowDefinitions: orgDefinitions{org.DefaultOrgID: "id: deploy\nrepository: none\nsteps:\n - type: workflow.finish\n"},
|
||||
}
|
||||
})
|
||||
for range tc.cycles {
|
||||
var listed, done sync.WaitGroup
|
||||
listed.Add(tc.concurrent)
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/binding"
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
)
|
||||
@@ -20,57 +18,6 @@ type bindingTarget struct {
|
||||
declared bool
|
||||
}
|
||||
|
||||
// activeWorkflows returns the definitions a dispatch cycle checks bindings
|
||||
// against, the same set a task is later pinned from.
|
||||
func (d *Daemon) activeWorkflows(ctx context.Context) (workflow.WorkflowDefinitionCollection, error) {
|
||||
if d.WorkflowDefinitions == nil {
|
||||
return workflow.ShippedDefinitions(), nil
|
||||
}
|
||||
collection, _, err := d.WorkflowDefinitions.WorkflowDefinitions(ctx)
|
||||
return collection, err
|
||||
}
|
||||
|
||||
// workflowEnablement returns which workflows each org has disabled; with no
|
||||
// source wired every workflow is enabled.
|
||||
func (d *Daemon) workflowEnablement(ctx context.Context) (workflowtask.WorkflowEnablement, error) {
|
||||
if d.WorkflowEnablement == nil {
|
||||
return workflowtask.WorkflowEnablement{}, nil
|
||||
}
|
||||
return d.WorkflowEnablement.WorkflowEnablement(ctx)
|
||||
}
|
||||
|
||||
// disabledRoutedWorkflow reports the workflow an issue's labels route to and
|
||||
// whether the identity's org has that workflow disabled, matching the
|
||||
// enablement filter binding dispatch applies. A read failure (org,
|
||||
// definitions, enablement) is an error so the caller falls through to
|
||||
// publishing rather than dropping the issue; no routable definition is not an
|
||||
// error, it is reported as not disabled and the pin path parks the task.
|
||||
func (d *Daemon) disabledRoutedWorkflow(ctx context.Context, labels, identityID string) (routed string, disabled bool, err error) {
|
||||
orgID, err := d.identityOrg(ctx, identity.IdentityID(identityID))
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
definitions, err := d.activeWorkflows(ctx)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
available := make(map[string]struct{}, len(definitions.Definitions))
|
||||
for _, definition := range definitions.Definitions {
|
||||
available[definition.ID] = struct{}{}
|
||||
}
|
||||
routed, resolveErr := workflow.ResolveWorkflowID(&workflow.Task{Labels: labels}, available)
|
||||
if resolveErr == nil {
|
||||
enablement, err := d.workflowEnablement(ctx)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return routed, !enablement.Enabled(orgID, routed), nil
|
||||
}
|
||||
// No routable definition is a normal outcome here, not a read failure:
|
||||
// keep the publish behaviour and let the pin path park the task.
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// resolveBindingTarget checks a matched binding against the workflow it
|
||||
// targets and resolves the task's repository and inputs. A non-empty reason
|
||||
// means the binding does not dispatch and the reason is recorded on it.
|
||||
|
||||
@@ -21,7 +21,7 @@ const callWaitBuffer = 1024
|
||||
// carry the caller's task id.
|
||||
func (d *Daemon) watchCallWaits(ctx context.Context) {
|
||||
reader, ok := d.Store.(storecontract.StepReader)
|
||||
if d.Bus == nil || !ok {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
sub := d.Bus.Subscribe(callWaitBuffer)
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/container"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/eventbus"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
|
||||
"github.com/samcharles93/archie-core/internal/storage"
|
||||
"github.com/samcharles93/archie-core/internal/taskrun"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// acquireTaskContainer prepares and acquires the sandbox container for a
|
||||
// task, reporting false when it could not -- having already parked the
|
||||
// task and logged why. The caller releases the container it returns.
|
||||
func (d *Daemon) acquireTaskContainer(
|
||||
ctx context.Context,
|
||||
task *workflow.Task,
|
||||
repo config.Repo,
|
||||
workDir string,
|
||||
image string,
|
||||
) (*container.Container, string, func(), bool) {
|
||||
park := func(reason string, err error) {
|
||||
d.Log.ErrorContext(ctx, reason, "err", err)
|
||||
d.parkBlockedTask(ctx, task, reason+": "+err.Error(), taskstate.ParkTransient, parkBlocker(err))
|
||||
}
|
||||
|
||||
// Resolve the PR number from inputs before the brief is written.
|
||||
if task.Workflow == "pr-review" {
|
||||
task.PRNumber = task.EffectivePRNumber()
|
||||
}
|
||||
|
||||
if err := writeTaskBrief(workDir, task); err != nil {
|
||||
park("task.json write failed", err)
|
||||
return nil, "", nil, false
|
||||
}
|
||||
|
||||
// Fetch the pull request with the daemon's forge credential into workDir,
|
||||
// so the container reads it without credentials.
|
||||
if task.Workflow == "pr-review" {
|
||||
if err := prefetchPRReview(ctx, d.forgeFor(ctx, task), task.Owner, task.Repo, task.PRNumber, workDir); err != nil {
|
||||
park("pr-review prefetch failed", err)
|
||||
return nil, "", nil, false
|
||||
}
|
||||
}
|
||||
|
||||
// Guard: Storage may be nil if the daemon was wired incorrectly. In
|
||||
// normal operation, Storage is always set when ContainerPool is set.
|
||||
if d.Storage == nil {
|
||||
d.Log.ErrorContext(ctx, "storage backend is nil -- cannot acquire container")
|
||||
d.parkRunningTask(ctx, task, "storage backend not configured", taskstate.ParkTransient)
|
||||
return nil, "", nil, false
|
||||
}
|
||||
|
||||
mounts, err := d.Storage.Setup(ctx, storage.TaskRef{
|
||||
WorktreeDir: workDir,
|
||||
Ecosystem: repo.Ecosystem,
|
||||
PersistentStorage: repo.PersistentStorage,
|
||||
Owner: task.Owner,
|
||||
Repo: task.Repo,
|
||||
})
|
||||
if err != nil {
|
||||
park("storage setup failed", err)
|
||||
return nil, "", nil, false
|
||||
}
|
||||
|
||||
credential, revokeCredential, err := d.runCredential(ctx, task)
|
||||
if err != nil {
|
||||
d.teardownStorage(ctx, task, repo, workDir)
|
||||
park("run credential failed", err)
|
||||
return nil, "", nil, false
|
||||
}
|
||||
|
||||
model, closeModel, err := d.openModelEgress(ctx, task, credential)
|
||||
if err != nil {
|
||||
revokeCredential()
|
||||
d.teardownStorage(ctx, task, repo, workDir)
|
||||
park("model egress failed", err)
|
||||
return nil, "", nil, false
|
||||
}
|
||||
revokeRun := revokeCredential
|
||||
revokeCredential = func() { closeModel(); revokeRun() }
|
||||
|
||||
// The container lives in its own network namespace, so a loopback State
|
||||
// Store target names the container itself. Give this task an address on
|
||||
// the host bridge that forwards to it, closed with the task.
|
||||
stateStoreURL, relay, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential)
|
||||
if err != nil {
|
||||
revokeCredential()
|
||||
d.teardownStorage(ctx, task, repo, workDir)
|
||||
park("state store relay failed", err)
|
||||
return nil, "", nil, false
|
||||
}
|
||||
revokeCredential = release
|
||||
|
||||
ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(d.containerEnv(task, credential, stateStoreURL), model.Env...))
|
||||
if err != nil {
|
||||
revokeCredential()
|
||||
// Roll back the storage we just set up: the mounts were created
|
||||
// for this task but no container will use them. Leaking them until
|
||||
// a later TTL sweep is not acceptable on a backend that allocates
|
||||
// real resources (volumes, NFS leases).
|
||||
d.teardownStorage(ctx, task, repo, workDir)
|
||||
park("container acquire failed", err)
|
||||
return nil, "", nil, false
|
||||
}
|
||||
d.admitToRelay(ctx, relay, task, ctr)
|
||||
return ctr, credential, revokeCredential, true
|
||||
}
|
||||
|
||||
// admitToRelay lets ctr, and only ctr, through relay. A container whose
|
||||
// addresses cannot be read is refused rather than let anyone through.
|
||||
func (d *Daemon) admitToRelay(ctx context.Context, relay *stateStoreRelay, task *workflow.Task, ctr *container.Container) {
|
||||
if relay == nil {
|
||||
return
|
||||
}
|
||||
addrs, err := d.ContainerPool.ContainerAddrs(ctx, ctr.ID)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "state store relay: container addresses unknown, refusing its connections", "task", task.ID, "err", err)
|
||||
}
|
||||
relay.Admit(addrs)
|
||||
}
|
||||
|
||||
// taskStateStoreEndpoint is the State Store address one task container is
|
||||
// handed, the relay carrying it (nil when none is needed) and the release that
|
||||
// ends that relay. The container lives
|
||||
// in its own network namespace, so a loopback target names the container
|
||||
// itself; behind a relay, the address is the host bridge's.
|
||||
func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, *stateStoreRelay, func(), error) {
|
||||
relay, err := d.openTaskStateStoreRelay(ctx)
|
||||
if err != nil {
|
||||
return "", nil, release, err
|
||||
}
|
||||
if relay == nil {
|
||||
return d.ConnectedStateStore.URL, nil, release, nil
|
||||
}
|
||||
return relay.Addr(), relay, func() { relay.Close(); release() }, nil
|
||||
}
|
||||
|
||||
// openTaskStateStoreRelay starts the relay that carries one task container's
|
||||
// State Store traffic to the daemon's target. The target stays bound where the
|
||||
// operator put it; the container is handed an address on the host bridge, the
|
||||
// same way it is handed NATS's. Nil when the target already reaches the
|
||||
// containers, or when no bridge address is known.
|
||||
func (d *Daemon) openTaskStateStoreRelay(ctx context.Context) (*stateStoreRelay, error) {
|
||||
addr, loopback := loopbackStateStoreTarget(d.ConnectedStateStore.URL)
|
||||
if !loopback {
|
||||
return nil, nil
|
||||
}
|
||||
gateway := ""
|
||||
if d.ContainerPool != nil {
|
||||
gateway = d.ContainerPool.HostGateway()
|
||||
}
|
||||
if gateway == "" {
|
||||
d.Log.WarnContext(ctx, "state store target is loopback and the container pool has no host bridge address; task containers cannot reach it",
|
||||
"target", d.ConnectedStateStore.URL)
|
||||
return nil, nil
|
||||
}
|
||||
return openStateStoreRelay(ctx, gateway, addr)
|
||||
}
|
||||
|
||||
// containerEnv is the environment a task container starts with. stateStoreURL
|
||||
// is the address the container can reach the State Store by, which is the
|
||||
// relay's rather than the daemon's own target.
|
||||
func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken, stateStoreURL string) []string {
|
||||
var env []string
|
||||
// The endpoint the daemon's own client connected with at startup, not
|
||||
// the live config: a reloaded [nats] section must not point new
|
||||
// containers at a server the daemon is not publishing on.
|
||||
env = append(env, "NATS_URL="+d.ConnectedNATS.URL)
|
||||
if token := d.ConnectedNATS.ContainerToken(); token != "" {
|
||||
env = append(env, "NATS_TOKEN="+token)
|
||||
}
|
||||
// Pass the container-reachable endpoint with this task's scoped token,
|
||||
// never the daemon's own.
|
||||
if stateStoreURL != "" {
|
||||
env = append(env, "STATE_STORE_URL="+stateStoreURL)
|
||||
if len(d.ConnectedStateStore.CA) > 0 {
|
||||
env = append(env, "STATE_STORE_CA="+string(d.ConnectedStateStore.CA))
|
||||
}
|
||||
if stateStoreToken != "" {
|
||||
env = append(env, "STATE_STORE_TOKEN="+stateStoreToken)
|
||||
}
|
||||
}
|
||||
// The agent runs as root in the container, so its commits land owned by
|
||||
// UID 0 on the host. The daemon's UID/GID let the agent hand the worktree
|
||||
// back before the daemon reads it to push.
|
||||
env = append(env, fmt.Sprintf("WORKTREE_UID=%d", os.Getuid()))
|
||||
env = append(env, fmt.Sprintf("WORKTREE_GID=%d", os.Getgid()))
|
||||
return env
|
||||
}
|
||||
|
||||
// runCredential issues the task's run credential, or "" when no State Store
|
||||
// is configured.
|
||||
func (d *Daemon) runCredential(ctx context.Context, task *workflow.Task) (string, func(), error) {
|
||||
if d.ConnectedStateStore.URL == "" {
|
||||
return "", func() {}, nil
|
||||
}
|
||||
return d.RunCredentials.Issue(ctx, task, d.configFor(task).Budgets.TaskWallClock.Std())
|
||||
}
|
||||
|
||||
// runViaAgent hands a task to archie-agent and waits for its result. It parks
|
||||
// the task only when the agent never answered or failed before recording an
|
||||
// outcome.
|
||||
//
|
||||
// credential is the task's run credential; the agent presents it to push the
|
||||
// task's branch.
|
||||
func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo config.Repo, profile config.AgentProfile, credential string) {
|
||||
cfg, err := d.taskConfig(ctx, task)
|
||||
if err != nil {
|
||||
d.parkRunningTask(ctx, task, err.Error(), taskstate.ParkTransient)
|
||||
return
|
||||
}
|
||||
taskCfg := cfg.ForTask()
|
||||
d.captureAttemptConfig(ctx, task, taskCfg)
|
||||
var instructions workflow.Instructions
|
||||
instructions.Instance, instructions.Organisation, err = d.TaskInstructions.TaskInstructions(ctx, task.ID)
|
||||
if err != nil {
|
||||
d.parkRunningTask(ctx, task, "resolve the task's instructions: "+err.Error(), taskstate.ParkTransient)
|
||||
return
|
||||
}
|
||||
req := taskrun.Request{
|
||||
Instructions: instructions,
|
||||
Task: task,
|
||||
Repo: repo,
|
||||
Cfg: taskCfg,
|
||||
Providers: agentexec.ProvidersFromConfig(cfg.Providers),
|
||||
MCPServers: cfg.Tools.MCPServers,
|
||||
RunCredential: credential,
|
||||
WorkflowDefinition: task.WorkflowDefinitionYAML,
|
||||
Tools: d.toolAllowlist(ctx, task, profile.Tools),
|
||||
}
|
||||
req.Trace = map[string]string{}
|
||||
otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(req.Trace))
|
||||
data, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "taskrun encode failed", "task", task.ID, "err", err)
|
||||
d.parkRunningTask(ctx, task, "taskrun encode failed: "+err.Error(), taskstate.ParkTransient)
|
||||
return
|
||||
}
|
||||
|
||||
reply, err := d.requestTaskRun(ctx, task.ID, data)
|
||||
if err != nil {
|
||||
if stoppingOnly(ctx) {
|
||||
d.Log.InfoContext(ctx, "daemon stopping; task left running for the next daemon to re-queue", "task", task.ID)
|
||||
return
|
||||
}
|
||||
err = runFailure(ctx, err)
|
||||
d.Log.ErrorContext(ctx, "taskrun request failed", "task", task.ID, "err", err)
|
||||
d.parkRunningTask(ctx, task, "taskrun request failed: "+err.Error(), taskstate.ParkTransient)
|
||||
return
|
||||
}
|
||||
|
||||
var resp taskrun.Response
|
||||
if err := json.Unmarshal(reply, &resp); err != nil {
|
||||
d.Log.ErrorContext(ctx, "taskrun decode response failed", "task", task.ID, "err", err)
|
||||
d.parkRunningTask(ctx, task, "taskrun decode response failed: "+err.Error(), taskstate.ParkTransient)
|
||||
return
|
||||
}
|
||||
if resp.Error != "" {
|
||||
d.Log.ErrorContext(ctx, "taskrun run failed", "task", task.ID, "err", resp.Error)
|
||||
d.parkRunningTask(ctx, task, "taskrun run failed: "+resp.Error, taskstate.ParkTransient)
|
||||
return
|
||||
}
|
||||
// The agent applies the task's own outcome transition, so a status the
|
||||
// task cannot reach from running means nothing landed: empty, unknown, or
|
||||
// still running. Transient, because the cause is environmental -- a State
|
||||
// Store that refused the transition -- and a retry can record one.
|
||||
if !taskstate.CanTransition(taskstate.Running, resp.Status) {
|
||||
reason := noOutcomeReason(resp.Status)
|
||||
d.Log.ErrorContext(ctx, "taskrun run recorded no outcome", "task", task.ID, "status", resp.Status)
|
||||
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
|
||||
return
|
||||
}
|
||||
|
||||
// Reached only past every failure return above, so this is always a
|
||||
// fully decoded, error-free response -- a version is never recorded off
|
||||
// a request that never reached an agent, or a response describing a
|
||||
// parked/errored task.
|
||||
if resp.AgentVersion != "" {
|
||||
d.AgentStatus.Observe(resp.AgentVersion, resp.AgentInstallType)
|
||||
}
|
||||
|
||||
d.Log.InfoContext(ctx, "taskrun complete", "task", task.ID, "status", resp.Status)
|
||||
}
|
||||
|
||||
// requestTaskRun publishes the taskrun request, retrying on
|
||||
// nats.ErrNoResponders until the container subscribes. Other errors return
|
||||
// at once.
|
||||
func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte) ([]byte, error) {
|
||||
subject := agentnats.SubjectForTask(taskID)
|
||||
deadline := time.Now().Add(taskRunReadyTimeout)
|
||||
backoff := taskRunRetryBackoff
|
||||
for {
|
||||
reply, err := d.Tasks.Request(ctx, subject, data)
|
||||
if err == nil {
|
||||
return reply, nil
|
||||
}
|
||||
if !errors.Is(err, eventbus.ErrNoResponders) || !time.Now().Before(deadline) {
|
||||
return nil, err
|
||||
}
|
||||
wait := min(backoff, time.Until(deadline))
|
||||
if wait <= 0 {
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(wait):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// captureAttemptConfig records the attempt's effective configuration as an
|
||||
// event. Failures are logged.
|
||||
func (d *Daemon) captureAttemptConfig(ctx context.Context, task *workflow.Task, cfg config.TaskConfig) {
|
||||
document, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err)
|
||||
return
|
||||
}
|
||||
event := events.Event{
|
||||
At: time.Now().UTC(),
|
||||
Kind: events.KindConfigCaptured,
|
||||
TaskID: task.ID,
|
||||
Repo: task.Owner + "/" + task.Repo,
|
||||
Issue: task.IssueNumber,
|
||||
Workflow: task.Workflow,
|
||||
Attempt: task.Attempt,
|
||||
Data: map[string]any{
|
||||
"schema": events.ConfigCapturedSchema,
|
||||
// Raw so the document is embedded once, as it was encoded, rather
|
||||
// than as a string a reader would have to decode again.
|
||||
"document": json.RawMessage(document),
|
||||
},
|
||||
}
|
||||
id, err := d.Store.InsertEvent(ctx, event)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err)
|
||||
return
|
||||
}
|
||||
// The assigned ID tells the event sink to broadcast without inserting a
|
||||
// duplicate row.
|
||||
event.ID = id
|
||||
d.emit(ctx, event)
|
||||
}
|
||||
|
||||
// writeTaskBrief writes task.json, the container's boot-time brief.
|
||||
func writeTaskBrief(workDir string, task *workflow.Task) error {
|
||||
return container.WriteTaskJSON(workDir, container.TaskPayload{
|
||||
ID: task.ID, Owner: task.Owner, Repo: task.Repo,
|
||||
Number: task.IssueNumber, Title: task.Title, Body: task.Body,
|
||||
Labels: strings.Split(task.Labels, ","),
|
||||
Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan,
|
||||
Inputs: task.Inputs,
|
||||
})
|
||||
}
|
||||
|
||||
// noOutcomeReason names why a taskrun reply left the task without an outcome,
|
||||
// quoting the status so the log line points at what the agent actually said.
|
||||
func noOutcomeReason(status string) string {
|
||||
if status == "" {
|
||||
return "agent returned no status: the run could not record an outcome"
|
||||
}
|
||||
return fmt.Sprintf("agent returned unrecognised status %q: the run could not record an outcome", status)
|
||||
}
|
||||
|
||||
const (
|
||||
// taskRunReadyTimeout is how long requestTaskRun retries a
|
||||
// nats.ErrNoResponders while a new container has not subscribed yet.
|
||||
taskRunReadyTimeout = 20 * time.Second
|
||||
// taskRunRetryBackoff is the delay between those retries.
|
||||
taskRunRetryBackoff = 250 * time.Millisecond
|
||||
)
|
||||
|
||||
// runFailure names why a taskrun request failed: the container's exit or the
|
||||
// time limit when either cancelled it, else the request's own error.
|
||||
func runFailure(ctx context.Context, err error) error {
|
||||
if cause := context.Cause(ctx); errors.Is(cause, errContainerExited) || errors.Is(cause, errTaskTimeLimit) {
|
||||
return cause
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
// The systemd watchdog beats only while StalestLoop is fresh, so a cycle that
|
||||
// doesn't stamp it gets archied killed shortly after every start.
|
||||
func TestCycleMarksPass(t *testing.T) {
|
||||
d := &Daemon{}
|
||||
d := wired(&Daemon{Store: &retryStore{}})
|
||||
d.Cycle(context.Background())
|
||||
if loop, at := d.StalestLoop(); loop != "cycle" || at.IsZero() {
|
||||
t.Fatalf("StalestLoop() = %q, %v; want a cycle pass", loop, at)
|
||||
|
||||
+19
-2527
File diff suppressed because it is too large.
Load diff
@@ -413,13 +413,6 @@ func TestResumeAfterTheRunReturnedLeavesNoCapacity(t *testing.T) {
|
||||
d.Wait()
|
||||
}
|
||||
|
||||
// A daemon with no bus still starts and stops: the watcher is optional.
|
||||
func TestCallWaitsWithoutABus(t *testing.T) {
|
||||
d := &Daemon{Cfg: config.NewHolder(config.Config{}), Log: slog.New(slog.DiscardHandler)}
|
||||
d.watchCallWaits(t.Context())
|
||||
d.WaitForTasks()
|
||||
}
|
||||
|
||||
type callWaitStore struct {
|
||||
storecontract.TaskStore
|
||||
mu sync.Mutex
|
||||
@@ -442,7 +435,7 @@ func TestCallWaitsRecoverDroppedStart(t *testing.T) {
|
||||
}}
|
||||
cfg := config.Config{}
|
||||
cfg.Containers.MaxConcurrency = 1
|
||||
d := &Daemon{Cfg: config.NewHolder(cfg), Store: store, Bus: events.NewBus(), Log: slog.New(slog.DiscardHandler)}
|
||||
d := wired(&Daemon{Cfg: config.NewHolder(cfg), Store: store, Bus: events.NewBus(), Log: slog.New(slog.DiscardHandler)})
|
||||
dispatcher := d.taskDispatcher()
|
||||
started := make(chan int64, 2)
|
||||
a, b := make(chan struct{}), make(chan struct{})
|
||||
@@ -532,12 +525,12 @@ func TestDrainPassDoesNotWaitForItsTasks(t *testing.T) {
|
||||
<-release
|
||||
},
|
||||
}
|
||||
d := &Daemon{
|
||||
d := wired(&Daemon{
|
||||
Cfg: config.NewHolder(config.Config{}),
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
Store: store,
|
||||
Tasks: stubTasks{},
|
||||
}
|
||||
})
|
||||
|
||||
drained := make(chan struct{})
|
||||
go func() {
|
||||
@@ -611,12 +604,12 @@ func TestDrainPassClaimsABacklogWithoutWaitingPerTask(t *testing.T) {
|
||||
store.queue = append(store.queue, &workflow.Task{ID: id + 1, Owner: "acme", Repo: "app"})
|
||||
}
|
||||
tasks := &countingTasks{}
|
||||
d := &Daemon{
|
||||
d := wired(&Daemon{
|
||||
Cfg: config.NewHolder(config.Config{}),
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
Store: store,
|
||||
Tasks: tasks,
|
||||
}
|
||||
})
|
||||
d.drainNATS(context.Background())
|
||||
d.WaitForTasks()
|
||||
if len(store.queue) != 0 {
|
||||
@@ -639,12 +632,12 @@ func TestDrainPassClaimsOnlyWhatTheDispatcherCanRun(t *testing.T) {
|
||||
}
|
||||
cfg := config.Config{}
|
||||
cfg.Containers.MaxConcurrency = 2
|
||||
d := &Daemon{
|
||||
d := wired(&Daemon{
|
||||
Cfg: config.NewHolder(cfg),
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
Store: store,
|
||||
Tasks: &countingTasks{},
|
||||
}
|
||||
})
|
||||
d.drainNATS(context.Background())
|
||||
if got := len(store.queue); got != 8 {
|
||||
t.Fatalf("pass left %d tasks unclaimed, want 8", got)
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/org"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// identityOrg resolves an identity's org. No PrincipalSource means the
|
||||
// default org; a store error is returned.
|
||||
// orgResolver is the identity-to-org surface the identity checks read through.
|
||||
type orgResolver interface {
|
||||
identityOrg(ctx context.Context, id identity.IdentityID) (org.OrgID, error)
|
||||
identityMayAct(ctx context.Context, name string) (bool, string, taskstate.ParkClass)
|
||||
identityActive(ctx context.Context, id identity.IdentityID) bool
|
||||
}
|
||||
|
||||
var _ orgResolver = (*Daemon)(nil)
|
||||
|
||||
func (d *Daemon) identityOrg(ctx context.Context, id identity.IdentityID) (org.OrgID, error) {
|
||||
principal, err := d.Principals.PrincipalFor(ctx, id)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return principal.Org, nil
|
||||
}
|
||||
|
||||
func (d *Daemon) identityActive(ctx context.Context, id identity.IdentityID) bool {
|
||||
if id == "" {
|
||||
d.Log.ErrorContext(ctx, "identity lifecycle check failed", "err", "identity ID is empty")
|
||||
return false
|
||||
}
|
||||
value, err := d.IdentityRepository.Get(ctx, id)
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "identity lifecycle check failed", "identity", id, "err", err)
|
||||
return false
|
||||
}
|
||||
return value.Lifecycle == identity.LifecycleActive
|
||||
}
|
||||
|
||||
// identityMayAct reports whether a task's identity may act, and the park
|
||||
// class if not. An empty, unknown or inactive identity never falls back to
|
||||
// the System forge.
|
||||
func (d *Daemon) identityMayAct(ctx context.Context, name string) (bool, string, taskstate.ParkClass) {
|
||||
if name == "" {
|
||||
return false, "task has no identity", taskstate.ParkNeedsDecision
|
||||
}
|
||||
value, err := d.IdentityRepository.Get(ctx, identity.IdentityID(name))
|
||||
if err != nil {
|
||||
return false, "execution identity unavailable: " + err.Error(), taskstate.ParkTransient
|
||||
}
|
||||
if value.Lifecycle != identity.LifecycleActive {
|
||||
return false, "identity may not act", taskstate.ParkNeedsDecision
|
||||
}
|
||||
return true, "", ""
|
||||
}
|
||||
@@ -55,7 +55,7 @@ func TestPublishDiscoveredCap(t *testing.T) {
|
||||
for _, n := range tt.existing {
|
||||
have[n] = true
|
||||
}
|
||||
d := &Daemon{Log: slog.New(slog.DiscardHandler), Store: taskLookup{have: have}, Tasks: bus}
|
||||
d := wired(&Daemon{Log: slog.New(slog.DiscardHandler), Store: taskLookup{have: have}, Tasks: bus})
|
||||
var issues []forge.Issue
|
||||
for n := 1; n <= tt.issues; n++ {
|
||||
issues = append(issues, forge.Issue{Number: n, Labels: []string{"archie"}})
|
||||
|
||||
@@ -90,11 +90,11 @@ func TestPollNATSSkipsDisabledRoutedWorkflow(t *testing.T) {
|
||||
bus := &publishCounter{}
|
||||
fg := &reactCounter{}
|
||||
var logs bytes.Buffer
|
||||
d := &Daemon{
|
||||
d := wired(&Daemon{
|
||||
Log: slog.New(slog.NewTextHandler(&logs, nil)), Store: taskLookup{}, Tasks: bus,
|
||||
WorkflowDefinitions: tt.definitions, WorkflowEnablement: tt.enablement,
|
||||
Principals: tt.principals,
|
||||
}
|
||||
})
|
||||
is := forge.Issue{Number: 42, Title: "crash on startup"}
|
||||
if created := d.pollNATS(t.Context(), fg, cfg, repo, is, tt.labels, "acme-bot"); created != tt.wantCreated {
|
||||
t.Errorf("pollNATS created %v, want %v", created, tt.wantCreated)
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workintake"
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
func (d *Daemon) issueSources(ctx context.Context) []IssueSource {
|
||||
return d.IssueSources(ctx)
|
||||
}
|
||||
|
||||
// pollIssuesWithConfig returns the repo's issues the dispatch rule makes
|
||||
// work, using the given forge client and dispatch config. complete is false when any part of the poll failed or was skipped, so
|
||||
// the result must not be read as the whole eligible set.
|
||||
func (d *Daemon) pollIssuesWithConfig(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo) (issues []forge.Issue, complete bool) {
|
||||
labelled := func() ([]forge.Issue, bool) {
|
||||
// A label trigger needs a non-empty label or it would match every open issue.
|
||||
if cfg.Label == "" {
|
||||
d.Log.ErrorContext(ctx, "label trigger configured with an empty label; refusing to poll (an empty label matches every open issue)", "repo", repo.FullName())
|
||||
return nil, false
|
||||
}
|
||||
issues, err := fg.IssuesWithLabel(ctx, repo.Owner, repo.Name, cfg.Label)
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "label poll failed", "repo", repo.FullName(), "err", err)
|
||||
return nil, false
|
||||
}
|
||||
return issues, true
|
||||
}
|
||||
assigned := func() ([]forge.Issue, bool) {
|
||||
if cfg.BotUser == "" {
|
||||
d.Log.ErrorContext(ctx, "assignee trigger needs the connection's account; refusing to poll", "repo", repo.FullName())
|
||||
return nil, false
|
||||
}
|
||||
issues, err := fg.AssignedIssues(ctx, repo.Owner, repo.Name, cfg.BotUser)
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "assigned poll failed", "repo", repo.FullName(), "err", err)
|
||||
return nil, false
|
||||
}
|
||||
return issues, true
|
||||
}
|
||||
switch cfg.Dispatch.Trigger {
|
||||
case "label":
|
||||
return labelled()
|
||||
case "either":
|
||||
byAssignee, assignedOK := assigned()
|
||||
byLabel, labelledOK := labelled()
|
||||
seen := map[int]bool{}
|
||||
for _, is := range byAssignee {
|
||||
seen[is.Number] = true
|
||||
}
|
||||
issues = byAssignee
|
||||
for _, is := range byLabel {
|
||||
if !seen[is.Number] {
|
||||
issues = append(issues, is)
|
||||
}
|
||||
}
|
||||
return issues, assignedOK && labelledOK
|
||||
default: // "assignee" (default)
|
||||
return assigned()
|
||||
}
|
||||
}
|
||||
|
||||
// poll discovers work through every organisation connection, resolved
|
||||
// afresh each cycle so a rotated credential or a new connection applies to
|
||||
// the next poll.
|
||||
func (d *Daemon) poll(ctx context.Context) error {
|
||||
var failed []string
|
||||
room := d.Cfg.Get().IntakePerPass
|
||||
for _, source := range d.issueSources(ctx) {
|
||||
if !d.identityActive(ctx, source.Identity) {
|
||||
continue
|
||||
}
|
||||
cfg := d.Cfg.Get()
|
||||
cfg.BotUser = source.BotUser
|
||||
if source.Dispatch.Trigger != "" {
|
||||
cfg.Dispatch, cfg.Label = source.Dispatch, source.Label
|
||||
}
|
||||
tag := string(source.Identity)
|
||||
for _, repo := range source.Repos {
|
||||
issues, complete := d.pollIssuesWithConfig(ctx, source.Forge, cfg, repo)
|
||||
if complete {
|
||||
d.withdrawUnpolled(ctx, repo, tag, dispatchRule(cfg), issues)
|
||||
} else {
|
||||
failed = append(failed, repo.FullName())
|
||||
}
|
||||
d.publishDiscovered(ctx, source.Forge, cfg, repo, issues, tag, &room)
|
||||
}
|
||||
}
|
||||
return passFailed("issue poll", failed)
|
||||
}
|
||||
|
||||
// rememberPoll records a repo's latest eligible set and returns the previous
|
||||
// one, reporting whether it was taken under the same rule.
|
||||
func (d *Daemon) rememberPoll(key string, current polledIssues) (polledIssues, bool) {
|
||||
d.lastPolledMu.Lock()
|
||||
defer d.lastPolledMu.Unlock()
|
||||
previous, seen := d.lastPolled[key]
|
||||
if d.lastPolled == nil {
|
||||
d.lastPolled = map[string]polledIssues{}
|
||||
}
|
||||
d.lastPolled[key] = current
|
||||
return previous, seen && previous.rule == current.rule
|
||||
}
|
||||
|
||||
// withdrawUnpolled declines the queued or running work of an identity's
|
||||
// issues in repo that a complete poll no longer returns: the issue was
|
||||
// closed, unlabelled or unassigned.
|
||||
func (d *Daemon) withdrawUnpolled(ctx context.Context, repo config.Repo, identity, rule string, polled []forge.Issue) {
|
||||
eligible := make(map[int]bool, len(polled))
|
||||
for _, is := range polled {
|
||||
eligible[is.Number] = true
|
||||
}
|
||||
// Only an issue the same rule found eligible last time and no longer
|
||||
// finds was withdrawn. A changed label, trigger or bot user, or the first
|
||||
// poll after a restart, only records the new set: tasks picked up under
|
||||
// the old rule were not withdrawn by anyone.
|
||||
previous, sameRule := d.rememberPoll(identity+"\x00"+repo.FullName(), polledIssues{rule: rule, eligible: eligible})
|
||||
if !sameRule {
|
||||
return
|
||||
}
|
||||
active, err := d.Store.TasksPage(ctx, storecontract.TaskPage{
|
||||
Statuses: []string{taskstate.Queued, taskstate.Running}, Limit: 500,
|
||||
})
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "withdrawal check: active tasks unavailable", "repo", repo.FullName(), "err", err)
|
||||
return
|
||||
}
|
||||
for _, t := range active {
|
||||
if t.Owner != repo.Owner || t.Repo != repo.Name || t.Identity != identity ||
|
||||
!t.IsForgeBacked() || t.PRNumber > 0 || eligible[t.IssueNumber] || !previous.eligible[t.IssueNumber] {
|
||||
continue
|
||||
}
|
||||
if err := d.WithdrawTask(ctx, t.ID, "the issue is no longer assigned, labelled or open"); err != nil {
|
||||
d.Log.WarnContext(ctx, "withdraw task", "task", t.ID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// publishDiscovered publishes a repo's discovered issues until the pass has
|
||||
// created its cap of tasks. room is what is left of the cap; the issues left
|
||||
// over are found again by the next poll, because an issue with a task is
|
||||
// skipped and one without is not. A cap of zero or less is no cap.
|
||||
func (d *Daemon) publishDiscovered(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, issues []forge.Issue, identity string, room *int) {
|
||||
capped := *room > 0
|
||||
for _, is := range issues {
|
||||
if capped && *room == 0 {
|
||||
d.Log.InfoContext(ctx, "intake cap reached; remaining issues wait for the next poll", "repo", repo.FullName())
|
||||
return
|
||||
}
|
||||
if d.pollNATS(ctx, fg, cfg, repo, is, strings.Join(is.Labels, ","), identity) && capped {
|
||||
*room--
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pollNATS publishes a discovered issue to NATS and reports whether it created
|
||||
// a task.
|
||||
func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) bool {
|
||||
// Read-only existence check prevents rediscovering an existing task.
|
||||
existing, err := d.Store.TaskByIssue(ctx, repo.Owner, repo.Name, is.Number)
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "task lookup failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
|
||||
return false
|
||||
}
|
||||
if existing != nil {
|
||||
return false
|
||||
}
|
||||
// An issue whose label-routed workflow the identity's org disabled must
|
||||
// not queue a task at all. Skip without acknowledging, so a later poll
|
||||
// picks the issue up again if the workflow is re-enabled.
|
||||
if routed, disabled, err := d.disabledRoutedWorkflow(ctx, labels, identity); err != nil {
|
||||
d.Log.WarnContext(ctx, "routed workflow check failed; publishing anyway", "repo", repo.FullName(), "issue", is.Number, "err", err)
|
||||
} else if disabled {
|
||||
d.Log.InfoContext(ctx, "issue skipped: its routed workflow is disabled for this org", "repo", repo.FullName(), "issue", is.Number, "workflow", routed)
|
||||
return false
|
||||
}
|
||||
// Classify and encode here: the envelope, its routing kind and its
|
||||
// subject belong to the work-intake domain, not to the transport.
|
||||
parsed := workintake.SplitLabels(labels)
|
||||
task := workintake.TaskEnvelope{
|
||||
Owner: repo.Owner,
|
||||
Repo: repo.Name,
|
||||
Number: is.Number,
|
||||
Title: is.Title,
|
||||
Body: is.Body,
|
||||
Labels: parsed,
|
||||
Identity: identity,
|
||||
Kind: workintake.KindForLabels(parsed),
|
||||
}
|
||||
if err := d.PublishTask(ctx, task); err != nil {
|
||||
d.Log.ErrorContext(ctx, "task publish failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
|
||||
return false
|
||||
}
|
||||
d.acknowledge(ctx, fg, cfg, repo, is)
|
||||
return true
|
||||
}
|
||||
|
||||
// acknowledge posts the pickup reaction
|
||||
// using fg -- the forge client that owns repo (identity-scoped or root) --
|
||||
// and the ack reaction from cfg, the dispatch config that owns this poll.
|
||||
func (d *Daemon) acknowledge(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue) {
|
||||
d.Log.InfoContext(ctx, "issue queued", "repo", repo.FullName(), "issue", is.Number, "title", is.Title)
|
||||
if ack := cfg.Dispatch.AckReaction; ack != "" {
|
||||
if err := fg.React(ctx, repo.Owner, repo.Name, is.Number, ack); err != nil {
|
||||
d.Log.WarnContext(ctx, "ack reaction failed", "issue", is.Number, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dispatchRule identifies the rule a poll selects issues by.
|
||||
func dispatchRule(cfg config.Config) string {
|
||||
return cfg.Dispatch.Trigger + "\x00" + cfg.Label + "\x00" + cfg.BotUser
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
)
|
||||
|
||||
// Startup runs crash recovery and access verification once.
|
||||
func (d *Daemon) Startup(ctx context.Context) error {
|
||||
d.wireReactions()
|
||||
if err := d.cleanupExpiredStorage(ctx); err != nil {
|
||||
d.Log.WarnContext(ctx, "startup volume cleanup failed", "err", err)
|
||||
}
|
||||
n, err := d.Store.RecoverStale(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n > 0 {
|
||||
d.Log.InfoContext(ctx, "re-queued tasks left running by a previous daemon", "count", n)
|
||||
}
|
||||
|
||||
d.sweepAccess(ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
// wireReactions builds the review-reaction consumer and publisher. Startup
|
||||
// calls it before the run loop and the scheduler start, so nothing builds them
|
||||
// concurrently.
|
||||
func (d *Daemon) wireReactions() {
|
||||
d.reactionConsumer = newReactionConsumer(d.Store, d.Store, d.Store, d.botUserForTask, d.Log)
|
||||
d.reactionPublisher = d.PublishReaction
|
||||
}
|
||||
|
||||
// sweepAccess accepts pending invitations and verifies push access through
|
||||
// every organisation connection, each in its own goroutine.
|
||||
func (d *Daemon) sweepAccess(ctx context.Context) {
|
||||
sweep := func(source IssueSource) {
|
||||
log := d.Log.With("identity", source.Identity)
|
||||
ctx, cancel := context.WithTimeout(ctx, sweepTimeout)
|
||||
defer cancel()
|
||||
if err := source.Forge.AcceptInvitations(ctx, repoRefs(source.Repos)); err != nil {
|
||||
log.WarnContext(ctx, "invitation sweep failed", "err", err)
|
||||
}
|
||||
for _, r := range source.Repos {
|
||||
if err := source.Forge.VerifyPush(ctx, r.Owner, r.Name); err != nil {
|
||||
// A sweep cut short by its own context is not a repo failure: name
|
||||
// the budget when it expired, stay quiet on a shutdown.
|
||||
switch {
|
||||
case errors.Is(ctx.Err(), context.DeadlineExceeded):
|
||||
// Every later call would return the same error, so say it once.
|
||||
log.WarnContext(ctx, "push check abandoned: sweep budget exhausted", "repo", r.FullName(), "err", err)
|
||||
return
|
||||
case ctx.Err() != nil:
|
||||
return
|
||||
}
|
||||
log.WarnContext(ctx, "repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for _, source := range d.issueSources(ctx) {
|
||||
wg.Go(func() { sweep(source) })
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// repoRefs is a source's configured repositories as the allowlist the forge
|
||||
// accepts invitations against.
|
||||
func repoRefs(repos []config.Repo) []forge.RepoRef {
|
||||
out := make([]forge.RepoRef, 0, len(repos))
|
||||
for _, r := range repos {
|
||||
out = append(out, forge.RepoRef{Owner: r.Owner, Repo: r.Name})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sweepTimeout bounds one forge's boot sweep. Startup gates the gateways and
|
||||
// the run loop, and a forge client carries no request deadline of its own, so
|
||||
// an unreachable host would otherwise hold boot open indefinitely.
|
||||
const sweepTimeout = 2 * time.Minute
|
||||
|
||||
// Run polls until the context ends.
|
||||
func (d *Daemon) Run(ctx context.Context) error {
|
||||
// A pass no longer waits for the tasks it submitted, so the run does, once
|
||||
// the ctx is cancelled.
|
||||
defer d.WaitForTasks()
|
||||
d.watchCallWaits(ctx)
|
||||
drained := make(chan struct{})
|
||||
defer func() { <-drained }()
|
||||
go func() {
|
||||
defer close(drained)
|
||||
d.drainLoop(ctx)
|
||||
}()
|
||||
interval := d.Cfg.Get().PollInterval.Std()
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
d.Cycle(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
// PollInterval may have been reloaded; reset the ticker so a
|
||||
// change takes effect on the next cycle.
|
||||
if iv := d.Cfg.Get().PollInterval.Std(); iv != interval {
|
||||
interval = iv
|
||||
ticker.Reset(iv)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RunOnce runs one cycle and drains until no task is in flight. A run waiting
|
||||
// on a callee stays in flight while the callee sits queued, so the queue is
|
||||
// drained again until every task, callees included, has returned.
|
||||
func (d *Daemon) RunOnce(ctx context.Context) {
|
||||
defer d.WaitForTasks()
|
||||
d.watchCallWaits(ctx)
|
||||
// One pass has no scheduler, so it runs the maintenance jobs itself.
|
||||
for _, job := range d.BuiltinJobs() {
|
||||
if err := job.Run(ctx); err != nil {
|
||||
d.Log.WarnContext(ctx, "built-in job failed", "job", job.ID, "err", err)
|
||||
}
|
||||
}
|
||||
d.dispatchBindings(ctx)
|
||||
d.Cycle(ctx)
|
||||
d.drainNATS(ctx)
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for d.taskDispatcher().inFlight() > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
d.drainNATS(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cycle is one poll pass: turn review reactions into tasks; maintenance runs
|
||||
// as built-in schedules. Captured events are dispatched by the caller
|
||||
// (drainLoop in Run, once in RunOnce) so they start within about a second.
|
||||
func (d *Daemon) Cycle(ctx context.Context) {
|
||||
d.markPass("cycle")
|
||||
d.drainReactions(ctx)
|
||||
d.retryTransientParks(ctx)
|
||||
}
|
||||
|
||||
// markPass stamps the start of one loop's pass, so a hung pass shows as stale.
|
||||
func (d *Daemon) markPass(loop string) {
|
||||
d.loopPasses.Store(loop, time.Now().UnixNano())
|
||||
}
|
||||
|
||||
// StalestLoop reports the run loop that least recently began a pass and when,
|
||||
// or the zero time when no loop has started yet. The systemd watchdog beats
|
||||
// only while this is fresh, so every loop Run starts must mark its passes.
|
||||
func (d *Daemon) StalestLoop() (string, time.Time) {
|
||||
var loop string
|
||||
var oldest int64
|
||||
d.loopPasses.Range(func(k, v any) bool {
|
||||
name, okName := k.(string)
|
||||
if ns, ok := v.(int64); ok && okName && (oldest == 0 || ns < oldest) {
|
||||
loop, oldest = name, ns
|
||||
}
|
||||
return true
|
||||
})
|
||||
if oldest == 0 {
|
||||
return "", time.Time{}
|
||||
}
|
||||
return loop, time.Unix(0, oldest).UTC()
|
||||
}
|
||||
|
||||
// emit publishes an observability event.
|
||||
func (d *Daemon) emit(ctx context.Context, e events.Event) {
|
||||
d.Bus.Publish(ctx, e)
|
||||
}
|
||||
@@ -36,13 +36,11 @@ type TaskModels interface {
|
||||
// never touch it.
|
||||
func (d *Daemon) taskConfig(ctx context.Context, task *workflow.Task) (config.Config, error) {
|
||||
cfg := d.configFor(task)
|
||||
if d.TaskModels != nil {
|
||||
models, err := d.TaskModels.TaskModels(ctx, task.ID)
|
||||
if err != nil {
|
||||
return config.Config{}, fmt.Errorf("resolve the task's models: %w", err)
|
||||
}
|
||||
cfg = cfg.WithOrgModels(models)
|
||||
models, err := d.TaskModels.TaskModels(ctx, task.ID)
|
||||
if err != nil {
|
||||
return config.Config{}, fmt.Errorf("resolve the task's models: %w", err)
|
||||
}
|
||||
cfg = cfg.WithOrgModels(models)
|
||||
cfg.Providers = cfg.ModelProviders()
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ func TestOpenModelEgressCarriesOnlyReferencedProviders(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
egressProxy := &recordingModelEgress{}
|
||||
d := &Daemon{
|
||||
d := wired(&Daemon{
|
||||
Cfg: config.NewHolder(config.Config{
|
||||
Models: map[string]string{"default": "used/model-one"},
|
||||
Providers: map[string]config.Provider{
|
||||
@@ -45,7 +45,7 @@ func TestOpenModelEgressCarriesOnlyReferencedProviders(t *testing.T) {
|
||||
},
|
||||
}),
|
||||
ModelEgress: egressProxy,
|
||||
}
|
||||
})
|
||||
task := &workflow.Task{ID: 7, Identity: string(sys), WorkflowDefinitionYAML: "id: w\nsteps:\n - type: agent.run\n"}
|
||||
|
||||
if _, _, err := d.openModelEgress(t.Context(), task, "run-credential"); err != nil {
|
||||
@@ -89,7 +89,7 @@ func TestRunViaAgentCarriesProviderCredentialVariable(t *testing.T) {
|
||||
t.Fatalf("marshal response: %v", err)
|
||||
}
|
||||
bus := &capturingBus{reply: reply}
|
||||
d := &Daemon{
|
||||
d := wired(&Daemon{
|
||||
Cfg: config.NewHolder(config.Config{
|
||||
Models: map[string]string{"default": "used/model-one"},
|
||||
Providers: map[string]config.Provider{
|
||||
@@ -98,7 +98,7 @@ func TestRunViaAgentCarriesProviderCredentialVariable(t *testing.T) {
|
||||
}),
|
||||
Tasks: bus,
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
}
|
||||
})
|
||||
task := &workflow.Task{ID: 7, Identity: string(sys), Attempt: 1, Workflow: "implement", WorkflowDefinitionYAML: "id: w\nsteps:\n - type: agent.run\n"}
|
||||
|
||||
d.runViaAgent(t.Context(), task, config.Repo{}, config.AgentProfile{}, "run-credential")
|
||||
|
||||
@@ -23,13 +23,11 @@ func (d *Daemon) toolAllowlist(ctx context.Context, task *workflow.Task, profile
|
||||
return storepkg.AllowedTools(profileTools, false, nil)
|
||||
}
|
||||
var accepted []string
|
||||
if d.PackageAuthorities != nil {
|
||||
authority, err := d.PackageAuthorities.AcceptedAuthority(ctx, name)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "package authority unavailable; offering no tools", "task", task.ID, "package", name, "err", err)
|
||||
} else {
|
||||
accepted = authority.Tools
|
||||
}
|
||||
authority, err := d.PackageAuthorities.AcceptedAuthority(ctx, name)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "package authority unavailable; offering no tools", "task", task.ID, "package", name, "err", err)
|
||||
} else {
|
||||
accepted = authority.Tools
|
||||
}
|
||||
return storepkg.AllowedTools(profileTools, true, accepted)
|
||||
}
|
||||
@@ -24,10 +24,10 @@ func (a acceptedTools) AcceptedAuthority(_ context.Context, name string) (storep
|
||||
// offered only the tools accepted for its package, and none when the package
|
||||
// is missing or unaccepted, while an operator workflow keeps its profile's list.
|
||||
func TestToolAllowlistFollowsPackageAuthority(t *testing.T) {
|
||||
d := &Daemon{Log: slog.Default(), PackageAuthorities: acceptedTools{
|
||||
d := wired(&Daemon{Log: slog.Default(), PackageAuthorities: acceptedTools{
|
||||
"triage": {Tools: []string{"search", "fetch"}},
|
||||
"unsigned": {},
|
||||
}}
|
||||
}})
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
|
||||
@@ -42,9 +42,6 @@ func (d *Daemon) checkPackageTrigger(ctx context.Context, task *workflow.Task) e
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
if d.PackageAuthorities == nil {
|
||||
return fmt.Errorf("package %s: no accepted authority is available", name)
|
||||
}
|
||||
authority, err := d.PackageAuthorities.AcceptedAuthority(ctx, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("package %s: accepted authority unavailable: %w", name, err)
|
||||
|
||||
@@ -12,11 +12,11 @@ import (
|
||||
// trigger accepted for its package, failing closed when none is accepted or
|
||||
// the package is unknown, while operator workflows are unbounded.
|
||||
func TestCheckPackageTrigger(t *testing.T) {
|
||||
d := &Daemon{PackageAuthorities: acceptedTools{
|
||||
d := wired(&Daemon{PackageAuthorities: acceptedTools{
|
||||
"triage": storepkg.Authority{Triggers: []string{"label:triage", "binding:b1", "call"}},
|
||||
"chatty": storepkg.Authority{Triggers: []string{"source:chat"}},
|
||||
"unsigned": storepkg.Authority{},
|
||||
}}
|
||||
}})
|
||||
tests := []struct {
|
||||
name string
|
||||
task workflow.Task
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/logging"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// parkCapabilityUnavailable parks a task whose daemon-side capability is
|
||||
// missing (container pool or task transport): both are environmental, so
|
||||
// the park class is transient and the KindParked event carries the class
|
||||
// for the dashboard. The reason distinguishes the two in logs and events.
|
||||
func (d *Daemon) parkCapabilityUnavailable(ctx context.Context, task *workflow.Task) {
|
||||
var reason string
|
||||
if d.ContainerPool == nil {
|
||||
reason = "managed agent container pool is unavailable; refusing to run this task on the host"
|
||||
d.Log.ErrorContext(ctx, "task parked: managed worker unavailable", "task", task.ID,
|
||||
"hint", "enable containers and make the archie-agent image available, then retry the task")
|
||||
} else {
|
||||
reason = "agent task transport is unavailable; refusing to run this task on the host"
|
||||
d.Log.ErrorContext(ctx, "task parked: agent task transport unavailable", "task", task.ID)
|
||||
}
|
||||
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
|
||||
}
|
||||
|
||||
// parkRunningTask moves a running task to parked with class, using a
|
||||
// bounded context that ignores cancellation. ErrStaleTransition is ignored;
|
||||
// other errors are logged.
|
||||
func (d *Daemon) parkRunningTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass) {
|
||||
d.parkBlockedTask(ctx, task, reason, class, "")
|
||||
}
|
||||
|
||||
// parkBlockedTask is parkRunningTask recording the blocker the park site
|
||||
// knows, so the dashboard can link to its fix.
|
||||
func (d *Daemon) parkBlockedTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass, blocker taskstate.Blocker) {
|
||||
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := d.Store.ParkTask(writeCtx, task.ID, workflow.StatusRunning, reason, class, blocker); err != nil {
|
||||
if !errors.Is(err, storecontract.ErrStaleTransition) {
|
||||
d.Log.WarnContext(ctx, "terminal park transition failed", "task", task.ID, "reason", reason, "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
// Recorded only once the transition landed. An entry claiming the task
|
||||
// parked would be a lie when another actor moved it out of running first
|
||||
// -- the same race the parked event is withheld for.
|
||||
d.recordPark(ctx, task.ID, reason)
|
||||
d.emit(ctx, events.Event{
|
||||
Kind: events.KindParked, TaskID: task.ID, Attempt: task.Attempt,
|
||||
Repo: strings.Trim(task.Owner+"/"+task.Repo, "/"), Issue: task.IssueNumber, Workflow: task.Workflow,
|
||||
Detail: reason, Data: map[string]any{"park_class": class},
|
||||
})
|
||||
}
|
||||
|
||||
// recordPark writes a park reason to the attempt's log. Dropped when no sink
|
||||
// is open.
|
||||
func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) {
|
||||
d.TaskLogs.Write(ctx, taskID, logging.Entry{
|
||||
Time: time.Now(),
|
||||
Level: slog.LevelError.String(),
|
||||
Message: "task parked: " + reason,
|
||||
Fields: map[string]any{"component": "daemon", "task": taskID},
|
||||
})
|
||||
}
|
||||
|
||||
// parkIfIdentityUnresolvable parks task when its identity cannot be acted on,
|
||||
// reporting whether it did.
|
||||
func (d *Daemon) parkIfIdentityUnresolvable(ctx context.Context, task *workflow.Task) bool {
|
||||
ok, reason, class := d.identityMayAct(ctx, task.Identity)
|
||||
if ok {
|
||||
return false
|
||||
}
|
||||
d.parkRunningTask(ctx, task, reason, class)
|
||||
return true
|
||||
}
|
||||
|
||||
// retryTransientParks requeues tasks parked for an environmental cause once
|
||||
// their backoff has passed. The store parks a task needs_decision, which
|
||||
// raises the alert, when its retries run out, so this never loops on one.
|
||||
func (d *Daemon) retryTransientParks(ctx context.Context) {
|
||||
now := time.Now()
|
||||
var after storecontract.TaskCursor
|
||||
for {
|
||||
page, err := d.Store.TasksPage(ctx, storecontract.TaskPage{Statuses: []string{workflow.StatusParked}, After: after, Limit: 200})
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "list parked tasks for retry failed", "err", err)
|
||||
return
|
||||
}
|
||||
for _, t := range page {
|
||||
if taskstate.TransientRetryDue(t.ParkClass, t.RetryCount, t.UpdatedAt, now) {
|
||||
d.retryTransientPark(ctx, t.ID)
|
||||
}
|
||||
}
|
||||
if len(page) < 200 {
|
||||
return
|
||||
}
|
||||
last := page[len(page)-1]
|
||||
after = storecontract.TaskCursor{UpdatedAt: last.UpdatedAt, ID: last.ID}
|
||||
}
|
||||
}
|
||||
|
||||
// retryTransientPark requeues one task on its stored retry mode, which the
|
||||
// list projection omits: a remediation must keep continuing its branch.
|
||||
func (d *Daemon) retryTransientPark(ctx context.Context, id int64) {
|
||||
t, err := d.Store.TaskByID(ctx, id)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "load task for transient retry failed", "task", id, "err", err)
|
||||
return
|
||||
}
|
||||
if err := d.Store.RetryTask(ctx, id, workflow.StatusParked, "", t.RetryMode, workflowtask.Resume{}, false); err != nil {
|
||||
if !errors.Is(err, storecontract.ErrStaleTransition) {
|
||||
d.Log.WarnContext(ctx, "transient park retry failed", "task", id, "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
d.emit(ctx, events.Event{
|
||||
Kind: events.KindTaskRetried, TaskID: id, Workflow: t.Workflow,
|
||||
Repo: strings.Trim(t.Owner+"/"+t.Repo, "/"), Issue: t.IssueNumber,
|
||||
Detail: "retried automatically after a transient park",
|
||||
Data: map[string]any{"retry_count": t.RetryCount + 1, "previous_reason": t.ParkReason},
|
||||
})
|
||||
}
|
||||
|
||||
// parkRefusedRun parks a task whose workflow identity check did not permit it.
|
||||
// A denial is the operator's to fix, but a chain that could not be read is a
|
||||
// store that will answer again, so it parks transient rather than demanding an
|
||||
// operator for a blip.
|
||||
func (d *Daemon) parkRefusedRun(ctx context.Context, task *workflow.Task, reason string, err error) {
|
||||
class, blocker := taskstate.ParkNeedsDecision, taskstate.BlockerPolicyDenied
|
||||
if err != nil {
|
||||
class, blocker = pinParkClass(err), parkBlocker(err)
|
||||
}
|
||||
d.parkBlockedTask(ctx, task, reason, class, blocker)
|
||||
}
|
||||
|
||||
// openTaskLog opens the task's log sink for one process() call and returns
|
||||
// its closer. Failure to open is logged, not fatal.
|
||||
func (d *Daemon) openTaskLog(task *workflow.Task) func() {
|
||||
if err := d.TaskLogs.Open(task.ID, task.Attempt); err != nil {
|
||||
d.Log.Warn("task log sink unavailable", "task", task.ID, "attempt", task.Attempt, "err", err)
|
||||
}
|
||||
return func() {
|
||||
if err := d.TaskLogs.Close(task.ID); err != nil {
|
||||
d.Log.Warn("task log sink close failed", "task", task.ID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -31,7 +31,7 @@ func TestEarlyParkRecordsAttempt(t *testing.T) {
|
||||
defer bus.Close()
|
||||
sub := bus.Subscribe(1)
|
||||
defer sub.Close()
|
||||
d := &Daemon{Store: parkStore{err: err}, Bus: bus, Log: slog.New(slog.DiscardHandler)}
|
||||
d := wired(&Daemon{Store: parkStore{err: err}, Bus: bus, Log: slog.New(slog.DiscardHandler)})
|
||||
task := &workflow.Task{ID: 7, Attempt: 3, Owner: "owner", Repo: "repo", Workflow: "implement"}
|
||||
d.parkRunningTask(t.Context(), task, "clone authentication failed", taskstate.ParkNeedsDecision)
|
||||
select {
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/eda/playbook"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workintake"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// pinTaskProfile pins the task's workflow definition and resolves its agent
|
||||
// profile, parking the task on failure.
|
||||
func (d *Daemon) pinTaskProfile(ctx context.Context, task *workflow.Task) (config.AgentProfile, bool) {
|
||||
if err := d.pinWorkflowDefinition(ctx, task); err != nil {
|
||||
d.Log.ErrorContext(ctx, "pin workflow definition failed", "task", task.ID, "err", err)
|
||||
d.parkBlockedTask(ctx, task, "pin workflow definition: "+err.Error(), pinParkClass(err), parkBlocker(err))
|
||||
return config.AgentProfile{}, false
|
||||
}
|
||||
iface, err := workflowtask.ParseWorkflowInterface(task.WorkflowDefinitionYAML)
|
||||
if err == nil {
|
||||
// Every producer meets the declared interface here, including the
|
||||
// ones that cannot read definitions when they enqueue (chat and
|
||||
// dashboard spawns go through the Gateway, which never does).
|
||||
err = iface.CheckInputs(task.Inputs)
|
||||
}
|
||||
if err == nil && iface.RepositoryMode() == workflowtask.RepositoryRequired && !task.HasRepository() {
|
||||
// An optional workflow may hand off to one that requires a
|
||||
// repository, because its task may carry one; a task that does not
|
||||
// is parked here rather than run the target's steps in a scratch
|
||||
// workspace.
|
||||
err = fmt.Errorf("requires a repository and this task has none")
|
||||
}
|
||||
if err == nil {
|
||||
var profile config.AgentProfile
|
||||
if profile, err = d.configFor(task).Containers.Profile(iface.Profile); err == nil {
|
||||
return profile, true
|
||||
}
|
||||
}
|
||||
d.Log.ErrorContext(ctx, "agent profile unavailable", "task", task.ID, "err", err)
|
||||
d.parkRunningTask(ctx, task, "workflow "+task.Workflow+": "+err.Error(), taskstate.ParkNeedsDecision)
|
||||
return config.AgentProfile{}, false
|
||||
}
|
||||
|
||||
// pinWorkflowDefinition ensures the task carries a valid definition pin for
|
||||
// the workflow it names and that its org has the workflow enabled. Every run
|
||||
// passes through here, whatever started it, so this is the one enablement
|
||||
// check for issue labels, handoffs, approvals, calls and playbooks alike.
|
||||
func (d *Daemon) pinWorkflowDefinition(ctx context.Context, task *workflow.Task) error {
|
||||
ctx, err := d.workflowContext(ctx, task)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := d.pinDefinition(ctx, task); err != nil {
|
||||
return err
|
||||
}
|
||||
enablement, err := d.workflowEnablement(ctx)
|
||||
if err != nil {
|
||||
return pinFailure{taskstate.ParkTransient, fmt.Errorf("workflow enablement: %w", err)}
|
||||
}
|
||||
if !enablement.Enabled(task.Org, task.Workflow) {
|
||||
return blockedPark{taskstate.BlockerWorkflowDisabled, pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("workflow %s is disabled", task.Workflow)}}
|
||||
}
|
||||
if err := d.checkPackageTrigger(ctx, task); err != nil {
|
||||
return pinFailure{taskstate.ParkNeedsDecision, err}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pinDefinition keeps a valid pin and resolves a missing or mismatched one
|
||||
// again from the active definitions.
|
||||
func (d *Daemon) pinDefinition(ctx context.Context, task *workflow.Task) error {
|
||||
pinned, ok, err := workflow.PinnedDefinitionID(task)
|
||||
if err != nil {
|
||||
return pinFailure{taskstate.ParkNeedsDecision, err}
|
||||
}
|
||||
if ok && pinned == task.Workflow {
|
||||
return nil
|
||||
}
|
||||
d.runActionPlaybooks(ctx, task)
|
||||
collection, version, err := d.WorkflowDefinitions.WorkflowDefinitions(ctx)
|
||||
if err != nil {
|
||||
return pinFailure{taskstate.ParkTransient, err}
|
||||
}
|
||||
return d.pinWorkflowFromCollection(ctx, task, collection, version)
|
||||
}
|
||||
|
||||
func (d *Daemon) pinWorkflowFromCollection(ctx context.Context, task *workflow.Task, collection workflow.WorkflowDefinitionCollection, version int64) error {
|
||||
available := make(map[string]struct{}, len(collection.Definitions))
|
||||
for _, definition := range collection.Definitions {
|
||||
available[definition.ID] = struct{}{}
|
||||
}
|
||||
id, err := d.resolveWorkflowID(ctx, task, available)
|
||||
if err != nil {
|
||||
return pinFailure{taskstate.ParkNeedsDecision, err}
|
||||
}
|
||||
definition, ok := collection.DefinitionByID(id)
|
||||
if !ok {
|
||||
return pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("workflow definition %q disappeared", id)}
|
||||
}
|
||||
task.Workflow = id
|
||||
task.WorkflowDefinitionVersion = version
|
||||
task.WorkflowDefinitionYAML = definition.YAML
|
||||
task.WorkflowDefinitionDigest = workflow.DigestDefinition(definition.YAML)
|
||||
if err := d.Store.Update(ctx, task); err != nil {
|
||||
return pinFailure{taskstate.ParkTransient, fmt.Errorf("persist workflow definition pin: %w", err)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveWorkflowID picks the definition to pin: a matching playbook, then
|
||||
// the kind defaults. A task already naming a workflow keeps it.
|
||||
func (d *Daemon) resolveWorkflowID(ctx context.Context, task *workflow.Task, available map[string]struct{}) (string, error) {
|
||||
if task.Workflow != "" {
|
||||
return workflow.ResolveWorkflowID(task, available)
|
||||
}
|
||||
decision, matched := d.Playbooks.Dispatch(ctx, playbookInput(task))
|
||||
if !matched {
|
||||
return workflow.ResolveWorkflowID(task, available)
|
||||
}
|
||||
if _, defined := available[decision.Workflow]; !defined {
|
||||
// Reported, not silently downgraded to the binding's choice: the
|
||||
// operator bound this trigger to a named workflow, and running a
|
||||
// different one under their rule is worse than parking the task.
|
||||
return "", fmt.Errorf("playbook %q binds this trigger to workflow %q, which is not defined", decision.PlaybookID, decision.Workflow)
|
||||
}
|
||||
d.Log.InfoContext(ctx, "playbook selected workflow",
|
||||
"task", task.ID, "playbook", decision.PlaybookID,
|
||||
"playbook_version", decision.Version, "workflow", decision.Workflow)
|
||||
return decision.Workflow, nil
|
||||
}
|
||||
|
||||
// runActionPlaybooks runs the action playbooks matching a task before it is
|
||||
// pinned. Failures never fail the task; tasks already naming a workflow are
|
||||
// skipped.
|
||||
func (d *Daemon) runActionPlaybooks(ctx context.Context, task *workflow.Task) {
|
||||
if task.Workflow != "" {
|
||||
return
|
||||
}
|
||||
if err := d.Playbooks.Run(ctx, d.PlaybookLedger, d.Log, playbookInput(task)); err != nil {
|
||||
d.Log.WarnContext(ctx, "action playbook run failed", "task", task.ID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// playbookInput builds the coordinator's dispatch input from a task row. The
|
||||
// event surface is what the intake point actually knows about the originating
|
||||
// issue -- the fields a `when` condition can read; it is deliberately not
|
||||
// padded with values the daemon would have to invent.
|
||||
func playbookInput(task *workflow.Task) playbook.DispatchInput {
|
||||
labels := workintake.SplitLabels(task.Labels)
|
||||
kind := string(workintake.KindForLabels(labels))
|
||||
return playbook.DispatchInput{
|
||||
Labels: labels,
|
||||
Kind: kind,
|
||||
TaskID: workintake.TaskEnvelope{Org: task.Org, Identity: task.Identity, Owner: task.Owner, Repo: task.Repo, Number: task.IssueNumber}.IdempotencyKey(),
|
||||
Event: map[string]any{
|
||||
"kind": kind,
|
||||
"labels": labels,
|
||||
"owner": task.Owner,
|
||||
"repo": task.Repo,
|
||||
"number": task.IssueNumber,
|
||||
"title": task.Title,
|
||||
"body": task.Body,
|
||||
"source": task.Source,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,480 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
"github.com/samcharles93/archie-core/internal/storage"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
"github.com/samcharles93/archie-core/internal/worktree"
|
||||
)
|
||||
|
||||
// repoHub is the per-task repository surface the pipeline resolves through.
|
||||
type repoHub interface {
|
||||
configFor(task *workflow.Task) config.Config
|
||||
forgeFor(ctx context.Context, task *workflow.Task) forge.Forge
|
||||
treesFor(task *workflow.Task) *worktree.Manager
|
||||
Repository(ctx context.Context, task *workflow.Task, operation string) (RepositoryAccess, error)
|
||||
}
|
||||
|
||||
var _ repoHub = (*Daemon)(nil)
|
||||
|
||||
func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHold) {
|
||||
// A task off the NATS stream only has an id once the store has claimed its
|
||||
// row, inside this call, so its capacity is named here.
|
||||
hold.name(task.ID, task.Attempt)
|
||||
// The log opens before any check that can park, so every park reason
|
||||
// reaches the attempt log.
|
||||
defer d.openTaskLog(task)()
|
||||
// The workflow may name the identity the run acts as, which everything
|
||||
// below is chosen by, so it is settled before anything else.
|
||||
if !d.adoptWorkflowIdentity(ctx, task) {
|
||||
return
|
||||
}
|
||||
// Register first so the task can be stopped during setup.
|
||||
ctx, finished := d.running.begin(ctx, task.ID, task.Identity)
|
||||
defer finished()
|
||||
|
||||
if d.parkIfIdentityUnresolvable(ctx, task) {
|
||||
return
|
||||
}
|
||||
|
||||
trees, repo, err := d.taskWorkspace(ctx, task)
|
||||
if err != nil {
|
||||
d.parkBlockedTask(ctx, task, err.Error(), pinParkClass(err), parkBlocker(err))
|
||||
return
|
||||
}
|
||||
|
||||
if d.ContainerPool == nil {
|
||||
d.parkCapabilityUnavailable(ctx, task)
|
||||
return
|
||||
}
|
||||
|
||||
d.Log.InfoContext(ctx, "processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "attempt", task.Attempt)
|
||||
workDir, ok := d.prepareWorkspace(ctx, task, trees, repo)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !task.HasRepository() {
|
||||
defer func() {
|
||||
if err := trees.RemoveScratch(task.ID); err != nil {
|
||||
d.Log.WarnContext(ctx, "scratch workspace cleanup failed", "task", task.ID, "err", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// The workflow is pinned before the container starts, because the
|
||||
// profile it names decides the container's image.
|
||||
profile, ok := d.pinTaskProfile(ctx, task)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
ctr, credential, revokeCredential, ok := d.acquireTaskContainer(ctx, task, repo, workDir, profile.Image)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer d.ContainerPool.Release(ctx, ctr)
|
||||
defer revokeCredential()
|
||||
|
||||
// Run the whole task in archie-agent, which calls back to the daemon for
|
||||
// store, forge and push operations.
|
||||
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
|
||||
runCtx, stopWatch := withContainerExit(limitCtx, ctr)
|
||||
d.runViaAgent(runCtx, task, repo, profile, credential)
|
||||
stopWatch()
|
||||
stopLimit()
|
||||
|
||||
d.teardownStorage(ctx, task, repo, workDir)
|
||||
d.cleanupTerminalTaskWorktree(ctx, task, trees)
|
||||
}
|
||||
|
||||
func (d *Daemon) taskWorkspace(ctx context.Context, task *workflow.Task) (*worktree.Manager, config.Repo, error) {
|
||||
if task.HasRepository() {
|
||||
resolved, err := d.Repository(ctx, task, config.OperationClone)
|
||||
if err == nil && !task.IsTestRun() && workflow.DefinitionPushes(task.WorkflowDefinitionYAML) {
|
||||
// Checked before the run rather than at its commit step, so a
|
||||
// missing permit parks in seconds instead of after the agent's work.
|
||||
_, err = d.Repository(ctx, task, config.OperationPush)
|
||||
}
|
||||
return resolved.Trees, resolved.Repository, err
|
||||
}
|
||||
trees := d.treesFor(task)
|
||||
repo, ok := d.repoFor(task)
|
||||
if !ok && task.HasRepository() {
|
||||
return nil, repo, pinFailure{taskstate.ParkTerminal, errors.New("repository is no longer configured")}
|
||||
}
|
||||
return trees, repo, nil
|
||||
}
|
||||
|
||||
// prepareWorkspace makes the directory the container binds as its
|
||||
// workspace, which must exist before Acquire: a fresh clone of the task's
|
||||
// repository, or an empty scratch directory for a task with none. It parks
|
||||
// the task and reports false when it cannot.
|
||||
func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, trees *worktree.Manager, repo config.Repo) (string, bool) {
|
||||
if !task.HasRepository() {
|
||||
dir, err := trees.PrepareScratch(task.ID)
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "scratch workspace prepare failed", "err", err)
|
||||
d.parkRunningTask(ctx, task, "scratch workspace prepare failed: "+err.Error(), taskstate.ParkTransient)
|
||||
return "", false
|
||||
}
|
||||
return dir, true
|
||||
}
|
||||
// The retry mode on the task decides whether to resume the pushed branch or
|
||||
// reset onto base. Prepare clones a missing worktree either way.
|
||||
target := worktree.Fresh
|
||||
if taskstate.NormalizeRetryMode(task.RetryMode) == taskstate.RetryKeepWorktree {
|
||||
target = worktree.Keep
|
||||
// Consumed: a later plain requeue must not skip the reset.
|
||||
task.RetryMode = ""
|
||||
}
|
||||
if taskstate.NormalizeRetryMode(task.RetryMode) == taskstate.RetryContinuePushedWork {
|
||||
if task.Branch == "" {
|
||||
// The action refuses this combination, so an empty branch here is a
|
||||
// dispatch bug upstream and no retry can fix it.
|
||||
d.Log.ErrorContext(ctx, "task has no persisted branch to continue", "task", task.ID)
|
||||
d.parkRunningTask(ctx, task, "worktree resume failed: task has no branch to continue", taskstate.ParkNeedsDecision)
|
||||
return "", false
|
||||
}
|
||||
target = worktree.Target(task.Branch)
|
||||
}
|
||||
d.reclaimStaleClones(ctx, task, trees)
|
||||
if err := worktree.AwaitSpace(ctx, trees.FreeBytes, minCloneSpace, diskCheckEvery, diskWaitLimit); err != nil {
|
||||
d.Log.WarnContext(ctx, "worktree prepare delayed", "task", task.ID, "err", err)
|
||||
d.parkRunningTask(ctx, task, "worktree prepare failed: "+err.Error(), taskstate.ParkTransient)
|
||||
return "", false
|
||||
}
|
||||
// Every task gets an independent full clone.
|
||||
prepare := func() (string, string, error) {
|
||||
return trees.Prepare(ctx, task.Owner, task.Repo, repo.BaseBranch(), task.IssueNumber, task.Title, task.Body, task.Labels, target)
|
||||
}
|
||||
dir, branch, err := prepare()
|
||||
if d.reownDenied(ctx, trees.Dir(task.Owner, task.Repo, task.IssueNumber), err) {
|
||||
dir, branch, err = prepare()
|
||||
}
|
||||
if err != nil {
|
||||
reason := "worktree prepare failed: " + err.Error()
|
||||
if target != worktree.Fresh {
|
||||
reason = "worktree resume failed: " + err.Error()
|
||||
}
|
||||
d.Log.ErrorContext(ctx, "worktree prepare failed", "task", task.ID, "err", err)
|
||||
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
|
||||
return "", false
|
||||
}
|
||||
task.Branch = branch
|
||||
if err := d.Store.Update(ctx, task); err != nil {
|
||||
d.Log.WarnContext(ctx, "task branch not persisted", "task", task.ID, "err", err)
|
||||
}
|
||||
return dir, true
|
||||
}
|
||||
|
||||
// reclaimStaleClones frees the work volume before a clone waits on it, by
|
||||
// removing the oldest clones nothing has touched for cloneRetention. A clone
|
||||
// with uncommitted or unpushed work is kept.
|
||||
func (d *Daemon) reclaimStaleClones(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
|
||||
if have, err := trees.FreeBytes(); err != nil || have >= minCloneSpace {
|
||||
return
|
||||
}
|
||||
n, err := worktree.ReclaimStale(d.Cfg.Get().WorkDir, trees.Dir(task.Owner, task.Repo, task.IssueNumber), cloneRetention, trees.FreeBytes, minCloneSpace, func(dir string) bool { return trees.HoldsUnsharedWork(ctx, dir) })
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "stale clone reclaim failed", "task", task.ID, "err", err)
|
||||
}
|
||||
if n > 0 {
|
||||
d.Log.InfoContext(ctx, "reclaimed stale clones on a full work volume", "task", task.ID, "clones", n)
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupTerminalTaskWorktree removes a settled task's worktree, unless it
|
||||
// holds uncommitted work or its state cannot be read.
|
||||
func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
|
||||
if trees == nil || task == nil || !task.HasRepository() {
|
||||
return
|
||||
}
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
latest, err := d.Store.TaskByID(cleanupCtx, task.ID)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "terminal worktree cleanup task lookup failed", "task", task.ID, "err", err)
|
||||
return
|
||||
}
|
||||
// A run that handed off shares its work item's worktree with its
|
||||
// successor, which owns it from here.
|
||||
if latest == nil || latest.HandoffToTaskID != 0 {
|
||||
return
|
||||
}
|
||||
// Settled statuses. Keep the worktree while a PR exists.
|
||||
switch latest.Status {
|
||||
case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted, workflow.StatusDead:
|
||||
if latest.PRNumber != 0 {
|
||||
return
|
||||
}
|
||||
if d.worktreeHoldsUncapturedWork(cleanupCtx, trees, task) {
|
||||
return
|
||||
}
|
||||
if err := d.cleanupWorktree(cleanupCtx, trees, task.Owner, task.Repo, task.IssueNumber); err != nil {
|
||||
d.Log.WarnContext(ctx, "terminal worktree cleanup failed", "task", task.ID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// worktreeHoldsUncapturedWork reports whether a terminal task's worktree has
|
||||
// to survive cleanup because it may hold work no commit captured. A worktree
|
||||
// that is not on disk at all has nothing to keep, so a task whose clone was
|
||||
// already removed still cleans up whatever leftovers share its path.
|
||||
func (d *Daemon) worktreeHoldsUncapturedWork(ctx context.Context, trees *worktree.Manager, task *workflow.Task) bool {
|
||||
dir := trees.Dir(task.Owner, task.Repo, task.IssueNumber)
|
||||
if _, err := os.Stat(dir); err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return false
|
||||
}
|
||||
d.Log.WarnContext(ctx, "terminal worktree kept: it could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err)
|
||||
return true
|
||||
}
|
||||
uncommitted, err := trees.HasUncommittedChanges(ctx, dir)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "terminal worktree kept: uncommitted work could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err)
|
||||
return true
|
||||
}
|
||||
if uncommitted {
|
||||
d.Log.InfoContext(ctx, "terminal worktree kept: it holds uncommitted work; archive the task to remove it", "task", task.ID, "dir", dir)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// teardownStorage runs after the workflow completes. The Docker backend is
|
||||
// a no-op; future backends (temp volumes, NFS leases) use this hook.
|
||||
func (d *Daemon) teardownStorage(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string) {
|
||||
if d.Storage == nil {
|
||||
return
|
||||
}
|
||||
if err := d.Storage.Teardown(ctx, storage.TaskRef{
|
||||
WorktreeDir: workDir,
|
||||
Ecosystem: repo.Ecosystem,
|
||||
PersistentStorage: repo.PersistentStorage,
|
||||
Owner: task.Owner,
|
||||
Repo: task.Repo,
|
||||
}); err != nil {
|
||||
d.Log.WarnContext(ctx, "storage teardown failed", "task", task.ID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Daemon) cleanupExpiredStorage(ctx context.Context) error {
|
||||
ttl := d.Cfg.Get().Containers.VolumeTTL.Std()
|
||||
if ttl <= 0 || d.Storage == nil {
|
||||
return nil
|
||||
}
|
||||
n, err := d.Storage.CleanupExpired(ctx, ttl)
|
||||
if n > 0 {
|
||||
d.Log.InfoContext(ctx, "expired persistent volumes removed", "count", n)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("persistent volume cleanup: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgeFor returns the forge client that owns task: its organisation
|
||||
// connection, else the root d.Forge for the root alone. This is the safety
|
||||
// boundary that keeps one identity's forge token from being used against
|
||||
// another identity's repos.
|
||||
func (d *Daemon) forgeFor(ctx context.Context, task *workflow.Task) forge.Forge {
|
||||
if task.HasRepository() {
|
||||
resolved, err := d.Repository(ctx, task, config.OperationClone)
|
||||
if err != nil {
|
||||
return forge.NewNoop(d.Log)
|
||||
}
|
||||
return resolved.Forge
|
||||
}
|
||||
if !d.rootTask(task) {
|
||||
return forge.NewNoop(d.Log)
|
||||
}
|
||||
return d.Forge
|
||||
}
|
||||
|
||||
// treesFor returns the worktree manager that owns task, mirroring forgeFor.
|
||||
func (d *Daemon) treesFor(task *workflow.Task) *worktree.Manager {
|
||||
if task.HasRepository() {
|
||||
// Cleanup needs only the org's path, not a still-valid external credential.
|
||||
return &worktree.Manager{WorkDir: worktree.OrgWorkDir(d.Cfg.Get().WorkDir, task.OrgOrDefault())}
|
||||
}
|
||||
if !d.rootTask(task) {
|
||||
return &worktree.Manager{WorkDir: filepath.Join(d.Cfg.Get().WorkDir, "identity-"+string(identity.StableID(task.Identity)))}
|
||||
}
|
||||
return d.Trees
|
||||
}
|
||||
|
||||
// repoFor resolves a root task's repo config from Cfg.Repos.
|
||||
func (d *Daemon) repoFor(t *workflow.Task) (config.Repo, bool) {
|
||||
if !d.rootTask(t) {
|
||||
return config.Repo{}, false
|
||||
}
|
||||
for _, r := range d.Cfg.Get().Repos {
|
||||
if r.Owner == t.Owner && r.Name == t.Repo {
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
return config.Repo{}, false
|
||||
}
|
||||
|
||||
func (d *Daemon) rootTask(task *workflow.Task) bool {
|
||||
return identity.IdentityID(task.Identity) == identity.SystemID
|
||||
}
|
||||
|
||||
func (d *Daemon) configFor(task *workflow.Task) config.Config {
|
||||
cfg := d.Cfg.Get()
|
||||
if !d.rootTask(task) {
|
||||
cfg.BotUser, cfg.BotEmail, cfg.Forge = "", "", config.Forge{}
|
||||
cfg.Org, cfg.GrantedCredentials, cfg.Identities, cfg.IdentityGrants = task.OrgOrDefault(), nil, nil, nil
|
||||
cfg.Repos, cfg.Models, cfg.Providers = nil, nil, nil
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// allowConcurrentForTask reports whether the task's repo allows concurrent
|
||||
// tasks. Unknown repos do not.
|
||||
func (d *Daemon) allowConcurrentForTask(task *workflow.Task) bool {
|
||||
if !task.HasRepository() {
|
||||
return true
|
||||
}
|
||||
repo, ok := d.repoFor(task)
|
||||
return ok && repo.AllowConcurrent
|
||||
}
|
||||
|
||||
// Repository resolves current credentials and permissions for the run's
|
||||
// execution identity, never the identity that originally requested the task.
|
||||
func (d *Daemon) Repository(ctx context.Context, task *workflow.Task, operation string) (RepositoryAccess, error) {
|
||||
run := *task
|
||||
if run.WorkflowDefinitionYAML != "" {
|
||||
iface, err := workflowtask.ParseWorkflowInterface(run.WorkflowDefinitionYAML)
|
||||
if err != nil || iface.Identity == "" {
|
||||
return RepositoryAccess{}, fmt.Errorf("workflow execution identity unavailable")
|
||||
}
|
||||
_, name, err := d.declaredIdentity(ctx, &run, iface.Identity)
|
||||
if err != nil {
|
||||
return RepositoryAccess{}, err
|
||||
}
|
||||
run.Identity = name
|
||||
}
|
||||
ctx, err := d.workflowContext(ctx, &run)
|
||||
if err != nil {
|
||||
return RepositoryAccess{}, err
|
||||
}
|
||||
if _, err := d.databaseExecutionIdentity(ctx, &run, identity.IdentityID(run.Identity)); err != nil {
|
||||
return RepositoryAccess{}, err
|
||||
}
|
||||
return d.ResolveRepository(ctx, &run, operation)
|
||||
}
|
||||
|
||||
// botUserForTask resolves the bot login that owns task, so the reaction
|
||||
// consumer can exclude Archie's own comments: the global forge bot user for
|
||||
// root tasks.
|
||||
func (d *Daemon) botUserForTask(task *workflowtask.Task) string {
|
||||
if !d.rootTask(task) {
|
||||
return ""
|
||||
}
|
||||
return d.Cfg.Get().BotUser
|
||||
}
|
||||
|
||||
// RemoveWorktree deletes a task's clone from the trees of the identity that
|
||||
// owns it. Archiving calls it: terminal cleanup keeps a worktree that may hold
|
||||
// uncaptured work, and the operator's archive is the decision to discard it.
|
||||
func (d *Daemon) RemoveWorktree(task *workflow.Task) error {
|
||||
trees := d.treesFor(task)
|
||||
if trees == nil {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
return d.cleanupWorktree(ctx, trees, task.Owner, task.Repo, task.IssueNumber)
|
||||
}
|
||||
|
||||
// cleanupWorktree removes a task's clone, first handing back any root-owned
|
||||
// files a killed container left in it.
|
||||
func (d *Daemon) cleanupWorktree(ctx context.Context, trees *worktree.Manager, owner, repo string, issue int) error {
|
||||
err := trees.Cleanup(owner, repo, issue)
|
||||
if d.reownDenied(ctx, trees.Dir(owner, repo, issue), err) {
|
||||
return trees.Cleanup(owner, repo, issue)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// reownDenied repairs dir when err says the daemon was refused access to it,
|
||||
// and reports whether the caller should retry. A SIGKILLed task container
|
||||
// leaves root-owned files that only a container can chown back.
|
||||
func (d *Daemon) reownDenied(ctx context.Context, dir string, err error) bool {
|
||||
if err == nil || !errors.Is(err, fs.ErrPermission) || d.ContainerPool == nil {
|
||||
return false
|
||||
}
|
||||
if rerr := d.ContainerPool.Reown(ctx, dir, os.Getuid(), os.Getgid()); rerr != nil {
|
||||
d.Log.WarnContext(ctx, "worktree ownership repair failed", "dir", dir, "err", rerr)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// A clone waits, holding its slot, until the work volume has minCloneSpace
|
||||
// free, and parks as transient if it does not within diskWaitLimit. Clones
|
||||
// share objects with the repository's mirror, so the floor covers a checkout
|
||||
// and its build output, not a full clone.
|
||||
const (
|
||||
minCloneSpace = 1 << 30
|
||||
diskCheckEvery = 15 * time.Second
|
||||
diskWaitLimit = 10 * time.Minute
|
||||
// cloneRetention is how long an untouched clone, such as a parked task's,
|
||||
// is kept before a full work volume may reclaim it.
|
||||
cloneRetention = 72 * time.Hour
|
||||
)
|
||||
|
||||
// withContainerExit derives a context cancelled when exited closes. A dead
|
||||
// container never answers its taskrun request, so without this the request
|
||||
// blocks forever and the task row stays running.
|
||||
func withContainerExit(ctx context.Context, ctr exitWatcher) (context.Context, func()) {
|
||||
runCtx, cancel := context.WithCancelCause(ctx)
|
||||
go func() {
|
||||
select {
|
||||
case <-ctr.Exited():
|
||||
if detail := ctr.ExitDetail(); detail != "" {
|
||||
cancel(fmt.Errorf("%w: %s", errContainerExited, detail))
|
||||
return
|
||||
}
|
||||
cancel(errContainerExited)
|
||||
case <-runCtx.Done():
|
||||
}
|
||||
}()
|
||||
return runCtx, func() { cancel(nil) }
|
||||
}
|
||||
|
||||
// exitWatcher is a running agent container, seen only for how it ends.
|
||||
type exitWatcher interface {
|
||||
Exited() <-chan struct{}
|
||||
ExitDetail() string
|
||||
}
|
||||
|
||||
// errContainerExited is the cancellation cause when a task's agent container
|
||||
// stops before answering its taskrun request.
|
||||
var errContainerExited = errors.New("agent container exited before the task finished")
|
||||
|
||||
// withTaskTimeLimit bounds a whole task run. A limit of zero or less applies
|
||||
// no bound.
|
||||
func withTaskTimeLimit(ctx context.Context, limit time.Duration) (context.Context, func()) {
|
||||
if limit <= 0 {
|
||||
return ctx, func() {}
|
||||
}
|
||||
return context.WithTimeoutCause(ctx, limit, fmt.Errorf("%w (%s)", errTaskTimeLimit, limit))
|
||||
}
|
||||
|
||||
// errTaskTimeLimit is the cancellation cause when a task outruns its overall
|
||||
// time limit (the execution settings' max task runtime).
|
||||
var errTaskTimeLimit = errors.New("task exceeded its time limit")
|
||||
@@ -54,7 +54,7 @@ func TestPolledTaskKeepsItsCreatingIdentity(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
bus := &publishedKeys{}
|
||||
d := &Daemon{
|
||||
d := wired(&Daemon{
|
||||
Log: slog.New(slog.DiscardHandler), Store: taskLookup{}, Tasks: bus,
|
||||
Cfg: config.NewHolder(config.Config{}),
|
||||
IssueSources: func(context.Context) []IssueSource {
|
||||
@@ -65,7 +65,7 @@ func TestPolledTaskKeepsItsCreatingIdentity(t *testing.T) {
|
||||
Repos: []config.Repo{{Owner: "acme", Name: "widget"}},
|
||||
}}
|
||||
},
|
||||
}
|
||||
})
|
||||
if err := d.poll(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -1,13 +1,7 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
"github.com/samcharles93/archie-core/internal/worktree"
|
||||
)
|
||||
@@ -20,31 +14,3 @@ type RepositoryAccess struct {
|
||||
Trees *worktree.Manager
|
||||
Forge forge.Forge
|
||||
}
|
||||
|
||||
// Repository resolves current credentials and permissions for the run's
|
||||
// execution identity, never the identity that originally requested the task.
|
||||
func (d *Daemon) Repository(ctx context.Context, task *workflow.Task, operation string) (RepositoryAccess, error) {
|
||||
if d.ResolveRepository == nil {
|
||||
return RepositoryAccess{}, fmt.Errorf("repository connections unavailable")
|
||||
}
|
||||
run := *task
|
||||
if run.WorkflowDefinitionYAML != "" {
|
||||
iface, err := workflowtask.ParseWorkflowInterface(run.WorkflowDefinitionYAML)
|
||||
if err != nil || iface.Identity == "" {
|
||||
return RepositoryAccess{}, fmt.Errorf("workflow execution identity unavailable")
|
||||
}
|
||||
_, name, err := d.declaredIdentity(ctx, &run, iface.Identity)
|
||||
if err != nil {
|
||||
return RepositoryAccess{}, err
|
||||
}
|
||||
run.Identity = name
|
||||
}
|
||||
ctx, err := d.workflowContext(ctx, &run)
|
||||
if err != nil {
|
||||
return RepositoryAccess{}, err
|
||||
}
|
||||
if _, err := d.databaseExecutionIdentity(ctx, &run, identity.IdentityID(run.Identity)); err != nil {
|
||||
return RepositoryAccess{}, err
|
||||
}
|
||||
return d.ResolveRepository(ctx, &run, operation)
|
||||
}
|
||||
@@ -5,9 +5,12 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workintake"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
)
|
||||
|
||||
@@ -122,17 +125,11 @@ func scanPRReviewReactions(
|
||||
// scanPRReviews scans every open PR task for review activity with its
|
||||
// identity's forge. Forges without review reading are skipped.
|
||||
func (d *Daemon) scanPRReviews(ctx context.Context) error {
|
||||
if d.Store == nil || d.Tasks == nil {
|
||||
return nil
|
||||
}
|
||||
tasks, err := d.Store.OpenPRs(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("review scan query: %w", err)
|
||||
}
|
||||
var failed []string
|
||||
if d.reactionPublisher == nil {
|
||||
d.reactionPublisher = d.PublishReaction
|
||||
}
|
||||
for i := range tasks {
|
||||
t := &tasks[i]
|
||||
reader, ok := d.forgeFor(ctx, t).(forge.PullRequestReviewReader)
|
||||
@@ -184,3 +181,112 @@ func (d *Daemon) scanOnePR(ctx context.Context, reader forge.PullRequestReviewRe
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// drainReactions turns available review reactions into queued remediate
|
||||
// runs. It runs before the task drain so a reaction queued in this pass can
|
||||
// be claimed by it. The consumer is built by Startup, so its drop counter
|
||||
// spans the process lifetime.
|
||||
func (d *Daemon) drainReactions(ctx context.Context) {
|
||||
if _, err := d.reactionConsumer.drain(ctx, d.Reactions, maxReactionsPerCycle); err != nil {
|
||||
d.Log.ErrorContext(ctx, "reaction drain failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// PublishReaction stamps the org on a review reaction, defaulting to the
|
||||
// root identity's, and publishes it.
|
||||
func (d *Daemon) PublishReaction(ctx context.Context, reaction workintake.ReviewCommentEnvelope) error {
|
||||
if reaction.Org == "" {
|
||||
resolved, err := d.identityOrg(ctx, identity.SystemID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("publish reaction %s: resolve org: %w", reaction.Ref(), err)
|
||||
}
|
||||
reaction.Org = resolved
|
||||
}
|
||||
payload, err := reaction.Encode()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return d.Tasks.PublishUnique(ctx, reaction.Subject(), reaction.IdempotencyKey(), payload)
|
||||
}
|
||||
|
||||
// reconcilePRs moves pr_open tasks to merged/rejected from GitHub state
|
||||
// and cleans up their worktrees.
|
||||
func (d *Daemon) reconcilePRs(ctx context.Context) error {
|
||||
tasks, err := d.Store.OpenPRs(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reconcile query: %w", err)
|
||||
}
|
||||
var failed []string
|
||||
for _, t := range tasks {
|
||||
fg := d.forgeFor(ctx, &t)
|
||||
if _, unavailable := fg.(*forge.NoopForge); unavailable {
|
||||
continue
|
||||
}
|
||||
trees := d.treesFor(&t)
|
||||
// PRState/PR reconciliation is valid regardless of Source -- a
|
||||
// chat-sourced task can still open a real PR against a real
|
||||
// branch. Only the issue-label call (tied to the synthetic
|
||||
// issue number) is forge-only.
|
||||
state, err := fg.PRState(ctx, t.Owner, t.Repo, t.PRNumber)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "PR state check failed", "pr", t.PRNumber, "err", err)
|
||||
failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber))
|
||||
continue
|
||||
}
|
||||
attempt := t.Attempt
|
||||
switch state {
|
||||
case "merged":
|
||||
if err := d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusMerged, ""); err != nil {
|
||||
d.Log.ErrorContext(ctx, "PR merged but task transition failed", "task", t.ID, "pr", t.PRNumber, "err", err)
|
||||
failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber))
|
||||
continue
|
||||
}
|
||||
if err := trees.Cleanup(t.Owner, t.Repo, t.IssueNumber); err != nil {
|
||||
d.Log.WarnContext(ctx, "worktree cleanup failed", "task", t.ID, "err", err)
|
||||
}
|
||||
// Close the issue explicitly; nothing else does.
|
||||
d.closeResolvedIssue(ctx, &t, fmt.Sprintf(
|
||||
"Closed by #%d, merged by archie.", t.PRNumber,
|
||||
))
|
||||
d.Log.InfoContext(ctx, "PR merged", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber)
|
||||
d.emit(ctx, events.Event{
|
||||
Kind: events.KindPRMerged, TaskID: t.ID, Attempt: attempt,
|
||||
Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber,
|
||||
Data: map[string]any{"pr": t.PRNumber},
|
||||
})
|
||||
case "closed":
|
||||
if err := d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusRejected, "PR closed without merge"); err != nil {
|
||||
d.Log.ErrorContext(ctx, "PR rejected but task transition failed", "task", t.ID, "pr", t.PRNumber, "err", err)
|
||||
failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber))
|
||||
continue
|
||||
}
|
||||
if err := trees.Cleanup(t.Owner, t.Repo, t.IssueNumber); err != nil {
|
||||
d.Log.WarnContext(ctx, "worktree cleanup failed", "task", t.ID, "err", err)
|
||||
}
|
||||
d.Log.InfoContext(ctx, "PR rejected", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber)
|
||||
d.emit(ctx, events.Event{
|
||||
Kind: events.KindPRRejected, TaskID: t.ID, Attempt: attempt,
|
||||
Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber,
|
||||
Data: map[string]any{"pr": t.PRNumber},
|
||||
})
|
||||
}
|
||||
}
|
||||
return passFailed("PR reconcile", failed)
|
||||
}
|
||||
|
||||
// closeResolvedIssue closes a finished task's forge issue, skipping chat
|
||||
// tasks. Failures are logged.
|
||||
func (d *Daemon) closeResolvedIssue(ctx context.Context, task *workflow.Task, comment string) {
|
||||
if !task.IsForgeBacked() {
|
||||
return
|
||||
}
|
||||
resolved, err := d.Repository(ctx, task, config.OperationPR)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "issue closure not permitted", "task", task.ID)
|
||||
return
|
||||
}
|
||||
if err := resolved.Forge.CloseIssue(ctx, task.Owner, task.Repo, task.IssueNumber, comment); err != nil {
|
||||
d.Log.WarnContext(ctx, "closing resolved issue failed; it stays open and will be re-polled",
|
||||
"repo", task.Owner+"/"+task.Repo, "issue", task.IssueNumber, "err", err)
|
||||
}
|
||||
}
|
||||
@@ -85,10 +85,10 @@ func TestContainerEnvStateStore(t *testing.T) {
|
||||
}
|
||||
t.Cleanup(relay.Close)
|
||||
|
||||
d := &Daemon{
|
||||
d := wired(&Daemon{
|
||||
ConnectedNATS: NATSEndpoint{URL: "nats://10.0.0.5:4222"},
|
||||
ConnectedStateStore: StateStoreEndpoint{URL: upstream.Addr().String(), CA: []byte("ca-pem")},
|
||||
}
|
||||
})
|
||||
env := envValues(d.containerEnv(&workflow.Task{ID: 7}, "task-token", relay.Addr()))
|
||||
|
||||
if got := env["STATE_STORE_URL"]; got != relay.Addr() {
|
||||
|
||||
@@ -0,0 +1,482 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workintake"
|
||||
"github.com/samcharles93/archie-core/internal/eventbus"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/natsrpc"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// drainLoop is the only task drainer and capture dispatcher while Run is
|
||||
// active. It runs apart from the poll tick so a captured event becomes a
|
||||
// running task within about a second regardless of poll_interval or forge
|
||||
// latency. Submit never blocks, so a full dispatcher
|
||||
// just leaves tasks waiting on slots and the loop stays paced by the ticker.
|
||||
func (d *Daemon) drainLoop(ctx context.Context) {
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
d.markPass("drain")
|
||||
d.dispatchBindings(ctx)
|
||||
d.drainNATS(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// drainNATS processes tasks from NATS, falling back to the task store's ClaimNext for
|
||||
// requeued tasks (waiting_approval approval, retry-parked) that didn't come
|
||||
// through a NATS publish. It returns once the stream and the claim fallback
|
||||
// are both empty: in-flight tasks keep running, so a task waiting on a callee
|
||||
// does not hold up the pass that would claim it. Shutdown waits for them
|
||||
// through WaitForTasks.
|
||||
func (d *Daemon) drainNATS(ctx context.Context) {
|
||||
dispatcher := d.taskDispatcher()
|
||||
// A full dispatcher leaves work in the stream and the store, not parked
|
||||
// on slots where it already reads as running.
|
||||
for ctx.Err() == nil && dispatcher.hasRoom() {
|
||||
msg, err := d.Tasks.Fetch(ctx)
|
||||
if err != nil && !errors.Is(err, eventbus.ErrNoMessage) {
|
||||
d.Log.ErrorContext(ctx, "nats fetch failed", "err", err)
|
||||
break
|
||||
}
|
||||
if err == nil {
|
||||
d.submitNATSTask(ctx, dispatcher, msg)
|
||||
d.markPass("drain")
|
||||
continue
|
||||
}
|
||||
|
||||
// ErrNoMessage means the stream is drained. That is precisely when
|
||||
// the SQLite fallback must run: requeued tasks (waiting_approval
|
||||
// approval, retry-parked) never went through a NATS publish, so
|
||||
// nothing will ever arrive on the stream to trigger them.
|
||||
d.claimRequeued(ctx, dispatcher)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// claimRequeued submits tasks from the store's claim fallback while the
|
||||
// dispatcher has room; the rest stay unclaimed for a later pass. It does
|
||||
// not go back to the stream between claims: each empty fetch waits a full poll
|
||||
// timeout, so a backlog left by a restart would outlast the systemd watchdog.
|
||||
func (d *Daemon) claimRequeued(ctx context.Context, dispatcher *taskDispatcher) {
|
||||
for ctx.Err() == nil && dispatcher.hasRoom() {
|
||||
task, err := d.Store.ClaimNext(ctx)
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "sqlite claim failed", "err", err)
|
||||
return
|
||||
}
|
||||
if task == nil {
|
||||
return
|
||||
}
|
||||
dispatcher.Submit(ctx, task, d.process)
|
||||
d.markPass("drain")
|
||||
}
|
||||
}
|
||||
|
||||
// taskDispatcher returns the daemon's long-lived global dispatcher, building
|
||||
// it from the running config on first use. It outlives a single drain pass so
|
||||
// ResizeTaskDispatcher can resize the running dispatcher.
|
||||
func (d *Daemon) taskDispatcher() *taskDispatcher {
|
||||
d.dispatcherOnce.Do(func() {
|
||||
d.dispatcher = newTaskDispatcher(d.Cfg.Get().Containers.MaxConcurrency, d.allowConcurrentForTask)
|
||||
})
|
||||
return d.dispatcher
|
||||
}
|
||||
|
||||
// WaitForTasks blocks until every submitted task's process has returned. A
|
||||
// drain pass no longer waits for its own tasks, so shutdown is where in-flight
|
||||
// work is waited for.
|
||||
func (d *Daemon) WaitForTasks() {
|
||||
d.taskDispatcher().Wait()
|
||||
}
|
||||
|
||||
// ResizeTaskDispatcher applies a new containers.max_concurrency to the
|
||||
// running global task dispatcher without a restart. Running tasks keep their
|
||||
// slots; a lowered limit is honoured as they finish, and a raised limit
|
||||
// admits queued work immediately.
|
||||
func (d *Daemon) ResizeTaskDispatcher(maxConcurrency int) {
|
||||
d.taskDispatcher().SetMaxConcurrency(maxConcurrency)
|
||||
}
|
||||
|
||||
// submitNATSTask decodes a fetched message and queues it for processing. A
|
||||
// message that cannot be decoded is acked rather than redelivered: it will
|
||||
// never decode on a retry, so leaving it unacked would block the queue.
|
||||
func (d *Daemon) submitNATSTask(ctx context.Context, dispatcher *taskDispatcher, msg eventbus.Message) {
|
||||
envelope, err := workintake.DecodeTask(msg.Data())
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "nats decode failed", "subject", msg.Subject(), "err", err)
|
||||
if err := msg.Ack(); err != nil {
|
||||
d.Log.WarnContext(ctx, "nats ack failed", "subject", msg.Subject(), "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
task := &workflow.Task{Owner: envelope.Owner, Repo: envelope.Repo, Identity: envelope.Identity}
|
||||
dispatcher.Submit(ctx, task, func(ctx context.Context, _ *workflow.Task, hold *taskHold) {
|
||||
d.processNATSTask(ctx, msg, hold)
|
||||
})
|
||||
}
|
||||
|
||||
// processNATSTask decodes a NATS message, writes it to SQLite, claims it,
|
||||
// and runs the workflow. The message is ack'd on terminal (park is a valid
|
||||
// outcome). Nak on transient errors so NATS redelivers.
|
||||
func (d *Daemon) processNATSTask(ctx context.Context, msg eventbus.Message, hold *taskHold) {
|
||||
ctx, span := natsrpc.StartConsumer(ctx, msg.Subject(), msg.Header())
|
||||
defer span.End()
|
||||
tm, err := workintake.DecodeTask(msg.Data())
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "nats decode failed", "err", err)
|
||||
if err := msg.Ack(); err != nil {
|
||||
d.Log.WarnContext(ctx, "ack failed", "err", err)
|
||||
}
|
||||
// bad message, don't retry
|
||||
return
|
||||
}
|
||||
|
||||
if !d.acceptTaskPublisher(ctx, msg, tm.Identity) {
|
||||
return
|
||||
}
|
||||
|
||||
inserted, err := d.Store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity)
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "nats enqueue failed", "err", err)
|
||||
if err := msg.Nak(); err != nil {
|
||||
d.Log.WarnContext(ctx, "nats nak failed", "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !inserted {
|
||||
// Already tracked -- dedup. Ack and move on.
|
||||
if err := msg.Ack(); err != nil {
|
||||
d.Log.WarnContext(ctx, "ack failed", "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
task, err := d.Store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number)
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "nats claim failed", "err", err)
|
||||
if err := msg.Nak(); err != nil {
|
||||
d.Log.WarnContext(ctx, "nats nak failed", "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if task == nil {
|
||||
// Claimed by another consumer, or task is not queued. Ack.
|
||||
if err := msg.Ack(); err != nil {
|
||||
d.Log.WarnContext(ctx, "ack failed", "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
d.emitTaskQueued(ctx, task)
|
||||
d.process(ctx, task, hold)
|
||||
if err := msg.Ack(); err != nil {
|
||||
d.Log.WarnContext(ctx, "ack failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Daemon) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, name string) bool {
|
||||
if ok, reason, class := d.identityMayAct(ctx, name); !ok {
|
||||
d.Log.ErrorContext(ctx, "nats intake rejected: "+reason, "subject", msg.Subject())
|
||||
if class == taskstate.ParkTransient {
|
||||
if err := msg.Nak(); err != nil {
|
||||
d.Log.WarnContext(ctx, "nak failed", "err", err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
if err := msg.Ack(); err != nil {
|
||||
d.Log.WarnContext(ctx, "ack failed", "err", err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// emitTaskQueued records the queued task under the identity that polled it,
|
||||
// which is both the actor and the authority the task was created with.
|
||||
func (d *Daemon) emitTaskQueued(ctx context.Context, task *workflow.Task) {
|
||||
e := events.Event{
|
||||
Kind: events.KindTaskQueued, TaskID: task.ID, Repo: task.Owner + "/" + task.Repo,
|
||||
Issue: task.IssueNumber, Detail: task.Title, Attempt: task.Attempt,
|
||||
ActorID: task.Identity, PrincipalID: task.Identity,
|
||||
}
|
||||
if value, err := d.IdentityRepository.Get(ctx, identity.IdentityID(task.Identity)); err == nil {
|
||||
e.ActorKind = string(value.Kind)
|
||||
}
|
||||
d.emit(ctx, e)
|
||||
}
|
||||
|
||||
// PublishTask validates a task envelope, stamps the producing identity's
|
||||
// org and publishes it. The idempotency key dedups rediscovered issues.
|
||||
func (d *Daemon) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error {
|
||||
if err := task.Kind.Validate(); err != nil {
|
||||
return fmt.Errorf("publish task %s: %w", task.Ref(), err)
|
||||
}
|
||||
if task.Identity == "" {
|
||||
return fmt.Errorf("publish task %s: no identity", task.Ref())
|
||||
}
|
||||
resolved, err := d.identityOrg(ctx, identity.IdentityID(task.Identity))
|
||||
if err != nil {
|
||||
return fmt.Errorf("publish task %s: resolve org: %w", task.Ref(), err)
|
||||
}
|
||||
task.Org = resolved
|
||||
payload, err := task.Encode()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return d.Tasks.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload)
|
||||
}
|
||||
|
||||
// InFlight counts the same submitted work that graceful shutdown waits for,
|
||||
// including tasks waiting for a repository or concurrency slot.
|
||||
func (d *Daemon) InFlight() int {
|
||||
dispatcher := d.taskDispatcher()
|
||||
dispatcher.mu.Lock()
|
||||
defer dispatcher.mu.Unlock()
|
||||
return dispatcher.pending
|
||||
}
|
||||
|
||||
// taskDispatcher limits concurrent tasks globally and runs one task per repo
|
||||
// at a time, in submit order, unless the repo allows concurrency.
|
||||
type taskDispatcher struct {
|
||||
allowConcurrent func(task *workflow.Task) bool
|
||||
|
||||
mu sync.Mutex
|
||||
limit int
|
||||
active int
|
||||
slotsFree *sync.Cond
|
||||
repoTail map[string]chan struct{}
|
||||
pending int
|
||||
// waiting counts submitted tasks that hold no slot yet, so a drain pass
|
||||
// stops claiming once queued work alone fills the limit.
|
||||
waiting int
|
||||
// holds maps a task id to the capacity its run holds, so a run waiting on
|
||||
// a callee can give that capacity back and take it again by task id.
|
||||
holds map[int64]*taskHold
|
||||
}
|
||||
|
||||
// SetMaxConcurrency accepts a new global limit without a restart. Running
|
||||
// tasks keep their slots; a lowered limit is enforced as they finish, and a
|
||||
// raised limit admits queued work immediately.
|
||||
func (d *taskDispatcher) SetMaxConcurrency(maxConcurrency int) {
|
||||
d.mu.Lock()
|
||||
d.limit = maxConcurrency
|
||||
d.mu.Unlock()
|
||||
d.slotsFree.Broadcast()
|
||||
}
|
||||
|
||||
func newTaskDispatcher(maxConcurrency int, allowConcurrent func(task *workflow.Task) bool) *taskDispatcher {
|
||||
if allowConcurrent == nil {
|
||||
allowConcurrent = func(*workflow.Task) bool { return false }
|
||||
}
|
||||
d := &taskDispatcher{
|
||||
limit: maxConcurrency,
|
||||
allowConcurrent: allowConcurrent,
|
||||
repoTail: make(map[string]chan struct{}),
|
||||
holds: make(map[int64]*taskHold),
|
||||
}
|
||||
d.slotsFree = sync.NewCond(&d.mu)
|
||||
return d
|
||||
}
|
||||
|
||||
// joinRepoLocked appends baton to the tail of repo's chain and returns the
|
||||
// baton ahead of it, which the joining task is blocked on. Callers hold d.mu.
|
||||
func (d *taskDispatcher) joinRepoLocked(repo string, baton chan struct{}) chan struct{} {
|
||||
previous := d.repoTail[repo]
|
||||
d.repoTail[repo] = baton
|
||||
return previous
|
||||
}
|
||||
|
||||
// takeSlotLocked blocks until the global limit has room, then counts the run
|
||||
// in. limit <= 0 means unlimited. The limit is re-read on every wake, so
|
||||
// SetMaxConcurrency reaches waiting tasks without recreating the dispatcher.
|
||||
// Callers hold d.mu.
|
||||
func (d *taskDispatcher) takeSlotLocked() {
|
||||
for d.limit > 0 && d.active >= d.limit {
|
||||
d.slotsFree.Wait()
|
||||
}
|
||||
d.active++
|
||||
}
|
||||
|
||||
// giveBackLocked gives back everything the run holds: its slot under the
|
||||
// global limit, and its place in its repo's chain, which releases the task
|
||||
// queued behind it. Callers hold d.mu.
|
||||
func (d *taskDispatcher) giveBackLocked(hold *taskHold) {
|
||||
if hold.slot {
|
||||
d.active--
|
||||
hold.slot = false
|
||||
}
|
||||
if hold.baton == nil {
|
||||
return
|
||||
}
|
||||
close(hold.baton)
|
||||
if d.repoTail[hold.repo] == hold.baton {
|
||||
delete(d.repoTail, hold.repo)
|
||||
}
|
||||
hold.baton = nil
|
||||
}
|
||||
|
||||
// finishRun retires a returned run: it holds no capacity any more, and no
|
||||
// Resume may take any for it. Whatever an in-flight resume takes later is
|
||||
// given back by reacquire, which sees finished.
|
||||
func (d *taskDispatcher) finishRun(hold *taskHold) {
|
||||
d.mu.Lock()
|
||||
hold.finished = true
|
||||
if hold.id != 0 && d.holds[hold.id] == hold {
|
||||
delete(d.holds, hold.id)
|
||||
}
|
||||
d.giveBackLocked(hold)
|
||||
d.pending--
|
||||
d.mu.Unlock()
|
||||
d.slotsFree.Broadcast()
|
||||
}
|
||||
|
||||
// setCallWaits applies a durable snapshot without accumulating duplicate events.
|
||||
func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) {
|
||||
d.mu.Lock()
|
||||
if d.holds[hold.id] != hold || hold.finished {
|
||||
d.mu.Unlock()
|
||||
return
|
||||
}
|
||||
previous := hold.waits
|
||||
hold.waits = waits
|
||||
if waits > 0 {
|
||||
d.giveBackLocked(hold)
|
||||
}
|
||||
d.mu.Unlock()
|
||||
if waits > 0 {
|
||||
d.slotsFree.Broadcast()
|
||||
} else if previous > 0 {
|
||||
// Reacquiring may block behind another task; the caller keeps running.
|
||||
go d.reacquire(hold)
|
||||
}
|
||||
}
|
||||
|
||||
// reacquire returns a resumed run's capacity: its place in its repo's chain
|
||||
// and a slot under the global limit. The run keeps executing meanwhile -- a
|
||||
// wait on a callee is not a pause. A hold records one slot and one place, so a
|
||||
// re-acquire queued while another is still waiting gives back whichever of the
|
||||
// two it finds already recorded instead of taking a second.
|
||||
func (d *taskDispatcher) reacquire(hold *taskHold) {
|
||||
d.mu.Lock()
|
||||
if hold.finished || hold.waits > 0 {
|
||||
// The run returned, or started waiting again, after the reconciliation that
|
||||
// queued this: it needs no capacity.
|
||||
d.mu.Unlock()
|
||||
return
|
||||
}
|
||||
var previous chan struct{}
|
||||
// A place another re-acquire already queued is the run's one place; joining
|
||||
// behind it would queue a second, which the task behind it would wait on
|
||||
// forever.
|
||||
if hold.chain && hold.baton == nil {
|
||||
hold.baton = make(chan struct{})
|
||||
previous = d.joinRepoLocked(hold.repo, hold.baton)
|
||||
}
|
||||
d.mu.Unlock()
|
||||
|
||||
if previous != nil {
|
||||
<-previous
|
||||
}
|
||||
|
||||
d.mu.Lock()
|
||||
d.takeSlotLocked()
|
||||
if hold.slot {
|
||||
// Another re-acquire took the run's one slot while this waited: give
|
||||
// back the slot this one just took. The place is the other's, so it is
|
||||
// left alone.
|
||||
d.active--
|
||||
d.mu.Unlock()
|
||||
d.slotsFree.Broadcast()
|
||||
return
|
||||
}
|
||||
hold.slot = true
|
||||
if hold.finished || hold.waits > 0 {
|
||||
// The run ended, or started waiting again, while this was queued for
|
||||
// capacity: give back what it took, so no slot or place in the repo
|
||||
// chain leaks.
|
||||
d.giveBackLocked(hold)
|
||||
d.mu.Unlock()
|
||||
d.slotsFree.Broadcast()
|
||||
return
|
||||
}
|
||||
d.mu.Unlock()
|
||||
}
|
||||
|
||||
// Submit queues a task's process behind the global limit and, unless the repo
|
||||
// allows concurrency, behind the task already holding that repo. process is
|
||||
// handed the run's capacity so it can name it once it knows the task's id and
|
||||
// give it back while it waits on a callee.
|
||||
func (d *taskDispatcher) Submit(
|
||||
ctx context.Context,
|
||||
task *workflow.Task,
|
||||
process func(context.Context, *workflow.Task, *taskHold),
|
||||
) {
|
||||
repo := task.Owner + "/" + task.Repo
|
||||
hold := &taskHold{dispatcher: d, repo: repo}
|
||||
|
||||
var previous chan struct{}
|
||||
if !d.allowConcurrent(task) {
|
||||
hold.chain = true
|
||||
hold.baton = make(chan struct{})
|
||||
d.mu.Lock()
|
||||
previous = d.joinRepoLocked(repo, hold.baton)
|
||||
d.mu.Unlock()
|
||||
}
|
||||
|
||||
d.mu.Lock()
|
||||
d.pending++
|
||||
d.waiting++
|
||||
d.mu.Unlock()
|
||||
go func() {
|
||||
defer d.finishRun(hold)
|
||||
if previous != nil {
|
||||
<-previous
|
||||
}
|
||||
d.mu.Lock()
|
||||
d.takeSlotLocked()
|
||||
d.waiting--
|
||||
hold.slot = true
|
||||
d.mu.Unlock()
|
||||
process(ctx, task, hold)
|
||||
}()
|
||||
}
|
||||
|
||||
// hasRoom reports whether a newly submitted task could take a slot without
|
||||
// queueing behind work already submitted. A run waiting on a callee holds no
|
||||
// slot, so its callee still finds room.
|
||||
func (d *taskDispatcher) hasRoom() bool {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.limit <= 0 || d.active+d.waiting < d.limit
|
||||
}
|
||||
|
||||
// inFlight is how many submitted tasks have not returned yet.
|
||||
func (d *taskDispatcher) inFlight() int {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.pending
|
||||
}
|
||||
|
||||
func (d *taskDispatcher) Wait() {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
for d.pending > 0 {
|
||||
d.slotsFree.Wait()
|
||||
}
|
||||
}
|
||||
@@ -39,7 +39,7 @@ func TestRetryTransientParks(t *testing.T) {
|
||||
{ID: 3, ParkClass: taskstate.ParkNeedsDecision, UpdatedAt: old},
|
||||
{ID: 4, ParkClass: taskstate.ParkTransient, RetryCount: taskstate.MaxTransientRetries, UpdatedAt: old},
|
||||
}}
|
||||
d := &Daemon{Store: store, Log: slog.New(slog.DiscardHandler)}
|
||||
d := wired(&Daemon{Store: store, Log: slog.New(slog.DiscardHandler)})
|
||||
d.retryTransientParks(t.Context())
|
||||
if len(store.retried) != 1 || store.retried[0] != 1 {
|
||||
t.Fatalf("retried = %v, want only task 1 (due, transient, retries left)", store.retried)
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/access"
|
||||
"github.com/samcharles93/archie-core/internal/domain/eda/playbook"
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/org"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storepkg"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/eventbus"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
)
|
||||
|
||||
// wired gives d the dependencies production always sets, with neutral
|
||||
// stand-ins for any the test left nil. A test that does not exercise one of
|
||||
// them then never reaches a nil branch in the code under test. A test that
|
||||
// does exercise one sets its own fake.
|
||||
func wired(d *Daemon) *Daemon {
|
||||
if d.Tasks == nil {
|
||||
d.Tasks = neutralTaskBus{}
|
||||
}
|
||||
if d.Store == nil {
|
||||
d.Store = neutralStore{}
|
||||
}
|
||||
if d.Reactions == nil {
|
||||
d.Reactions = neutralReactions{}
|
||||
}
|
||||
if d.Bus == nil {
|
||||
d.Bus = events.NewBus()
|
||||
}
|
||||
if d.RunCredentials == nil {
|
||||
d.RunCredentials = neutralCredentials{}
|
||||
}
|
||||
if d.TaskModels == nil {
|
||||
d.TaskModels = neutralTaskConfig{}
|
||||
}
|
||||
if d.TaskInstructions == nil {
|
||||
d.TaskInstructions = neutralTaskConfig{}
|
||||
}
|
||||
if d.Access == nil {
|
||||
d.Access = allowAll{}
|
||||
}
|
||||
if d.Principals == nil {
|
||||
d.Principals = neutralPrincipals{}
|
||||
}
|
||||
if d.Denials == nil {
|
||||
d.Denials = neutralDenials{}
|
||||
}
|
||||
if d.IdentityRepository == nil {
|
||||
d.IdentityRepository = neutralIdentities{}
|
||||
}
|
||||
if d.AgentAssignments == nil {
|
||||
d.AgentAssignments = neutralAssignments{}
|
||||
}
|
||||
if d.PackageAuthorities == nil {
|
||||
d.PackageAuthorities = neutralAuthorities{}
|
||||
}
|
||||
if d.Playbooks == nil {
|
||||
d.Playbooks = neutralPlaybooks{}
|
||||
}
|
||||
if d.WorkflowDefinitions == nil {
|
||||
d.WorkflowDefinitions = neutralDefinitions{}
|
||||
}
|
||||
if d.WorkflowEnablement == nil {
|
||||
d.WorkflowEnablement = neutralEnablement{}
|
||||
}
|
||||
if d.IssueSources == nil {
|
||||
d.IssueSources = func(context.Context) []IssueSource { return nil }
|
||||
}
|
||||
if d.OrgPrincipals == nil {
|
||||
d.OrgPrincipals = func(context.Context) []access.Principal {
|
||||
return []access.Principal{{Org: org.DefaultOrgID}}
|
||||
}
|
||||
}
|
||||
if d.MappingMatches == nil {
|
||||
d.MappingMatches = neutralMatches{}
|
||||
}
|
||||
if d.WithdrawTask == nil {
|
||||
d.WithdrawTask = func(context.Context, int64, string) error { return nil }
|
||||
}
|
||||
if d.AgentStatus == nil {
|
||||
d.AgentStatus = &AgentStatus{}
|
||||
}
|
||||
if d.Log == nil {
|
||||
d.Log = slog.New(slog.DiscardHandler)
|
||||
}
|
||||
d.wireReactions()
|
||||
return d
|
||||
}
|
||||
|
||||
// neutralTaskBus takes no work and accepts every publish.
|
||||
type neutralTaskBus struct{}
|
||||
|
||||
func (neutralTaskBus) PublishUnique(context.Context, string, string, []byte) error { return nil }
|
||||
|
||||
func (neutralTaskBus) Fetch(context.Context) (eventbus.Message, error) {
|
||||
return nil, eventbus.ErrNoMessage
|
||||
}
|
||||
|
||||
func (neutralTaskBus) Request(context.Context, string, []byte) ([]byte, error) { return nil, nil }
|
||||
|
||||
// neutralStore records no events. Any other store call panics, so a test that
|
||||
// reaches one sets its own store.
|
||||
type neutralStore struct{ storecontract.TaskStore }
|
||||
|
||||
func (neutralStore) InsertEvent(context.Context, events.Event) (int64, error) { return 0, nil }
|
||||
|
||||
// neutralReactions has no review reactions waiting.
|
||||
type neutralReactions struct{}
|
||||
|
||||
func (neutralReactions) Fetch(context.Context) (eventbus.Message, error) {
|
||||
return nil, eventbus.ErrNoMessage
|
||||
}
|
||||
|
||||
// neutralCredentials issues an empty run credential.
|
||||
type neutralCredentials struct{}
|
||||
|
||||
func (neutralCredentials) Issue(context.Context, *workflow.Task, time.Duration) (string, func(), error) {
|
||||
return "", func() {}, nil
|
||||
}
|
||||
|
||||
// neutralTaskConfig has no org model aliases and no instructions.
|
||||
type neutralTaskConfig struct{}
|
||||
|
||||
func (neutralTaskConfig) TaskModels(context.Context, int64) (config.OrgModels, error) {
|
||||
return config.OrgModels{}, nil
|
||||
}
|
||||
|
||||
func (neutralTaskConfig) TaskInstructions(context.Context, int64) (string, string, error) {
|
||||
return "", "", nil
|
||||
}
|
||||
|
||||
// allowAll permits every policy request.
|
||||
type allowAll struct{}
|
||||
|
||||
func (allowAll) Authorize(context.Context, access.Principal, access.Action, access.Resource, access.Context) access.Decision {
|
||||
return access.Decision{Allowed: true}
|
||||
}
|
||||
|
||||
// neutralPrincipals resolves every identity to the default org with no role.
|
||||
type neutralPrincipals struct{}
|
||||
|
||||
func (neutralPrincipals) PrincipalFor(_ context.Context, id identity.IdentityID) (access.Principal, error) {
|
||||
return access.Principal{IdentityID: id, Org: org.DefaultOrgID}, nil
|
||||
}
|
||||
|
||||
type neutralDenials struct{}
|
||||
|
||||
func (neutralDenials) RecordDenial(context.Context, access.Denial) error { return nil }
|
||||
|
||||
// neutralIdentities resolves every identity as an active agent.
|
||||
type neutralIdentities struct{}
|
||||
|
||||
func (neutralIdentities) List(context.Context) ([]identity.Identity, error) { return nil, nil }
|
||||
|
||||
func (neutralIdentities) Get(_ context.Context, id identity.IdentityID) (identity.Identity, error) {
|
||||
return identity.Identity{ID: id, Kind: identity.KindBot, Lifecycle: identity.LifecycleActive}, nil
|
||||
}
|
||||
|
||||
func (neutralIdentities) ResolveLegacyName(context.Context, string) (identity.Identity, error) {
|
||||
return identity.Identity{}, identity.ErrNotFound
|
||||
}
|
||||
|
||||
func (neutralIdentities) Create(_ context.Context, v identity.Identity, _ identity.Audit) (identity.Identity, error) {
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (neutralIdentities) Apply(_ context.Context, _ identity.IdentityID, _ int64, _ identity.Command, _ identity.Audit) (identity.Identity, error) {
|
||||
return identity.Identity{}, nil
|
||||
}
|
||||
|
||||
type neutralAssignments struct{}
|
||||
|
||||
func (neutralAssignments) ListOrgAgents(context.Context, org.OrgID) ([]org.AgentAssignment, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// neutralAuthorities accepts nothing for any package.
|
||||
type neutralAuthorities struct{}
|
||||
|
||||
func (neutralAuthorities) AcceptedAuthority(context.Context, string) (storepkg.Authority, error) {
|
||||
return storepkg.Authority{}, nil
|
||||
}
|
||||
|
||||
// neutralPlaybooks matches no playbook.
|
||||
type neutralPlaybooks struct{}
|
||||
|
||||
func (neutralPlaybooks) Dispatch(context.Context, playbook.DispatchInput) (playbook.Decision, bool) {
|
||||
return playbook.Decision{}, false
|
||||
}
|
||||
|
||||
func (neutralPlaybooks) Run(context.Context, storecontract.PlaybookDispatcher, *slog.Logger, playbook.DispatchInput) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// neutralMatches records no mapping match.
|
||||
type neutralMatches struct{}
|
||||
|
||||
func (neutralMatches) RecordMappingMatch(context.Context, string, string) error { return nil }
|
||||
|
||||
// neutralDefinitions serves the shipped workflow definitions at version 0.
|
||||
type neutralDefinitions struct{}
|
||||
|
||||
func (neutralDefinitions) WorkflowDefinitions(context.Context) (workflow.WorkflowDefinitionCollection, int64, error) {
|
||||
return workflow.ShippedDefinitions(), 0, nil
|
||||
}
|
||||
|
||||
// neutralEnablement disables no workflow.
|
||||
type neutralEnablement struct{}
|
||||
|
||||
func (neutralEnablement) WorkflowEnablement(context.Context) (workflowtask.WorkflowEnablement, error) {
|
||||
return workflowtask.WorkflowEnablement{}, nil
|
||||
}
|
||||
@@ -37,7 +37,7 @@ func TestWithdrawUnpolled(t *testing.T) {
|
||||
{"eligible again under the new rule", "label:bot", []int{3}, false},
|
||||
{"dropped under the same rule: withdrawn", "label:bot", nil, true},
|
||||
}
|
||||
d := &Daemon{Log: slog.New(slog.DiscardHandler), Store: activeTasks{tasks: []workflow.Task{task}}}
|
||||
d := wired(&Daemon{Log: slog.New(slog.DiscardHandler), Store: activeTasks{tasks: []workflow.Task{task}}})
|
||||
for _, step := range steps {
|
||||
withdrawn := false
|
||||
d.WithdrawTask = func(context.Context, int64, string) error { withdrawn = true; return nil }
|
||||
|
||||
@@ -60,9 +60,6 @@ func (d *Daemon) adoptWorkflowIdentity(ctx context.Context, task *workflow.Task)
|
||||
// declaredIdentity resolves a workflow's declared identity to the ID it is
|
||||
// authorized as and the name the task carries while it runs.
|
||||
func (d *Daemon) declaredIdentity(ctx context.Context, task *workflow.Task, ref string) (identity.IdentityID, string, error) {
|
||||
if d.IdentityRepository == nil {
|
||||
return "", "", pinFailure{taskstate.ParkTransient, fmt.Errorf("identity authority unavailable for %s", ref)}
|
||||
}
|
||||
value, err := d.databaseExecutionIdentity(ctx, task, identity.IdentityID(ref))
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
@@ -83,13 +80,7 @@ func (d *Daemon) databaseExecutionIdentity(ctx context.Context, task *workflow.T
|
||||
if !value.Kind.Agent() || value.Lifecycle != identity.LifecycleActive {
|
||||
return identity.Identity{}, blockedPark{taskstate.BlockerAgentAssignment, pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("execution identity %s must be an active bot or service account", id)}}
|
||||
}
|
||||
if d.AgentAssignments == nil {
|
||||
return identity.Identity{}, pinFailure{taskstate.ParkTransient, fmt.Errorf("agent assignment authority unavailable")}
|
||||
}
|
||||
owner := org.OrgID(task.OrgOrDefault())
|
||||
if d.Principals == nil {
|
||||
return identity.Identity{}, pinFailure{taskstate.ParkTransient, fmt.Errorf("principal authority unavailable")}
|
||||
}
|
||||
principal, err := d.Principals.PrincipalFor(ctx, value.ID)
|
||||
if err != nil {
|
||||
return identity.Identity{}, pinFailure{taskstate.ParkTransient, fmt.Errorf("resolve execution principal: %w", err)}
|
||||
@@ -112,9 +103,6 @@ func (d *Daemon) databaseExecutionIdentity(ctx context.Context, task *workflow.T
|
||||
// an install that has not built one. The error reports a chain that could not
|
||||
// be consulted, which is not a denial: nothing was evaluated.
|
||||
func (d *Daemon) mayRunWorkflow(ctx context.Context, task *workflow.Task, id identity.IdentityID) (bool, error) {
|
||||
if d.Access == nil || d.Principals == nil {
|
||||
return true, nil
|
||||
}
|
||||
principal, err := d.principalIn(ctx, id, task.Org)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "workflow identity: principal unavailable", "task", task.ID, "identity", id, "err", err)
|
||||
@@ -133,26 +121,12 @@ func (d *Daemon) mayRunWorkflow(ctx context.Context, task *workflow.Task, id ide
|
||||
// operator to involve: the chain will answer once the store does.
|
||||
return false, pinFailure{taskstate.ParkTransient, decision.Err}
|
||||
}
|
||||
if d.Denials != nil {
|
||||
denial := access.Denial{
|
||||
Principal: principal.IdentityID, Org: principal.Org, Action: access.ActionRun,
|
||||
Kind: resource.Kind, ResourceID: resource.ID, Level: decision.Level, Policies: decision.Policies,
|
||||
}
|
||||
if err := d.Denials.RecordDenial(ctx, denial); err != nil {
|
||||
d.Log.WarnContext(ctx, "workflow identity: record denial", "task", task.ID, "err", err)
|
||||
}
|
||||
denial := access.Denial{
|
||||
Principal: principal.IdentityID, Org: principal.Org, Action: access.ActionRun,
|
||||
Kind: resource.Kind, ResourceID: resource.ID, Level: decision.Level, Policies: decision.Policies,
|
||||
}
|
||||
if err := d.Denials.RecordDenial(ctx, denial); err != nil {
|
||||
d.Log.WarnContext(ctx, "workflow identity: record denial", "task", task.ID, "err", err)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// parkRefusedRun parks a task whose workflow identity check did not permit it.
|
||||
// A denial is the operator's to fix, but a chain that could not be read is a
|
||||
// store that will answer again, so it parks transient rather than demanding an
|
||||
// operator for a blip.
|
||||
func (d *Daemon) parkRefusedRun(ctx context.Context, task *workflow.Task, reason string, err error) {
|
||||
class, blocker := taskstate.ParkNeedsDecision, taskstate.BlockerPolicyDenied
|
||||
if err != nil {
|
||||
class, blocker = pinParkClass(err), parkBlocker(err)
|
||||
}
|
||||
d.parkBlockedTask(ctx, task, reason, class, blocker)
|
||||
}
|
||||
@@ -129,11 +129,11 @@ func TestWorkflowIdentity(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
store := &identityStore{}
|
||||
var denied denials
|
||||
d := &Daemon{
|
||||
d := wired(&Daemon{
|
||||
Store: store, Log: slog.New(slog.DiscardHandler),
|
||||
Access: tt.allowed, Principals: principalOf{}, Denials: &denied,
|
||||
IdentityRepository: agentDirectory{}, AgentAssignments: agentDirectory{},
|
||||
}
|
||||
})
|
||||
if tt.disabled {
|
||||
d.WorkflowEnablement = enablementOf(workflowtask.WorkflowEnablement{}.SetEnabled("", "deploy", false))
|
||||
}
|
||||
@@ -177,11 +177,11 @@ func TestWorkflowRunCarriesEventContext(t *testing.T) {
|
||||
want bool
|
||||
}{{"valid", true}, {"unsigned", false}, {"", false}} {
|
||||
t.Run(tt.signature, func(t *testing.T) {
|
||||
d := &Daemon{
|
||||
d := wired(&Daemon{
|
||||
Store: &identityStore{}, Log: slog.New(slog.DiscardHandler),
|
||||
Access: signedOnly{runAllowed{sys: true, "id-deployer": true}},
|
||||
Principals: principalOf{}, IdentityRepository: agentDirectory{}, AgentAssignments: agentDirectory{},
|
||||
}
|
||||
})
|
||||
task := &workflow.Task{
|
||||
ID: 1, Identity: string(sys), Workflow: "deploy", BindingSignature: tt.signature,
|
||||
WorkflowDefinitionYAML: deploy, WorkflowDefinitionDigest: workflow.DigestDefinition(deploy),
|
||||
@@ -236,11 +236,11 @@ func TestDashboardExecutionIdentity(t *testing.T) {
|
||||
}
|
||||
store := &identityStore{}
|
||||
cfg := config.Config{BotUser: "root-bot", GrantedCredentials: []string{"root-secret"}, Forge: config.Forge{Token: config.SecretRef{Engine: "env", Key: "ROOT"}}, Repos: []config.Repo{{Owner: "acme", Name: "app"}}}
|
||||
d := &Daemon{
|
||||
d := wired(&Daemon{
|
||||
Store: store, Log: slog.New(slog.DiscardHandler), Cfg: config.NewHolder(cfg),
|
||||
IdentityRepository: repository, AgentAssignments: assignedAgents{{IdentityID: value.ID, OrgID: tt.assigned}},
|
||||
Principals: workflowPrincipals{sys: "org-sys", "dashboard-agent": "org-sys"}, Access: runAllowed{sys: true, "dashboard-agent": true},
|
||||
}
|
||||
})
|
||||
definition := "id: deploy\nidentity: dashboard-agent\nrepository: none\n"
|
||||
newTask := func() *workflow.Task {
|
||||
return &workflow.Task{ID: 1, Identity: string(sys), Workflow: "deploy", WorkflowDefinitionYAML: definition, WorkflowDefinitionDigest: workflow.DigestDefinition(definition)}
|
||||
|
||||
@@ -18,13 +18,6 @@ func (d *Daemon) workflowContext(ctx context.Context, task *workflow.Task) (cont
|
||||
if owner == "" {
|
||||
owner = org.DefaultOrgID
|
||||
}
|
||||
if d.Principals == nil {
|
||||
principal, forwarded := access.PrincipalFromContext(ctx)
|
||||
if owner != org.DefaultOrgID || org.OrgFromContext(ctx) != owner || (forwarded && principal.Org != "" && principal.Org != owner) {
|
||||
return nil, pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("workflow org %s: principal unavailable", owner)}
|
||||
}
|
||||
return ctx, nil
|
||||
}
|
||||
principal, err := d.principalIn(ctx, identity.IdentityID(task.Identity), owner)
|
||||
if err != nil {
|
||||
return nil, pinFailure{taskstate.ParkTransient, fmt.Errorf("workflow principal: %w", err)}
|
||||
|
||||
@@ -88,12 +88,12 @@ func TestDispatchOrgWorkflows(t *testing.T) {
|
||||
{ID: "root", MappingID: "root", Matcher: binding.Matcher{Source: "root"}, OrgID: org.DefaultOrgID, Workflow: "deploy", Status: binding.StatusArmed, Owner: "owner", Repo: "repo"},
|
||||
{ID: "acme", MappingID: "acme", Matcher: binding.Matcher{Source: "acme"}, OrgID: "acme", Workflow: "deploy", Status: binding.StatusArmed},
|
||||
}}
|
||||
d := &Daemon{Cfg: config.NewHolder(config.Config{}), Store: store, Log: slog.New(slog.DiscardHandler), Bindings: store, BindingDispatcher: store, BindingTaskCreator: store, Mappings: store, OrgPrincipals: func(context.Context) []access.Principal {
|
||||
d := wired(&Daemon{Cfg: config.NewHolder(config.Config{}), Store: store, Log: slog.New(slog.DiscardHandler), Bindings: store, BindingDispatcher: store, BindingTaskCreator: store, Mappings: store, OrgPrincipals: func(context.Context) []access.Principal {
|
||||
return []access.Principal{{IdentityID: identity.SystemID, Org: org.DefaultOrgID}, {IdentityID: "acme-bot", Org: "acme"}}
|
||||
}, Principals: workflowPrincipals{sys: org.DefaultOrgID, "acme-bot": "acme"}, WorkflowDefinitions: orgDefinitions{
|
||||
org.DefaultOrgID: "id: deploy\nrepository: required\nsteps:\n - type: workflow.finish\n",
|
||||
"acme": "id: deploy\nrepository: none\nsteps:\n - type: workflow.finish\n",
|
||||
}}
|
||||
}})
|
||||
if disabled {
|
||||
d.WorkflowEnablement = enablementOf(workflowtask.WorkflowEnablement{}.SetEnabled("acme", "deploy", false))
|
||||
}
|
||||
@@ -135,27 +135,16 @@ func TestPinWorkflowInTaskOrg(t *testing.T) {
|
||||
principals access.PrincipalSource
|
||||
want string
|
||||
refused bool
|
||||
contextOrg org.OrgID
|
||||
}{
|
||||
{"legacy default org", "", nil, shared, false, ""},
|
||||
{"non-default org definition", "acme", workflowPrincipals{"bot": "acme"}, custom, false, ""},
|
||||
{"another org cannot supply the definition", "acme", workflowPrincipals{"bot": org.DefaultOrgID}, "", true, ""},
|
||||
{"a member of the task's org may start it from another", "acme", memberOf{home: org.DefaultOrgID, other: "acme"}, custom, false, ""},
|
||||
{"a non-member cannot start a task in another org", "acme", memberOf{home: org.DefaultOrgID, other: "beta"}, "", true, ""},
|
||||
{"unresolved org is refused", "acme", nil, "", true, ""},
|
||||
{"unresolved context cannot select another org", "", nil, "", true, "acme"},
|
||||
{"non-default org definition", "acme", workflowPrincipals{"bot": "acme"}, custom, false},
|
||||
{"another org cannot supply the definition", "acme", workflowPrincipals{"bot": org.DefaultOrgID}, "", true},
|
||||
{"a member of the task's org may start it from another", "acme", memberOf{home: org.DefaultOrgID, other: "acme"}, custom, false},
|
||||
{"a non-member cannot start a task in another org", "acme", memberOf{home: org.DefaultOrgID, other: "beta"}, "", true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
d := &Daemon{Store: listsWorkspaces{&identityStore{}}, WorkflowDefinitions: orgDefinitions{org.DefaultOrgID: shared, "acme": custom}}
|
||||
if tc.principals != nil {
|
||||
d.Principals = tc.principals
|
||||
}
|
||||
d := wired(&Daemon{Store: listsWorkspaces{&identityStore{}}, Principals: tc.principals, WorkflowDefinitions: orgDefinitions{org.DefaultOrgID: shared, "acme": custom}})
|
||||
task := &workflow.Task{ID: 1, Identity: "bot", Org: tc.owner, Workflow: "deploy"}
|
||||
ctx := t.Context()
|
||||
if tc.contextOrg != "" {
|
||||
ctx = org.WithOrg(access.WithPrincipal(ctx, access.Principal{IdentityID: "foreign", Org: tc.contextOrg}), tc.contextOrg)
|
||||
}
|
||||
err := d.pinWorkflowDefinition(ctx, task)
|
||||
err := d.pinWorkflowDefinition(t.Context(), task)
|
||||
if tc.refused {
|
||||
if err == nil || pinParkClass(err) != taskstate.ParkNeedsDecision || task.WorkflowDefinitionYAML != "" {
|
||||
t.Fatalf("cross-org pin accepted: task %#v err %v", task, err)
|
||||
|
||||
Reference in new issue
Block a user