refactor(daemon): issue polling is its own type (#1931)

This commit is contained in:
sam committed 2026-11-13 04:12:29 +11:00
1 parent 63a1750936
commit 0d13b919da
10 files changed
+122 -85

No files matched your search

+1 -2
View File
@@ -621,7 +621,6 @@ func (b *boot) buildDaemon(ctx context.Context) {
ModelEgress: b.modelEgress,
TaskModels: b.taskModels,
TaskInstructions: b.taskInstructions,
IssueSources: b.issueSources,
OrgPrincipals: b.orgPrincipals,
TaskLogs: b.taskLogs,
AgentStatus: b.agentStatus,
@@ -630,7 +629,7 @@ func (b *boot) buildDaemon(ctx context.Context) {
Playbooks: b.livePlaybooks(),
}
b.d.WithdrawTask = b.taskActions().Withdraw
b.d.IssuePoller = daemon.NewIssuePoller(b.cfgHolder, b.stateStore, log, b.issueSources, b.taskActions().Withdraw, b.d.PublishTask, b.d)
b.d.Access = b.accessChain
}
+1 -1
View File
@@ -29,7 +29,7 @@ const mirrorIdle = 7 * 24 * time.Hour
func (d *Daemon) BuiltinJobs() []BuiltinJob {
return []BuiltinJob{
{ID: "volume-cleanup", Name: "Remove expired volumes", DefaultInterval: time.Hour, Run: d.cleanupExpiredStorage},
{ID: "issue-discovery", Name: "Discover forge issues", DefaultInterval: time.Minute, Run: d.poll},
{ID: "issue-discovery", Name: "Discover forge issues", DefaultInterval: time.Minute, Run: d.IssuePoller.poll},
{ID: "pr-reconcile", Name: "Reconcile pull requests", DefaultInterval: time.Minute, Run: d.reconcilePRs},
{ID: "pr-review-scan", Name: "Scan pull request reviews", DefaultInterval: time.Minute, Run: d.scanPRReviews},
{ID: "mirror-prune", Name: "Prune repository mirrors", DefaultInterval: 24 * time.Hour, Run: d.pruneMirrors},
+2 -18
View File
@@ -118,11 +118,6 @@ type Daemon struct {
// counter.
reactionConsumer *reactionConsumer
// lastPolled remembers each repo's eligible issues from its previous
// complete poll, under the dispatch rule that produced them, so only an
// issue that was eligible and dropped out reads as withdrawn.
lastPolledMu sync.Mutex
lastPolled map[string]polledIssues
// reactionPublisher delivers review reactions. Startup sets it to
// PublishReaction; tests swap it to deliver without the bus.
reactionPublisher ReactionPublisher
@@ -157,9 +152,8 @@ type Daemon struct {
// Storage is the pluggable storage backend for container mounts.
// A runnable task requires it; acquireTaskContainer parks on nil.
Storage storage.Backend
// IssueSources resolves, each poll cycle, every organisation connection
// to poll and the identity whose permit reaches it.
IssueSources func(context.Context) []IssueSource
// IssuePoller discovers forge issues and publishes them as tasks.
IssuePoller *IssuePoller
// OrgPrincipals is one acting principal per organisation, system org
// first.
OrgPrincipals func(context.Context) []access.Principal
@@ -185,10 +179,6 @@ type Daemon struct {
WorkflowDefinitions interface {
WorkflowDefinitions(context.Context) (workflow.WorkflowDefinitionCollection, int64, error)
}
// WithdrawTask declines a task whose issue a complete poll no longer
// returns.
WithdrawTask func(ctx context.Context, taskID int64, reason string) error
// WorkflowEnablement supplies which workflows each org has disabled. A
// binding targeting a workflow its org disabled does not dispatch.
WorkflowEnablement interface {
@@ -273,12 +263,6 @@ func passFailed(pass string, failed []string) error {
return fmt.Errorf("%s failed for %d item(s): %s", pass, len(failed), strings.Join(failed, ", "))
}
// polledIssues is one repo's eligible issues under one dispatch rule.
type polledIssues struct {
rule string
eligible map[int]bool
}
// pinFailure carries the park class for a pin failure: operator-actionable
// for a bad pin or missing workflow, transient for a store error.
type pinFailure struct {
+1 -1
View File
@@ -62,7 +62,7 @@ func TestPublishDiscoveredCap(t *testing.T) {
}
room := tt.cap
repo := config.Repo{Owner: "acme", Name: "widget"}
d.publishDiscovered(t.Context(), nil, config.Config{}, repo, issues, string(sys), &room)
d.IssuePoller.publishDiscovered(t.Context(), nil, config.Config{}, repo, issues, string(sys), &room)
if bus.published != tt.want {
t.Fatalf("published %d tasks, want %d", bus.published, tt.want)
}
+1 -1
View File
@@ -96,7 +96,7 @@ func TestPollNATSSkipsDisabledRoutedWorkflow(t *testing.T) {
Principals: tt.principals,
})
is := forge.Issue{Number: 42, Title: "crash on startup"}
if created := d.pollNATS(t.Context(), fg, cfg, repo, is, tt.labels, "acme-bot"); created != tt.wantCreated {
if created := d.IssuePoller.pollNATS(t.Context(), fg, cfg, repo, is, tt.labels, "acme-bot"); created != tt.wantCreated {
t.Errorf("pollNATS created %v, want %v", created, tt.wantCreated)
}
if bus.published != tt.wantPublished {
+100 -44
View File
@@ -2,44 +2,97 @@ package daemon
import (
"context"
"log/slog"
"strings"
"sync"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/identity"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workintake"
"github.com/samcharles93/archie-core/internal/forge"
"github.com/samcharles93/archie-core/internal/taskstate"
)
func (d *Daemon) issueSources(ctx context.Context) []IssueSource {
return d.IssueSources(ctx)
// intakeGate is the identity and binding checks a poll reads through. The
// Daemon satisfies it; the poller keeps no state of its own for either.
type intakeGate interface {
identityActive(ctx context.Context, id identity.IdentityID) bool
disabledRoutedWorkflow(ctx context.Context, labels, identityID string) (routed string, disabled bool, err error)
}
// IssuePoller discovers the forge issues that dispatch rules make work and
// publishes them as tasks, and withdraws tasks whose issue drops out.
type IssuePoller struct {
cfg *config.Holder
store storecontract.TaskStore
log *slog.Logger
sources func(context.Context) []IssueSource
withdrawTask func(ctx context.Context, taskID int64, reason string) error
publishTask func(context.Context, workintake.TaskEnvelope) error
gate intakeGate
// lastPolled remembers each repo's eligible issues from its previous
// complete poll, under the dispatch rule that produced them, so only an
// issue that was eligible and dropped out reads as withdrawn.
lastPolledMu sync.Mutex
lastPolled map[string]polledIssues
}
// NewIssuePoller builds the poller over the running configuration, the task
// store and the task publisher. sources resolves each poll cycle's connections,
// withdrawTask declines a task whose issue a complete poll no longer returns,
// and gate answers the identity and routed-workflow checks.
func NewIssuePoller(
cfg *config.Holder,
store storecontract.TaskStore,
log *slog.Logger,
sources func(context.Context) []IssueSource,
withdrawTask func(ctx context.Context, taskID int64, reason string) error,
publishTask func(context.Context, workintake.TaskEnvelope) error,
gate intakeGate,
) *IssuePoller {
return &IssuePoller{
cfg: cfg,
store: store,
log: log,
sources: sources,
withdrawTask: withdrawTask,
publishTask: publishTask,
gate: gate,
lastPolled: map[string]polledIssues{},
}
}
func (p *IssuePoller) issueSources(ctx context.Context) []IssueSource {
return p.sources(ctx)
}
// pollIssuesWithConfig returns the repo's issues the dispatch rule makes
// work, using the given forge client and dispatch config. complete is false when any part of the poll failed or was skipped, so
// the result must not be read as the whole eligible set.
func (d *Daemon) pollIssuesWithConfig(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo) (issues []forge.Issue, complete bool) {
func (p *IssuePoller) pollIssuesWithConfig(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo) (issues []forge.Issue, complete bool) {
labelled := func() ([]forge.Issue, bool) {
// A label trigger needs a non-empty label or it would match every open issue.
if cfg.Label == "" {
d.Log.ErrorContext(ctx, "label trigger configured with an empty label; refusing to poll (an empty label matches every open issue)", "repo", repo.FullName())
p.log.ErrorContext(ctx, "label trigger configured with an empty label; refusing to poll (an empty label matches every open issue)", "repo", repo.FullName())
return nil, false
}
issues, err := fg.IssuesWithLabel(ctx, repo.Owner, repo.Name, cfg.Label)
if err != nil {
d.Log.ErrorContext(ctx, "label poll failed", "repo", repo.FullName(), "err", err)
p.log.ErrorContext(ctx, "label poll failed", "repo", repo.FullName(), "err", err)
return nil, false
}
return issues, true
}
assigned := func() ([]forge.Issue, bool) {
if cfg.BotUser == "" {
d.Log.ErrorContext(ctx, "assignee trigger needs the connection's account; refusing to poll", "repo", repo.FullName())
p.log.ErrorContext(ctx, "assignee trigger needs the connection's account; refusing to poll", "repo", repo.FullName())
return nil, false
}
issues, err := fg.AssignedIssues(ctx, repo.Owner, repo.Name, cfg.BotUser)
if err != nil {
d.Log.ErrorContext(ctx, "assigned poll failed", "repo", repo.FullName(), "err", err)
p.log.ErrorContext(ctx, "assigned poll failed", "repo", repo.FullName(), "err", err)
return nil, false
}
return issues, true
@@ -69,27 +122,27 @@ func (d *Daemon) pollIssuesWithConfig(ctx context.Context, fg forge.Forge, cfg c
// poll discovers work through every organisation connection, resolved
// afresh each cycle so a rotated credential or a new connection applies to
// the next poll.
func (d *Daemon) poll(ctx context.Context) error {
func (p *IssuePoller) poll(ctx context.Context) error {
var failed []string
room := d.Cfg.Get().IntakePerPass
for _, source := range d.issueSources(ctx) {
if !d.identityActive(ctx, source.Identity) {
room := p.cfg.Get().IntakePerPass
for _, source := range p.issueSources(ctx) {
if !p.gate.identityActive(ctx, source.Identity) {
continue
}
cfg := d.Cfg.Get()
cfg := p.cfg.Get()
cfg.BotUser = source.BotUser
if source.Dispatch.Trigger != "" {
cfg.Dispatch, cfg.Label = source.Dispatch, source.Label
}
tag := string(source.Identity)
for _, repo := range source.Repos {
issues, complete := d.pollIssuesWithConfig(ctx, source.Forge, cfg, repo)
issues, complete := p.pollIssuesWithConfig(ctx, source.Forge, cfg, repo)
if complete {
d.withdrawUnpolled(ctx, repo, tag, dispatchRule(cfg), issues)
p.withdrawUnpolled(ctx, repo, tag, dispatchRule(cfg), issues)
} else {
failed = append(failed, repo.FullName())
}
d.publishDiscovered(ctx, source.Forge, cfg, repo, issues, tag, &room)
p.publishDiscovered(ctx, source.Forge, cfg, repo, issues, tag, &room)
}
}
return passFailed("issue poll", failed)
@@ -97,21 +150,18 @@ func (d *Daemon) poll(ctx context.Context) error {
// rememberPoll records a repo's latest eligible set and returns the previous
// one, reporting whether it was taken under the same rule.
func (d *Daemon) rememberPoll(key string, current polledIssues) (polledIssues, bool) {
d.lastPolledMu.Lock()
defer d.lastPolledMu.Unlock()
previous, seen := d.lastPolled[key]
if d.lastPolled == nil {
d.lastPolled = map[string]polledIssues{}
}
d.lastPolled[key] = current
func (p *IssuePoller) rememberPoll(key string, current polledIssues) (polledIssues, bool) {
p.lastPolledMu.Lock()
defer p.lastPolledMu.Unlock()
previous, seen := p.lastPolled[key]
p.lastPolled[key] = current
return previous, seen && previous.rule == current.rule
}
// withdrawUnpolled declines the queued or running work of an identity's
// issues in repo that a complete poll no longer returns: the issue was
// closed, unlabelled or unassigned.
func (d *Daemon) withdrawUnpolled(ctx context.Context, repo config.Repo, identity, rule string, polled []forge.Issue) {
func (p *IssuePoller) withdrawUnpolled(ctx context.Context, repo config.Repo, identity, rule string, polled []forge.Issue) {
eligible := make(map[int]bool, len(polled))
for _, is := range polled {
eligible[is.Number] = true
@@ -120,15 +170,15 @@ func (d *Daemon) withdrawUnpolled(ctx context.Context, repo config.Repo, identit
// finds was withdrawn. A changed label, trigger or bot user, or the first
// poll after a restart, only records the new set: tasks picked up under
// the old rule were not withdrawn by anyone.
previous, sameRule := d.rememberPoll(identity+"\x00"+repo.FullName(), polledIssues{rule: rule, eligible: eligible})
previous, sameRule := p.rememberPoll(identity+"\x00"+repo.FullName(), polledIssues{rule: rule, eligible: eligible})
if !sameRule {
return
}
active, err := d.Store.TasksPage(ctx, storecontract.TaskPage{
active, err := p.store.TasksPage(ctx, storecontract.TaskPage{
Statuses: []string{taskstate.Queued, taskstate.Running}, Limit: 500,
})
if err != nil {
d.Log.WarnContext(ctx, "withdrawal check: active tasks unavailable", "repo", repo.FullName(), "err", err)
p.log.WarnContext(ctx, "withdrawal check: active tasks unavailable", "repo", repo.FullName(), "err", err)
return
}
for _, t := range active {
@@ -136,8 +186,8 @@ func (d *Daemon) withdrawUnpolled(ctx context.Context, repo config.Repo, identit
!t.IsForgeBacked() || t.PRNumber > 0 || eligible[t.IssueNumber] || !previous.eligible[t.IssueNumber] {
continue
}
if err := d.WithdrawTask(ctx, t.ID, "the issue is no longer assigned, labelled or open"); err != nil {
d.Log.WarnContext(ctx, "withdraw task", "task", t.ID, "err", err)
if err := p.withdrawTask(ctx, t.ID, "the issue is no longer assigned, labelled or open"); err != nil {
p.log.WarnContext(ctx, "withdraw task", "task", t.ID, "err", err)
}
}
}
@@ -146,14 +196,14 @@ func (d *Daemon) withdrawUnpolled(ctx context.Context, repo config.Repo, identit
// created its cap of tasks. room is what is left of the cap; the issues left
// over are found again by the next poll, because an issue with a task is
// skipped and one without is not. A cap of zero or less is no cap.
func (d *Daemon) publishDiscovered(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, issues []forge.Issue, identity string, room *int) {
func (p *IssuePoller) publishDiscovered(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, issues []forge.Issue, identity string, room *int) {
capped := *room > 0
for _, is := range issues {
if capped && *room == 0 {
d.Log.InfoContext(ctx, "intake cap reached; remaining issues wait for the next poll", "repo", repo.FullName())
p.log.InfoContext(ctx, "intake cap reached; remaining issues wait for the next poll", "repo", repo.FullName())
return
}
if d.pollNATS(ctx, fg, cfg, repo, is, strings.Join(is.Labels, ","), identity) && capped {
if p.pollNATS(ctx, fg, cfg, repo, is, strings.Join(is.Labels, ","), identity) && capped {
*room--
}
}
@@ -161,11 +211,11 @@ func (d *Daemon) publishDiscovered(ctx context.Context, fg forge.Forge, cfg conf
// pollNATS publishes a discovered issue to NATS and reports whether it created
// a task.
func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) bool {
func (p *IssuePoller) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) bool {
// Read-only existence check prevents rediscovering an existing task.
existing, err := d.Store.TaskByIssue(ctx, repo.Owner, repo.Name, is.Number)
existing, err := p.store.TaskByIssue(ctx, repo.Owner, repo.Name, is.Number)
if err != nil {
d.Log.ErrorContext(ctx, "task lookup failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
p.log.ErrorContext(ctx, "task lookup failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
return false
}
if existing != nil {
@@ -174,10 +224,10 @@ func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config
// An issue whose label-routed workflow the identity's org disabled must
// not queue a task at all. Skip without acknowledging, so a later poll
// picks the issue up again if the workflow is re-enabled.
if routed, disabled, err := d.disabledRoutedWorkflow(ctx, labels, identity); err != nil {
d.Log.WarnContext(ctx, "routed workflow check failed; publishing anyway", "repo", repo.FullName(), "issue", is.Number, "err", err)
if routed, disabled, err := p.gate.disabledRoutedWorkflow(ctx, labels, identity); err != nil {
p.log.WarnContext(ctx, "routed workflow check failed; publishing anyway", "repo", repo.FullName(), "issue", is.Number, "err", err)
} else if disabled {
d.Log.InfoContext(ctx, "issue skipped: its routed workflow is disabled for this org", "repo", repo.FullName(), "issue", is.Number, "workflow", routed)
p.log.InfoContext(ctx, "issue skipped: its routed workflow is disabled for this org", "repo", repo.FullName(), "issue", is.Number, "workflow", routed)
return false
}
// Classify and encode here: the envelope, its routing kind and its
@@ -193,26 +243,32 @@ func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config
Identity: identity,
Kind: workintake.KindForLabels(parsed),
}
if err := d.PublishTask(ctx, task); err != nil {
d.Log.ErrorContext(ctx, "task publish failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
if err := p.publishTask(ctx, task); err != nil {
p.log.ErrorContext(ctx, "task publish failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
return false
}
d.acknowledge(ctx, fg, cfg, repo, is)
p.acknowledge(ctx, fg, cfg, repo, is)
return true
}
// acknowledge posts the pickup reaction
// using fg -- the forge client that owns repo (identity-scoped or root) --
// and the ack reaction from cfg, the dispatch config that owns this poll.
func (d *Daemon) acknowledge(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue) {
d.Log.InfoContext(ctx, "issue queued", "repo", repo.FullName(), "issue", is.Number, "title", is.Title)
func (p *IssuePoller) acknowledge(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue) {
p.log.InfoContext(ctx, "issue queued", "repo", repo.FullName(), "issue", is.Number, "title", is.Title)
if ack := cfg.Dispatch.AckReaction; ack != "" {
if err := fg.React(ctx, repo.Owner, repo.Name, is.Number, ack); err != nil {
d.Log.WarnContext(ctx, "ack reaction failed", "issue", is.Number, "err", err)
p.log.WarnContext(ctx, "ack reaction failed", "issue", is.Number, "err", err)
}
}
}
// polledIssues is one repo's eligible issues under one dispatch rule.
type polledIssues struct {
rule string
eligible map[int]bool
}
// dispatchRule identifies the rule a poll selects issues by.
func dispatchRule(cfg config.Config) string {
return cfg.Dispatch.Trigger + "\x00" + cfg.Label + "\x00" + cfg.BotUser
+1 -1
View File
@@ -65,7 +65,7 @@ func (d *Daemon) sweepAccess(ctx context.Context) {
}
var wg sync.WaitGroup
for _, source := range d.issueSources(ctx) {
for _, source := range d.IssuePoller.issueSources(ctx) {
wg.Go(func() { sweep(source) })
}
wg.Wait()
+9 -9
View File
@@ -57,16 +57,16 @@ func TestPolledTaskKeepsItsCreatingIdentity(t *testing.T) {
d := wired(&Daemon{
Log: slog.New(slog.DiscardHandler), Store: taskLookup{}, Tasks: bus,
Cfg: config.NewHolder(config.Config{}),
IssueSources: func(context.Context) []IssueSource {
return []IssueSource{{
Identity: tt.source,
Forge: labelledIssues{issues: []forge.Issue{{Number: 12, Title: "t"}}},
Dispatch: config.Dispatch{Trigger: "label"}, Label: "archie",
Repos: []config.Repo{{Owner: "acme", Name: "widget"}},
}}
},
})
if err := d.poll(t.Context()); err != nil {
d.IssuePoller.sources = func(context.Context) []IssueSource {
return []IssueSource{{
Identity: tt.source,
Forge: labelledIssues{issues: []forge.Issue{{Number: 12, Title: "t"}}},
Dispatch: config.Dispatch{Trigger: "label"}, Label: "archie",
Repos: []config.Repo{{Owner: "acme", Name: "widget"}},
}}
}
if err := d.IssuePoller.poll(t.Context()); err != nil {
t.Fatal(err)
}
if len(bus.payloads) != 1 {
+4 -6
View File
@@ -71,9 +71,6 @@ func wired(d *Daemon) *Daemon {
if d.WorkflowEnablement == nil {
d.WorkflowEnablement = neutralEnablement{}
}
if d.IssueSources == nil {
d.IssueSources = func(context.Context) []IssueSource { return nil }
}
if d.OrgPrincipals == nil {
d.OrgPrincipals = func(context.Context) []access.Principal {
return []access.Principal{{Org: org.DefaultOrgID}}
@@ -82,15 +79,16 @@ func wired(d *Daemon) *Daemon {
if d.MappingMatches == nil {
d.MappingMatches = neutralMatches{}
}
if d.WithdrawTask == nil {
d.WithdrawTask = func(context.Context, int64, string) error { return nil }
}
if d.AgentStatus == nil {
d.AgentStatus = &AgentStatus{}
}
if d.Log == nil {
d.Log = slog.New(slog.DiscardHandler)
}
d.IssuePoller = NewIssuePoller(d.Cfg, d.Store, d.Log,
func(context.Context) []IssueSource { return nil },
func(context.Context, int64, string) error { return nil },
d.PublishTask, d)
d.wireReactions()
return d
}
+2 -2
View File
@@ -40,12 +40,12 @@ func TestWithdrawUnpolled(t *testing.T) {
d := wired(&Daemon{Log: slog.New(slog.DiscardHandler), Store: activeTasks{tasks: []workflow.Task{task}}})
for _, step := range steps {
withdrawn := false
d.WithdrawTask = func(context.Context, int64, string) error { withdrawn = true; return nil }
d.IssuePoller.withdrawTask = func(context.Context, int64, string) error { withdrawn = true; return nil }
var issues []forge.Issue
for _, n := range step.polls {
issues = append(issues, forge.Issue{Number: n})
}
d.withdrawUnpolled(t.Context(), repo, "", step.rule, issues)
d.IssuePoller.withdrawUnpolled(t.Context(), repo, "", step.rule, issues)
if withdrawn != step.want {
t.Fatalf("%s: withdrawn = %v, want %v", step.name, withdrawn, step.want)
}