fix(daemon): issue poll and PR passes report failed items to the scheduler
This commit is contained in:
1 parent
f727d54d75
commit
dc26daeebe
3 files changed
+43
-24
No files matched your search
@@ -29,21 +29,13 @@ const mirrorIdle = 7 * 24 * time.Hour
|
||||
func (d *Daemon) BuiltinJobs() []BuiltinJob {
|
||||
return []BuiltinJob{
|
||||
{ID: "volume-cleanup", Name: "Remove expired volumes", DefaultInterval: time.Hour, Run: d.cleanupExpiredStorage},
|
||||
{ID: "issue-discovery", Name: "Discover forge issues", DefaultInterval: time.Minute, Run: logsOwnErrors(d.poll)},
|
||||
{ID: "pr-reconcile", Name: "Reconcile pull requests", DefaultInterval: time.Minute, Run: logsOwnErrors(d.reconcilePRs)},
|
||||
{ID: "pr-review-scan", Name: "Scan pull request reviews", DefaultInterval: time.Minute, Run: logsOwnErrors(d.scanPRReviews)},
|
||||
{ID: "issue-discovery", Name: "Discover forge issues", DefaultInterval: time.Minute, Run: d.poll},
|
||||
{ID: "pr-reconcile", Name: "Reconcile pull requests", DefaultInterval: time.Minute, Run: d.reconcilePRs},
|
||||
{ID: "pr-review-scan", Name: "Scan pull request reviews", DefaultInterval: time.Minute, Run: d.scanPRReviews},
|
||||
{ID: "mirror-prune", Name: "Prune repository mirrors", DefaultInterval: 24 * time.Hour, Run: d.pruneMirrors},
|
||||
}
|
||||
}
|
||||
|
||||
// logsOwnErrors adapts a pass that reports its per-item failures itself.
|
||||
func logsOwnErrors(pass func(context.Context)) func(context.Context) error {
|
||||
return func(ctx context.Context) error {
|
||||
pass(ctx)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Daemon) pruneMirrors(ctx context.Context) error {
|
||||
removed, err := worktree.PruneMirrors(ctx, d.Cfg.Get().WorkDir, mirrorIdle, time.Now())
|
||||
for _, mirror := range removed {
|
||||
|
||||
@@ -1190,8 +1190,13 @@ func (d *taskDispatcher) Wait() {
|
||||
// poll discovers work through every organisation connection, resolved
|
||||
// afresh each cycle so a rotated credential or a new connection applies to
|
||||
// the next poll.
|
||||
func (d *Daemon) poll(ctx context.Context) {
|
||||
func (d *Daemon) poll(ctx context.Context) error {
|
||||
d.markPoll("")
|
||||
// Webhook intake replaces polling; an empty pass is not a failure.
|
||||
if d.Cfg.Get().Forge.Intake == config.ForgeIntakeWebhook {
|
||||
return nil
|
||||
}
|
||||
var failed []string
|
||||
for _, source := range d.issueSources(ctx) {
|
||||
if !d.identityActive(ctx, source.Identity) {
|
||||
continue
|
||||
@@ -1203,6 +1208,8 @@ func (d *Daemon) poll(ctx context.Context) {
|
||||
issues, complete := d.pollIssuesWithConfig(ctx, source.Forge, cfg, repo)
|
||||
if complete {
|
||||
d.withdrawUnpolled(ctx, repo, tag, dispatchRule(cfg), issues)
|
||||
} else {
|
||||
failed = append(failed, repo.FullName())
|
||||
}
|
||||
for _, is := range issues {
|
||||
if d.Tasks != nil {
|
||||
@@ -1211,6 +1218,16 @@ func (d *Daemon) poll(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
}
|
||||
return passFailed("issue poll", failed)
|
||||
}
|
||||
|
||||
// passFailed reports a maintenance pass whose items failed; each item's cause
|
||||
// is already logged where it happened.
|
||||
func passFailed(pass string, failed []string) error {
|
||||
if len(failed) == 0 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s failed for %d item(s): %s", pass, len(failed), strings.Join(failed, ", "))
|
||||
}
|
||||
|
||||
// polledIssues is one repo's eligible issues under one dispatch rule.
|
||||
@@ -1482,12 +1499,12 @@ func (d *Daemon) closeResolvedIssue(ctx context.Context, fg forge.Forge, task *w
|
||||
|
||||
// reconcilePRs moves pr_open tasks to merged/rejected from GitHub state
|
||||
// and cleans up their worktrees.
|
||||
func (d *Daemon) reconcilePRs(ctx context.Context) {
|
||||
func (d *Daemon) reconcilePRs(ctx context.Context) error {
|
||||
tasks, err := d.Store.OpenPRs(ctx)
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "reconcile query failed", "err", err)
|
||||
return
|
||||
return fmt.Errorf("reconcile query: %w", err)
|
||||
}
|
||||
var failed []string
|
||||
for _, t := range tasks {
|
||||
fg := d.forgeFor(ctx, &t)
|
||||
if _, unavailable := fg.(*forge.NoopForge); unavailable {
|
||||
@@ -1501,6 +1518,7 @@ func (d *Daemon) reconcilePRs(ctx context.Context) {
|
||||
state, err := fg.PRState(ctx, t.Owner, t.Repo, t.PRNumber)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "PR state check failed", "pr", t.PRNumber, "err", err)
|
||||
failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber))
|
||||
continue
|
||||
}
|
||||
attempt := t.Attempt
|
||||
@@ -1508,6 +1526,7 @@ func (d *Daemon) reconcilePRs(ctx context.Context) {
|
||||
case "merged":
|
||||
if err := d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusMerged, ""); err != nil {
|
||||
d.Log.ErrorContext(ctx, "PR merged but task transition failed", "task", t.ID, "pr", t.PRNumber, "err", err)
|
||||
failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber))
|
||||
continue
|
||||
}
|
||||
if err := trees.Cleanup(t.Owner, t.Repo, t.IssueNumber); err != nil {
|
||||
@@ -1526,6 +1545,7 @@ func (d *Daemon) reconcilePRs(ctx context.Context) {
|
||||
case "closed":
|
||||
if err := d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusRejected, "PR closed without merge"); err != nil {
|
||||
d.Log.ErrorContext(ctx, "PR rejected but task transition failed", "task", t.ID, "pr", t.PRNumber, "err", err)
|
||||
failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber))
|
||||
continue
|
||||
}
|
||||
if err := trees.Cleanup(t.Owner, t.Repo, t.IssueNumber); err != nil {
|
||||
@@ -1539,6 +1559,7 @@ func (d *Daemon) reconcilePRs(ctx context.Context) {
|
||||
})
|
||||
}
|
||||
}
|
||||
return passFailed("PR reconcile", failed)
|
||||
}
|
||||
|
||||
func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHold) {
|
||||
|
||||
@@ -79,15 +79,15 @@ func scanPRReviewReactions(
|
||||
|
||||
// scanPRReviews scans every open PR task for review activity with its
|
||||
// identity's forge. Forges without review reading are skipped.
|
||||
func (d *Daemon) scanPRReviews(ctx context.Context) {
|
||||
func (d *Daemon) scanPRReviews(ctx context.Context) error {
|
||||
if d.Store == nil || d.Tasks == nil {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
tasks, err := d.Store.OpenPRs(ctx)
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "review scan query failed", "err", err)
|
||||
return
|
||||
return fmt.Errorf("review scan query: %w", err)
|
||||
}
|
||||
var failed []string
|
||||
if d.reactionPublisher == nil {
|
||||
d.reactionPublisher = d.PublishReaction
|
||||
}
|
||||
@@ -97,17 +97,21 @@ func (d *Daemon) scanPRReviews(ctx context.Context) {
|
||||
if !ok {
|
||||
continue // the noop forge and any forge without the capability
|
||||
}
|
||||
d.scanOnePR(ctx, reader, t)
|
||||
if !d.scanOnePR(ctx, reader, t) {
|
||||
failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber))
|
||||
}
|
||||
}
|
||||
return passFailed("review scan", failed)
|
||||
}
|
||||
|
||||
func (d *Daemon) scanOnePR(ctx context.Context, reader forge.PullRequestReviewReader, t *workflowtask.Task) {
|
||||
// scanOnePR reports false when the scan failed; the cause is logged here.
|
||||
func (d *Daemon) scanOnePR(ctx context.Context, reader forge.PullRequestReviewReader, t *workflowtask.Task) bool {
|
||||
// Resolve the org; on failure skip this task for now.
|
||||
if t.Org == "" {
|
||||
resolved, err := d.identityOrg(ctx, d.publisherIdentity(t.Identity))
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "review scan org resolve failed", "task", t.ID, "err", err)
|
||||
return
|
||||
return false
|
||||
}
|
||||
t.Org = resolved
|
||||
}
|
||||
@@ -119,15 +123,17 @@ func (d *Daemon) scanOnePR(ctx context.Context, reader forge.PullRequestReviewRe
|
||||
t.ReviewCursor, t.WatchCommentID, d.reactionPublisher)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "review scan failed", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber, "err", err)
|
||||
return
|
||||
return false
|
||||
}
|
||||
if reviewCursor == t.ReviewCursor && commentCursor == t.WatchCommentID {
|
||||
return
|
||||
return true
|
||||
}
|
||||
if err := d.Store.SetReviewCursors(ctx, t.ID, reviewCursor, commentCursor); err != nil {
|
||||
// A stale guard here means the task was claimed between the scan and
|
||||
// the write; the next scan re-reads and re-lists from the old cursor,
|
||||
// and PublishUnique plus the consumer's dedup absorb the repeats.
|
||||
d.Log.WarnContext(ctx, "review cursor persist failed", "task", t.ID, "err", err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
Reference in new issue
Block a user