175 lines
5.4 KiB
Go
175 lines
5.4 KiB
Go
package daemon
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/config"
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
"github.com/samcharles93/archie-core/internal/forge"
|
|
)
|
|
|
|
// Startup runs crash recovery and access verification once.
|
|
func (d *Daemon) Startup(ctx context.Context) error {
|
|
if err := d.cleanupExpiredStorage(ctx); err != nil {
|
|
d.Log.WarnContext(ctx, "startup volume cleanup failed", "err", err)
|
|
}
|
|
n, err := d.Store.RecoverStale(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n > 0 {
|
|
d.Log.InfoContext(ctx, "re-queued tasks left running by a previous daemon", "count", n)
|
|
}
|
|
|
|
d.sweepAccess(ctx)
|
|
return nil
|
|
}
|
|
|
|
// sweepAccess accepts pending invitations and verifies push access through
|
|
// every organisation connection, each in its own goroutine.
|
|
func (d *Daemon) sweepAccess(ctx context.Context) {
|
|
sweep := func(source IssueSource) {
|
|
log := d.Log.With("identity", source.Identity)
|
|
ctx, cancel := context.WithTimeout(ctx, sweepTimeout)
|
|
defer cancel()
|
|
if err := source.Forge.AcceptInvitations(ctx, repoRefs(source.Repos)); err != nil {
|
|
log.WarnContext(ctx, "invitation sweep failed", "err", err)
|
|
}
|
|
for _, r := range source.Repos {
|
|
if err := source.Forge.VerifyPush(ctx, r.Owner, r.Name); err != nil {
|
|
// A sweep cut short by its own context is not a repo failure: name
|
|
// the budget when it expired, stay quiet on a shutdown.
|
|
switch {
|
|
case errors.Is(ctx.Err(), context.DeadlineExceeded):
|
|
// Every later call would return the same error, so say it once.
|
|
log.WarnContext(ctx, "push check abandoned: sweep budget exhausted", "repo", r.FullName(), "err", err)
|
|
return
|
|
case ctx.Err() != nil:
|
|
return
|
|
}
|
|
log.WarnContext(ctx, "repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
for _, source := range d.IssuePoller.issueSources(ctx) {
|
|
wg.Go(func() { sweep(source) })
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
// repoRefs is a source's configured repositories as the allowlist the forge
|
|
// accepts invitations against.
|
|
func repoRefs(repos []config.Repo) []forge.RepoRef {
|
|
out := make([]forge.RepoRef, 0, len(repos))
|
|
for _, r := range repos {
|
|
out = append(out, forge.RepoRef{Owner: r.Owner, Repo: r.Name})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// sweepTimeout bounds one forge's boot sweep. Startup gates the gateways and
|
|
// the run loop, and a forge client carries no request deadline of its own, so
|
|
// an unreachable host would otherwise hold boot open indefinitely.
|
|
const sweepTimeout = 2 * time.Minute
|
|
|
|
// Run polls until the context ends.
|
|
func (d *Daemon) Run(ctx context.Context) error {
|
|
// A pass no longer waits for the tasks it submitted, so the run does, once
|
|
// the ctx is cancelled.
|
|
defer d.TaskBus.WaitForTasks()
|
|
d.watchCallWaits(ctx)
|
|
drained := make(chan struct{})
|
|
defer func() { <-drained }()
|
|
go func() {
|
|
defer close(drained)
|
|
d.TaskBus.drainLoop(ctx, d.BindingDispatch.dispatchBindings)
|
|
}()
|
|
interval := d.Cfg.Get().PollInterval.Std()
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
d.Cycle(ctx)
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-ticker.C:
|
|
// PollInterval may have been reloaded; reset the ticker so a
|
|
// change takes effect on the next cycle.
|
|
if iv := d.Cfg.Get().PollInterval.Std(); iv != interval {
|
|
interval = iv
|
|
ticker.Reset(iv)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// RunOnce runs one cycle and drains until no task is in flight. A run waiting
|
|
// on a callee stays in flight while the callee sits queued, so the queue is
|
|
// drained again until every task, callees included, has returned.
|
|
func (d *Daemon) RunOnce(ctx context.Context) {
|
|
defer d.TaskBus.WaitForTasks()
|
|
d.watchCallWaits(ctx)
|
|
// One pass has no scheduler, so it runs the maintenance jobs itself.
|
|
for _, job := range d.BuiltinJobs() {
|
|
if err := job.Run(ctx); err != nil {
|
|
d.Log.WarnContext(ctx, "built-in job failed", "job", job.ID, "err", err)
|
|
}
|
|
}
|
|
d.BindingDispatch.dispatchBindings(ctx)
|
|
d.Cycle(ctx)
|
|
d.TaskBus.drainNATS(ctx)
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
for d.TaskBus.InFlight() > 0 {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
d.TaskBus.drainNATS(ctx)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Cycle is one poll pass: turn review reactions into tasks; maintenance runs
|
|
// as built-in schedules. Captured events are dispatched by the caller
|
|
// (drainLoop in Run, once in RunOnce) so they start within about a second.
|
|
func (d *Daemon) Cycle(ctx context.Context) {
|
|
d.markPass("cycle")
|
|
d.ReviewScanner.drainReactions(ctx)
|
|
d.retryTransientParks(ctx)
|
|
}
|
|
|
|
// markPass stamps the start of one loop's pass, so a hung pass shows as stale.
|
|
func (d *Daemon) markPass(loop string) {
|
|
d.loopPasses.Store(loop, time.Now().UnixNano())
|
|
}
|
|
|
|
// StalestLoop reports the run loop that least recently began a pass and when,
|
|
// or the zero time when no loop has started yet. The systemd watchdog beats
|
|
// only while this is fresh, so every loop Run starts must mark its passes.
|
|
func (d *Daemon) StalestLoop() (string, time.Time) {
|
|
var loop string
|
|
var oldest int64
|
|
d.loopPasses.Range(func(k, v any) bool {
|
|
name, okName := k.(string)
|
|
if ns, ok := v.(int64); ok && okName && (oldest == 0 || ns < oldest) {
|
|
loop, oldest = name, ns
|
|
}
|
|
return true
|
|
})
|
|
if oldest == 0 {
|
|
return "", time.Time{}
|
|
}
|
|
return loop, time.Unix(0, oldest).UTC()
|
|
}
|
|
|
|
// emit publishes an observability event.
|
|
func (d *Daemon) emit(ctx context.Context, e events.Event) {
|
|
d.Bus.Publish(ctx, e)
|
|
}
|