Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bca1f3b412 | ||
|
|
3af96fd818
|
||
|
|
1043982529
|
No files matched your search
@@ -5,6 +5,13 @@ release's per-component sections are labelled beneath its heading.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [1.58.1] - 2026-11-01
|
||||
|
||||
### archied
|
||||
|
||||
- The systemd watchdog now also watches the task intake loop, and its warning names the loop that stalled.
|
||||
- archied no longer restarts about a minute after it starts. Under systemd, the watchdog never saw a poll pass and killed it on every start since 1.56.0.
|
||||
|
||||
## [1.58.0] - 2026-11-01
|
||||
|
||||
### archied
|
||||
|
||||
@@ -87,6 +87,7 @@
|
||||
"/news/archie/1.56.0/": "/news/1.56.0/",
|
||||
"/news/archie/1.57.0/": "/news/1.57.0/",
|
||||
"/news/archie/1.58.0/": "/news/1.58.0/",
|
||||
"/news/archie/1.58.1/": "/news/1.58.1/",
|
||||
"/news/archie/1.6.0/": "/news/1.6.0/",
|
||||
"/news/archie/1.7.0/": "/news/1.7.0/",
|
||||
"/news/archie/1.8.0/": "/news/1.8.0/",
|
||||
@@ -190,6 +191,7 @@
|
||||
"/news/archied/1.56.0/": "/news/1.56.0/",
|
||||
"/news/archied/1.57.0/": "/news/1.57.0/",
|
||||
"/news/archied/1.58.0/": "/news/1.58.0/",
|
||||
"/news/archied/1.58.1/": "/news/1.58.1/",
|
||||
"/news/archied/1.6.0/": "/news/1.6.0/",
|
||||
"/news/archied/1.7.0/": "/news/1.7.0/",
|
||||
"/news/archied/1.8.0/": "/news/1.8.0/",
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
[
|
||||
{
|
||||
"version": "1.58.1",
|
||||
"date": "2026-11-01",
|
||||
"body": "### archied\n\n- The systemd watchdog now also watches the task intake loop, and its warning names the loop that stalled.\n- archied no longer restarts about a minute after it starts. Under systemd, the watchdog never saw a poll pass and killed it on every start since 1.56.0."
|
||||
},
|
||||
{
|
||||
"version": "1.58.0",
|
||||
"date": "2026-11-01",
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
// because archie-messaging, a separate process that cannot import this
|
||||
// package, is the next process that must be able to announce itself.
|
||||
// What stays here is what is the daemon's own: which
|
||||
// marker counts as progress (daemon.LastPollAt) and how long the loop may go
|
||||
// marker counts as progress (daemon.StalestLoop) and how long a loop may go
|
||||
// without beginning a poll pass.
|
||||
//
|
||||
// Nothing here is fatal -- a process with no NOTIFY_SOCKET (a plain terminal,
|
||||
@@ -53,7 +53,7 @@ func (b *boot) startWatchdog(ctx context.Context) {
|
||||
sampler := sdnotify.Watchdog{
|
||||
Interval: interval,
|
||||
Cadence: b.loopCadence,
|
||||
Progress: b.d.LastPollAt,
|
||||
Progress: b.d.StalestLoop,
|
||||
Beat: func() { n.Send(sdnotify.WatchdogState) },
|
||||
Log: b.log,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The systemd watchdog beats only while StalestLoop is fresh, so a cycle that
|
||||
// doesn't stamp it gets archied killed shortly after every start.
|
||||
func TestCycleMarksPass(t *testing.T) {
|
||||
d := &Daemon{}
|
||||
d.Cycle(context.Background())
|
||||
if loop, at := d.StalestLoop(); loop != "cycle" || at.IsZero() {
|
||||
t.Fatalf("StalestLoop() = %q, %v; want a cycle pass", loop, at)
|
||||
}
|
||||
}
|
||||
+20
-17
@@ -240,10 +240,10 @@ type Daemon struct {
|
||||
// to use.
|
||||
running runningTasks
|
||||
|
||||
// lastPollAt maps each poll loop (an identity ID, or "" for the root
|
||||
// loop) to when its latest pass began, as Unix nanoseconds. LastPollAt
|
||||
// reports the oldest, so one wedged loop is not hidden by a healthy one.
|
||||
lastPollAt sync.Map
|
||||
// loopPasses maps each run loop ("cycle", "drain") to when its latest
|
||||
// pass began, as Unix nanoseconds. StalestLoop reports the oldest, so one
|
||||
// wedged loop isn't hidden by a healthy one.
|
||||
loopPasses sync.Map
|
||||
}
|
||||
|
||||
// IssueSource is one forge connection's repositories as the identity
|
||||
@@ -389,6 +389,7 @@ func (d *Daemon) drainLoop(ctx context.Context) {
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
d.markPass("drain")
|
||||
d.drainNATS(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -489,6 +490,7 @@ func (d *Daemon) RunOnce(ctx context.Context) {
|
||||
// tasks; maintenance runs as built-in schedules. Draining queued tasks is the caller's job
|
||||
// (drainLoop in Run, drainNATS in RunOnce) so there is one drainer.
|
||||
func (d *Daemon) Cycle(ctx context.Context) {
|
||||
d.markPass("cycle")
|
||||
d.dispatchBindings(ctx)
|
||||
d.drainReactions(ctx)
|
||||
}
|
||||
@@ -1205,7 +1207,6 @@ func (d *taskDispatcher) Wait() {
|
||||
// afresh each cycle so a rotated credential or a new connection applies to
|
||||
// the next poll.
|
||||
func (d *Daemon) poll(ctx context.Context) error {
|
||||
d.markPoll("")
|
||||
// Webhook intake replaces polling; an empty pass is not a failure.
|
||||
if d.Cfg.Get().Forge.Intake == config.ForgeIntakeWebhook {
|
||||
return nil
|
||||
@@ -2603,26 +2604,28 @@ func (d *Daemon) allowConcurrentForTask(task *workflow.Task) bool {
|
||||
return ok && repo.AllowConcurrent
|
||||
}
|
||||
|
||||
// LastPollAt reports when the least recently polled loop last began a pass,
|
||||
// or the zero time when no pass has started yet.
|
||||
func (d *Daemon) LastPollAt() time.Time {
|
||||
// StalestLoop reports the run loop that least recently began a pass and when,
|
||||
// or the zero time when no loop has started yet. The systemd watchdog beats
|
||||
// only while this is fresh, so every loop Run starts must mark its passes.
|
||||
func (d *Daemon) StalestLoop() (string, time.Time) {
|
||||
var loop string
|
||||
var oldest int64
|
||||
d.lastPollAt.Range(func(_, v any) bool {
|
||||
if ns, ok := v.(int64); ok && (oldest == 0 || ns < oldest) {
|
||||
oldest = ns
|
||||
d.loopPasses.Range(func(k, v any) bool {
|
||||
name, okName := k.(string)
|
||||
if ns, ok := v.(int64); ok && okName && (oldest == 0 || ns < oldest) {
|
||||
loop, oldest = name, ns
|
||||
}
|
||||
return true
|
||||
})
|
||||
if oldest == 0 {
|
||||
return time.Time{}
|
||||
return "", time.Time{}
|
||||
}
|
||||
return time.Unix(0, oldest).UTC()
|
||||
return loop, time.Unix(0, oldest).UTC()
|
||||
}
|
||||
|
||||
// markPoll stamps the start of a poll pass for one loop, so a hung pass shows
|
||||
// as stale.
|
||||
func (d *Daemon) markPoll(loop string) {
|
||||
d.lastPollAt.Store(loop, time.Now().UnixNano())
|
||||
// markPass stamps the start of one loop's pass, so a hung pass shows as stale.
|
||||
func (d *Daemon) markPass(loop string) {
|
||||
d.loopPasses.Store(loop, time.Now().UnixNano())
|
||||
}
|
||||
|
||||
// errContainerExited is the cancellation cause when a task's agent container
|
||||
|
||||
@@ -114,9 +114,9 @@ type Watchdog struct {
|
||||
Interval time.Duration
|
||||
// Cadence reports how long the loop may go between passes.
|
||||
Cadence func() time.Duration
|
||||
// Progress reports the start of the loop's most recent pass, or the zero
|
||||
// time when no pass has begun yet.
|
||||
Progress func() time.Time
|
||||
// Progress reports the stalest loop and the start of its most recent
|
||||
// pass, or the zero time when no pass has begun yet.
|
||||
Progress func() (string, time.Time)
|
||||
// Beat emits one WATCHDOG=1.
|
||||
Beat func()
|
||||
Log *slog.Logger
|
||||
@@ -138,7 +138,7 @@ func (w Watchdog) Run(ctx context.Context) error {
|
||||
case <-ticker.C:
|
||||
}
|
||||
|
||||
last := w.Progress()
|
||||
loop, last := w.Progress()
|
||||
if last.IsZero() {
|
||||
// No pass has begun at all. Measure staleness from arming, so a
|
||||
// loop that never starts polling is caught like one that stopped.
|
||||
@@ -149,7 +149,7 @@ func (w Watchdog) Run(ctx context.Context) error {
|
||||
case age > cadence:
|
||||
if !stalled {
|
||||
w.Log.WarnContext(ctx, "systemd watchdog: run loop has not begun a poll pass within its interval; withholding the heartbeat until it does",
|
||||
"stale_for", age.String(), "loop_cadence", cadence.String())
|
||||
"loop", loop, "stale_for", age.String(), "loop_cadence", cadence.String())
|
||||
stalled = true
|
||||
}
|
||||
default:
|
||||
|
||||
Reference in new issue
Block a user