Compare commits

...
3 Commits
Author SHA1 Message Date
sam bca1f3b412 chore(release): v1.58.1
Quality / check (push) Successful in 3m43s
deploy / gate (push) Successful in 4m36s
deploy / archied-image (push) Successful in 2m52s
deploy / agent-image (push) Successful in 3m18s
deploy / dist-zip (push) Successful in 2m37s
2026-11-02 00:27:14 +11:00
sam 3af96fd818 fix(daemon): watch the task intake loop too and name the stalled loop
Quality / check (push) Canceled after 26s
deploy / gate (push) Successful in 4m30s
deploy / archied-image (push) Successful in 2m50s
deploy / agent-image (push) Successful in 12s
deploy / dist-zip (push) Skipped
2026-11-02 00:26:50 +11:00
sam 1043982529 fix(daemon): stamp each run-loop cycle so the systemd watchdog keeps beating
Quality / check (push) Successful in 3m37s
deploy / gate (push) Successful in 4m27s
deploy / archied-image (push) Successful in 2m54s
deploy / agent-image (push) Successful in 11s
deploy / dist-zip (push) Skipped
2026-11-02 00:22:24 +11:00
7 changed files with 57 additions and 24 deletions

No files matched your search

+7
View File
@@ -5,6 +5,13 @@ release's per-component sections are labelled beneath its heading.
## [Unreleased]
## [1.58.1] - 2026-11-01
### archied
- The systemd watchdog now also watches the task intake loop, and its warning names the loop that stalled.
- archied no longer restarts about a minute after it starts. Under systemd, the watchdog never saw a poll pass and killed it on every start since 1.56.0.
## [1.58.0] - 2026-11-01
### archied
+2
View File
@@ -87,6 +87,7 @@
"/news/archie/1.56.0/": "/news/1.56.0/",
"/news/archie/1.57.0/": "/news/1.57.0/",
"/news/archie/1.58.0/": "/news/1.58.0/",
"/news/archie/1.58.1/": "/news/1.58.1/",
"/news/archie/1.6.0/": "/news/1.6.0/",
"/news/archie/1.7.0/": "/news/1.7.0/",
"/news/archie/1.8.0/": "/news/1.8.0/",
@@ -190,6 +191,7 @@
"/news/archied/1.56.0/": "/news/1.56.0/",
"/news/archied/1.57.0/": "/news/1.57.0/",
"/news/archied/1.58.0/": "/news/1.58.0/",
"/news/archied/1.58.1/": "/news/1.58.1/",
"/news/archied/1.6.0/": "/news/1.6.0/",
"/news/archied/1.7.0/": "/news/1.7.0/",
"/news/archied/1.8.0/": "/news/1.8.0/",
+5
View File
@@ -1,4 +1,9 @@
[
{
"version": "1.58.1",
"date": "2026-11-01",
"body": "### archied\n\n- The systemd watchdog now also watches the task intake loop, and its warning names the loop that stalled.\n- archied no longer restarts about a minute after it starts. Under systemd, the watchdog never saw a poll pass and killed it on every start since 1.56.0."
},
{
"version": "1.58.0",
"date": "2026-11-01",
+2 -2
View File
@@ -7,7 +7,7 @@
// because archie-messaging, a separate process that cannot import this
// package, is the next process that must be able to announce itself.
// What stays here is what is the daemon's own: which
// marker counts as progress (daemon.LastPollAt) and how long the loop may go
// marker counts as progress (daemon.StalestLoop) and how long a loop may go
// without beginning a poll pass.
//
// Nothing here is fatal -- a process with no NOTIFY_SOCKET (a plain terminal,
@@ -53,7 +53,7 @@ func (b *boot) startWatchdog(ctx context.Context) {
sampler := sdnotify.Watchdog{
Interval: interval,
Cadence: b.loopCadence,
Progress: b.d.LastPollAt,
Progress: b.d.StalestLoop,
Beat: func() { n.Send(sdnotify.WatchdogState) },
Log: b.log,
}
+16
View File
@@ -0,0 +1,16 @@
package daemon
import (
"context"
"testing"
)
// The systemd watchdog beats only while StalestLoop is fresh, so a cycle that
// doesn't stamp it gets archied killed shortly after every start.
func TestCycleMarksPass(t *testing.T) {
d := &Daemon{}
d.Cycle(context.Background())
if loop, at := d.StalestLoop(); loop != "cycle" || at.IsZero() {
t.Fatalf("StalestLoop() = %q, %v; want a cycle pass", loop, at)
}
}
+20 -17
View File
@@ -240,10 +240,10 @@ type Daemon struct {
// to use.
running runningTasks
// lastPollAt maps each poll loop (an identity ID, or "" for the root
// loop) to when its latest pass began, as Unix nanoseconds. LastPollAt
// reports the oldest, so one wedged loop is not hidden by a healthy one.
lastPollAt sync.Map
// loopPasses maps each run loop ("cycle", "drain") to when its latest
// pass began, as Unix nanoseconds. StalestLoop reports the oldest, so one
// wedged loop isn't hidden by a healthy one.
loopPasses sync.Map
}
// IssueSource is one forge connection's repositories as the identity
@@ -389,6 +389,7 @@ func (d *Daemon) drainLoop(ctx context.Context) {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
d.markPass("drain")
d.drainNATS(ctx)
select {
case <-ctx.Done():
@@ -489,6 +490,7 @@ func (d *Daemon) RunOnce(ctx context.Context) {
// tasks; maintenance runs as built-in schedules. Draining queued tasks is the caller's job
// (drainLoop in Run, drainNATS in RunOnce) so there is one drainer.
func (d *Daemon) Cycle(ctx context.Context) {
d.markPass("cycle")
d.dispatchBindings(ctx)
d.drainReactions(ctx)
}
@@ -1205,7 +1207,6 @@ func (d *taskDispatcher) Wait() {
// afresh each cycle so a rotated credential or a new connection applies to
// the next poll.
func (d *Daemon) poll(ctx context.Context) error {
d.markPoll("")
// Webhook intake replaces polling; an empty pass is not a failure.
if d.Cfg.Get().Forge.Intake == config.ForgeIntakeWebhook {
return nil
@@ -2603,26 +2604,28 @@ func (d *Daemon) allowConcurrentForTask(task *workflow.Task) bool {
return ok && repo.AllowConcurrent
}
// LastPollAt reports when the least recently polled loop last began a pass,
// or the zero time when no pass has started yet.
func (d *Daemon) LastPollAt() time.Time {
// StalestLoop reports the run loop that least recently began a pass and when,
// or the zero time when no loop has started yet. The systemd watchdog beats
// only while this is fresh, so every loop Run starts must mark its passes.
func (d *Daemon) StalestLoop() (string, time.Time) {
var loop string
var oldest int64
d.lastPollAt.Range(func(_, v any) bool {
if ns, ok := v.(int64); ok && (oldest == 0 || ns < oldest) {
oldest = ns
d.loopPasses.Range(func(k, v any) bool {
name, okName := k.(string)
if ns, ok := v.(int64); ok && okName && (oldest == 0 || ns < oldest) {
loop, oldest = name, ns
}
return true
})
if oldest == 0 {
return time.Time{}
return "", time.Time{}
}
return time.Unix(0, oldest).UTC()
return loop, time.Unix(0, oldest).UTC()
}
// markPoll stamps the start of a poll pass for one loop, so a hung pass shows
// as stale.
func (d *Daemon) markPoll(loop string) {
d.lastPollAt.Store(loop, time.Now().UnixNano())
// markPass stamps the start of one loop's pass, so a hung pass shows as stale.
func (d *Daemon) markPass(loop string) {
d.loopPasses.Store(loop, time.Now().UnixNano())
}
// errContainerExited is the cancellation cause when a task's agent container
+5 -5
View File
@@ -114,9 +114,9 @@ type Watchdog struct {
Interval time.Duration
// Cadence reports how long the loop may go between passes.
Cadence func() time.Duration
// Progress reports the start of the loop's most recent pass, or the zero
// time when no pass has begun yet.
Progress func() time.Time
// Progress reports the stalest loop and the start of its most recent
// pass, or the zero time when no pass has begun yet.
Progress func() (string, time.Time)
// Beat emits one WATCHDOG=1.
Beat func()
Log *slog.Logger
@@ -138,7 +138,7 @@ func (w Watchdog) Run(ctx context.Context) error {
case <-ticker.C:
}
last := w.Progress()
loop, last := w.Progress()
if last.IsZero() {
// No pass has begun at all. Measure staleness from arming, so a
// loop that never starts polling is caught like one that stopped.
@@ -149,7 +149,7 @@ func (w Watchdog) Run(ctx context.Context) error {
case age > cadence:
if !stalled {
w.Log.WarnContext(ctx, "systemd watchdog: run loop has not begun a poll pass within its interval; withholding the heartbeat until it does",
"stale_for", age.String(), "loop_cadence", cadence.String())
"loop", loop, "stale_for", age.String(), "loop_cadence", cadence.String())
stalled = true
}
default: