158 lines
6.5 KiB
Go
158 lines
6.5 KiB
Go
package daemon
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"log/slog"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
|
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
"github.com/samcharles93/archie-core/internal/logging"
|
|
"github.com/samcharles93/archie-core/internal/taskstate"
|
|
)
|
|
|
|
// parkCapabilityUnavailable parks a task whose daemon-side capability is
|
|
// missing (container pool or task transport): both are environmental, so
|
|
// the park class is transient and the KindParked event carries the class
|
|
// for the dashboard. The reason distinguishes the two in logs and events.
|
|
func (d *Daemon) parkCapabilityUnavailable(ctx context.Context, task *workflow.Task) {
|
|
var reason string
|
|
if d.Runtime == nil {
|
|
reason = "managed agent container pool is unavailable; refusing to run this task on the host"
|
|
d.Log.ErrorContext(ctx, "task parked: managed worker unavailable", "task", task.ID,
|
|
"hint", "enable containers and make the archie-agent image available, then retry the task")
|
|
} else {
|
|
reason = "agent task transport is unavailable; refusing to run this task on the host"
|
|
d.Log.ErrorContext(ctx, "task parked: agent task transport unavailable", "task", task.ID)
|
|
}
|
|
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
|
|
}
|
|
|
|
// parkRunningTask moves a running task to parked with class, using a
|
|
// bounded context that ignores cancellation. ErrStaleTransition is ignored;
|
|
// other errors are logged.
|
|
func (d *Daemon) parkRunningTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass) {
|
|
d.parkBlockedTask(ctx, task, reason, class, "")
|
|
}
|
|
|
|
// parkBlockedTask is parkRunningTask recording the blocker the park site
|
|
// knows, so the dashboard can link to its fix.
|
|
func (d *Daemon) parkBlockedTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass, blocker taskstate.Blocker) {
|
|
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer cancel()
|
|
|
|
if err := d.Store.ParkTask(writeCtx, task.ID, workflow.StatusRunning, reason, class, blocker); err != nil {
|
|
if !errors.Is(err, storecontract.ErrStaleTransition) {
|
|
d.Log.WarnContext(ctx, "terminal park transition failed", "task", task.ID, "reason", reason, "err", err)
|
|
}
|
|
return
|
|
}
|
|
// Recorded only once the transition landed. An entry claiming the task
|
|
// parked would be a lie when another actor moved it out of running first
|
|
// -- the same race the parked event is withheld for.
|
|
d.recordPark(ctx, task.ID, reason)
|
|
d.emit(ctx, events.Event{
|
|
Kind: events.KindParked, TaskID: task.ID, Attempt: task.Attempt,
|
|
Repo: strings.Trim(task.Owner+"/"+task.Repo, "/"), Issue: task.IssueNumber, Workflow: task.Workflow,
|
|
Detail: reason, Data: map[string]any{"park_class": class},
|
|
})
|
|
}
|
|
|
|
// recordPark writes a park reason to the attempt's log. Dropped when no sink
|
|
// is open.
|
|
func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) {
|
|
d.TaskLogs.Write(ctx, taskID, logging.Entry{
|
|
Time: time.Now(),
|
|
Level: slog.LevelError.String(),
|
|
Message: "task parked: " + reason,
|
|
Fields: map[string]any{"component": "daemon", "task": taskID},
|
|
})
|
|
}
|
|
|
|
// parkIfIdentityUnresolvable parks task when its identity cannot be acted on,
|
|
// reporting whether it did.
|
|
func (d *Daemon) parkIfIdentityUnresolvable(ctx context.Context, task *workflow.Task) bool {
|
|
ok, reason, class := d.Identity.identityMayAct(ctx, task.Identity)
|
|
if ok {
|
|
return false
|
|
}
|
|
d.parkRunningTask(ctx, task, reason, class)
|
|
return true
|
|
}
|
|
|
|
// retryTransientParks requeues tasks parked for an environmental cause once
|
|
// their backoff has passed. The store parks a task needs_decision, which
|
|
// raises the alert, when its retries run out, so this never loops on one.
|
|
func (d *Daemon) retryTransientParks(ctx context.Context) {
|
|
now := time.Now()
|
|
var after storecontract.TaskCursor
|
|
for {
|
|
page, err := d.Store.TasksPage(ctx, storecontract.TaskPage{Statuses: []string{workflow.StatusParked}, After: after, Limit: 200})
|
|
if err != nil {
|
|
d.Log.WarnContext(ctx, "list parked tasks for retry failed", "err", err)
|
|
return
|
|
}
|
|
for _, t := range page {
|
|
if taskstate.TransientRetryDue(t.ParkClass, t.RetryCount, t.UpdatedAt, now) {
|
|
d.retryTransientPark(ctx, t.ID)
|
|
}
|
|
}
|
|
if len(page) < 200 {
|
|
return
|
|
}
|
|
last := page[len(page)-1]
|
|
after = storecontract.TaskCursor{UpdatedAt: last.UpdatedAt, ID: last.ID}
|
|
}
|
|
}
|
|
|
|
// retryTransientPark requeues one task on its stored retry mode, which the
|
|
// list projection omits: a remediation must keep continuing its branch.
|
|
func (d *Daemon) retryTransientPark(ctx context.Context, id int64) {
|
|
t, err := d.Store.TaskByID(ctx, id)
|
|
if err != nil {
|
|
d.Log.WarnContext(ctx, "load task for transient retry failed", "task", id, "err", err)
|
|
return
|
|
}
|
|
if err := d.Store.RetryTask(ctx, id, workflow.StatusParked, "", t.RetryMode, workflowtask.Resume{}, false); err != nil {
|
|
if !errors.Is(err, storecontract.ErrStaleTransition) {
|
|
d.Log.WarnContext(ctx, "transient park retry failed", "task", id, "err", err)
|
|
}
|
|
return
|
|
}
|
|
d.emit(ctx, events.Event{
|
|
Kind: events.KindTaskRetried, TaskID: id, Workflow: t.Workflow,
|
|
Repo: strings.Trim(t.Owner+"/"+t.Repo, "/"), Issue: t.IssueNumber,
|
|
Detail: "retried automatically after a transient park",
|
|
Data: map[string]any{"retry_count": t.RetryCount + 1, "previous_reason": t.ParkReason},
|
|
})
|
|
}
|
|
|
|
// parkRefusedRun parks a task whose workflow identity check did not permit it.
|
|
// A denial is the operator's to fix, but a chain that could not be read is a
|
|
// store that will answer again, so it parks transient rather than demanding an
|
|
// operator for a blip.
|
|
func (d *Daemon) parkRefusedRun(ctx context.Context, task *workflow.Task, reason string, err error) {
|
|
class, blocker := taskstate.ParkNeedsDecision, taskstate.BlockerPolicyDenied
|
|
if err != nil {
|
|
class, blocker = pinParkClass(err), parkBlocker(err)
|
|
}
|
|
d.parkBlockedTask(ctx, task, reason, class, blocker)
|
|
}
|
|
|
|
// openTaskLog opens the task's log sink for one process() call and returns
|
|
// its closer. Failure to open is logged, not fatal.
|
|
func (d *Daemon) openTaskLog(task *workflow.Task) func() {
|
|
if err := d.TaskLogs.Open(task.ID, task.Attempt); err != nil {
|
|
d.Log.Warn("task log sink unavailable", "task", task.ID, "attempt", task.Attempt, "err", err)
|
|
}
|
|
return func() {
|
|
if err := d.TaskLogs.Close(task.ID); err != nil {
|
|
d.Log.Warn("task log sink close failed", "task", task.ID, "err", err)
|
|
}
|
|
}
|
|
}
|