Files

158 lines
6.5 KiB
Go

package daemon
import (
"context"
"errors"
"log/slog"
"strings"
"time"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/logging"
"github.com/samcharles93/archie-core/internal/taskstate"
)
// parkCapabilityUnavailable parks a task whose daemon-side capability is
// missing (container pool or task transport): both are environmental, so
// the park class is transient and the KindParked event carries the class
// for the dashboard. The reason distinguishes the two in logs and events.
func (d *Daemon) parkCapabilityUnavailable(ctx context.Context, task *workflow.Task) {
var reason string
if d.Runtime == nil {
reason = "managed agent container pool is unavailable; refusing to run this task on the host"
d.Log.ErrorContext(ctx, "task parked: managed worker unavailable", "task", task.ID,
"hint", "enable containers and make the archie-agent image available, then retry the task")
} else {
reason = "agent task transport is unavailable; refusing to run this task on the host"
d.Log.ErrorContext(ctx, "task parked: agent task transport unavailable", "task", task.ID)
}
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
}
// parkRunningTask moves a running task to parked with class, using a
// bounded context that ignores cancellation. ErrStaleTransition is ignored;
// other errors are logged.
func (d *Daemon) parkRunningTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass) {
d.parkBlockedTask(ctx, task, reason, class, "")
}
// parkBlockedTask is parkRunningTask recording the blocker the park site
// knows, so the dashboard can link to its fix.
func (d *Daemon) parkBlockedTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass, blocker taskstate.Blocker) {
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if err := d.Store.ParkTask(writeCtx, task.ID, workflow.StatusRunning, reason, class, blocker); err != nil {
if !errors.Is(err, storecontract.ErrStaleTransition) {
d.Log.WarnContext(ctx, "terminal park transition failed", "task", task.ID, "reason", reason, "err", err)
}
return
}
// Recorded only once the transition landed. An entry claiming the task
// parked would be a lie when another actor moved it out of running first
// -- the same race the parked event is withheld for.
d.recordPark(ctx, task.ID, reason)
d.emit(ctx, events.Event{
Kind: events.KindParked, TaskID: task.ID, Attempt: task.Attempt,
Repo: strings.Trim(task.Owner+"/"+task.Repo, "/"), Issue: task.IssueNumber, Workflow: task.Workflow,
Detail: reason, Data: map[string]any{"park_class": class},
})
}
// recordPark writes a park reason to the attempt's log. Dropped when no sink
// is open.
func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) {
d.TaskLogs.Write(ctx, taskID, logging.Entry{
Time: time.Now(),
Level: slog.LevelError.String(),
Message: "task parked: " + reason,
Fields: map[string]any{"component": "daemon", "task": taskID},
})
}
// parkIfIdentityUnresolvable parks task when its identity cannot be acted on,
// reporting whether it did.
func (d *Daemon) parkIfIdentityUnresolvable(ctx context.Context, task *workflow.Task) bool {
ok, reason, class := d.Identity.identityMayAct(ctx, task.Identity)
if ok {
return false
}
d.parkRunningTask(ctx, task, reason, class)
return true
}
// retryTransientParks requeues tasks parked for an environmental cause once
// their backoff has passed. The store parks a task needs_decision, which
// raises the alert, when its retries run out, so this never loops on one.
func (d *Daemon) retryTransientParks(ctx context.Context) {
now := time.Now()
var after storecontract.TaskCursor
for {
page, err := d.Store.TasksPage(ctx, storecontract.TaskPage{Statuses: []string{workflow.StatusParked}, After: after, Limit: 200})
if err != nil {
d.Log.WarnContext(ctx, "list parked tasks for retry failed", "err", err)
return
}
for _, t := range page {
if taskstate.TransientRetryDue(t.ParkClass, t.RetryCount, t.UpdatedAt, now) {
d.retryTransientPark(ctx, t.ID)
}
}
if len(page) < 200 {
return
}
last := page[len(page)-1]
after = storecontract.TaskCursor{UpdatedAt: last.UpdatedAt, ID: last.ID}
}
}
// retryTransientPark requeues one task on its stored retry mode, which the
// list projection omits: a remediation must keep continuing its branch.
func (d *Daemon) retryTransientPark(ctx context.Context, id int64) {
t, err := d.Store.TaskByID(ctx, id)
if err != nil {
d.Log.WarnContext(ctx, "load task for transient retry failed", "task", id, "err", err)
return
}
if err := d.Store.RetryTask(ctx, id, workflow.StatusParked, "", t.RetryMode, workflowtask.Resume{}, false); err != nil {
if !errors.Is(err, storecontract.ErrStaleTransition) {
d.Log.WarnContext(ctx, "transient park retry failed", "task", id, "err", err)
}
return
}
d.emit(ctx, events.Event{
Kind: events.KindTaskRetried, TaskID: id, Workflow: t.Workflow,
Repo: strings.Trim(t.Owner+"/"+t.Repo, "/"), Issue: t.IssueNumber,
Detail: "retried automatically after a transient park",
Data: map[string]any{"retry_count": t.RetryCount + 1, "previous_reason": t.ParkReason},
})
}
// parkRefusedRun parks a task whose workflow identity check did not permit it.
// A denial is the operator's to fix, but a chain that could not be read is a
// store that will answer again, so it parks transient rather than demanding an
// operator for a blip.
func (d *Daemon) parkRefusedRun(ctx context.Context, task *workflow.Task, reason string, err error) {
class, blocker := taskstate.ParkNeedsDecision, taskstate.BlockerPolicyDenied
if err != nil {
class, blocker = pinParkClass(err), parkBlocker(err)
}
d.parkBlockedTask(ctx, task, reason, class, blocker)
}
// openTaskLog opens the task's log sink for one process() call and returns
// its closer. Failure to open is logged, not fatal.
func (d *Daemon) openTaskLog(task *workflow.Task) func() {
if err := d.TaskLogs.Open(task.ID, task.Attempt); err != nil {
d.Log.Warn("task log sink unavailable", "task", task.ID, "attempt", task.Attempt, "err", err)
}
return func() {
if err := d.TaskLogs.Close(task.ID); err != nil {
d.Log.Warn("task log sink close failed", "task", task.ID, "err", err)
}
}
}