refactor(daemon): tasks run through a TaskRuntime; the Docker pool is its first backend (#1864)
This commit is contained in:
6 files changed
+72
-21
No files matched your search
@@ -617,7 +617,7 @@ func (b *boot) buildDaemon(ctx context.Context) {
|
||||
Log: log,
|
||||
TaskTransport: b.natsClient,
|
||||
RunCredentials: b.stateStoreGrants,
|
||||
ContainerPool: b.containerPool,
|
||||
Runtime: daemon.DockerRuntime(b.containerPool),
|
||||
ModelEgress: b.modelEgress,
|
||||
TaskModels: b.taskModels,
|
||||
TaskInstructions: b.taskInstructions,
|
||||
|
||||
@@ -34,7 +34,7 @@ func (d *Daemon) acquireTaskContainer(
|
||||
repo config.Repo,
|
||||
workDir string,
|
||||
image string,
|
||||
) (*container.Container, string, func(), bool) {
|
||||
) (TaskContainer, string, func(), bool) {
|
||||
park := func(reason string, err error) {
|
||||
d.Log.ErrorContext(ctx, reason, "err", err)
|
||||
d.parkBlockedTask(ctx, task, reason+": "+err.Error(), taskstate.ParkTransient, parkBlocker(err))
|
||||
@@ -60,7 +60,7 @@ func (d *Daemon) acquireTaskContainer(
|
||||
}
|
||||
|
||||
// Guard: Storage may be nil if the daemon was wired incorrectly. In
|
||||
// normal operation, Storage is always set when ContainerPool is set.
|
||||
// normal operation, Storage is always set when Runtime is set.
|
||||
if d.Storage == nil {
|
||||
d.Log.ErrorContext(ctx, "storage backend is nil -- cannot acquire container")
|
||||
d.parkRunningTask(ctx, task, "storage backend not configured", taskstate.ParkTransient)
|
||||
@@ -104,7 +104,7 @@ func (d *Daemon) acquireTaskContainer(
|
||||
return nil, "", nil, false
|
||||
}
|
||||
|
||||
ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(env, model.Env...))
|
||||
ctr, err := d.Runtime.Acquire(ctx, image, append(mounts, model.Mounts...), append(env, model.Env...))
|
||||
if err != nil {
|
||||
revokeCredential()
|
||||
// Roll back the storage we just set up: the mounts were created
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/container"
|
||||
"github.com/samcharles93/archie-core/internal/domain/eda/playbook"
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
@@ -116,10 +115,9 @@ type Daemon struct {
|
||||
// AgentStatus records agent versions for update verification.
|
||||
AgentStatus *AgentStatus
|
||||
|
||||
// ContainerPool manages Docker container lifecycle. Nil means autonomous
|
||||
// execution is unavailable and process parks tasks; every runnable task gets
|
||||
// a fresh container.
|
||||
ContainerPool *container.Pool
|
||||
// Runtime runs each task's agent in a fresh isolated container. Nil means
|
||||
// autonomous execution is unavailable and process parks tasks.
|
||||
Runtime TaskRuntime
|
||||
// Storage is the pluggable storage backend for container mounts.
|
||||
// A runnable task requires it; acquireTaskContainer parks on nil.
|
||||
Storage storage.Backend
|
||||
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
// for the dashboard. The reason distinguishes the two in logs and events.
|
||||
func (d *Daemon) parkCapabilityUnavailable(ctx context.Context, task *workflow.Task) {
|
||||
var reason string
|
||||
if d.ContainerPool == nil {
|
||||
if d.Runtime == nil {
|
||||
reason = "managed agent container pool is unavailable; refusing to run this task on the host"
|
||||
d.Log.ErrorContext(ctx, "task parked: managed worker unavailable", "task", task.ID,
|
||||
"hint", "enable containers and make the archie-agent image available, then retry the task")
|
||||
|
||||
@@ -55,7 +55,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHol
|
||||
return
|
||||
}
|
||||
|
||||
if d.ContainerPool == nil {
|
||||
if d.Runtime == nil {
|
||||
d.parkCapabilityUnavailable(ctx, task)
|
||||
return
|
||||
}
|
||||
@@ -83,7 +83,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHol
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer d.ContainerPool.Release(ctx, ctr)
|
||||
defer d.Runtime.Release(ctx, ctr)
|
||||
defer revokeCredential()
|
||||
|
||||
// Run the whole task in archie-agent, which calls back to the daemon for
|
||||
@@ -414,10 +414,10 @@ func (d *Daemon) cleanupWorktree(ctx context.Context, trees *worktree.Manager, o
|
||||
// and reports whether the caller should retry. A SIGKILLed task container
|
||||
// leaves root-owned files that only a container can chown back.
|
||||
func (d *Daemon) reownDenied(ctx context.Context, dir string, err error) bool {
|
||||
if err == nil || !errors.Is(err, fs.ErrPermission) || d.ContainerPool == nil {
|
||||
if err == nil || !errors.Is(err, fs.ErrPermission) || d.Runtime == nil {
|
||||
return false
|
||||
}
|
||||
if rerr := d.ContainerPool.Reown(ctx, dir, os.Getuid(), os.Getgid()); rerr != nil {
|
||||
if rerr := d.Runtime.Reown(ctx, dir, os.Getuid(), os.Getgid()); rerr != nil {
|
||||
d.Log.WarnContext(ctx, "worktree ownership repair failed", "dir", dir, "err", rerr)
|
||||
return false
|
||||
}
|
||||
@@ -440,7 +440,7 @@ const (
|
||||
// withContainerExit derives a context cancelled when exited closes. A dead
|
||||
// container never answers its taskrun request, so without this the request
|
||||
// blocks forever and the task row stays running.
|
||||
func withContainerExit(ctx context.Context, ctr exitWatcher) (context.Context, func()) {
|
||||
func withContainerExit(ctx context.Context, ctr TaskContainer) (context.Context, func()) {
|
||||
runCtx, cancel := context.WithCancelCause(ctx)
|
||||
go func() {
|
||||
select {
|
||||
@@ -456,12 +456,6 @@ func withContainerExit(ctx context.Context, ctr exitWatcher) (context.Context, f
|
||||
return runCtx, func() { cancel(nil) }
|
||||
}
|
||||
|
||||
// exitWatcher is a running agent container, seen only for how it ends.
|
||||
type exitWatcher interface {
|
||||
Exited() <-chan struct{}
|
||||
ExitDetail() string
|
||||
}
|
||||
|
||||
// errContainerExited is the cancellation cause when a task's agent container
|
||||
// stops before answering its taskrun request.
|
||||
var errContainerExited = errors.New("agent container exited before the task finished")
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/container"
|
||||
"github.com/samcharles93/archie-core/internal/storage"
|
||||
)
|
||||
|
||||
// TaskRuntime starts and stops the isolated environment a task's agent runs
|
||||
// in. The Docker container pool is one implementation.
|
||||
type TaskRuntime interface {
|
||||
// Acquire starts the agent from image with mounts and env.
|
||||
Acquire(ctx context.Context, image string, mounts []storage.Mount, env []string) (TaskContainer, error)
|
||||
// Release stops and removes a container Acquire returned.
|
||||
Release(ctx context.Context, c TaskContainer)
|
||||
// Reown hands dir back to uid:gid when a killed container left files the
|
||||
// daemon cannot touch.
|
||||
Reown(ctx context.Context, dir string, uid, gid int) error
|
||||
}
|
||||
|
||||
// TaskContainer is one running task environment.
|
||||
type TaskContainer interface {
|
||||
// Exited closes when the agent stops running for any reason.
|
||||
Exited() <-chan struct{}
|
||||
// ExitDetail says how it ended, once Exited has closed.
|
||||
ExitDetail() string
|
||||
}
|
||||
|
||||
// DockerRuntime runs tasks in pool's containers. A nil pool is no runtime, so
|
||||
// the daemon parks tasks rather than holding a runtime that cannot start one.
|
||||
func DockerRuntime(pool *container.Pool) TaskRuntime {
|
||||
if pool == nil {
|
||||
return nil
|
||||
}
|
||||
return dockerRuntime{pool: pool}
|
||||
}
|
||||
|
||||
type dockerRuntime struct{ pool *container.Pool }
|
||||
|
||||
func (r dockerRuntime) Acquire(ctx context.Context, image string, mounts []storage.Mount, env []string) (TaskContainer, error) {
|
||||
c, err := r.pool.Acquire(ctx, image, mounts, env)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Release releases a container this runtime acquired; any other value has
|
||||
// nothing of this pool's to release.
|
||||
func (r dockerRuntime) Release(ctx context.Context, c TaskContainer) {
|
||||
if ctr, ok := c.(*container.Container); ok {
|
||||
r.pool.Release(ctx, ctr)
|
||||
}
|
||||
}
|
||||
|
||||
func (r dockerRuntime) Reown(ctx context.Context, dir string, uid, gid int) error {
|
||||
return r.pool.Reown(ctx, dir, uid, gid)
|
||||
}
|
||||
Reference in new issue
Block a user