// Package storage defines the pluggable storage backend interface for agent // container execution. The Docker backend (DockerBackend) is the MVP // implementation -- no NFS/SMB/S3 yet, just Docker volumes + bind mounts. // // /data/ volume layout (implemented): // // /data/ // worktree/ -- bind mount of host worktree // .git/task.json -- boot-time brief (WriteTaskJSON). Under .git so // the agent's own commit cannot sweep it onto the // task branch: go-git's Add ignores .gitignore. // repo/ -- per-repo persistent volume (optional, on request) // session.jsonl -- agent session transcript, appended per stage // memory.jsonl -- cross-session project memory // plugins/ -- daemon-staged bundled plugins // cache/ // go/ -- GOMODCACHE (shared across all tasks) // node/ -- npm cache (shared) // pnpm/ -- pnpm store (shared) // deno/ -- Deno module cache (shared) // bun/ -- Bun package cache (shared) // pip/ -- pip cache (shared) // cargo/ -- Rust cargo cache (shared) // mcp-npm/ -- npm cache for npx-launched per-task MCP // servers (shared, mounted regardless of the // task repo's ecosystem -- see MCPNPMCacheMountDir) // // Cache volumes are named Docker volumes created once and shared across all // tasks. They are never deleted -- the daemon operator manages them. package storage import ( "bufio" "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "sort" "strings" "time" "github.com/moby/moby/api/types/mount" "github.com/moby/moby/client" ) // ── types ────────────────────────────────────────────────────────────── // TaskRef identifies a task for storage setup. type TaskRef struct { Owner, Repo string IssueNumber int Ecosystem string // "go", "node", "python", "rust", "custom" WorktreeDir string // host path to the git worktree PersistentStorage bool // if true, attach a per-repo Docker volume at /data/repo } // Mount describes a container mount point. type Mount struct { Type string // "bind" or "volume" Source string // host path (bind) or volume name (volume) Destination string // container path } // Mount type constants. const ( MountTypeBind = "bind" MountTypeVolume = "volume" ) // AppendJSONLine appends one complete JSON value to a JSONL file. The value is // marshalled before opening the file so a marshal failure cannot leave a // partial record behind. func AppendJSONLine(path string, value any) error { record, err := json.Marshal(value) if err != nil { return fmt.Errorf("marshal JSONL record: %w", err) } record = append(record, '\n') if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return fmt.Errorf("create JSONL directory: %w", err) } f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { return fmt.Errorf("open JSONL file: %w", err) } if _, err := f.Write(record); err != nil { _ = f.Close() return fmt.Errorf("append JSONL record: %w", err) } if err := f.Close(); err != nil { return fmt.Errorf("close JSONL file: %w", err) } return nil } // ReadJSONLines decodes each line of the JSONL file at path as a T, calling // fn for each. A path that does not exist yet is not an error -- no log // written and an empty log mean the same thing to a caller -- so fn is // simply never called. A malformed line, or an error fn itself returns, // stops the scan and is returned wrapped. // // This is AppendJSONLine's read-side counterpart, kept in the package that // owns the format: see organisation.md's "a package owns its own format end // to end." func ReadJSONLines[T any](path string, fn func(T) error) error { f, err := os.Open(path) if errors.Is(err, os.ErrNotExist) { return nil } if err != nil { return fmt.Errorf("open JSONL file: %w", err) } defer f.Close() scanner := bufio.NewScanner(f) scanner.Buffer(make([]byte, 64*1024), 8*1024*1024) for scanner.Scan() { var record T if err := json.Unmarshal(scanner.Bytes(), &record); err != nil { return fmt.Errorf("decode JSONL record: %w", err) } if err := fn(record); err != nil { return fmt.Errorf("handle JSONL record: %w", err) } } if err := scanner.Err(); err != nil { return fmt.Errorf("scan JSONL file: %w", err) } return nil } // StagePlugin writes a bundled plugin unless persistent project storage // already contains a plugin with the same filename. Persistent copies are // operator-owned overrides and therefore win over daemon-staged source. func StagePlugin(dir, name string, source []byte) error { if filepath.Base(name) != name || name == "." || name == "" { return fmt.Errorf("invalid plugin filename %q", name) } if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("create plugin directory: %w", err) } path := filepath.Join(dir, name) f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) if errors.Is(err, os.ErrExist) { return nil } if err != nil { return fmt.Errorf("create staged plugin: %w", err) } if _, err := f.Write(source); err != nil { _ = f.Close() return fmt.Errorf("write staged plugin: %w", err) } if err := f.Close(); err != nil { return fmt.Errorf("close staged plugin: %w", err) } return nil } // WorktreeMountDir is the fixed container path where a task's worktree is // bind-mounted. archie-agent uses this path directly rather than the host // path archied's worktree.Manager reports -- the two processes see the same // files at different paths. const WorktreeMountDir = "/data/worktree" // Persistent storage paths share the per-repository volume so session output, // project memory, and staged plugins survive individual task containers. const ( PersistentMountDir = "/data/repo" SessionPath = PersistentMountDir + "/session.jsonl" MemoryPath = PersistentMountDir + "/memory.jsonl" PluginsDir = PersistentMountDir + "/plugins" ) // MCPNPMCacheMountDir is the fixed container path where the npm package // cache for npx-launched per-task MCP servers is mounted. Unlike the // ecosystem caches below, this is mounted on every task container // regardless of the task repo's language: per-task MCP servers come from // daemon-wide config (config.MCPServer), not from anything specific to the // checked-out repo, so an npx-based server needs the cache warm whether the // task is a Go repo, a Python repo, or anything else. const MCPNPMCacheMountDir = "/data/cache/mcp-npm" // Backend prepares container storage. Each container runtime backend // (Docker, future containerd, etc.) implements this interface. type Backend interface { // Setup returns the mount specs for a task container. Must include // at minimum a bind mount for the worktree at /data/worktree. Setup(ctx context.Context, task TaskRef) ([]Mount, error) // Teardown cleans up ephemeral storage. Cache volumes survive. Teardown(ctx context.Context, task TaskRef) error // CleanupExpired removes persistent per-repo storage older than ttl. // Shared ecosystem caches and storage not owned by Archie survive. CleanupExpired(ctx context.Context, ttl time.Duration) (int, error) } // ── cache mounts per ecosystem ───────────────────────────────────────── // cacheVolume describes a named Docker volume that is shared across tasks. type cacheVolume struct { Name string // Docker volume name MountPath string // container destination path } // alwaysOnCacheVolumes are shared cache volumes mounted on every task // container regardless of the task repo's ecosystem, because what they // cache isn't tied to the repo being worked on. The npx package cache for // per-task MCP servers is the only current instance: MCP servers are // daemon-wide config, not per-repo, so an npx-based one needs a warm cache // no matter which repo the task happens to be against. var alwaysOnCacheVolumes = []cacheVolume{ {Name: "archie-cache-mcp-npm", MountPath: MCPNPMCacheMountDir}, } // cacheVolumesByEcosystem maps ecosystem names to their shared cache volumes. // Volumes are created once by the pool and never deleted. var cacheVolumesByEcosystem = map[string][]cacheVolume{ "go": { {Name: "archie-cache-go", MountPath: "/data/cache/go"}, }, "node": { {Name: "archie-cache-node", MountPath: "/data/cache/node"}, {Name: "archie-cache-pnpm", MountPath: "/data/cache/pnpm"}, }, "typescript": { {Name: "archie-cache-node", MountPath: "/data/cache/node"}, {Name: "archie-cache-pnpm", MountPath: "/data/cache/pnpm"}, }, "deno": { {Name: "archie-cache-deno", MountPath: "/data/cache/deno"}, }, "bun": { {Name: "archie-cache-bun", MountPath: "/data/cache/bun"}, }, "python": { {Name: "archie-cache-pip", MountPath: "/data/cache/pip"}, }, "rust": { {Name: "archie-cache-cargo", MountPath: "/data/cache/cargo"}, }, } // cacheMounts returns the cache mount specs for an ecosystem -- the // always-on volumes plus any matched by ecosystem -- sorted by destination // path for deterministic container configuration. Ecosystem is // case-insensitive -- "Python" and "python" are equivalent. An unknown or // empty ecosystem still returns the always-on volumes. func cacheMounts(ecosystem string) []Mount { vols := append([]cacheVolume(nil), alwaysOnCacheVolumes...) vols = append(vols, cacheVolumesByEcosystem[strings.ToLower(ecosystem)]...) mounts := make([]Mount, len(vols)) for i, v := range vols { mounts[i] = Mount{ Type: MountTypeVolume, Source: v.Name, Destination: v.MountPath, } } sort.Slice(mounts, func(i, j int) bool { return mounts[i].Destination < mounts[j].Destination }) return mounts } // ── error types ──────────────────────────────────────────────────────── // ErrVolumeCreate is returned when a Docker volume cannot be created. type ErrVolumeCreate struct { Volume string Cause error } func (e *ErrVolumeCreate) Error() string { return "create volume " + e.Volume + ": " + e.Cause.Error() } func (e *ErrVolumeCreate) Unwrap() error { return e.Cause } // ── Docker backend ───────────────────────────────────────────────────── // DockerBackend implements Backend for Docker via the moby client. type DockerBackend struct { cli *client.Client now func() time.Time } const ( storageLabel = "com.samcharles93.archie.storage" storageKindRepo = "repo" storageKindCache = "cache" ) // NewDockerBackend creates a Docker storage backend. func NewDockerBackend(cli *client.Client) *DockerBackend { return &DockerBackend{cli: cli} } // ensureVolume creates a Docker volume if it doesn't already exist. // It is idempotent: concurrent callers racing to create the same volume // will not see an error -- "already exists" is treated as success. // // When d.cli is nil (unit tests), ensureVolume returns nil without // creating a volume. The returned mounts will fail at container create // time if Docker cannot find the volume. Production code always sets // cli via NewDockerBackend. func (d *DockerBackend) ensureVolume(ctx context.Context, name string, labels map[string]string) error { if d.cli == nil { return nil // nil client for testing -- volumes must be pre-created } // VolumeInspectOptions{} is the empty struct form (requires moby v25+). // The go.mod pins a compatible version; zero value is always valid. _, err := d.cli.VolumeInspect(ctx, name, client.VolumeInspectOptions{}) if err == nil { return nil // already exists } _, err = d.cli.VolumeCreate(ctx, client.VolumeCreateOptions{ Name: name, Driver: "local", Labels: labels, }) if err != nil { // TOCTOU: a concurrent caller may have created the volume between // our inspect and create calls. Docker returns an error for // duplicate volumes -- treat that as success. if strings.Contains(err.Error(), "already exists") || strings.Contains(err.Error(), "already in use") { return nil } return &ErrVolumeCreate{Volume: name, Cause: err} } return nil } // repoVolumeName returns the Docker volume name for a repo's persistent storage. func repoVolumeName(owner, repo string) string { return fmt.Sprintf("archie-repo-%s-%s", owner, repo) } // Setup returns the mount list for a task container. Mounts are ordered: // 1. /data/worktree bind mount (always first) // 2. /data/repo volume mount (only when PersistentStorage is true) // 3. cache volume mounts: the always-on volumes plus /data/cache/ // (sorted together by destination) func (d *DockerBackend) Setup(ctx context.Context, task TaskRef) ([]Mount, error) { mounts := []Mount{ { Type: MountTypeBind, Source: task.WorktreeDir, Destination: WorktreeMountDir, }, } // Per-repo persistent volume. if task.PersistentStorage && task.Owner != "" && task.Repo != "" { volName := repoVolumeName(task.Owner, task.Repo) if err := d.ensureVolume(ctx, volName, map[string]string{storageLabel: storageKindRepo}); err != nil { return nil, err } mounts = append(mounts, Mount{ Type: MountTypeVolume, Source: volName, Destination: PersistentMountDir, }) } caches := cacheMounts(task.Ecosystem) for _, m := range caches { if err := d.ensureVolume(ctx, m.Source, map[string]string{storageLabel: storageKindCache}); err != nil { return nil, err } } mounts = append(mounts, caches...) return mounts, nil } // Teardown is a no-op for Docker backend. Cache volumes survive task // completion so subsequent tasks benefit from warmed caches. func (d *DockerBackend) Teardown(ctx context.Context, task TaskRef) error { return nil } // CleanupExpired removes Archie-owned per-repo volumes whose creation time is // at least ttl old. Docker volume labels are immutable, so the TTL is a hard // maximum age rather than an idle lease. Removal is non-forced: a volume still // attached to a running container is retained and retried on the next sweep. func (d *DockerBackend) CleanupExpired(ctx context.Context, ttl time.Duration) (int, error) { if d.cli == nil || ttl <= 0 { return 0, nil } result, err := d.cli.VolumeList(ctx, client.VolumeListOptions{ Filters: client.Filters{}.Add("label", storageLabel+"="+storageKindRepo), }) if err != nil { return 0, fmt.Errorf("list persistent volumes: %w", err) } now := time.Now() if d.now != nil { now = d.now() } var ( removed int errs []error ) for _, vol := range result.Items { // Defend against Docker implementations that ignore label filters. if vol.Labels[storageLabel] != storageKindRepo { continue } created, err := time.Parse(time.RFC3339Nano, vol.CreatedAt) if err != nil { errs = append(errs, fmt.Errorf("persistent volume %s created_at %q: %w", vol.Name, vol.CreatedAt, err)) continue } if now.Sub(created) < ttl { continue } if _, err := d.cli.VolumeRemove(ctx, vol.Name, client.VolumeRemoveOptions{}); err != nil { errs = append(errs, fmt.Errorf("remove persistent volume %s: %w", vol.Name, err)) continue } removed++ } return removed, errors.Join(errs...) } // ConvertMounts converts []Mount to moby mount.Mount for container creation. func ConvertMounts(mounts []Mount) []mount.Mount { out := make([]mount.Mount, len(mounts)) for i, m := range mounts { out[i] = mount.Mount{ Type: mount.Type(m.Type), Source: m.Source, Target: m.Destination, } } return out }