From c2c8e9230472e75833af071508ea4728730469a4 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Tue, 21 Jul 2026 03:46:06 +1000 Subject: [PATCH] =?UTF-8?q?feat:=20archied=20walking=20skeleton=20?= =?UTF-8?q?=E2=80=94=20poll,=20store,=20worktree,=20workflow=20engine,=20b?= =?UTF-8?q?ootstrap=20PR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Repurposes archie-core as the archie orchestrator daemon (the old Pocketbase PM-backend plan is archived under .references/). - cmd/archied: -config, -once (single cycle for timers/testing) - internal/config: TOML; per-repo gate command lists; role->model refs - internal/store: SQLite (modernc, WAL) task state machine with transitions audit, idempotent enqueue, crash recovery (running -> queued, attempt++) - internal/forge: go-github v78 — labelled-issue polling, invitation auto-accept, push verification, PR create/state, comments, close - internal/worktree: fresh clone per task, bot identity, askpass token handling, diff-size stats, cleanup on merge/reject - internal/workflow: extensible engine (stages over TaskContext, shared step library, label routing bug->tdd feature->feasibility default-> implement); parks always comment on the issue. Bootstrap workflow (deterministic marker-file PR) proves the plumbing without LLM spend. Store and worktree round-trips covered by tests (local bare remote). --- .gitignore | 2 + .references/PLAN-pm-backend.md | 965 +++++++++++++++++++++++++++ .references/frontend.md | 4 + .references/skills-implementation.md | 367 ++++++++++ DESIGN.md | 52 ++ config.example.toml | 38 ++ go.mod | 22 + go.sum | 61 ++ internal/config/config.go | 132 ++++ internal/daemon/daemon.go | 168 +++++ internal/forge/github.go | 124 ++++ internal/store/store.go | 204 ++++++ internal/store/store_test.go | 73 ++ internal/workflow/steps.go | 105 +++ internal/workflow/workflow.go | 147 ++++ internal/worktree/worktree.go | 151 +++++ internal/worktree/worktree_test.go | 122 ++++ 17 files changed, 2737 insertions(+) create mode 100644 .gitignore create mode 100644 .references/PLAN-pm-backend.md create mode 100644 .references/frontend.md create mode 100644 .references/skills-implementation.md create mode 100644 DESIGN.md create mode 100644 config.example.toml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/config/config.go create mode 100644 internal/daemon/daemon.go create mode 100644 internal/forge/github.go create mode 100644 internal/store/store.go create mode 100644 internal/store/store_test.go create mode 100644 internal/workflow/steps.go create mode 100644 internal/workflow/workflow.go create mode 100644 internal/worktree/worktree.go create mode 100644 internal/worktree/worktree_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..53f7ceaf5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +archied +*.db diff --git a/.references/PLAN-pm-backend.md b/.references/PLAN-pm-backend.md new file mode 100644 index 000000000..2a9bffd4d --- /dev/null +++ b/.references/PLAN-pm-backend.md @@ -0,0 +1,965 @@ +# Archie Backend — Implementation Plan + +## 1. Project Overview + +**Archie** is an "Agentic AI for everyone" platform — a mobile-first app where users manage AI agent tasks, converse with AI models (Archie 1.6 Max/Pro/Lite), configure AI providers (OpenAI, Anthropic, Ollama), manage skill bundles (AgentSkills.io compliant), store knowledge entries, and control their agent's capabilities. + +### Architecture + +``` +┌─────────────────────┐ REST/WS ┌─────────────────────┐ +│ Flutter Frontend │ ◄──────────────► │ Go Backend │ +│ (Riverpod + Dio) │ │ (stdlib net/http) │ +└─────────────────────┘ │ │ + │ ┌─────────────┐ │ + │ │ Pocketbase │ │ + │ │ (embedded) │ │ + │ └─────────────┘ │ + │ ┌─────────────┐ │ + │ │ AI Provider │ │ + │ │ Proxy (LLM) │ │ + │ └─────────────┘ │ + └─────────────────────┘ +``` + +- **Pocketbase** provides the storage layer (embedded SQLite), auto-generates CRUD REST APIs, handles auth, and offers real-time subscriptions. +- **Go stdlib** (`net/http`, `gorilla/websocket`) serves: + - Custom REST handlers that augment Pocketbase (business logic, validation, auth middleware) + - WebSocket hub for real-time chat streaming + - AI provider proxy (routes model requests to OpenAI/Anthropic/Ollama with unified API) +- **WebSocket** streams chat messages token-by-token for real-time AI responses. + +--- + +## 2. Phased Implementation Plan + +### Phase 0: Project Scaffolding & Infrastructure (Day 1) + +**Goal**: Runnable Go server with Pocketbase embedded, health check, configuration, and clean project structure. + +#### 2.1 Project Layout + +``` +/work/apps/archie-core/ +├── cmd/ +│ └── server/ +│ └── main.go # Entrypoint: init Pocketbase, mount routes, start server +├── internal/ +│ ├── config/ +│ │ └── config.go # Env-based config (PB data dir, port, JWT secret, provider API keys) +│ ├── server/ +│ │ ├── server.go # Server struct, HTTP mux setup, graceful shutdown +│ │ ├── middleware.go # CORS, request logging, auth middleware, rate limiting +│ │ └── routes.go # Route registration +│ ├── auth/ +│ │ └── auth_handler.go # Custom auth endpoints (login, register, verify, token refresh) +│ ├── tasks/ +│ │ ├── task_model.go # DB schema helpers / Pocketbase collection mapping +│ │ ├── task_handler.go # REST handlers for tasks +│ │ └── task_service.go # Business logic for tasks (scheduling, filtering) +│ ├── chat/ +│ │ ├── chat_handler.go # REST handlers for conversations / messages +│ │ ├── chat_service.go # Business logic: create conversation, add messages +│ │ └── websocket.go # WebSocket hub: connect/disconnect, streaming, broadcast +│ ├── skills/ +│ │ ├── skill_handler.go # REST handlers for skills +│ │ ├── skill_service.go # Skill discovery, AgentSkills.io spec loading +│ │ └── skill_model.go # Skill bundle model +│ ├── knowledge/ +│ │ ├── knowledge_handler.go # REST handlers for knowledge entries +│ │ └── knowledge_service.go # CRUD + full-text search +│ ├── providers/ +│ │ ├── provider_handler.go # REST handlers for AI provider config +│ │ ├── provider_service.go # Provider config management, key validation +│ │ └── proxy.go # AI provider proxy (OpenAI, Anthropic, Ollama) +│ ├── models/ +│ │ └── model_handler.go # REST handlers for available models listing +│ └── common/ +│ ├── response.go # Standard JSON response helpers +│ ├── errors.go # Error types, error response formatting +│ └── validation.go # Input validation helpers +├── migrations/ +│ └── 001_initial_schema.go # Pocketbase collection definitions +├── pb_data/ # Pocketbase data directory (gitignored) +├── pb_public/ # Admin UI assets (if using PB's built-in) +├── go.mod +├── go.sum +├── .env.example +├── Makefile +└── PLAN.md +``` + +#### 2.2 Dependencies (go.mod) + +| Module | Purpose | +|---|---| +| `github.com/pocketbase/pocketbase` | Embedded Pocketbase server | +| `github.com/gorilla/websocket` | WebSocket implementation | +| `github.com/go-chi/chi/v5` (or stdlib `net/http` mux in Go 1.22+) | HTTP routing — since Go 1.22 supports method-based routing (`GET /tasks`, `POST /tasks`) with stdlib, we can avoid chi | +| `github.com/golang-jwt/jwt/v5` | JWT token generation/validation | +| `golang.org/x/crypto` | Password hashing (bcrypt) | +| `github.com/sashabaranov/go-openai` | OpenAI API client | +| `github.com/liushuangls/go-anthropic` | Anthropic API client | +| `github.com/joho/godotenv` | .env file loading | + +#### 2.3 Pocketbase Schema — Collection Definitions + +**Collections** (each created via Pocketbase migration or admin API): + +| Collection | Fields | Auth? | API Rules | +|-----------------------|------------------------------------------------------------------------|--------|--------------------------| +| `users` | Built-in Pocketbase users collection (email, password, name, avatar) | Yes | PB-managed auth | +| `conversations` | `user` (relation→users), `title` (text), `model` (text), `provider` (text), `created`, `updated` | No | Owner-only CRUD | +| `messages` | `conversation` (relation→conversations), `user` (relation→users), `role` (select: user/assistant/system), `content` (text/json), `tokens_used` (number), `created` | No | Owner read, system write | +| `tasks` | `user` (relation→users), `title` (text), `subtitle` (text), `type` (select: Agent/Scheduled/Favorites), `icon` (text), `scheduled_at` (date, optional), `status` (select: pending/in_progress/completed/failed), `created`, `updated` | No | Owner-only CRUD | +| `skills` | `user` (relation→users, optional — null = official), `name` (text, unique per user), `description` (text), `enabled` (bool), `official` (bool), `source_path` (text, optional), `metadata` (json, for SKILL.md frontmatter), `created`, `updated` | No | Owner read/write for user skills, read-only for official | +| `knowledge` | `user` (relation→users), `title` (text), `content` (text), `enabled` (bool), `created`, `updated` | No | Owner-only CRUD | +| `provider_configs` | `user` (relation→users), `provider` (select: openai/anthropic/ollama), `api_key_encrypted` (text), `base_url` (text, optional), `is_active` (bool), `created`, `updated` | No | Owner-only CRUD | +| `settings` | `user` (relation→users, unique), `theme` (select: light/dark/system), `language` (text, default "en"), `selected_model` (text), `created`, `updated` | No | Owner-only CRUD | + +> **Note:** For MVP, we can skip encryption on `api_key_encrypted` and use reversible encryption with a server-side key. For V1, we'll use Pocketbase's built-in encryption or envelope encryption. + +#### 2.4 Configuration (.env / env vars) + +```env +PORT=8090 +PB_DATA_DIR=./pb_data +JWT_SECRET=your-256-bit-secret +ENCRYPTION_KEY=32-byte-key-for-provider-api-keys + +# Provider defaults (used if user hasn't configured their own) +OPENAI_API_KEY= +ANTHROPIC_API_KEY= +OLLAMA_BASE_URL=http://localhost:11434 + +CORS_ALLOWED_ORIGINS=http://localhost:3000,http://localhost:5173 +``` + +--- + +### Phase 1: Core Server Foundation (Day 2-3) + +**Goal**: Server boots, Pocketbase initializes with schema, health endpoint works, CORS + auth middleware operational. + +#### 1.1 Server Bootstrap (`cmd/server/main.go`) + +```go +func main() { + cfg := config.Load() + pbApp := pocketbase.NewWithConfig(pocketbase.Config{ + DefaultDataDir: cfg.PBDataDir, + }) + + srv := server.New(cfg, pbApp) + srv.RegisterRoutes() + + if err := pbApp.Start(); err != nil { + log.Fatal(err) + } +} +``` + +#### 1.2 Auth Middleware (`internal/server/middleware.go`) + +- **JWT verification middleware** — extracts `Authorization: Bearer `, validates JWT, injects `user_id` into context +- **CORS middleware** — allows configurable origins, methods, headers +- **Request logging middleware** — method, path, status, duration +- **Rate limiting** — per-IP / per-user rate limiting (optional in MVP) + +#### 1.3 Route Registration (`internal/server/routes.go`) + +``` +GET /api/health → health check (no auth) + +# Auth (Pocketbase built-in + custom endpoints) +POST /api/auth/register → register new user (Pocketbase users) +POST /api/auth/login → login, returns JWT +POST /api/auth/refresh → refresh JWT +GET /api/auth/me → current user profile + +# Tasks (all require auth) +GET /api/tasks → list tasks (filterable by type, status) +POST /api/tasks → create task +GET /api/tasks/:id → get task details +PATCH /api/tasks/:id → update task +DELETE /api/tasks/:id → delete task + +# Conversations & Messages (all require auth) +GET /api/conversations → list user's conversations +POST /api/conversations → create conversation +GET /api/conversations/:id → get conversation with messages +DELETE /api/conversations/:id → delete conversation +POST /api/conversations/:id/messages → send message (returns via WS via REST) +WS /api/ws/chat/:conversation_id → WebSocket: stream AI responses + +# Skills (all require auth) +GET /api/skills → list skills (official + user's) +POST /api/skills → create custom skill +GET /api/skills/:id → get skill details +PATCH /api/skills/:id → update skill (toggle enabled, edit) +DELETE /api/skills/:id → delete custom skill + +# Knowledge (all require auth) +GET /api/knowledge → list knowledge entries +POST /api/knowledge → create entry +GET /api/knowledge/:id → get entry details +PATCH /api/knowledge/:id → update entry +DELETE /api/knowledge/:id → delete entry + +# Providers (all require auth) +GET /api/providers → list configured providers +POST /api/providers → add/update provider config +DELETE /api/providers/:provider → remove provider config +POST /api/providers/:provider/validate → test API key connectivity + +# Models (all require auth) +GET /api/models → list available models (aggregated from all active providers) + +# Settings (all require auth) +GET /api/settings → get user settings +PATCH /api/settings → update user settings + +# Static files for Pocketbase admin UI +/* → serve pb_public/ +``` + +--- + +### Phase 2: Authentication & User Management (Day 3-4) + +**Goal**: Full auth flow working — register, login, JWT, token refresh, middleware enforcement. + +#### 2.1 Pocketbase Auth Integration + +Pocketbase provides: +- Built-in `users` collection with email/password auth +- JWT token generation + verification +- Token refresh endpoints + +We expose our own handlers (`/api/auth/register`, `/api/auth/login`, `/api/auth/refresh`) that: +1. Delegate to Pocketbase's auth API internally +2. Add custom response shaping (include user profile data) +3. Support additional fields (name, avatar) + +#### 2.2 Custom Auth Handler (`internal/auth/auth_handler.go`) + +- **POST /api/auth/register**: validates input, calls `pb.Users().Create()`, returns JWT + user +- **POST /api/auth/login**: calls `pb.Users().AuthWithPassword()`, returns JWT + user +- **POST /api/auth/refresh**: validates existing JWT, returns new JWT +- **GET /api/auth/me**: returns current user profile from `pb.Users().GetOne()` + +#### 2.3 Middleware Integration + +Auth middleware extracts `user_id` from validated JWT and sets it on context. All subsequent handlers read `user_id` from context for record ownership filtering. + +--- + +### Phase 3: Task Management (Day 4-5) + +**Goal**: Full CRUD for agent tasks, filtering by type (All/Agent/Scheduled/Favorites), icon mapping. + +#### 3.1 Task Schema Mapping + +Mapping frontend model to Pocketbase: +```json +{ + "id": "abc123", + "user": "user_rel_id", + "title": "Finishing Home Hub Website with ...", + "subtitle": "I sincerely apologize for the previous mistake...", + "type": "Agent", // "Agent" | "Scheduled" | "Favorites" + "icon": "auto_awesome_motion", // string key, frontend maps to IconData + "status": "in_progress", // "pending" | "in_progress" | "completed" | "failed" + "scheduled_at": null, // ISO date or null + "created": "2025-10-28T00:00:00Z", + "updated": "2025-10-28T00:00:00Z" +} +``` + +#### 3.2 Task Handlers + +| Method | Path | Behavior | +|--------|------|----------| +| GET | `/api/tasks` | List user's tasks. Query params: `type` (Agent/Scheduled/Favorites), `status`, `search` (text search on title/subtitle) | +| POST | `/api/tasks` | Create task. Validates required fields. Auto-sets `user` from JWT. | +| GET | `/api/tasks/:id` | Get single task (ownership check) | +| PATCH | `/api/tasks/:id` | Update task fields (title, subtitle, type, status, icon) | +| DELETE | `/api/tasks/:id` | Soft-delete or hard-delete task | + +#### 3.3 Filter Logic + +The frontend's `FilterRow` sends the selected filter as a query param: +- `?type=Agent` — return tasks where type = "Agent" +- `?type=Scheduled` — return tasks where type = "Scheduled" +- `?type=Favorites` — return tasks where type = "Favorites" +- No filter or `?type=All` — return all tasks for user + +--- + +### Phase 4: Chat System — REST + WebSocket (Day 5-8) + +**Goal**: Full conversational AI with real-time token streaming via WebSocket, conversation history via REST. + +#### 4.1 Data Model + +**Conversation:** +```json +{ + "id": "conv_001", + "user": "user_rel_id", + "title": "Building a dashboard", + "model": "Archie 1.6 Pro", + "provider": "anthropic", + "created": "2025-10-28T00:00:00Z", + "updated": "2025-10-28T01:00:00Z" +} +``` + +**Message:** +```json +{ + "id": "msg_001", + "conversation": "conv_001", + "user": "user_rel_id", + "role": "user", // "user" | "assistant" | "system" + "content": "Hello! Can you help me build a dashboard?", + "tokens_used": 15, + "created": "2025-10-28T00:00:00Z" +} +``` + +#### 4.2 REST Endpoints for Conversations + +| Method | Path | Behavior | +|--------|------|----------| +| GET | `/api/conversations` | List user's conversations, ordered by `updated` desc. Supports `?search=` to filter by title. | +| POST | `/api/conversations` | Create conversation. Body: `{ "title": "...", "model": "...", "provider": "..." }`. Auto-sets `user`. | +| GET | `/api/conversations/:id` | Get conversation metadata + all messages. Supports pagination (`?page=1&perPage=50`). | +| PATCH | `/api/conversations/:id` | Update conversation (rename title, change model) | +| DELETE | `/api/conversations/:id` | Delete conversation + all messages | +| POST | `/api/conversations/:id/messages` | Send a message. Body: `{ "content": "..." }`. Creates user message, triggers AI, creates assistant message. Returns the assistant message once complete (synchronous fallback). | + +#### 4.3 WebSocket Hub (`internal/chat/websocket.go`) + +**Architecture:** + +``` +Client (Flutter) ──WS──► Hub (per-server singleton) + │ + ┌───────┴───────┐ + │ Connection │ (per-client, per-conversation) + │ Manager │ + └───────┬───────┘ + │ + ┌─────────────┼─────────────┐ + │ │ │ + OpenAI Proxy Anthropic Proxy Ollama Proxy +``` + +**WebSocket Protocol:** + +**Client → Server messages:** +```json +// Send a chat message +{ + "type": "message", + "conversation_id": "conv_001", + "content": "Build me a dashboard", + "model": "Archie 1.6 Pro" +} + +// Join a conversation room (subscribe to streaming) +{ + "type": "join", + "conversation_id": "conv_001" +} + +// Leave a conversation room +{ + "type": "leave", + "conversation_id": "conv_001" +} + +// Ping (keep-alive) +{ + "type": "ping" +} +``` + +**Server → Client messages:** +```json +// Streaming token from AI +{ + "type": "token", + "conversation_id": "conv_001", + "token": "build", + "index": 0 +} + +// Stream complete +{ + "type": "done", + "conversation_id": "conv_001", + "message_id": "msg_002", + "tokens_used": 412 +} + +// Error during streaming +{ + "type": "error", + "conversation_id": "conv_001", + "code": "provider_error", + "message": "OpenAI API rate limit exceeded" +} + +// System notification +{ + "type": "system", + "conversation_id": "conv_001", + "message": "Model switched to Archie 1.6 Lite" +} + +// Pong (keep-alive response) +{ + "type": "pong" +} +``` + +**Hub Design:** + +```go +type Hub struct { + mu sync.RWMutex + connections map[string]*Client // clientID → Client + rooms map[string]map[string]*Client // conversationID → { clientID → Client } +} + +type Client struct { + id string + userID string + conn *websocket.Conn + send chan []byte + hub *Hub +} + +type ProxyService struct { + openai *openai.Client + anthropic *anthropic.Client + ollama *ollama.Client +} +``` + +- Clients register on connect, unregister on disconnect +- `join` message adds client to a room (conversation) +- `message` type triggers AI provider call with streaming +- Hub broadcasts tokens to all clients in the conversation room +- Only the sender's client receives the token stream by default (single-user conversations in MVP) + +#### 4.4 AI Provider Proxy (`internal/providers/proxy.go`) + +**Unified interface:** + +```go +type AIProvider interface { + ChatStream(ctx context.Context, req ChatRequest) (<-chan Token, error) + Chat(ctx context.Context, req ChatRequest) (string, int, error) +} + +type ChatRequest struct { + Model string + Messages []Message + Config ProviderConfig +} + +type Token struct { + Content string + Index int + Done bool + Err error +} +``` + +**Provider implementations:** + +| Provider | Client Library | Model Mapping | +|----------|---------------|---------------| +| OpenAI | `go-openai` | `gpt-4o`, `gpt-4o-mini`, etc | +| Anthropic | `go-anthropic` | `claude-sonnet-4`, `claude-haiku-3.5`, etc | +| Ollama | Raw HTTP | Maps to local `OLLAMA_BASE_URL` + model name | + +**Model Name Resolution:** +``` +Frontend Model Name → Provider + Backend Model +────────────────────────────────────────────────────── +Archie 1.6 Max → anthropic / claude-sonnet-4-20250514 +Archie 1.6 Pro → openai / gpt-4o +Archie 1.6 Lite → openai / gpt-4o-mini (or ollama/llama3) +``` + +The resolution is configurable via `settings` so the user can map Archie model tiers to their preferred providers. + +#### 4.5 Message Flow (WebSocket Path) + +``` +1. Client sends WS message: { "type": "message", "conversation_id": "abc", "content": "Hello" } +2. Server validates JWT, verifies ownership +3. Server saves user message to Pocketbase (messages collection) +4. Server loads conversation history from Pocketbase (last N messages for context) +5. Server constructs system prompt (includes enabled skills + knowledge context) +6. Server calls AIProvider.ChatStream() +7. For each token received from provider → WS broadcast to room +8. On stream complete → save assistant message to Pocketbase, broadcast "done" with message_id + tokens_used +``` + +**Fallback (REST-only path without WS):** + +The `POST /api/conversations/:id/messages` endpoint works synchronously, returning the full assistant response as JSON after the AI completes. This is used when the client cannot establish WebSocket (e.g., initial development, degraded mode). + +--- + +### Phase 5: Skills Management (Day 8-10) + +**Goal**: CRUD for skills, official library (read-only), custom skills, AgentSkills.io SKILL.md spec integration. + +#### 5.1 Skill Model + +```json +{ + "id": "skill_001", + "user": null, // null = official skill + "name": "automation-and-scheduling", + "description": "MUST read before requests involving automated execution...", + "enabled": true, + "official": true, + "source_path": null, // filesystem path for local skills + "metadata": { + "version": "1.0.0", + "author": "Archie", + "tags": ["automation", "scheduling"] + }, + "created": "2025-05-08T00:00:00Z", + "updated": "2025-05-08T00:00:00Z" +} +``` + +#### 5.2 REST Endpoints + +| Method | Path | Behavior | +|--------|------|----------| +| GET | `/api/skills` | List skills: official skills + user's custom skills. Query: `?enabled=true` to filter. | +| POST | `/api/skills` | Create custom skill. Body: `{ "name": "...", "description": "...", "metadata": {...} }` | +| GET | `/api/skills/:id` | Get skill details | +| PATCH | `/api/skills/:id` | Update skill (toggle enabled, edit name/description). Official skills: only `enabled` toggleable. | +| DELETE | `/api/skills/:id` | Delete custom skill only | + +#### 5.3 AgentSkills.io Integration + +- **Official skills** are seeded into Pocketbase via migration (JSON from AgentSkills.io registry) +- **Custom skills** can be created via REST API +- When a conversation starts, the server builds a system prompt that includes enabled skills' instructions (progressive disclosure: only skill names/descriptions in base context, full SKILL.md content loaded on demand) +- The `GET /api/skills/:id/instructions` endpoint returns the full SKILL.md content of a skill for the AI to reference + +#### 5.4 Skill Loading in Chat Context + +When a user starts a conversation: + +1. Server loads all `enabled=true` skills for the user +2. Injects **catalog** (name + description) into system prompt (~50-100 tokens each) +3. When the AI decides a skill is relevant, it can request the full instructions via a tool call (future) or the server includes all enabled skill instructions in the prompt (MVP simplification) + +--- + +### Phase 6: Knowledge Management (Day 10-11) + +**Goal**: CRUD for knowledge entries, text search, injection into AI context. + +#### 6.1 Knowledge Model + +```json +{ + "id": "knowledge_001", + "user": "user_rel_id", + "title": "Design and feature preferences for AI-powered family dashboard", + "content": "When building AI-powered family dashboard applications...", + "enabled": true, + "created": "2025-10-28T00:00:00Z", + "updated": "2025-10-28T00:00:00Z" +} +``` + +#### 6.2 REST Endpoints + +| Method | Path | Behavior | +|--------|------|----------| +| GET | `/api/knowledge` | List knowledge entries. Query: `?search=` for text search on title+content, `?enabled=true` | +| POST | `/api/knowledge` | Create entry. Body: `{ "title": "...", "content": "..." }` | +| GET | `/api/knowledge/:id` | Get entry | +| PATCH | `/api/knowledge/:id` | Update entry | +| DELETE | `/api/knowledge/:id` | Delete entry | + +#### 6.3 Knowledge Injection + +Enabled knowledge entries are included in the AI's system prompt during chat (similar to skills but with full content for small entries, or summarized for large ones). + +--- + +### Phase 7: Provider Configuration & Model Management (Day 11-12) + +**Goal**: Users configure AI providers (API keys), see available models, switch models in chat. + +#### 7.1 Provider Config Model + +```json +{ + "id": "provider_001", + "user": "user_rel_id", + "provider": "openai", + "api_key_encrypted": "enc:AES256:base64...", + "base_url": null, // optional custom URL (e.g., for reverse proxy) + "is_active": true, + "created": "2025-10-28T00:00:00Z", + "updated": "2025-10-28T00:00:00Z" +} +``` + +#### 7.2 API Key Encryption + +- Use AES-256-GCM with a server-side `ENCRYPTION_KEY` +- Keys are encrypted at rest, decrypted in memory when proxying AI requests +- Never log or expose API keys in responses (return `has_key: true` instead) + +#### 7.3 REST Endpoints + +| Method | Path | Behavior | +|--------|------|----------| +| GET | `/api/providers` | List configured providers (sanitized: no key in response, just `has_key: bool`) | +| POST | `/api/providers` | Add/update provider config. Body: `{ "provider": "openai", "api_key": "sk-...", "base_url": null }`. Encrypts key before storing. | +| DELETE | `/api/providers/:provider` | Remove provider config | +| POST | `/api/providers/:provider/validate` | Test connectivity by making a small API call to the provider | + +#### 7.4 Model Listing + +`GET /api/models` aggregates available models from all active providers: + +```json +{ + "models": [ + { + "id": "gpt-4o", + "name": "Archie 1.6 Pro", + "provider": "openai", + "tier": "pro", + "capabilities": ["chat", "function_calling", "vision"], + "available": true + }, + { + "id": "claude-sonnet-4-20250514", + "name": "Archie 1.6 Max", + "provider": "anthropic", + "tier": "max", + "capabilities": ["chat", "function_calling", "extended_thinking"], + "available": true + }, + { + "id": "gpt-4o-mini", + "name": "Archie 1.6 Lite", + "provider": "openai", + "tier": "lite", + "available": true + } + ] +} +``` + +This endpoint powers the frontend's `ModelSelector` dropdown. + +--- + +### Phase 8: User Settings (Day 12) + +**Goal**: User preferences (theme, language, selected model) stored and fetched. + +#### 8.1 Settings Model + +```json +{ + "id": "settings_001", + "user": "user_rel_id", + "theme": "system", // "light" | "dark" | "system" + "language": "en", + "selected_model": "Archie 1.6 Lite", + "created": "2025-10-28T00:00:00Z", + "updated": "2025-10-28T00:00:00Z" +} +``` + +#### 8.2 REST Endpoints + +| Method | Path | Behavior | +|--------|------|----------| +| GET | `/api/settings` | Get user settings. Creates default settings if none exist. | +| PATCH | `/api/settings` | Update settings. Partial updates supported (only send fields to change). | + +--- + +## 3. Database Migrations + +Pocketbase supports programmatic collection creation. We'll write a migration in Go that runs on server start: + +```go +// internal/migrations/001_initial_schema.go +func init() { + migrations.Register(func(app core.App) error { + // Create conversations collection + // Create messages collection + // Create tasks collection + // Create skills collection + // Create knowledge collection + // Create provider_configs collection + // Create settings collection + // Seed official skills + return nil + }, func(app core.App) error { + // Rollback: delete collections + return nil + }) +} +``` + +Each collection defines: +- Fields with types and validation rules +- List/detail/create/update/delete API rules (ownership-based: `@request.auth.id = user`) +- Indexes (user_id + created for conversations, user_id + type for tasks, full-text search indexes for knowledge) + +--- + +## 4. WebSocket Architecture (Detailed) + +### 4.1 Connection Lifecycle + +``` +Client connects to WS /api/ws/chat + │ + ▼ +Server upgrades HTTP → WS + │ + ▼ +Client sends auth token (first message or query param) + │ + ▼ +Server validates JWT, creates Client struct, registers with Hub + │ + ▼ +Client sends { "type": "join", "conversation_id": "abc" } + │ + ▼ +Server validates conversation ownership, adds to room + │ + ▼ +Client sends { "type": "message", ... } + │ + ▼ +Server processes → streams tokens → broadcasts to room + │ + ▼ (connection drops / client disconnects) +Server removes from room + hub, cleans up +``` + +### 4.2 gorilla/websocket Setup + +```go +var upgrader = websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, + CheckOrigin: func(r *http.Request) bool { + // In dev, allow all origins. In prod, check against CORS config. + return true + }, +} +``` + +### 4.3 Concurrency Model + +- **Hub**: single goroutine with channel-based operations (register, unregister, broadcast) +- **Per-client read pump**: goroutine that reads from WS and sends to hub +- **Per-client write pump**: goroutine that reads from hub and writes to WS +- **Per-stream goroutine**: spawned when AI streaming starts, sends tokens to client's send channel + +### 4.4 Error Handling & Recovery + +- Auto-reconnect: if WS drops, client can rejoin with `{ "type": "reconnect", "conversation_id": "...", "last_token_index": 42 }` +- Rate limiting: max N messages per second per user +- Provider errors: broadcast `{ "type": "error" }` to room, don't crash the connection +- Dead connection cleanup: ping/pong with timeout (30s) + +--- + +## 5. API Conventions + +### 5.1 Standard Response Format + +**Success:** +```json +{ + "data": { ... }, // single object or array + "meta": { + "page": 1, + "per_page": 50, + "total": 120 + } +} +``` + +**Error:** +```json +{ + "error": { + "code": "not_found", + "message": "Task not found", + "details": null + } +} +``` + +### 5.2 HTTP Status Codes + +| Code | Usage | +|------|-------| +| 200 | Success (GET, PATCH) | +| 201 | Created (POST) | +| 204 | No Content (DELETE) | +| 400 | Bad Request (validation error) | +| 401 | Unauthorized (missing/invalid token) | +| 403 | Forbidden (wrong ownership) | +| 404 | Not Found | +| 429 | Rate Limited | +| 500 | Internal Server Error | + +### 5.3 Pagination + +All list endpoints support: +- `?page=1&perPage=50` (pagination) +- `?sort=-created` (sort by field, `-` prefix for descending) +- `?filter=field:value` (field filtering) + +These are passed through to Pocketbase's built-in filtering where possible, or handled in custom logic. + +--- + +## 6. Frontend ↔ Backend Contract Notes + +### 6.1 Existing Frontend Patterns to Support + +| Frontend Widget | Backend API Dependency | +|----------------|----------------------| +| `HomeScreen` (task list) | `GET /api/tasks?type=...` | +| `FilterRow` | `?type=Agent|Scheduled|Favorites` query param | +| `ChatScreen` (suggestions) | `GET /api/chat/suggestions` (or static JSON → seed in DB) | +| `ModelSelector` | `GET /api/models` | +| `ChatInputArea` | `POST /api/conversations/:id/messages` + WS streaming | +| `SuggestionCarousel` | `GET /api/chat/suggestions` | +| `LibraryScreen` (Skills tab) | `GET /api/skills`, `PATCH /api/skills/:id`, `POST /api/skills` | +| `LibraryScreen` (Knowledge tab) | `GET /api/knowledge`, `POST /api/knowledge`, `PATCH /api/knowledge/:id` | +| `ProfileScreen` (providers) | `GET /api/providers`, `POST /api/providers` | +| `ProfileScreen` (settings) | `GET /api/settings`, `PATCH /api/settings` | +| `ProfileScreen` (logout) | Client-side token clear + `POST /api/auth/logout` | +| `ArchieFab` → ChatScreen | Navigation + `POST /api/conversations` | + +### 6.2 Frontend Mock Data That Should Be Replaced + +| Current Mock | Replace With | +|---|---| +| `assets/data/dashboard.json` | `GET /api/tasks` | +| `assets/data/chat_suggestions.json` | Seeded in DB or `GET /api/chat/suggestions` | +| `mockTasks` in `agent_task.dart` | `GET /api/tasks` | +| Hardcoded skills in `Notifier` | `GET /api/skills` | +| Hardcoded knowledge in `Notifier` | `GET /api/knowledge` | +| Hardcoded providers in `profile_screen.dart` | `GET /api/providers` | + +--- + +## 7. Testing Strategy + +### 7.1 Unit Tests + +| Package | Tests | +|---------|-------| +| `internal/auth` | Token generation, validation, refresh | +| `internal/tasks` | CRUD logic, filtering, ownership checks | +| `internal/chat` | Message creation, conversation management | +| `internal/providers` | Request building, response parsing, model name resolution | +| `internal/server` | Middleware (CORS, auth), route registration | +| `internal/common` | Validation helpers, error formatting | + +### 7.2 Integration Tests + +- HTTP tests using `httptest.Server` with a test Pocketbase instance +- Test each endpoint: happy path, auth errors, validation errors, ownership enforcement +- WebSocket tests with gorilla/websocket test helpers + +### 7.3 Test Data + +- Seed a test Pocketbase instance with known data +- Use `testify/assert` + `testify/require` for assertions +- Table-driven tests for handler logic + +--- + +## 8. Development Workflow + +### 8.1 Makefile Targets + +```makefile +run # Start server with hot-reload (air) +build # Build binary +test # Run all tests +test-watch # Run tests on file changes +migrate # Run Pocketbase migrations +seed # Seed development data (official skills, sample tasks) +lint # golangci-lint +``` + +### 8.2 Environment Files + +``` +.env.example # Template with placeholder values +.env.dev # Development overrides (auto-loaded if exists) +.env # Not committed +``` + +### 8.3 First Run Sequence + +1. `cp .env.example .env.dev` +2. `make run` → starts server on `:8090` +3. Server auto-creates Pocketbase data directory +4. Migrations create all collections +5. Seed script populates official skills + sample data +6. Visit `http://localhost:8090/_/` for Pocketbase Admin UI + +--- + +## 9. Security Considerations + +| Concern | Mitigation | +|---------|-----------| +| API key exposure | Encrypted at rest (AES-256-GCM), never in response payloads | +| JWT theft | Short-lived tokens (15 min) + refresh tokens (7 day). HTTPS enforced. | +| Auth bypass | Middleware checks JWT on all protected routes. Ownership checks on all CRUD operations. | +| SQL injection | Pocketbase uses parameterized queries internally. Custom queries use prepared statements. | +| Rate limiting | Per-IP and per-user rate limits on auth endpoints and chat messages. | +| CORS | Restricted to configured origins. Not wildcard in production. | +| WS origin check | `CheckOrigin` verifies against CORS whitelist in production. | +| Input validation | All inputs validated server-side. Pocketbase field rules as second layer. | + +--- + +## 10. Future Considerations (Post-MVP) + +- **File uploads**: Use Pocketbase's built-in file storage for attachments +- **Push notifications**: Send via WebSocket when tasks complete or require attention +- **Multi-user conversations**: Expand WebSocket rooms to support shared conversations +- **Admin dashboard**: Pocketbase Admin UI or custom admin panel for user management +- **Skill registry sync**: Periodically fetch official skills from AgentSkills.io API +- **Knowledge RAG**: Use embeddings + vector search for more relevant knowledge injection +- **Usage tracking**: Track tokens used per user, enforce tier limits +- **Offline support**: Frontend caches tasks/conversations locally via Flutter Hive diff --git a/.references/frontend.md b/.references/frontend.md new file mode 100644 index 000000000..835fcdb40 --- /dev/null +++ b/.references/frontend.md @@ -0,0 +1,4 @@ +# Reference details for Frontend + +The Frontend is being developed in `/work/apps/StudioProjects/archie` and is a flutter application. + diff --git a/.references/skills-implementation.md b/.references/skills-implementation.md new file mode 100644 index 000000000..07dff24ab --- /dev/null +++ b/.references/skills-implementation.md @@ -0,0 +1,367 @@ +We would need to implement the agentskills spec, I'm not sure if it needs to be implemented in the Frontend app or the backend, let me know. here's the client implementation details: + +> ## Documentation Index +> Fetch the complete documentation index at: https://agentskills.io/llms.txt +> Use this file to discover all available pages before exploring further. + +# How to add skills support to your agent + +> A guide for adding Agent Skills support to an AI agent or development tool. + +This guide walks through how to add Agent Skills support to an AI agent or development tool. It covers the full lifecycle: discovering skills, telling the model about them, loading their content into context, and keeping that content effective over time. + +The core integration is the same regardless of your agent's architecture. The implementation details vary based on two factors: + +* **Where do skills live?** A locally-running agent can scan the user's filesystem for skill directories. A cloud-hosted or sandboxed agent will need an alternative discovery mechanism — an API, a remote registry, or bundled assets. +* **How does the model access skill content?** If the model has file-reading capabilities, it can read `SKILL.md` files directly. Otherwise, you'll provide a dedicated tool or inject skill content into the prompt programmatically. + +The guide notes where these differences matter. You don't need to support every scenario — follow the path that fits your agent. + +**Prerequisites**: Familiarity with the [Agent Skills specification](/specification), which defines the `SKILL.md` file format, frontmatter fields, and directory conventions. + +## The core principle: progressive disclosure + +Every skills-compatible agent follows the same three-tier loading strategy: + +| Tier | What's loaded | When | Token cost | +| --------------- | --------------------------- | ------------------------------------ | --------------------------- | +| 1. Catalog | Name + description | Session start | \~50-100 tokens per skill | +| 2. Instructions | Full `SKILL.md` body | When the skill is activated | \<5000 tokens (recommended) | +| 3. Resources | Scripts, references, assets | When the instructions reference them | Varies | + +The model sees the catalog from the start, so it knows what skills are available. When it decides a skill is relevant, it loads the full instructions. If those instructions reference supporting files, the model loads them individually as needed. + +This keeps the base context small while giving the model access to specialized knowledge on demand. An agent with 20 installed skills doesn't pay the token cost of 20 full instruction sets upfront — only the ones actually used in a given conversation. + +## Step 1: Discover skills + +At session startup, find all available skills and load their metadata. + +### Where to scan + +Which directories you scan depends on your agent's environment. Most locally-running agents scan at least two scopes: + +* **Project-level** (relative to the working directory): Skills specific to a project or repository. +* **User-level** (relative to the home directory): Skills available across all projects for a given user. + +Other scopes are possible too — for example, organization-wide skills deployed by an admin, or skills bundled with the agent itself. The right set of scopes depends on your agent's deployment model. + +Within each scope, consider scanning both a **client-specific directory** and the **`.agents/skills/` convention**: + +| Scope | Path | Purpose | +| ------- | ---------------------------------- | ----------------------------- | +| Project | `/./skills/` | Your client's native location | +| Project | `/.agents/skills/` | Cross-client interoperability | +| User | `~/./skills/` | Your client's native location | +| User | `~/.agents/skills/` | Cross-client interoperability | + +The `.agents/skills/` paths have emerged as a widely-adopted convention for cross-client skill sharing. While the Agent Skills specification does not mandate where skill directories live (it only defines what goes inside them), scanning `.agents/skills/` means skills installed by other compliant clients are automatically visible to yours, and vice versa. + + + Some implementations also scan `.claude/skills/` (both project-level and user-level) for pragmatic compatibility, since many existing skills are installed there. Other additional locations include ancestor directories up to the git root (useful for monorepos), [XDG](https://specifications.freedesktop.org/basedir-spec/latest/) config directories, and user-configured paths. + + +### What to scan for + +Within each skills directory, look for **subdirectories containing a file named exactly `SKILL.md`**: + +``` +~/.agents/skills/ +├── pdf-processing/ +│ ├── SKILL.md ← discovered +│ └── scripts/ +│ └── extract.py +├── data-analysis/ +│ └── SKILL.md ← discovered +└── README.md ← ignored (not a skill directory) +``` + +Practical scanning rules: + +* Skip directories that won't contain skills, such as `.git/` and `node_modules/` +* Optionally respect `.gitignore` to avoid scanning build artifacts +* Set reasonable bounds (e.g., max depth of 4-6 levels, max 2000 directories) to prevent runaway scanning in large directory trees + +### Handling name collisions + +When two skills share the same `name`, apply a deterministic precedence rule. + +The universal convention across existing implementations: **project-level skills override user-level skills.** + +Within the same scope (e.g., two skills named `code-review` found under both `/.agents/skills/` and `/./skills/`), either first-found or last-found is acceptable — pick one and be consistent. Log a warning when a collision occurs so the user knows a skill was shadowed. + +### Trust considerations + +Project-level skills come from the repository being worked on, which may be untrusted (e.g., a freshly cloned open-source project). Consider gating project-level skill loading on a trust check — only load them if the user has marked the project folder as trusted. This prevents untrusted repositories from silently injecting instructions into the agent's context. + +### Cloud-hosted and sandboxed agents + +If your agent runs in a container or on a remote server, it won't have access to the user's local filesystem. Discovery needs to work differently depending on the skill scope: + +* **Project-level skills** are often the easiest case. If the agent operates on a cloned repository (even inside a sandbox), project-level skills travel with the code and can be scanned from the repo's directory tree. +* **User-level and organization-level skills** don't exist in the sandbox. You'll need to provision them from an external source — for example, cloning a configuration repository, accepting skill URLs or packages through your agent's settings, or letting users upload skill directories through a web UI. +* **Built-in skills** can be packaged as static assets within the agent's deployment artifact, making them available in every session without external fetching. + +Once skills are available to the agent, the rest of the lifecycle — parsing, disclosure, activation — works the same. + +## Step 2: Parse `SKILL.md` files + +For each discovered `SKILL.md`, extract the metadata and body content. + +### Frontmatter extraction + +A `SKILL.md` file has two parts: YAML frontmatter between `---` delimiters, and a markdown body after the closing delimiter. To parse: + +1. Find the opening `---` at the start of the file and the closing `---` after it. +2. Parse the YAML block between them. Extract `name` and `description` (required), plus any optional fields. +3. Everything after the closing `---`, trimmed, is the skill's body content. + +See the [specification](/specification) for the full set of frontmatter fields and their constraints. + +### Handling malformed YAML + +Skill files authored for other clients may contain technically invalid YAML that their parsers happen to accept. The most common issue is unquoted values containing colons: + +```yaml theme={null} +# Technically invalid YAML — the colon breaks parsing +description: Use this skill when: the user asks about PDFs +``` + +Consider a fallback that wraps such values in quotes or converts them to YAML block scalars before retrying. This improves cross-client compatibility at minimal cost. + +### Lenient validation + +Warn on issues but still load the skill when possible: + +* Name doesn't match the parent directory name → warn, load anyway +* Name exceeds 64 characters → warn, load anyway +* Description is missing or empty → skip the skill (a description is essential for disclosure), log the error +* YAML is completely unparseable → skip the skill, log the error + +Record diagnostics so they can be surfaced to the user (in a debug command, log file, or UI), but don't block skill loading on cosmetic issues. + + + The [specification](/specification) defines strict constraints on the `name` field (matching the parent directory, character set, max length). The lenient approach above deliberately relaxes these to improve compatibility with skills authored for other clients. + + +### What to store + +At minimum, each skill record needs three fields: + +| Field | Description | +| ------------- | ------------------------------------ | +| `name` | From frontmatter | +| `description` | From frontmatter | +| `location` | Absolute path to the `SKILL.md` file | + +Store these in an in-memory map keyed by `name` for fast lookup during activation. + +You can also store the **body** (the markdown content after the frontmatter) at discovery time, or read it from `location` at activation time. Storing it makes activation faster; reading it at activation time uses less memory in aggregate and picks up changes to skill files between activations. + +The skill's **base directory** (the parent directory of `location`) is needed later to resolve relative paths and enumerate bundled resources — derive it from `location` when needed. + +## Step 3: Disclose available skills to the model + +Tell the model what skills exist without loading their full content. This is [tier 1 of progressive disclosure](#the-core-principle-progressive-disclosure). + +### Building the skill catalog + +For each discovered skill, include `name`, `description`, and optionally `location` (the path to the `SKILL.md` file) in whatever structured format suits your stack — XML, JSON, or a bulleted list all work: + +```xml theme={null} + + + pdf-processing + Extract PDF text, fill forms, merge files. Use when handling PDFs. + /home/user/.agents/skills/pdf-processing/SKILL.md + + + data-analysis + Analyze datasets, generate charts, and create summary reports. + /home/user/project/.agents/skills/data-analysis/SKILL.md + + +``` + +The `location` field serves two purposes: it enables file-read activation (see [Step 4](#step-4-activate-skills)), and it gives the model a base path for resolving relative references in the skill body (like `scripts/evaluate.py`). If your dedicated activation tool provides the skill directory path in its result (see [Structured wrapping](#structured-wrapping) in Step 4), you can omit `location` from the catalog. Otherwise, include it. + +Each skill adds roughly 50-100 tokens to the catalog. Even with dozens of skills installed, the catalog remains compact. + +### Where to place the catalog + +Two approaches are common: + +**System prompt section**: Add the catalog as a labeled section in the system prompt, preceded by brief instructions on how to use skills. This is the simplest approach and works with any model that has access to a file-reading tool. + +**Tool description**: Embed the catalog in the description of a dedicated skill-activation tool (see [Step 4](#step-4-activate-skills)). This keeps the system prompt clean and naturally couples discovery with activation. + +Both work. System prompt placement is simpler and more broadly compatible; tool description embedding is cleaner when you have a dedicated activation tool. + +### Behavioral instructions + +Include a short instruction block alongside the catalog telling the model how and when to use skills. The wording depends on which activation mechanism you support (see [Step 4](#step-4-activate-skills)): + +**If the model activates skills by reading files:** + +``` +The following skills provide specialized instructions for specific tasks. +When a task matches a skill's description, use your file-read tool to load +the SKILL.md at the listed location before proceeding. +When a skill references relative paths, resolve them against the skill's +directory (the parent of SKILL.md) and use absolute paths in tool calls. +``` + +**If the model activates skills via a dedicated tool:** + +``` +The following skills provide specialized instructions for specific tasks. +When a task matches a skill's description, call the activate_skill tool +with the skill's name to load its full instructions. +``` + +Keep these instructions concise. The goal is to tell the model that skills exist and how to load them — the skill content itself provides the detailed instructions once loaded. + +### Filtering + +Some skills should be excluded from the catalog. Common reasons: + +* The user has disabled the skill in settings +* A permission system denies access to the skill +* The skill has opted out of model-driven activation (e.g., via a `disable-model-invocation` flag) + +**Hide filtered skills entirely** from the catalog rather than listing them and blocking at activation time. This prevents the model from wasting turns attempting to load skills it can't use. + +### When no skills are available + +If no skills are discovered, omit the catalog and behavioral instructions entirely. Don't show an empty `` block or register a skill tool with no valid options — this would confuse the model. + +## Step 4: Activate skills + +When the model or user selects a skill, deliver the full instructions into the conversation context. This is [tier 2 of progressive disclosure](#the-core-principle-progressive-disclosure). + +### Model-driven activation + +Most implementations rely on the model's own judgment as the activation mechanism, rather than implementing harness-side trigger matching or keyword detection. The model reads the catalog (from [Step 3](#step-3-disclose-available-skills-to-the-model)), decides a skill is relevant to the current task, and loads it. + +Two implementation patterns: + +**File-read activation**: The model calls its standard file-read tool with the `SKILL.md` path from the catalog. No special infrastructure needed — the agent's existing file-reading capability is sufficient. The model receives the file content as a tool result. This is the simplest approach when the model has file access. + +**Dedicated tool activation**: Register a tool (e.g., `activate_skill`) that takes a skill name and returns the content. This is required when the model can't read files directly, and optional (but useful) even when it can. Advantages over raw file reads: + +* Control what content is returned — e.g., strip YAML frontmatter or preserve it (see [What the model receives](#what-the-model-receives) below) +* Wrap content in structured tags for identification during context management +* List bundled resources (e.g., `references/*`) alongside the instructions +* Enforce permissions or prompt for user consent +* Track activation for analytics + + + If you use a dedicated activation tool, constrain the `name` parameter to the set of valid skill names (e.g., as an enum in the tool schema). This prevents the model from hallucinating nonexistent skill names. If no skills are available, don't register the tool at all. + + +### User-explicit activation + +Users should also be able to activate skills directly, without waiting for the model to decide. The most common pattern is a **slash command or mention syntax** (`/skill-name` or `$skill-name`) that the harness intercepts. The specific syntax is up to you — the key idea is that the harness handles the lookup and injection, so the model receives skill content without needing to take an activation action itself. + +An autocomplete widget (listing available skills as the user types) can also make this discoverable. + +### What the model receives + +When a skill is activated, the model receives the skill's instructions. Two options for what exactly that content looks like: + +**Full file**: The model sees the entire `SKILL.md` including YAML frontmatter. This is the natural outcome with file-read activation, where the model reads the raw file. It's also a valid choice for dedicated tools. The frontmatter may contain fields useful at activation time — for example, [`compatibility`](/specification#compatibility-field) notes environment requirements that could inform how the model executes the skill's instructions. + +**Body only (frontmatter stripped)**: The harness parses and removes the YAML frontmatter, returning only the markdown instructions. Among existing implementations with dedicated activation tools, most take this approach — stripping the frontmatter after extracting `name` and `description` during discovery. + +Both approaches work in practice. + +### Structured wrapping + +If you use a dedicated activation tool, consider wrapping skill content in identifying tags. For example: + +```xml theme={null} + +# PDF Processing + +## When to use this skill +Use this skill when the user needs to work with PDF files... + +[rest of SKILL.md body] + +Skill directory: /home/user/.agents/skills/pdf-processing +Relative paths in this skill are relative to the skill directory. + + + scripts/extract.py + scripts/merge.py + references/pdf-spec-summary.md + + +``` + +This has practical benefits: + +* The model can clearly distinguish skill instructions from other conversation content +* The harness can identify skill content during context compaction ([Step 5](#step-5-manage-skill-context-over-time)) +* Bundled resources are surfaced to the model without being eagerly loaded + +### Listing bundled resources + +When a dedicated activation tool returns skill content, it can also enumerate supporting files (scripts, references, assets) in the skill directory — but it should **not eagerly read them**. The model loads specific files on demand using its file-read tools when the skill's instructions reference them. + +For large skill directories, consider capping the listing and noting that it may be incomplete. + +### Permission allowlisting + +If your agent has a permission system that gates file access, **allowlist skill directories** so the model can read bundled resources without triggering user confirmation prompts. Without this, every reference to a bundled script or reference file results in a permission dialog, breaking the flow for skills that include resources beyond the `SKILL.md` itself. + +## Step 5: Manage skill context over time + +Once skill instructions are in the conversation context, keep them effective for the duration of the session. + +### Protect skill content from context compaction + +If your agent truncates or summarizes older messages when the context window fills up, **exempt skill content from pruning**. Skill instructions are durable behavioral guidance — losing them mid-conversation silently degrades the agent's performance without any visible error. The model continues operating but without the specialized instructions the skill provided. + +Common approaches: + +* Flag skill tool outputs as protected so the pruning algorithm skips them +* Use the [structured tags](#structured-wrapping) from Step 4 to identify skill content and preserve it during compaction + +### Deduplicate activations + +Consider tracking which skills have been activated in the current session. If the model (or user) attempts to load a skill that's already in context, you can skip the re-injection to avoid the same instructions appearing multiple times in the conversation. + +### Subagent delegation (optional) + +This is an advanced pattern only supported by some clients. Instead of injecting skill instructions into the main conversation, the skill is run in a **separate subagent session**. The subagent receives the skill instructions, performs the task, and returns a summary of its work to the main conversation. + +This pattern is useful when a skill's workflow is complex enough to benefit from a dedicated, focused session. + +--- + +Based on the specification provided and Archie's architecture (a mobile/web Flutter app with a backend profile), the **Agent Skills implementation needs to be split, with the core engine living in the Backend**. + +Here is how the responsibilities break down for Archie: + +### 1. The Backend (The Agent Engine) +The backend is where the actual LLM lives (or is orchestrated) and where the "cloud-hosted or sandboxed agent" rules apply. +* **Storage & Discovery:** It stores the user's custom skills (which we create in the app) either in a database or a secure cloud bucket. When the user starts a chat session, the backend gathers the metadata (`name`, `description`) for all skills the user has enabled. +* **Disclosure (Tier 1):** The backend injects the `catalog` of available skills into the LLM's system prompt or tool descriptions. +* **Activation (Tier 2 & 3):** The backend provides an `activate_skill` tool to the LLM. When the LLM calls it, the backend fetches the `SKILL.md` content from the user's profile and returns it to the LLM, tracking the context so it isn't lost during conversation compaction. +* **Execution:** If a skill has a `scripts/` folder, the backend needs a secure sandbox to execute those scripts if the agent attempts to run them. + +### 2. The Frontend App (What we are building now) +Since Archie is a client app and doesn't run the LLM locally on the device (and lacks access to arbitrary system folders like `~/.agents/skills/`), its job is purely **Management and Presentation**: +* **Skill Creation (The Library Tab):** The UI we just planned. It lets users construct the `SKILL.md` and associated `scripts/` or assets via the visual file tree. The app then bundles this into a JSON payload and sends it to the backend to be saved to their profile. +* **Skill Toggling:** The toggles in the Library screen enable or disable which skills the backend should inject into the session catalog for that user. +* **Visibility:** Optionally showing a visual indicator in the Chat screen when the backend reports that the agent has activated a specific skill. + +### How this impacts our UI plan +The plan we just made is **perfect** for this architecture. By creating a `NewSkillEntry` widget that builds a visual file tree, we are simply constructing the package that the backend will eventually consume. + +To answer the two questions from my previous plan based on this context: +1. **For Knowledge Entry:** We should use a simple multi-line Material `TextField` (or basic Markdown editor) for now, as it just saves raw text knowledge to the backend database. +2. **For Skill Entry:** We don't need real device file uploads. We just need to give users text editors to define `SKILL.md` and any script strings directly in the app. The app constructs a structured JSON representing the folder and sends it to the backend. + +Should we proceed with building out the `NewSkillEntry` file tree UI and `NewKnowledgeEntry` modules in the `LibraryScreen` as planned? diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 000000000..e22443653 --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,52 @@ +# archied — design notes + +Archie is a resident daemon that keeps personal projects moving: it polls GitHub for issues labelled +`archie`, works each one in an isolated fresh-clone worktree through a routed **workflow**, and opens a +pull request for human review. Sam reviews and merges; the machine grinds in between. + +Guiding principle: **environmental constraints over prompt instructions.** Agents don't get asked nicely +to write good code — the quality gate (vet/build/test/lint, configurable per repo and per workflow stage) +refuses to let a run succeed until it passes. The gate mechanics live in `ai-sdk/agentloop`; this repo +owns orchestration only. + +## Workflows, not a pipeline + +`internal/workflow` is the engine: a Workflow is an ordered list of stages over a shared TaskContext; +stages are deterministic steps (worktree, gate, diff cap, PR, comments — the shared step library in +`steps.go`) or agent stages (`agentloop.Run` with a stage-specific mission/toolset/gate). Routing is +label-driven (`bug` → tdd, `feature` → feasibility, default → implement), refinable by an LLM triage +stage. Adding a workflow must never require touching the engine or reimplementing steps. + +Planned workflows: +- **bootstrap** (registered): deterministic marker-file PR proving invites/clone/push/PR plumbing. +- **implement**: planner (read-only agentloop, plan posted to the issue) → builder (full toolset, + repo gate) → diff cap → PR. +- **tdd** (bugfixes): analyse → problem surface → write failing repro tests (gate stage with + `ExpectFailure`) → fix (gate: suite passes) → PR. +- **feasibility** (features): read-only analysis → roadmap fit → won't-do close with reasons, or + plan + PRD → email Sam → `waiting_human` → LLM-judged reply → hand off to implement. + +## Task lifecycle + +`queued → running(workflow:stage) → waiting_human | pr_open → merged | parked | rejected | closed_wont_do` + +SQLite (`internal/store`), WAL, one row per issue with a transitions audit table. Crash recovery: +`running` tasks re-queue on startup with attempt++. Parks are never silent — every park comments on the +issue with the reason and gate output. + +## Rejected alternative: building on tau's peer-agent runtime + +tau has a working headless child (`internal/app/child.go RunChild()`, stdio-JSONL, shared SQLite): the +wrong substrate here. Its model is ephemeral child processes under an explicit "no resident process" +principle, and it drags tau's store/coordinator/session machinery into a batch worker. archied instead +shares code with tau at the module level: `ai-sdk/toolkit` (the promoted tool set) and `ai-sdk/agentloop` +(gate/budgets/loop-breaker, patterns harvested from tau's coordinator). + +## Security posture + +- Fine-grained PAT on the bot account only (Contents RW, PRs RW, Issues RW, Metadata R), via + `ARCHIE_GITHUB_TOKEN`; git auth through an askpass helper so the token stays out of argv and + `.git/config`. +- The model never runs git: `internal/worktree` owns all repo operations deterministically. +- Agent tools are jailed to the task worktree (toolkit path jail); intended deployment is a dedicated + Incus container so the blast radius is environmental, not prompt-negotiated. diff --git a/config.example.toml b/config.example.toml new file mode 100644 index 000000000..db6a92f9a --- /dev/null +++ b/config.example.toml @@ -0,0 +1,38 @@ +# archied configuration. Copy to ~/.config/archie/config.toml. +# The GitHub token is NOT configured here: export ARCHIE_GITHUB_TOKEN. + +# work_dir = "~/.local/share/archie/work" # default +# db_path = "~/.local/share/archie/archie.db" +poll_interval = "60s" +label = "archie" +bot_user = "vectis-labs-archie" +diff_cap_lines = 400 + +[models] # role -> ai-sdk runtime ref (used by agent workflows, Phase 4b) +triage = "openai/gpt-5.4" +planner = "openai/gpt-5.4" +builder = "openai/gpt-5.4" + +[providers.openai] +class = "openai" +api_key_env = "OPENAI_API_KEY" + +[providers.ollama] +class = "ollama" + +[budgets] +max_steps = 60 +max_tokens = 400000 +wall_clock = "30m" +gate_max_failures = 5 + +[[repos]] +owner = "samcharles93" +name = "todo" +base = "main" +gate = [ + ["go", "vet", "./..."], + ["go", "build", "./..."], + ["golangci-lint", "run"], + ["go", "test", "./...", "-count=1"], +] diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..c356bd0e3 --- /dev/null +++ b/go.mod @@ -0,0 +1,22 @@ +module github.com/samcharles93/archie-core + +go 1.26.2 + +require ( + github.com/BurntSushi/toml v1.6.0 + github.com/google/go-github/v78 v78.0.0 + modernc.org/sqlite v1.54.0 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.46.0 // indirect + modernc.org/libc v1.74.1 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..273290f23 --- /dev/null +++ b/go.sum @@ -0,0 +1,61 @@ +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-github/v78 v78.0.0 h1:b1tytzFE8i//lRVDx5Qh/EdJbtTPtSVD3nF7hraEs9w= +github.com/google/go-github/v78 v78.0.0/go.mod h1:Uxvdzy82AkNlC6JQ57se9TqvmgBT7RF0ouHDNg2jd6g= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= +modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ= +modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.54.0 h1:JCxR4qwkJvOaqAoYcgDoO25Nc+ROg6EJ2LfBVzdrgog= +modernc.org/sqlite v1.54.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 000000000..3379bf975 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,132 @@ +// Package config loads archied's TOML configuration. The GitHub token is +// deliberately not part of the file: it comes from ARCHIE_GITHUB_TOKEN. +package config + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/BurntSushi/toml" +) + +// Duration is a time.Duration that unmarshals from TOML strings ("60s"). +type Duration time.Duration + +func (d *Duration) UnmarshalText(b []byte) error { + v, err := time.ParseDuration(string(b)) + if err != nil { + return err + } + *d = Duration(v) + return nil +} + +func (d Duration) Std() time.Duration { return time.Duration(d) } + +// Repo is one managed repository. +type Repo struct { + Owner string `toml:"owner"` + Name string `toml:"name"` + // Base is the branch PRs target. Defaults to "main". + Base string `toml:"base"` + // Gate is the quality-gate command list for this repo, e.g. + // [["go","vet","./..."], ["task","check"]]. Workflow stages may + // extend or override it (a TDD repro stage inverts the test command). + Gate [][]string `toml:"gate"` +} + +func (r Repo) FullName() string { return r.Owner + "/" + r.Name } + +func (r Repo) BaseBranch() string { + if r.Base == "" { + return "main" + } + return r.Base +} + +// Budgets bound every agent stage. Zero disables a limit. +type Budgets struct { + MaxSteps int `toml:"max_steps"` + MaxTokens int `toml:"max_tokens"` + WallClock Duration `toml:"wall_clock"` + GateMaxFailures int `toml:"gate_max_failures"` +} + +// Provider configures one LLM provider for the runtime catalog. +type Provider struct { + Class string `toml:"class"` + APIKeyEnv string `toml:"api_key_env"` + BaseURL string `toml:"base_url"` +} + +// Config is the daemon configuration. +type Config struct { + WorkDir string `toml:"work_dir"` + DBPath string `toml:"db_path"` + PollInterval Duration `toml:"poll_interval"` + // Label marks issues archie should pick up. + Label string `toml:"label"` + BotUser string `toml:"bot_user"` + // BotEmail is the git author email; defaults to the GitHub noreply + // address for BotUser. + BotEmail string `toml:"bot_email"` + DiffCapLines int `toml:"diff_cap_lines"` + + // Models maps a role ("planner", "builder", "triage") to a runtime + // model ref ("provider/model"). + Models map[string]string `toml:"models"` + + Providers map[string]Provider `toml:"providers"` + + Budgets Budgets `toml:"budgets"` + Repos []Repo `toml:"repos"` +} + +// Load reads, parses, and defaults the configuration. +func Load(path string) (Config, error) { + var cfg Config + if _, err := toml.DecodeFile(path, &cfg); err != nil { + return cfg, fmt.Errorf("config %s: %w", path, err) + } + if cfg.WorkDir == "" { + cfg.WorkDir = filepath.Join(dataHome(), "archie", "work") + } + if cfg.DBPath == "" { + cfg.DBPath = filepath.Join(dataHome(), "archie", "archie.db") + } + if cfg.PollInterval == 0 { + cfg.PollInterval = Duration(60 * time.Second) + } + if cfg.Label == "" { + cfg.Label = "archie" + } + if cfg.DiffCapLines == 0 { + cfg.DiffCapLines = 400 + } + if cfg.BotUser == "" { + return cfg, errors.New("config: bot_user is required") + } + if cfg.BotEmail == "" { + cfg.BotEmail = cfg.BotUser + "@users.noreply.github.com" + } + if len(cfg.Repos) == 0 { + return cfg, errors.New("config: at least one [[repos]] entry is required") + } + for i, r := range cfg.Repos { + if r.Owner == "" || r.Name == "" { + return cfg, fmt.Errorf("config: repos[%d] needs owner and name", i) + } + } + return cfg, nil +} + +func dataHome() string { + if x := os.Getenv("XDG_DATA_HOME"); x != "" { + return x + } + home, _ := os.UserHomeDir() + return filepath.Join(home, ".local", "share") +} diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go new file mode 100644 index 000000000..d6013628c --- /dev/null +++ b/internal/daemon/daemon.go @@ -0,0 +1,168 @@ +// Package daemon is archied's resident loop: poll GitHub for labelled +// issues, enqueue them, and process one task at a time through its +// routed workflow. State lives in the store; the daemon is restartable +// at any point. +package daemon + +import ( + "context" + "log/slog" + "strings" + "time" + + "github.com/google/go-github/v78/github" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/forge" + "github.com/samcharles93/archie-core/internal/store" + "github.com/samcharles93/archie-core/internal/workflow" + "github.com/samcharles93/archie-core/internal/worktree" +) + +type Daemon struct { + Cfg config.Config + Store *store.Store + Forge *forge.Client + Trees *worktree.Manager + Workflows workflow.Registry + Log *slog.Logger +} + +// Startup runs crash recovery and access verification once. +func (d *Daemon) Startup(ctx context.Context) error { + n, err := d.Store.RecoverStale(ctx) + if err != nil { + return err + } + if n > 0 { + d.Log.Info("re-queued tasks left running by a previous daemon", "count", n) + } + if err := d.Forge.AcceptInvitations(ctx); err != nil { + d.Log.Warn("invitation sweep failed", "err", err) + } + for _, r := range d.Cfg.Repos { + if err := d.Forge.VerifyPush(ctx, r.Owner, r.Name); err != nil { + d.Log.Warn("repo not pushable — tasks from it will fail", "repo", r.FullName(), "err", err) + } + } + return nil +} + +// Run polls until the context ends. +func (d *Daemon) Run(ctx context.Context) error { + ticker := time.NewTicker(d.Cfg.PollInterval.Std()) + defer ticker.Stop() + for { + d.Cycle(ctx) + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +// Cycle is one poll-and-drain pass: enqueue new labelled issues, +// reconcile open PRs, then process every queued task sequentially. +func (d *Daemon) Cycle(ctx context.Context) { + d.poll(ctx) + d.reconcilePRs(ctx) + for { + if ctx.Err() != nil { + return + } + task, err := d.Store.ClaimNext(ctx) + if err != nil { + d.Log.Error("claim failed", "err", err) + return + } + if task == nil { + return + } + d.process(ctx, task) + } +} + +func (d *Daemon) poll(ctx context.Context) { + for _, repo := range d.Cfg.Repos { + issues, err := d.Forge.LabelledIssues(ctx, repo.Owner, repo.Name, d.Cfg.Label) + if err != nil { + d.Log.Error("poll failed", "repo", repo.FullName(), "err", err) + continue + } + for _, is := range issues { + inserted, err := d.Store.EnqueueIssue(ctx, + repo.Owner, repo.Name, is.GetNumber(), is.GetTitle(), is.GetBody(), labelNames(is)) + if err != nil { + d.Log.Error("enqueue failed", "repo", repo.FullName(), "issue", is.GetNumber(), "err", err) + continue + } + if inserted { + d.Log.Info("issue queued", "repo", repo.FullName(), "issue", is.GetNumber(), "title", is.GetTitle()) + } + } + } +} + +// reconcilePRs moves pr_open tasks to merged/rejected from GitHub state +// and cleans up their worktrees. +func (d *Daemon) reconcilePRs(ctx context.Context) { + tasks, err := d.Store.OpenPRs(ctx) + if err != nil { + d.Log.Error("reconcile query failed", "err", err) + return + } + for _, t := range tasks { + state, err := d.Forge.PRState(ctx, t.Owner, t.Repo, t.PRNumber) + if err != nil { + d.Log.Warn("PR state check failed", "pr", t.PRNumber, "err", err) + continue + } + switch state { + case "merged": + _ = d.Store.Transition(ctx, t.ID, store.StatusPROpen, store.StatusMerged, "") + _ = d.Trees.Cleanup(t.Owner, t.Repo, t.IssueNumber) + d.Log.Info("PR merged", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber) + case "closed": + _ = d.Store.Transition(ctx, t.ID, store.StatusPROpen, store.StatusRejected, "PR closed without merge") + _ = d.Trees.Cleanup(t.Owner, t.Repo, t.IssueNumber) + d.Log.Info("PR rejected", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber) + } + } +} + +func (d *Daemon) process(ctx context.Context, task *store.Task) { + repo, ok := d.repoFor(task) + if !ok { + _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "repo no longer in config") + return + } + wf := workflow.Route(task, d.Workflows) + d.Log.Info("processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "workflow", wf.Name, "attempt", task.Attempt) + workflow.Run(ctx, wf, &workflow.TaskContext{ + Task: task, + Repo: repo, + Cfg: d.Cfg, + Forge: d.Forge, + Store: d.Store, + Trees: d.Trees, + Log: d.Log, + }) +} + +func (d *Daemon) repoFor(t *store.Task) (config.Repo, bool) { + for _, r := range d.Cfg.Repos { + if r.Owner == t.Owner && r.Name == t.Repo { + return r, true + } + } + return config.Repo{}, false +} + +func labelNames(is *github.Issue) string { + var names []string + for _, l := range is.Labels { + names = append(names, l.GetName()) + } + return strings.Join(names, ",") +} diff --git a/internal/forge/github.go b/internal/forge/github.go new file mode 100644 index 000000000..825fbea3a --- /dev/null +++ b/internal/forge/github.go @@ -0,0 +1,124 @@ +// Package forge wraps the GitHub API surface archied needs: polling +// labelled issues, accepting collaborator invitations, commenting, and +// opening pull requests. Gitea support later lands as a second +// implementation behind the same methods. +package forge + +import ( + "context" + "fmt" + "log/slog" + + "github.com/google/go-github/v78/github" +) + +type Client struct { + gh *github.Client + log *slog.Logger +} + +func New(token string, log *slog.Logger) *Client { + return &Client{gh: github.NewClient(nil).WithAuthToken(token), log: log} +} + +// AcceptInvitations auto-accepts pending repository invitations so +// adding archie as a collaborator is all a human has to do. +func (c *Client) AcceptInvitations(ctx context.Context) error { + invites, _, err := c.gh.Users.ListInvitations(ctx, nil) + if err != nil { + return fmt.Errorf("list invitations: %w", err) + } + for _, inv := range invites { + if _, err := c.gh.Users.AcceptInvitation(ctx, inv.GetID()); err != nil { + c.log.Warn("accept invitation failed", "repo", inv.GetRepo().GetFullName(), "err", err) + continue + } + c.log.Info("accepted repository invitation", "repo", inv.GetRepo().GetFullName()) + } + return nil +} + +// LabelledIssues returns open issues carrying the label, excluding PRs. +func (c *Client) LabelledIssues(ctx context.Context, owner, repo, label string) ([]*github.Issue, error) { + var out []*github.Issue + opts := &github.IssueListByRepoOptions{ + State: "open", + Labels: []string{label}, + ListOptions: github.ListOptions{PerPage: 50}, + } + for { + issues, resp, err := c.gh.Issues.ListByRepo(ctx, owner, repo, opts) + if err != nil { + return nil, fmt.Errorf("list issues %s/%s: %w", owner, repo, err) + } + for _, is := range issues { + if !is.IsPullRequest() { + out = append(out, is) + } + } + if resp.NextPage == 0 { + break + } + opts.ListOptions.Page = resp.NextPage + } + return out, nil +} + +// Comment posts an issue (or PR) comment. +func (c *Client) Comment(ctx context.Context, owner, repo string, number int, body string) error { + _, _, err := c.gh.Issues.CreateComment(ctx, owner, repo, number, + &github.IssueComment{Body: github.Ptr(body)}) + return err +} + +// CreatePR opens a pull request and returns its number. +func (c *Client) CreatePR(ctx context.Context, owner, repo, title, head, base, body string) (int, error) { + pr, _, err := c.gh.PullRequests.Create(ctx, owner, repo, &github.NewPullRequest{ + Title: github.Ptr(title), + Head: github.Ptr(head), + Base: github.Ptr(base), + Body: github.Ptr(body), + }) + if err != nil { + return 0, fmt.Errorf("create PR %s/%s: %w", owner, repo, err) + } + return pr.GetNumber(), nil +} + +// PRState returns "open", "merged", or "closed" for a PR. +func (c *Client) PRState(ctx context.Context, owner, repo string, number int) (string, error) { + pr, _, err := c.gh.PullRequests.Get(ctx, owner, repo, number) + if err != nil { + return "", err + } + if pr.GetMerged() { + return "merged", nil + } + return pr.GetState(), nil +} + +// CloseIssue closes an issue with a final comment (feasibility "won't do"). +func (c *Client) CloseIssue(ctx context.Context, owner, repo string, number int, comment string) error { + if comment != "" { + if err := c.Comment(ctx, owner, repo, number, comment); err != nil { + return err + } + } + _, _, err := c.gh.Issues.Edit(ctx, owner, repo, number, + &github.IssueRequest{State: github.Ptr("closed")}) + return err +} + +// VerifyPush confirms the token can push to the repo (permission check +// at startup so misconfiguration surfaces before any work is claimed). +func (c *Client) VerifyPush(ctx context.Context, owner, repo string) error { + r, _, err := c.gh.Repositories.Get(ctx, owner, repo) + if err != nil { + return fmt.Errorf("get repo %s/%s: %w", owner, repo, err) + } + perms := r.GetPermissions() + if !perms["push"] { + return fmt.Errorf("no push permission on %s/%s", owner, repo) + } + return nil +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 000000000..d54f209de --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,204 @@ +// Package store is archied's SQLite state: one row per task (a GitHub +// issue picked up for work), with every lifecycle transition recorded. +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + + _ "modernc.org/sqlite" +) + +// Task lifecycle statuses. Workflows move tasks between them; the +// daemon owns queued→running claims and crash recovery. +const ( + StatusQueued = "queued" + StatusRunning = "running" + StatusWaitingHuman = "waiting_human" + StatusPROpen = "pr_open" + StatusMerged = "merged" + StatusParked = "parked" + StatusRejected = "rejected" + StatusClosedWontDo = "closed_wont_do" +) + +type Task struct { + ID int64 + Owner string + Repo string + IssueNumber int + Title string + Body string + Labels string // comma-separated, as seen at enqueue time + Status string + Workflow string + Stage string + Branch string + Plan string + Notes string + PRNumber int + TokensUsed int + Iterations int + Attempt int + ParkReason string +} + +type Store struct{ db *sql.DB } + +// Open opens (creating if needed) the SQLite database and its schema. +func Open(path string) (*Store, error) { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, err + } + db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + if err != nil { + return nil, err + } + if _, err := db.Exec(schema); err != nil { + db.Close() + return nil, fmt.Errorf("store: init schema: %w", err) + } + return &Store{db: db}, nil +} + +const schema = ` +CREATE TABLE IF NOT EXISTS tasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + owner TEXT NOT NULL, + repo TEXT NOT NULL, + issue_number INTEGER NOT NULL, + title TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL DEFAULT '', + labels TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'queued', + workflow TEXT NOT NULL DEFAULT '', + stage TEXT NOT NULL DEFAULT '', + branch TEXT NOT NULL DEFAULT '', + plan TEXT NOT NULL DEFAULT '', + notes TEXT NOT NULL DEFAULT '', + pr_number INTEGER NOT NULL DEFAULT 0, + tokens_used INTEGER NOT NULL DEFAULT 0, + iterations INTEGER NOT NULL DEFAULT 0, + attempt INTEGER NOT NULL DEFAULT 0, + park_reason TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(owner, repo, issue_number) +); +CREATE TABLE IF NOT EXISTS transitions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_id INTEGER NOT NULL, + at TEXT NOT NULL DEFAULT (datetime('now')), + from_status TEXT NOT NULL, + to_status TEXT NOT NULL, + detail TEXT NOT NULL DEFAULT '' +); +` + +func (s *Store) Close() error { return s.db.Close() } + +// EnqueueIssue inserts a new queued task for the issue; returns false if +// the issue is already tracked (the idempotency key is owner/repo/number). +func (s *Store) EnqueueIssue(ctx context.Context, owner, repo string, number int, title, body, labels string) (bool, error) { + res, err := s.db.ExecContext(ctx, ` + INSERT INTO tasks (owner, repo, issue_number, title, body, labels) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(owner, repo, issue_number) DO NOTHING`, + owner, repo, number, title, body, labels) + if err != nil { + return false, err + } + n, _ := res.RowsAffected() + return n > 0, nil +} + +// ClaimNext atomically moves the oldest queued task to running and +// returns it; nil when the queue is empty. +func (s *Store) ClaimNext(ctx context.Context) (*Task, error) { + row := s.db.QueryRowContext(ctx, ` + UPDATE tasks SET status='running', attempt=attempt+1, updated_at=datetime('now') + WHERE id = (SELECT id FROM tasks WHERE status='queued' ORDER BY id LIMIT 1) + RETURNING id, owner, repo, issue_number, title, body, labels, status, + workflow, stage, branch, plan, notes, pr_number, tokens_used, + iterations, attempt, park_reason`) + t, err := scanTask(row) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return t, err +} + +func scanTask(row *sql.Row) (*Task, error) { + var t Task + err := row.Scan(&t.ID, &t.Owner, &t.Repo, &t.IssueNumber, &t.Title, &t.Body, + &t.Labels, &t.Status, &t.Workflow, &t.Stage, &t.Branch, &t.Plan, &t.Notes, + &t.PRNumber, &t.TokensUsed, &t.Iterations, &t.Attempt, &t.ParkReason) + if err != nil { + return nil, err + } + return &t, nil +} + +// Transition moves a task to a new status, recording the change. +func (s *Store) Transition(ctx context.Context, taskID int64, from, to, detail string) error { + if _, err := s.db.ExecContext(ctx, + `UPDATE tasks SET status=?, updated_at=datetime('now') WHERE id=?`, to, taskID); err != nil { + return err + } + _, err := s.db.ExecContext(ctx, + `INSERT INTO transitions (task_id, from_status, to_status, detail) VALUES (?, ?, ?, ?)`, + taskID, from, to, clip(detail, 4000)) + return err +} + +// Update persists mutable task fields written by workflows. +func (s *Store) Update(ctx context.Context, t *Task) error { + _, err := s.db.ExecContext(ctx, ` + UPDATE tasks SET workflow=?, stage=?, branch=?, plan=?, notes=?, + pr_number=?, tokens_used=?, iterations=?, park_reason=?, + updated_at=datetime('now') + WHERE id=?`, + t.Workflow, t.Stage, t.Branch, t.Plan, t.Notes, + t.PRNumber, t.TokensUsed, t.Iterations, clip(t.ParkReason, 4000), t.ID) + return err +} + +// RecoverStale re-queues tasks left running by a crashed daemon. +func (s *Store) RecoverStale(ctx context.Context) (int64, error) { + res, err := s.db.ExecContext(ctx, + `UPDATE tasks SET status='queued', updated_at=datetime('now') WHERE status='running'`) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + +// OpenPRs returns tasks whose PR state should be reconciled with GitHub. +func (s *Store) OpenPRs(ctx context.Context) ([]Task, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT id, owner, repo, issue_number, pr_number, status FROM tasks WHERE status='pr_open'`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Task + for rows.Next() { + var t Task + if err := rows.Scan(&t.ID, &t.Owner, &t.Repo, &t.IssueNumber, &t.PRNumber, &t.Status); err != nil { + return nil, err + } + out = append(out, t) + } + return out, rows.Err() +} + +func clip(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 000000000..fdeaed044 --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,73 @@ +package store + +import ( + "context" + "path/filepath" + "testing" +) + +func openTest(t *testing.T) *Store { + t.Helper() + s, err := Open(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +func TestEnqueueIsIdempotent(t *testing.T) { + s := openTest(t) + ctx := context.Background() + + ins, err := s.EnqueueIssue(ctx, "sam", "todo", 1, "add tests", "body", "archie") + if err != nil || !ins { + t.Fatalf("first enqueue = (%v, %v)", ins, err) + } + ins, err = s.EnqueueIssue(ctx, "sam", "todo", 1, "add tests", "body", "archie") + if err != nil || ins { + t.Fatalf("duplicate enqueue must be a no-op, got (%v, %v)", ins, err) + } +} + +func TestClaimTransitionAndRecovery(t *testing.T) { + s := openTest(t) + ctx := context.Background() + + if _, err := s.EnqueueIssue(ctx, "sam", "todo", 7, "t", "", "archie,bug"); err != nil { + t.Fatal(err) + } + + task, err := s.ClaimNext(ctx) + if err != nil || task == nil { + t.Fatalf("claim = (%v, %v)", task, err) + } + if task.Status != StatusRunning || task.Attempt != 1 || task.Labels != "archie,bug" { + t.Fatalf("claimed task = %+v", task) + } + if next, _ := s.ClaimNext(ctx); next != nil { + t.Fatal("second claim must return nil while task is running") + } + + // Crash recovery: running goes back to queued, attempt increments on + // the next claim. + if n, err := s.RecoverStale(ctx); err != nil || n != 1 { + t.Fatalf("recover = (%d, %v)", n, err) + } + task, err = s.ClaimNext(ctx) + if err != nil || task == nil || task.Attempt != 2 { + t.Fatalf("re-claim after recovery = (%+v, %v)", task, err) + } + + if err := s.Transition(ctx, task.ID, StatusRunning, StatusPROpen, "PR #3"); err != nil { + t.Fatal(err) + } + task.PRNumber = 3 + if err := s.Update(ctx, task); err != nil { + t.Fatal(err) + } + open, err := s.OpenPRs(ctx) + if err != nil || len(open) != 1 || open[0].PRNumber != 3 { + t.Fatalf("open PRs = (%+v, %v)", open, err) + } +} diff --git a/internal/workflow/steps.go b/internal/workflow/steps.go new file mode 100644 index 000000000..5ff54a706 --- /dev/null +++ b/internal/workflow/steps.go @@ -0,0 +1,105 @@ +package workflow + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/samcharles93/archie-core/internal/store" +) + +// Shared step library. Every workflow composes these; workflow-specific +// stages live next to their workflow definition. + +// StagePrepareWorktree clones the repo fresh and checks out the task +// branch. +func StagePrepareWorktree() Stage { + return Stage{Name: "prepare", Run: func(ctx context.Context, tc *TaskContext) error { + dir, branch, err := tc.Trees.Prepare(ctx, tc.Task.Owner, tc.Task.Repo, tc.Repo.BaseBranch(), tc.Task.IssueNumber) + if err != nil { + return err + } + tc.Dir, tc.Branch = dir, branch + tc.Task.Branch = branch + return nil + }} +} + +// StageCommitPush commits everything in the worktree and pushes the +// branch; an empty tree parks (the workflow produced nothing). +func StageCommitPush(message func(*TaskContext) string) Stage { + return Stage{Name: "commit-push", Run: func(ctx context.Context, tc *TaskContext) error { + changed, err := tc.Trees.CommitAll(ctx, tc.Dir, message(tc)) + if err != nil { + return err + } + if !changed { + return fmt.Errorf("worktree has no changes to commit") + } + return tc.Trees.Push(ctx, tc.Dir, tc.Branch) + }} +} + +// StageDiffCap parks tasks whose diff exceeds the configured line cap — +// oversized changes need human pre-approval, not an auto-opened PR. +func StageDiffCap() Stage { + return Stage{Name: "diff-cap", Run: func(ctx context.Context, tc *TaskContext) error { + lines, err := tc.Trees.ChangedLines(ctx, tc.Dir, tc.Repo.BaseBranch()) + if err != nil { + return err + } + if lines > tc.Cfg.DiffCapLines { + tc.Outcome = Outcome{ + Status: store.StatusParked, + Detail: fmt.Sprintf("diff is %d changed lines (cap %d) — split the issue or approve manually", lines, tc.Cfg.DiffCapLines), + } + } + return nil + }} +} + +// StageOpenPR opens the pull request with the given body and records it. +func StageOpenPR(body func(*TaskContext) string) Stage { + return Stage{Name: "open-pr", Run: func(ctx context.Context, tc *TaskContext) error { + t := tc.Task + title := fmt.Sprintf("%s (archie)", t.Title) + num, err := tc.Forge.CreatePR(ctx, t.Owner, t.Repo, title, tc.Branch, tc.Repo.BaseBranch(), body(tc)) + if err != nil { + return err + } + t.PRNumber = num + tc.Outcome = Outcome{Status: store.StatusPROpen, Detail: fmt.Sprintf("PR #%d", num)} + return nil + }} +} + +// Bootstrap is the deterministic no-LLM workflow that proves the +// plumbing: it adds a marker file and opens a PR referencing the issue. +// It stays registered as a diagnostics workflow (label a test issue and +// you exercise invites, clone, push, and PR mechanics end to end). +func Bootstrap() Workflow { + return Workflow{ + Name: "bootstrap", + Stages: []Stage{ + StagePrepareWorktree(), + {Name: "apply", Run: func(ctx context.Context, tc *TaskContext) error { + dir := filepath.Join(tc.Dir, ".archie") + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + content := fmt.Sprintf("# archie bootstrap\n\nIssue: #%d — %s\nTime: %s\n\nThis file proves the archie pipeline (poll → worktree → push → PR) works for this repository.\n", + tc.Task.IssueNumber, tc.Task.Title, time.Now().UTC().Format(time.RFC3339)) + return os.WriteFile(filepath.Join(dir, "bootstrap.md"), []byte(content), 0o644) + }}, + StageCommitPush(func(tc *TaskContext) string { + return fmt.Sprintf("chore: archie bootstrap marker for #%d", tc.Task.IssueNumber) + }), + StageDiffCap(), + StageOpenPR(func(tc *TaskContext) string { + return fmt.Sprintf("Deterministic bootstrap PR from archie's plumbing walk-through.\n\nCloses #%d", tc.Task.IssueNumber) + }), + }, + } +} diff --git a/internal/workflow/workflow.go b/internal/workflow/workflow.go new file mode 100644 index 000000000..855ae7cb3 --- /dev/null +++ b/internal/workflow/workflow.go @@ -0,0 +1,147 @@ +// Package workflow is archied's extensible pipeline engine. A Workflow +// is an ordered list of stages over a shared TaskContext; stages are +// either deterministic steps (git, gate, PR, comments) or agent stages +// (agentloop runs). New workflows compose from the shared step library — +// adding one must never require reimplementing the engine or the steps. +package workflow + +import ( + "context" + "fmt" + "log/slog" + "strings" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/forge" + "github.com/samcharles93/archie-core/internal/store" + "github.com/samcharles93/archie-core/internal/worktree" +) + +// TaskContext carries everything a stage may need. Stages communicate +// forward by mutating Task (persisted after every stage) and the +// scratch fields below. +type TaskContext struct { + Task *store.Task + Repo config.Repo + Cfg config.Config + Forge *forge.Client + Store *store.Store + Trees *worktree.Manager + Log *slog.Logger + + // Dir/Branch are set by the prepare step. + Dir string + Branch string + // Outcome describes where the task ended up; the engine applies it. + Outcome Outcome +} + +// Outcome is a stage's terminal decision for the whole workflow. Stages +// that don't end the workflow leave it zero. +type Outcome struct { + Status string // store.Status* value; empty = continue to next stage + Detail string // park reason / close rationale / PR body context +} + +// Stage is one named step. Returning an error parks the task with the +// error text; setting ctx.Outcome ends the workflow with that status. +type Stage struct { + Name string + Run func(ctx context.Context, tc *TaskContext) error +} + +// Workflow is a named, ordered stage list. +type Workflow struct { + Name string + Stages []Stage +} + +// Registry maps workflow names to definitions. +type Registry map[string]Workflow + +// Route picks the workflow for a task. Label-driven first; an LLM +// triage stage inside a workflow can still re-route by returning an +// Outcome that re-enqueues under a different workflow (later phase). +func Route(t *store.Task, reg Registry) Workflow { + labels := strings.Split(t.Labels, ",") + for _, l := range labels { + switch strings.TrimSpace(l) { + case "bug": + if wf, ok := reg["tdd"]; ok { + return wf + } + case "feature": + if wf, ok := reg["feasibility"]; ok { + return wf + } + } + } + if wf, ok := reg["implement"]; ok { + return wf + } + // Registry always ships a default; this is a config error backstop. + return Workflow{Name: "none", Stages: []Stage{{ + Name: "fail", + Run: func(context.Context, *TaskContext) error { + return fmt.Errorf("no workflow registered for task") + }, + }}} +} + +// Run executes the workflow: stages in order, task persisted after each, +// errors park the task with a comment on the issue — never silently. +func Run(ctx context.Context, wf Workflow, tc *TaskContext) { + t := tc.Task + t.Workflow = wf.Name + log := tc.Log.With("workflow", wf.Name, "repo", tc.Repo.FullName(), "issue", t.IssueNumber) + + for _, stage := range wf.Stages { + t.Stage = stage.Name + _ = tc.Store.Update(ctx, t) + log.Info("stage starting", "stage", stage.Name) + + if err := stage.Run(ctx, tc); err != nil { + t.ParkReason = fmt.Sprintf("stage %s: %v", stage.Name, err) + log.Warn("stage failed — parking", "stage", stage.Name, "err", err) + park(ctx, tc, t.ParkReason) + return + } + if tc.Outcome.Status != "" { + finish(ctx, tc, log) + return + } + } + // A workflow must end with an explicit outcome; not doing so is a + // definition bug, which still must not vanish silently. + park(ctx, tc, "workflow ended without an outcome (definition bug)") +} + +func finish(ctx context.Context, tc *TaskContext, log *slog.Logger) { + t := tc.Task + if tc.Outcome.Status == store.StatusParked { + park(ctx, tc, tc.Outcome.Detail) + return + } + _ = tc.Store.Update(ctx, t) + _ = tc.Store.Transition(ctx, t.ID, store.StatusRunning, tc.Outcome.Status, tc.Outcome.Detail) + log.Info("workflow finished", "status", tc.Outcome.Status) +} + +func park(ctx context.Context, tc *TaskContext, reason string) { + t := tc.Task + t.ParkReason = reason + _ = tc.Store.Update(ctx, t) + _ = tc.Store.Transition(ctx, t.ID, store.StatusRunning, store.StatusParked, reason) + body := fmt.Sprintf("**archie parked this task.**\n\n```\n%s\n```\n\nThe worktree is kept for inspection. Re-add the `%s` label after addressing this to retry.", + clip(reason, 3000), tc.Cfg.Label) + if err := tc.Forge.Comment(ctx, t.Owner, t.Repo, t.IssueNumber, body); err != nil { + tc.Log.Error("failed to post park comment", "err", err) + } +} + +func clip(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "\n…(truncated)" +} diff --git a/internal/worktree/worktree.go b/internal/worktree/worktree.go new file mode 100644 index 000000000..da76931dd --- /dev/null +++ b/internal/worktree/worktree.go @@ -0,0 +1,151 @@ +// Package worktree owns archied's git operations: fresh clone per task, +// branch, commit as the bot identity, push, diff stats, cleanup. The +// daemon performs these deterministically — the model's shell tool never +// drives git. +package worktree + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +type Manager struct { + // WorkDir is the root under which task worktrees are created. + WorkDir string + // Token authenticates clone/push over HTTPS via an askpass helper, + // keeping it out of .git/config and process argv. + Token string + BotUser string + BotEmail string + // BaseURL overrides the forge host (tests use a local file:// remote). + // Empty means https://github.com. + BaseURL string +} + +// Dir is the worktree path for a task. +func (m *Manager) Dir(owner, repo string, issue int) string { + return filepath.Join(m.WorkDir, owner+"-"+repo, fmt.Sprintf("issue-%d", issue)) +} + +// askpass writes (once) a helper script that emits the token, so git +// authenticates without the token appearing in argv or .git/config. +func (m *Manager) askpass() (string, error) { + path := filepath.Join(m.WorkDir, ".git-askpass") + script := "#!/bin/sh\necho \"$ARCHIE_GIT_TOKEN\"\n" + if err := os.MkdirAll(m.WorkDir, 0o755); err != nil { + return "", err + } + if err := os.WriteFile(path, []byte(script), 0o700); err != nil { + return "", err + } + return path, nil +} + +func (m *Manager) git(ctx context.Context, dir string, args ...string) (string, error) { + helper, err := m.askpass() + if err != nil { + return "", err + } + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Dir = dir + cmd.Env = append( + os.Environ(), + "GIT_ASKPASS="+helper, + "ARCHIE_GIT_TOKEN="+m.Token, + "GIT_TERMINAL_PROMPT=0", + ) + out, err := cmd.CombinedOutput() + if err != nil { + return string(out), fmt.Errorf("git %s: %w\n%s", strings.Join(args, " "), err, out) + } + return string(out), nil +} + +// Prepare creates a fresh clone for the task and checks out its branch. +// Any leftover worktree from a prior attempt is removed first. +func (m *Manager) Prepare(ctx context.Context, owner, repo, base string, issue int) (dir, branch string, err error) { + dir = m.Dir(owner, repo, issue) + branch = fmt.Sprintf("archie/issue-%d", issue) + if err := os.RemoveAll(dir); err != nil { + return "", "", err + } + if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { + return "", "", err + } + url := m.cloneURL(owner, repo) + if _, err := m.git(ctx, filepath.Dir(dir), "clone", "--branch", base, url, dir); err != nil { + return "", "", err + } + for _, args := range [][]string{ + {"config", "user.name", m.BotUser}, + {"config", "user.email", m.BotEmail}, + {"checkout", "-b", branch}, + } { + if _, err := m.git(ctx, dir, args...); err != nil { + return "", "", err + } + } + return dir, branch, nil +} + +func (m *Manager) cloneURL(owner, repo string) string { + if m.BaseURL != "" { + return fmt.Sprintf("%s/%s/%s.git", m.BaseURL, owner, repo) + } + return fmt.Sprintf("https://%s@github.com/%s/%s.git", m.BotUser, owner, repo) +} + +// CommitAll stages everything and commits; returns false when the tree +// is clean (nothing to commit). +func (m *Manager) CommitAll(ctx context.Context, dir, message string) (bool, error) { + if _, err := m.git(ctx, dir, "add", "-A"); err != nil { + return false, err + } + status, err := m.git(ctx, dir, "status", "--porcelain") + if err != nil { + return false, err + } + if strings.TrimSpace(status) == "" { + return false, nil + } + if _, err := m.git(ctx, dir, "commit", "-m", message); err != nil { + return false, err + } + return true, nil +} + +// Push publishes the branch. +func (m *Manager) Push(ctx context.Context, dir, branch string) error { + _, err := m.git(ctx, dir, "push", "-u", "origin", branch) + return err +} + +// ChangedLines reports lines added+deleted vs the base branch — the +// input to the diff-size cap. +func (m *Manager) ChangedLines(ctx context.Context, dir, base string) (int, error) { + out, err := m.git(ctx, dir, "diff", "--shortstat", "origin/"+base+"...HEAD") + if err != nil { + return 0, err + } + total := 0 + for _, f := range strings.Fields(out) { + var n int + if _, err := fmt.Sscanf(f, "%d", &n); err == nil { + total += n + } + } + // The first field is the file count; subtract it back out. + var files int + fmt.Sscanf(strings.TrimSpace(out), "%d", &files) + return total - files, nil +} + +// Cleanup removes a task's worktree (merged/rejected); parked worktrees +// are kept for post-mortems. +func (m *Manager) Cleanup(owner, repo string, issue int) error { + return os.RemoveAll(m.Dir(owner, repo, issue)) +} diff --git a/internal/worktree/worktree_test.go b/internal/worktree/worktree_test.go new file mode 100644 index 000000000..d0babfded --- /dev/null +++ b/internal/worktree/worktree_test.go @@ -0,0 +1,122 @@ +package worktree + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// newLocalRemote creates a bare repo with one commit on main and +// returns the directory that serves as the fake forge host: the bare +// repo lives at //.git so cloneURL resolves it via +// file://. +func newLocalRemote(t *testing.T, owner, repo string) string { + t.Helper() + host := t.TempDir() + bare := filepath.Join(host, owner, repo+".git") + seed := filepath.Join(t.TempDir(), "seed") + + run := func(dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + + if err := os.MkdirAll(bare, 0o755); err != nil { + t.Fatal(err) + } + run(host, "init", "--bare", "-b", "main", bare) + if err := os.MkdirAll(seed, 0o755); err != nil { + t.Fatal(err) + } + run(seed, "init", "-b", "main") + run(seed, "config", "user.name", "seeder") + run(seed, "config", "user.email", "seed@example.com") + if err := os.WriteFile(filepath.Join(seed, "README.md"), []byte("seed\n"), 0o644); err != nil { + t.Fatal(err) + } + run(seed, "add", "-A") + run(seed, "commit", "-m", "seed") + run(seed, "push", "file://"+bare, "main") + return host +} + +func TestPrepareCommitPushRoundTrip(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + ctx := context.Background() + host := newLocalRemote(t, "sam", "todo") + + m := &Manager{ + WorkDir: t.TempDir(), + Token: "unused-for-file-remotes", + BotUser: "archie-bot", + BotEmail: "archie-bot@example.com", + BaseURL: "file://" + host, + } + + dir, branch, err := m.Prepare(ctx, "sam", "todo", "main", 42) + if err != nil { + t.Fatalf("prepare: %v", err) + } + if branch != "archie/issue-42" { + t.Fatalf("branch = %q", branch) + } + + // Clean tree commits nothing. + changed, err := m.CommitAll(ctx, dir, "noop") + if err != nil || changed { + t.Fatalf("clean CommitAll = (%v, %v)", changed, err) + } + + if err := os.WriteFile(filepath.Join(dir, "hello.txt"), []byte("one\ntwo\n"), 0o644); err != nil { + t.Fatal(err) + } + changed, err = m.CommitAll(ctx, dir, "add hello") + if err != nil || !changed { + t.Fatalf("CommitAll = (%v, %v)", changed, err) + } + + lines, err := m.ChangedLines(ctx, dir, "main") + if err != nil { + t.Fatalf("ChangedLines: %v", err) + } + if lines != 2 { + t.Fatalf("ChangedLines = %d, want 2", lines) + } + + if err := m.Push(ctx, dir, branch); err != nil { + t.Fatalf("push: %v", err) + } + + // The branch must exist on the remote after push. + cmd := exec.Command("git", "ls-remote", "--heads", + "file://"+filepath.Join(host, "sam", "todo.git"), branch) + out, err := cmd.CombinedOutput() + if err != nil || !strings.Contains(string(out), branch) { + t.Fatalf("branch not on remote: %v\n%s", err, out) + } + + // Committer identity must be the bot. + cmd = exec.Command("git", "log", "-1", "--format=%an <%ae>") + cmd.Dir = dir + out, _ = cmd.CombinedOutput() + if got := strings.TrimSpace(string(out)); got != "archie-bot " { + t.Fatalf("author = %q", got) + } + + if err := m.Cleanup("sam", "todo", 42); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Fatal("cleanup left the worktree behind") + } +}