diff --git a/internal/agentexec/inprocess.go b/internal/agentexec/inprocess.go index 2a4b6f439..1df139b80 100644 --- a/internal/agentexec/inprocess.go +++ b/internal/agentexec/inprocess.go @@ -12,7 +12,9 @@ import ( "github.com/samcharles93/ai-sdk/agentloop" "github.com/samcharles93/ai-sdk/core" "github.com/samcharles93/ai-sdk/runtime" + "github.com/samcharles93/ai-sdk/toolkit" + "github.com/samcharles93/archie-core/internal/skill" "github.com/samcharles93/archie-core/internal/skillscript" "github.com/samcharles93/archie-core/internal/tools" ) @@ -84,6 +86,12 @@ func (r *InProcessRunner) Run(ctx context.Context, workspace string, req Request if err != nil { return Result{}, fmt.Errorf("build central tool set: %w", err) } + captureTools := captureToolSet(req.CaptureTools, captures) + scriptTools := scriptToolSet(workspace) + pluginTools, err := pluginToolSet(req, workspace, centralTools, captureTools, scriptTools) + if err != nil { + return Result{}, err + } res, err := r.run(ctx, agentloop.Config{ Runtime: r.runtime, ModelRef: req.Model, @@ -98,8 +106,9 @@ func (r *InProcessRunner) Run(ctx context.Context, workspace string, req Request ProtectPaths: protectionMatcher(req.Protection, req.ReadOnly), Extra: mergeToolSets( centralTools, - captureToolSet(req.CaptureTools, captures), - scriptToolSet(workspace), + captureTools, + scriptTools, + pluginTools, ), Logger: r.logger(req), }) @@ -131,6 +140,48 @@ func (r *InProcessRunner) Run(ctx context.Context, workspace string, req Request return result, err } +func pluginToolSet(req Request, workspace string, occupied ...core.ToolSet) (core.ToolSet, error) { + if req.ReadOnly || len(req.Protection.Suffixes)+len(req.Protection.Globs) > 0 || len(req.Gate.Commands) > 0 { + return nil, nil + } + const params = `{"type":"object","properties":{"input":{"type":"string"}},"required":["input"]}` + builtins := toolkit.NewRegistry() + if err := toolkit.RegisterBuiltins(builtins, workspace); err != nil { + return nil, fmt.Errorf("register built-in tools for plugin collision check: %w", err) + } + reserved := map[string]struct{}{"finish": {}, "write_note": {}} + for _, name := range builtins.Names() { + reserved[name] = struct{}{} + } + set := make(core.ToolSet, len(req.Plugins)) + for _, spec := range req.Plugins { + if _, exists := reserved[spec.Name]; exists { + return nil, fmt.Errorf("plugin tool %q conflicts with an agent-loop tool", spec.Name) + } + for _, tools := range occupied { + if _, exists := tools[spec.Name]; exists { + return nil, fmt.Errorf("plugin tool %q conflicts with an existing tool", spec.Name) + } + } + plugin := skill.Plugin{Name: spec.Name, Src: spec.Src} + set[spec.Name] = core.NewTool( + spec.Name, + "Run the project-bundled "+spec.Name+" plugin.", + json.RawMessage(params), + func(_ context.Context, input string) (string, error) { + var args struct { + Input string `json:"input"` + } + if err := json.Unmarshal([]byte(input), &args); err != nil { + return spec.Name + " rejected: arguments must be a JSON object with an input field", nil //nolint:nilerr // rejection feedback lets the model retry with valid arguments + } + return plugin.Run(args.Input) + }, + ) + } + return set, nil +} + func (r *InProcessRunner) logger(req Request) *slog.Logger { if r.log == nil { return slog.New(slog.DiscardHandler) diff --git a/internal/agentexec/inprocess_test.go b/internal/agentexec/inprocess_test.go index 025640f77..a9171b5d4 100644 --- a/internal/agentexec/inprocess_test.go +++ b/internal/agentexec/inprocess_test.go @@ -15,6 +15,47 @@ import ( "github.com/samcharles93/archie-core/internal/tools" ) +func TestPluginToolSetExecutesBundledPlugin(t *testing.T) { + set, err := pluginToolSet(Request{Plugins: []PluginSpec{{Name: "echo", Src: "package main\nfunc Run(input string) string { return input + \"!\" }"}}}, t.TempDir()) + if err != nil { + t.Fatal(err) + } + tool := set["echo"] + got, err := tool.Execute(context.Background(), `{"input":"hello"}`) + if err != nil { + t.Fatal(err) + } + if got != "hello!" { + t.Errorf("plugin output = %q, want hello!", got) + } +} + +func TestPluginToolSetExcludedFromConstrainedStages(t *testing.T) { + requests := []Request{ + {ReadOnly: true, Plugins: []PluginSpec{{Name: "unsafe"}}}, + {Protection: Protection{Suffixes: []string{"_test.go"}}, Plugins: []PluginSpec{{Name: "unsafe"}}}, + {Gate: Gate{Commands: []Command{{Name: "test"}}}, Plugins: []PluginSpec{{Name: "unsafe"}}}, + } + for _, request := range requests { + set, err := pluginToolSet(request, t.TempDir()) + if err != nil { + t.Fatal(err) + } + if len(set) != 0 { + t.Error("plugin tool exposed in constrained stage") + } + } +} + +func TestPluginToolSetRejectsAgentLoopCollisions(t *testing.T) { + for _, name := range []string{"write", "finish"} { + _, err := pluginToolSet(Request{Plugins: []PluginSpec{{Name: name}}}, t.TempDir()) + if err == nil { + t.Errorf("plugin %q did not conflict with agent-loop tool", name) + } + } +} + func TestInProcessRunnerMapsRequestAndCapturesOutput(t *testing.T) { runner := &InProcessRunner{ runtime: runtime.NewRuntime(runtime.Config{}), diff --git a/internal/app/agentworker/persistent_runner.go b/internal/app/agentworker/persistent_runner.go new file mode 100644 index 000000000..b5d935b54 --- /dev/null +++ b/internal/app/agentworker/persistent_runner.go @@ -0,0 +1,110 @@ +package agentworker + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + + "github.com/samcharles93/archie-core/internal/agentexec" + "github.com/samcharles93/archie-core/internal/storage" +) + +type persistentRunner struct { + agentexec.Runner + enabled bool + sessionPath string + memoryPath string + pluginsDir string +} + +type memoryRecord struct { + TaskID int64 `json:"task_id"` + Attempt int `json:"attempt"` + Stage string `json:"stage"` + Content string `json:"content"` +} + +func readProjectMemory(path string) (string, error) { + const maxEntries = 100 + f, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil { + return "", err + } + defer f.Close() + + var entries []string + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 64*1024), 8*1024*1024) + for scanner.Scan() { + var record memoryRecord + if err := json.Unmarshal(scanner.Bytes(), &record); err != nil { + return "", fmt.Errorf("decode memory record: %w", err) + } + if strings.TrimSpace(record.Content) != "" { + entries = append(entries, "- "+record.Content) + if len(entries) > maxEntries { + entries = entries[len(entries)-maxEntries:] + } + } + } + if err := scanner.Err(); err != nil { + return "", fmt.Errorf("scan memory records: %w", err) + } + if len(entries) == 0 { + return "", nil + } + return "Project memory from earlier sessions:\n" + strings.Join(entries, "\n"), nil +} + +func (r persistentRunner) Run(ctx context.Context, workspace string, request agentexec.Request, report agentexec.ToolCallReporter) (agentexec.Result, error) { + if !r.enabled { + return r.Runner.Run(ctx, workspace, request, report) + } + sessionPath, memoryPath, pluginsDir := r.sessionPath, r.memoryPath, r.pluginsDir + if sessionPath == "" { + sessionPath = storage.SessionPath + } + if memoryPath == "" { + memoryPath = storage.MemoryPath + } + if pluginsDir == "" { + pluginsDir = storage.PluginsDir + } + for i := range request.Plugins { + plugin := &request.Plugins[i] + filename := plugin.Name + ".go" + if err := storage.StagePlugin(pluginsDir, filename, []byte(plugin.Src)); err != nil { + return agentexec.Result{}, fmt.Errorf("stage plugin %s: %w", plugin.Name, err) + } + source, err := os.ReadFile(pluginsDir + "/" + filename) + if err != nil { + return agentexec.Result{}, fmt.Errorf("read staged plugin %s: %w", plugin.Name, err) + } + plugin.Src = string(source) + } + + result, runErr := r.Runner.Run(ctx, workspace, request, report) + if runErr != nil { + return result, runErr + } + if err := result.ValidateFor(request); err != nil { + return result, fmt.Errorf("validate persistent result: %w", err) + } + if err := storage.AppendJSONLine(sessionPath, result); err != nil { + return result, fmt.Errorf("append session result: %w", err) + } + for _, note := range result.AppendedNotes { + record := memoryRecord{TaskID: request.TaskID, Attempt: request.Attempt, Stage: request.Stage, Content: note} + if err := storage.AppendJSONLine(memoryPath, record); err != nil { + return result, fmt.Errorf("append project memory: %w", err) + } + } + return result, nil +} diff --git a/internal/app/agentworker/persistent_runner_test.go b/internal/app/agentworker/persistent_runner_test.go new file mode 100644 index 000000000..eefb56853 --- /dev/null +++ b/internal/app/agentworker/persistent_runner_test.go @@ -0,0 +1,119 @@ +package agentworker + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/agentexec" +) + +type persistentRunnerFunc func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) + +func (f persistentRunnerFunc) Run(ctx context.Context, workspace string, request agentexec.Request, report agentexec.ToolCallReporter) (agentexec.Result, error) { + return f(ctx, workspace, request, report) +} + +func TestPersistentRunnerRecordsStageMemoryAndPluginOverride(t *testing.T) { + dir := t.TempDir() + pluginsDir := filepath.Join(dir, "plugins") + if err := os.MkdirAll(pluginsDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(pluginsDir, "check.go"), []byte("override"), 0o600); err != nil { + t.Fatal(err) + } + + runner := persistentRunner{ + Runner: persistentRunnerFunc(func(_ context.Context, _ string, request agentexec.Request, _ agentexec.ToolCallReporter) (agentexec.Result, error) { + if got := request.Plugins[0].Src; got != "override" { + t.Fatalf("plugin source = %q, want persistent override", got) + } + return agentexec.Result{Version: agentexec.ProtocolVersion, TaskID: request.TaskID, Attempt: request.Attempt, Stage: request.Stage, Status: agentexec.StatusPassed, AppendedNotes: []string{"remember me"}}, nil + }), + enabled: true, + sessionPath: filepath.Join(dir, "session.jsonl"), + memoryPath: filepath.Join(dir, "memory.jsonl"), + pluginsDir: pluginsDir, + } + _, err := runner.Run(context.Background(), t.TempDir(), agentexec.Request{ + TaskID: 7, Attempt: 2, Stage: "implement", + Plugins: []agentexec.PluginSpec{{Name: "check", Src: "bundled"}}, + }, nil) + if err != nil { + t.Fatal(err) + } + + for _, path := range []string{runner.sessionPath, runner.memoryPath} { + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(string(content), "\n") { + t.Errorf("%s is not newline-terminated JSONL", path) + } + } + memory, err := os.ReadFile(runner.memoryPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(memory), `"content":"remember me"`) { + t.Errorf("memory record = %s, want appended note", memory) + } +} + +func TestReadProjectMemoryRestoresEarlierNotes(t *testing.T) { + path := filepath.Join(t.TempDir(), "memory.jsonl") + if err := os.WriteFile(path, []byte("{\"content\":\"first fact\"}\n{\"content\":\"second fact\"}\n"), 0o600); err != nil { + t.Fatal(err) + } + got, err := readProjectMemory(path) + if err != nil { + t.Fatal(err) + } + for _, fact := range []string{"first fact", "second fact"} { + if !strings.Contains(got, fact) { + t.Errorf("memory prompt %q missing %q", got, fact) + } + } +} + +func TestReadProjectMemoryAcceptsLargePersistedNote(t *testing.T) { + path := filepath.Join(t.TempDir(), "memory.jsonl") + note := strings.Repeat("x", 70*1024) + encoded := `{"content":"` + note + `"}` + "\n" + if err := os.WriteFile(path, []byte(encoded), 0o600); err != nil { + t.Fatal(err) + } + got, err := readProjectMemory(path) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(got, note) { + t.Error("large persisted note was not restored") + } +} + +func TestPersistentRunnerRejectsFailedResultBeforePersistence(t *testing.T) { + dir := t.TempDir() + runner := persistentRunner{ + Runner: persistentRunnerFunc(func(_ context.Context, _ string, _ agentexec.Request, _ agentexec.ToolCallReporter) (agentexec.Result, error) { + return agentexec.Result{AppendedNotes: []string{"untrusted"}}, os.ErrPermission + }), + enabled: true, + sessionPath: filepath.Join(dir, "session.jsonl"), + memoryPath: filepath.Join(dir, "memory.jsonl"), + pluginsDir: filepath.Join(dir, "plugins"), + } + _, err := runner.Run(context.Background(), t.TempDir(), agentexec.Request{}, nil) + if err == nil { + t.Fatal("expected runner error") + } + for _, path := range []string{runner.sessionPath, runner.memoryPath} { + if _, statErr := os.Stat(path); !os.IsNotExist(statErr) { + t.Errorf("%s persisted rejected output", path) + } + } +} diff --git a/internal/app/agentworker/task_execution.go b/internal/app/agentworker/task_execution.go index 852697b86..cfc70303d 100644 --- a/internal/app/agentworker/task_execution.go +++ b/internal/app/agentworker/task_execution.go @@ -14,6 +14,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/workflow/skillbuild" "github.com/samcharles93/archie-core/internal/domain/workflow/wfeval" "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/storage" "github.com/samcharles93/archie-core/internal/store" "github.com/samcharles93/archie-core/internal/taskrun" "github.com/samcharles93/archie-core/internal/worktree" @@ -164,6 +165,7 @@ func runTask(ctx context.Context, req taskrun.Request, dependencies taskDependen if agent == nil { return nil, fmt.Errorf("no agent runner configured for task %d", req.Task.ID) } + agent = persistentRunner{Runner: agent, enabled: req.Repo.PersistentStorage} // A workflow run in this process publishes to an in-process *events.Bus // the daemon cannot see -- archied and archie-agent are separate @@ -194,6 +196,15 @@ func runTask(ctx context.Context, req taskrun.Request, dependencies taskDependen Log: log, CustomStages: wfeval.Discover, } + if req.Repo.PersistentStorage { + memory, readErr := readProjectMemory(storage.MemoryPath) + if readErr != nil { + return nil, fmt.Errorf("read project memory: %w", readErr) + } + if memory != "" { + tc.SystemPrompt = func() string { return memory } + } + } workflow.Run(ctx, wf, tc) diff --git a/internal/storage/storage.go b/internal/storage/storage.go index b3dfab827..01ab9c2b0 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -10,6 +10,9 @@ // the agent's own commit cannot sweep it onto the // task branch: go-git's Add ignores .gitignore. // repo/ -- per-repo persistent volume (optional, on request) +// session.jsonl -- agent session transcript, appended per stage +// memory.jsonl -- cross-session project memory +// plugins/ -- daemon-staged bundled plugins // cache/ // go/ -- GOMODCACHE (shared across all tasks) // node/ -- npm cache (shared) @@ -19,20 +22,17 @@ // pip/ -- pip cache (shared) // cargo/ -- Rust cargo cache (shared) // -// TODO(PRD ยง3): session.jsonl, memory.jsonl, plugins/ volume -- these -// require agent-side implementation before the storage layer can mount -// them. session.jsonl needs per-stage structured output from the agent; -// memory.jsonl needs cross-session persistence; plugins/ needs daemon- -// staged bundled plugins. -// // Cache volumes are named Docker volumes created once and shared across all // tasks. They are never deleted -- the daemon operator manages them. package storage import ( "context" + "encoding/json" "errors" "fmt" + "os" + "path/filepath" "sort" "strings" "time" @@ -65,12 +65,75 @@ const ( MountTypeVolume = "volume" ) +// AppendJSONLine appends one complete JSON value to a JSONL file. The value is +// marshalled before opening the file so a marshal failure cannot leave a +// partial record behind. +func AppendJSONLine(path string, value any) error { + record, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("marshal JSONL record: %w", err) + } + record = append(record, '\n') + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("create JSONL directory: %w", err) + } + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("open JSONL file: %w", err) + } + if _, err := f.Write(record); err != nil { + _ = f.Close() + return fmt.Errorf("append JSONL record: %w", err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("close JSONL file: %w", err) + } + return nil +} + +// StagePlugin writes a bundled plugin unless persistent project storage +// already contains a plugin with the same filename. Persistent copies are +// operator-owned overrides and therefore win over daemon-staged source. +func StagePlugin(dir, name string, source []byte) error { + if filepath.Base(name) != name || name == "." || name == "" { + return fmt.Errorf("invalid plugin filename %q", name) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create plugin directory: %w", err) + } + path := filepath.Join(dir, name) + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if errors.Is(err, os.ErrExist) { + return nil + } + if err != nil { + return fmt.Errorf("create staged plugin: %w", err) + } + if _, err := f.Write(source); err != nil { + _ = f.Close() + return fmt.Errorf("write staged plugin: %w", err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("close staged plugin: %w", err) + } + return nil +} + // WorktreeMountDir is the fixed container path where a task's worktree is // bind-mounted. archie-agent uses this path directly rather than the host // path archied's worktree.Manager reports -- the two processes see the same // files at different paths. const WorktreeMountDir = "/data/worktree" +// Persistent storage paths share the per-repository volume so session output, +// project memory, and staged plugins survive individual task containers. +const ( + PersistentMountDir = "/data/repo" + SessionPath = PersistentMountDir + "/session.jsonl" + MemoryPath = PersistentMountDir + "/memory.jsonl" + PluginsDir = PersistentMountDir + "/plugins" +) + // Backend prepares container storage. Each container runtime backend // (Docker, future containerd, etc.) implements this interface. type Backend interface { @@ -237,7 +300,7 @@ func (d *DockerBackend) Setup(ctx context.Context, task TaskRef) ([]Mount, error mounts = append(mounts, Mount{ Type: MountTypeVolume, Source: volName, - Destination: "/data/repo", + Destination: PersistentMountDir, }) } diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index 4cc486f67..cc2728991 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/httptest" "os" + "path/filepath" "slices" "sort" "strings" @@ -324,6 +325,60 @@ func TestDockerBackendSetupWithPersistentStorage(t *testing.T) { } } +func TestPersistentStoragePaths(t *testing.T) { + tests := []struct { + name string + got string + want string + }{ + {name: "root", got: PersistentMountDir, want: "/data/repo"}, + {name: "session", got: SessionPath, want: "/data/repo/session.jsonl"}, + {name: "memory", got: MemoryPath, want: "/data/repo/memory.jsonl"}, + {name: "plugins", got: PluginsDir, want: "/data/repo/plugins"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.got != tt.want { + t.Errorf("path = %q, want %q", tt.got, tt.want) + } + }) + } +} + +func TestAppendJSONLineAppendsCompleteRecords(t *testing.T) { + path := filepath.Join(t.TempDir(), "nested", "session.jsonl") + for _, value := range []map[string]int{{"stage": 1}, {"stage": 2}} { + if err := AppendJSONLine(path, value); err != nil { + t.Fatal(err) + } + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if want := "{\"stage\":1}\n{\"stage\":2}\n"; string(got) != want { + t.Errorf("JSONL = %q, want %q", got, want) + } +} + +func TestStagePluginPreservesPersistentOverride(t *testing.T) { + dir := t.TempDir() + if err := StagePlugin(dir, "check.go", []byte("bundled")); err != nil { + t.Fatal(err) + } + if err := StagePlugin(dir, "check.go", []byte("replacement")); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(filepath.Join(dir, "check.go")) + if err != nil { + t.Fatal(err) + } + if string(got) != "bundled" { + t.Errorf("plugin = %q, want persistent override", got) + } +} + func TestDockerBackendSetupWithoutPersistentStorage(t *testing.T) { dir := t.TempDir() if err := os.MkdirAll(dir, 0o755); err != nil {