package daemon import ( "context" "encoding/json" "errors" "fmt" "log/slog" "os" "slices" "sort" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/nats-server/v2/server" natssrv "github.com/nats-io/nats-server/v2/test" natsio "github.com/nats-io/nats.go" "github.com/samcharles93/archie-core/internal/agentexec" "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/container" "github.com/samcharles93/archie-core/internal/domain/workflow" "github.com/samcharles93/archie-core/internal/domain/workintake" "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/forge" agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats" arnats "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats" "github.com/samcharles93/archie-core/internal/infrastructure/kit" "github.com/samcharles93/archie-core/internal/infrastructure/kitrun" "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore" "github.com/samcharles93/archie-core/internal/logging" "github.com/samcharles93/archie-core/internal/secret" "github.com/samcharles93/archie-core/internal/storage" "github.com/samcharles93/archie-core/internal/taskrun" "github.com/samcharles93/archie-core/internal/taskstate" "github.com/samcharles93/archie-core/internal/worktree" ) type cleanupStorage struct { ttl time.Duration } func (s *cleanupStorage) Setup(context.Context, storage.TaskRef) ([]storage.Mount, error) { return nil, nil } func (s *cleanupStorage) Teardown(context.Context, storage.TaskRef) error { return nil } func (s *cleanupStorage) CleanupExpired(_ context.Context, ttl time.Duration) (int, error) { s.ttl = ttl return 2, nil } func TestCleanupExpiredStorageUsesConfiguredTTL(t *testing.T) { backend := &cleanupStorage{} d := &Daemon{ Cfg: config.NewHolder(config.Config{ Containers: config.ContainerConfig{VolumeTTL: config.Duration(6 * time.Hour)}, }), Storage: backend, Log: slog.New(slog.DiscardHandler), } d.cleanupExpiredStorage(context.Background()) if backend.ttl != 6*time.Hour { t.Fatalf("CleanupExpired ttl = %s, want 6h", backend.ttl) } } // TestOpenTaskLogOpensAndClosesTheRightSink guards process()'s wiring of // TaskLogs without the ~10-dependency harness a full process() test would // need (Store, Forge, Trees, ContainerPool, ...): openTaskLog only ever // touches d.TaskLogs and d.Log, so it's testable in complete isolation. // Uses TaskRegistry.Write's own true/false contract as the observable -- // true right after opening (proving Open ran under this task's actual // ID/attempt, not a typo'd constant), false after the returned closer runs // (proving it's a real Close, not a no-op). func TestOpenTaskLogOpensAndClosesTheRightSink(t *testing.T) { reg := logging.NewTaskRegistry(t.TempDir(), logging.NewFeed(10), logging.TaskSinkOptions{}) d := &Daemon{TaskLogs: reg, Log: slog.New(slog.DiscardHandler)} task := &workflow.Task{ID: 5, Attempt: 2} closeFn := d.openTaskLog(task) if ok := reg.Write(t.Context(), 5, logging.Entry{Message: "x"}); !ok { t.Fatal("Write() = false right after openTaskLog, want true (sink should be open)") } closeFn() if ok := reg.Write(t.Context(), 5, logging.Entry{Message: "y"}); ok { t.Error("Write() = true after openTaskLog's closer ran, want false (sink should be closed)") } } // TestParkRunningTaskRecordsItsReasonInTheTasksOwnLog pins the answer to the // question the task log exists for: "why did this attempt park?" Every park // used to be written only to the daemon-wide log, so an attempt that parked // before its container produced any output -- a worktree prepare failure, an // unavailable worker, a taskrun error -- left a task log that the sink had // created and nothing had written to. The dashboard offered that zero-byte // file as a download, and the reason lived somewhere the operator could not // hand to anyone. func TestParkRunningTaskRecordsItsReasonInTheTasksOwnLog(t *testing.T) { d, st, _ := testDaemon(t, 3, 0) d.Log = slog.New(slog.DiscardHandler) d.TaskLogs = logging.NewTaskRegistry(t.TempDir(), logging.NewFeed(10), logging.TaskSinkOptions{}) ctx := t.Context() if _, err := st.EnqueueIssue(ctx, "acme", "widget", 42, "task", "", "", ""); err != nil { t.Fatal(err) } task, err := st.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("ClaimNext = (%+v, %v)", task, err) } // The sink is opened for the attempt by process() and closed when the run // ends; the park happens in between, which is the only window it is read in. closeLog := d.openTaskLog(task) defer closeLog() const reason = "worktree prepare failed: checkout branch fix/42-x: reference not found" d.parkRunningTask(ctx, task.ID, reason, taskstate.ParkTransient) page, err := d.TaskLogs.TaskLog(ctx, task.ID, task.Attempt, logging.Query{}) if err != nil { t.Fatalf("TaskLog: %v", err) } if !page.Found { t.Fatalf("attempt %d has no log file, want the reason recorded in one", task.Attempt) } if len(page.Entries) != 1 { t.Fatalf("entries = %+v, want exactly the park reason", page.Entries) } entry := page.Entries[0] if !strings.Contains(entry.Message, reason) { t.Errorf("entry = %q, want it to carry the park reason %q", entry.Message, reason) } if entry.Level != slog.LevelError.String() { t.Errorf("entry level = %q, want %q", entry.Level, slog.LevelError.String()) } if entry.Fields["component"] != "daemon" { t.Errorf("entry component = %v, want the daemon to be identifiable in the attempt's own log", entry.Fields["component"]) } } // TestParkDoesNotRecordAParkThatLosesTheRace keeps the new task-log entry as // truthful as the parked event it accompanies. If another actor moved the task // out of running first, the guarded transition reports ErStoreStaleTransition // and nothing was parked -- so "task parked: " must not appear in the // attempt's log either, or the log claims an outcome the task board contradicts. func TestParkDoesNotRecordAParkThatLosesTheRace(t *testing.T) { d, st, _ := testDaemon(t, 3, 0) d.Log = slog.New(slog.DiscardHandler) d.TaskLogs = logging.NewTaskRegistry(t.TempDir(), logging.NewFeed(10), logging.TaskSinkOptions{}) ctx := t.Context() if _, err := st.EnqueueIssue(ctx, "acme", "widget", 42, "task", "", "", ""); err != nil { t.Fatal(err) } // Left queued, not claimed: the running -> parked transition cannot match. task, err := st.TaskByIssue(ctx, "acme", "widget", 42) if err != nil || task == nil { t.Fatalf("TaskByIssue = (%+v, %v)", task, err) } closeLog := d.openTaskLog(task) defer closeLog() d.parkRunningTask(ctx, task.ID, "worktree prepare failed: simulated", taskstate.ParkTransient) page, err := d.TaskLogs.TaskLog(ctx, task.ID, task.Attempt, logging.Query{}) if err != nil { t.Fatalf("TaskLog: %v", err) } if len(page.Entries) != 0 { t.Errorf("entries = %+v, want none: the task was never parked", page.Entries) } } func TestProcessParksWhenManagedWorkerCapabilityIsUnavailable(t *testing.T) { tests := []struct { name string pool *container.Pool wantReason string }{ { name: "container pool unavailable", wantReason: "managed agent container pool is unavailable; refusing to run this task on the host", }, { name: "task transport unavailable", pool: &container.Pool{}, wantReason: "agent task transport is unavailable; refusing to run this task on the host", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { d, st, _ := testDaemon(t, 3, 0) d.Log = slog.New(slog.DiscardHandler) d.Trees = nil // both guards must run before any worktree operation d.ContainerPool = tt.pool d.Cfg.Set(config.Config{ Repos: []config.Repo{{Owner: "acme", Name: "widget"}}, }) ctx := t.Context() if _, err := st.EnqueueIssue(ctx, "acme", "widget", 42, "task", "", "", ""); err != nil { t.Fatal(err) } task, err := st.ClaimNext(ctx) if err != nil { t.Fatal(err) } if task == nil { t.Fatal("ClaimNext = nil task") } d.process(ctx, task) got, err := st.TaskByID(ctx, task.ID) if err != nil { t.Fatal(err) } if got.Status != workflow.StatusParked { t.Fatalf("task status = %q, want parked", got.Status) } if got.ParkReason != tt.wantReason { t.Errorf("park reason = %q, want %q", got.ParkReason, tt.wantReason) } }) } } func TestProcessDoesNotEmitParkedWhenCapabilityTransitionLosesRace(t *testing.T) { d, st, _ := testDaemon(t, 3, 0) d.Log = slog.New(slog.DiscardHandler) d.Trees = nil d.Cfg.Set(config.Config{Repos: []config.Repo{{Owner: "acme", Name: "widget"}}}) d.Bus = events.NewBus() sub := d.Bus.Subscribe(1) t.Cleanup(sub.Close) ctx := t.Context() if _, err := st.EnqueueIssue(ctx, "acme", "widget", 42, "task", "", "", ""); err != nil { t.Fatal(err) } task, err := st.TaskByIssue(ctx, "acme", "widget", 42) if err != nil || task == nil { t.Fatalf("TaskByIssue = (%+v, %v)", task, err) } // The task is still queued, simulating another actor winning the guarded // running-state transition before this process reports its capability // failure. d.process(ctx, task) select { case event := <-sub.C: t.Fatalf("event = %+v, want no false parked event after stale transition", event) default: } } func TestTaskDispatcherEnforcesMaxConcurrency(t *testing.T) { t.Parallel() const maxConcurrency = 2 dispatcher := newTaskDispatcher(maxConcurrency, nil) release := make(chan struct{}) started := make(chan int, 3) var active atomic.Int32 var peak atomic.Int32 for i := 1; i <= 3; i++ { task := &workflow.Task{Owner: "acme", Repo: "repo-" + string(rune('0'+i))} dispatcher.Submit(context.Background(), task, func(context.Context, *workflow.Task) { n := active.Add(1) for { old := peak.Load() if n <= old || peak.CompareAndSwap(old, n) { break } } started <- i <-release active.Add(-1) }) } for range maxConcurrency { select { case <-started: case <-time.After(time.Second): t.Fatal("tasks did not start up to max_concurrency") } } select { case task := <-started: t.Fatalf("task %d started before a concurrency slot was available", task) case <-time.After(50 * time.Millisecond): } close(release) dispatcher.Wait() if got := peak.Load(); got != maxConcurrency { t.Fatalf("peak concurrency = %d, want %d", got, maxConcurrency) } select { case <-started: default: t.Fatal("queued task never started after a concurrency slot became available") } } func TestTaskDispatcherSerializesTasksForSameRepo(t *testing.T) { t.Parallel() dispatcher := newTaskDispatcher(3, nil) releaseFirst := make(chan struct{}) firstStarted := make(chan struct{}) secondStarted := make(chan struct{}) otherRepoStarted := make(chan struct{}) dispatcher.Submit(context.Background(), &workflow.Task{Owner: "acme", Repo: "widget"}, func(context.Context, *workflow.Task) { close(firstStarted) <-releaseFirst }) dispatcher.Submit(context.Background(), &workflow.Task{Owner: "acme", Repo: "widget"}, func(context.Context, *workflow.Task) { close(secondStarted) }) dispatcher.Submit(context.Background(), &workflow.Task{Owner: "acme", Repo: "gizmo"}, func(context.Context, *workflow.Task) { close(otherRepoStarted) }) for name, started := range map[string]<-chan struct{}{ "first same-repo task": firstStarted, "other-repo task": otherRepoStarted, } { select { case <-started: case <-time.After(time.Second): t.Fatalf("%s did not start", name) } } select { case <-secondStarted: t.Fatal("second task for the same repo ran concurrently") case <-time.After(50 * time.Millisecond): } close(releaseFirst) dispatcher.Wait() select { case <-secondStarted: default: t.Fatal("second same-repo task did not start after the first completed") } } func TestTaskDispatcherRunsConcurrentReposInParallel(t *testing.T) { t.Parallel() dispatcher := newTaskDispatcher(3, func(task *workflow.Task) bool { return task.Owner == "acme" && task.Repo == "widget" }) releaseFirst := make(chan struct{}) firstStarted := make(chan struct{}) secondStarted := make(chan struct{}) dispatcher.Submit(context.Background(), &workflow.Task{Owner: "acme", Repo: "widget"}, func(context.Context, *workflow.Task) { close(firstStarted) <-releaseFirst }) dispatcher.Submit(context.Background(), &workflow.Task{Owner: "acme", Repo: "widget"}, func(context.Context, *workflow.Task) { close(secondStarted) }) for name, started := range map[string]<-chan struct{}{ "first opted-in task": firstStarted, "second opted-in task": secondStarted, } { select { case <-started: case <-time.After(time.Second): t.Fatalf("%s did not start -- allow_concurrent repos should not be serialized", name) } } close(releaseFirst) dispatcher.Wait() } func TestTaskDispatcherWaitsForRunningTasks(t *testing.T) { t.Parallel() dispatcher := newTaskDispatcher(1, nil) release := make(chan struct{}) dispatcher.Submit(context.Background(), &workflow.Task{Owner: "acme", Repo: "widget"}, func(context.Context, *workflow.Task) { <-release }) waited := make(chan struct{}) go func() { dispatcher.Wait() close(waited) }() select { case <-waited: t.Fatal("Wait returned while a task was still running") case <-time.After(50 * time.Millisecond): } close(release) select { case <-waited: case <-time.After(time.Second): t.Fatal("Wait did not return after the running task completed") } } func TestDaemonAllowConcurrentForReadsRepoConfig(t *testing.T) { d := &Daemon{ Cfg: config.NewHolder(config.Config{ Repos: []config.Repo{ {Owner: "acme", Name: "widget", AllowConcurrent: true}, {Owner: "acme", Name: "todo"}, }, }), } if !d.allowConcurrentForTask(&workflow.Task{Owner: "acme", Repo: "widget"}) { t.Fatal("expected allow_concurrent=true repo to report concurrent-allowed") } if d.allowConcurrentForTask(&workflow.Task{Owner: "acme", Repo: "todo"}) { t.Fatal("expected repo without allow_concurrent to report false") } if d.allowConcurrentForTask(&workflow.Task{Owner: "acme", Repo: "unknown"}) { t.Fatal("expected unknown repo to report false") } } // allowConcurrentForTask must resolve the repo from the identity's own // repo list when the task names an identity -- root Cfg.Repos must not be // consulted, or an identity-only repo would silently fall back to the // serialized default and an overlapping identity repo would use the wrong // policy. func TestAllowConcurrentForTaskPrefersOwningIdentityRepo(t *testing.T) { shared := config.Repo{Owner: "acme", Name: "shared"} d := &Daemon{ Cfg: config.NewHolder(config.Config{ // Root (legacy) list: shared is NOT opted in. Repos: []config.Repo{shared}, }), Identities: []*IdentityRunner{ { Name: "archie", // The identity's own list opts shared in. Repos: []config.Repo{{Owner: "acme", Name: "shared", AllowConcurrent: true}}, }, }, } if !d.allowConcurrentForTask(&workflow.Task{Owner: "acme", Repo: "shared", Identity: "archie"}) { t.Fatal("identity-owned task must use the identity's repo allow_concurrent") } // A root-owned (identity-less) task must NOT inherit the identity's // opt-in: it resolves through the root repo list. if d.allowConcurrentForTask(&workflow.Task{Owner: "acme", Repo: "shared"}) { t.Fatal("identity-less task leaked the identity's allow_concurrent") } } func TestContainerEnvIncludesConfiguredNATSCredentials(t *testing.T) { // ConnectedNATS is the endpoint the daemon's own client connected with // at startup; containerEnv deliberately reads this frozen runtime value, // not live config or environment. d := &Daemon{ Cfg: config.NewHolder(config.Config{}), ConnectedNATS: NATSEndpoint{URL: "nats://nats.example:4222", Token: "test-nats-token"}, } // Runtime credentials are frozen when the daemon connects. Mutating the // source environment later must not send new workers to the same endpoint // with a credential the broker never adopted. t.Setenv("ARCHIE_NATS_SECRET", "changed-after-startup") got := d.containerEnv(nil, "") for _, want := range []string{ "NATS_URL=nats://nats.example:4222", "NATS_TOKEN=test-nats-token", } { if !slices.Contains(got, want) { t.Errorf("containerEnv() = %q, want %q", got, want) } } } // TestContainerEnvIncludesWorktreeOwnership: the agent process inside a // container runs as root (no USER in the Dockerfile, no userns-remap), so // commits it writes to the bind-mounted worktree land owned by UID 0 on the // host. The daemon then reads those same loose objects to push, running as // its own non-root host user -- a UID mismatch that surfaces as // "openat objects/../..: permission denied" (archie-core#520). Passing the // daemon's own UID/GID lets the agent chown the worktree back before // pushing. func TestContainerEnvIncludesWorktreeOwnership(t *testing.T) { d := &Daemon{ Cfg: config.NewHolder(config.Config{}), } got := d.containerEnv(nil, "") for _, want := range []string{ fmt.Sprintf("WORKTREE_UID=%d", os.Getuid()), fmt.Sprintf("WORKTREE_GID=%d", os.Getgid()), } { if !slices.Contains(got, want) { t.Errorf("containerEnv() = %q, want %q", got, want) } } } func TestContainerEnvUsesOnlyOwningIdentityProviderCredential(t *testing.T) { t.Setenv("ROOT_PROVIDER_KEY", "root-secret") t.Setenv("WORKER_PROVIDER_KEY", "worker-secret") d := &Daemon{ Cfg: config.NewHolder(config.Config{Providers: map[string]config.Provider{ "openai": {Class: "openai", APIKeyEnv: "ROOT_PROVIDER_KEY"}, }}), Identities: []*IdentityRunner{{ Name: "worker", Cfg: config.IdentityConfig{Name: "worker", Providers: map[string]config.Provider{ "openai": {Class: "openai", APIKeyEnv: "WORKER_PROVIDER_KEY"}, }}, }}, } got := d.containerEnv(&workflow.Task{Identity: "worker"}, "") if !slices.Contains(got, "WORKER_PROVIDER_KEY=worker-secret") { t.Fatalf("identity credential missing from container env: %q", got) } if slices.Contains(got, "ROOT_PROVIDER_KEY=root-secret") { t.Fatalf("root credential leaked into identity container env: %q", got) } } func startEmbeddedNATSForDaemon(t *testing.T) *server.Server { t.Helper() srv := natssrv.RunRandClientPortServer() t.Cleanup(srv.Shutdown) if err := srv.EnableJetStream(&server.JetStreamConfig{StoreDir: t.TempDir()}); err != nil { t.Fatalf("enable jetstream: %v", err) } return srv } // daemonWithNATS returns the daemon, its store, and the concrete bus // client -- tests that drive core-NATS subscriptions directly need the // connection, which the TaskBus contract deliberately does not expose. func daemonWithNATS(t *testing.T) (*Daemon, *pgstore.TaskDB, *arnats.Client) { t.Helper() srv := startEmbeddedNATSForDaemon(t) client, err := arnats.Connect(context.Background(), arnats.Config{URL: srv.ClientURL(), Subjects: []string{workintake.SubjectTaskWildcard}, FilterSubject: workintake.SubjectTaskWildcard}, slog.New(slog.DiscardHandler)) if err != nil { t.Fatalf("nats connect: %v", err) } t.Cleanup(client.Close) s := pgstore.Open(t) d := &Daemon{ Cfg: config.NewHolder(config.Config{ Dispatch: config.Dispatch{Labels: map[string]string{ "parked": "archie:parked", "queued": "archie:queued", "dead": "archie:dead", }}, }), Store: s, Tasks: client, WorktreeGrants: fixedGrantIssuer{token: "test-worktree-grant"}, Log: slog.New(slog.DiscardHandler), } return d, s, client } type fixedGrantIssuer struct{ token string } func (g fixedGrantIssuer) Issue(*workflow.Task) (string, func(), error) { return g.token, func() {}, nil } func TestRequestTaskRunDoesNotSleepPastReadyDeadline(t *testing.T) { d, _, _ := daemonWithNATS(t) d.TaskRunReadyTimeout = 10 * time.Millisecond d.TaskRunRetryBackoff = time.Second start := time.Now() _, err := d.requestTaskRun(context.Background(), 1, nil) elapsed := time.Since(start) if err == nil { t.Fatal("requestTaskRun() error = nil, want no responders") } if elapsed >= 200*time.Millisecond { t.Fatalf("requestTaskRun() took %s, want it to stop at the ready deadline", elapsed) } } func TestRunViaAgentParksOnRequestFailure(t *testing.T) { d, s, _ := daemonWithNATS(t) // Bound the no-responders retry window tightly so this test proves // "no archie-agent ever showed up" parks the task, without waiting out // the real production default (20s). d.TaskRunReadyTimeout = 50 * time.Millisecond d.TaskRunRetryBackoff = 10 * time.Millisecond ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 1, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } // No responder registered on the taskrun subject, ever -- the retry // window must exhaust and runViaAgent must park rather than leave the // task stuck running. runPinnedViaAgent(ctx, d, task, config.Repo{Owner: "acme", Name: "widget"}) got, err := s.TaskByIssue(ctx, "acme", "widget", 1) if err != nil || got == nil { t.Fatalf("TaskByIssue: (%+v, %v)", got, err) } if got.Status != workflow.StatusParked { t.Fatalf("status = %q, want %q", got.Status, workflow.StatusParked) } if !strings.Contains(got.ParkReason, "taskrun request failed") { t.Errorf("ParkReason = %q, want taskrun request failure", got.ParkReason) } } // TestRunViaAgentRetriesUntilResponderAppears is a regression test for the // deterministic race where a freshly spawned archie-agent container hasn't // finished connecting to NATS and subscribing yet when the daemon // publishes its very first taskrun request: ContainerPool.Acquire returns // as soon as Docker issues the start syscall, well before the container's // NATS/JetStream setup completes, so the request used to fail with // nats.ErrNoResponders and park the task on effectively every run. This // simulates that exact ordering -- no subscriber at request time, one // registers shortly after -- and asserts the retry recovers instead of // parking. func TestRunViaAgentRetriesUntilResponderAppears(t *testing.T) { d, s, busClient := daemonWithNATS(t) d.TaskRunReadyTimeout = time.Second d.TaskRunRetryBackoff = 20 * time.Millisecond ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 6, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } // Register the subscriber only after a short delay, simulating the // spawned container's NATS/JetStream startup lag -- the very race // this retry loop exists to survive. go func() { time.Sleep(80 * time.Millisecond) coreConn, connErr := busClient.CoreConn() if connErr != nil { // Fatalf from a non-test goroutine does not stop the test. t.Errorf("CoreConn: %v", connErr) return } sub, err := coreConn.Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { data, _ := json.Marshal(taskrun.Response{Status: workflow.StatusPROpen}) _ = msg.Respond(data) }) if err != nil { t.Errorf("late subscribe: %v", err) return } t.Cleanup(func() { _ = sub.Unsubscribe() }) }() runPinnedViaAgent(ctx, d, task, config.Repo{Owner: "acme", Name: "widget"}) got, err := s.TaskByIssue(ctx, "acme", "widget", 6) if err != nil || got == nil { t.Fatalf("TaskByIssue: (%+v, %v)", got, err) } if got.Status == workflow.StatusParked { t.Fatal("runViaAgent parked a task whose responder appeared within the retry window") } } // TestRunViaAgentDoesNotRetryOnContextCancellation proves requestTaskRun // doesn't confuse a caller-cancelled context with the "not ready yet" // no-responders case: it must return promptly (well under the retry // window) rather than looping through backoff attempts. func TestRunViaAgentDoesNotRetryOnContextCancellation(t *testing.T) { d, s, _ := daemonWithNATS(t) d.TaskRunReadyTimeout = 5 * time.Second d.TaskRunRetryBackoff = 500 * time.Millisecond ctx, cancel := context.WithCancel(context.Background()) cancel() if _, err := s.EnqueueIssue(context.Background(), "acme", "widget", 7, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(context.Background()) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } start := time.Now() runPinnedViaAgent(ctx, d, task, config.Repo{Owner: "acme", Name: "widget"}) if elapsed := time.Since(start); elapsed > 200*time.Millisecond { t.Fatalf("runViaAgent took %s with an already-cancelled context, want it to return immediately without retrying", elapsed) } } func TestRunViaAgentParksOnRunError(t *testing.T) { d, s, busClient := daemonWithNATS(t) ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 2, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { data, _ := json.Marshal(taskrun.Response{Error: "registry build failed"}) _ = msg.Respond(data) }) if err != nil { t.Fatalf("subscribe: %v", err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) runPinnedViaAgent(ctx, d, task, config.Repo{Owner: "acme", Name: "widget"}) got, err := s.TaskByIssue(ctx, "acme", "widget", 2) if err != nil || got == nil { t.Fatalf("TaskByIssue: (%+v, %v)", got, err) } if got.Status != workflow.StatusParked { t.Fatalf("status = %q, want %q", got.Status, workflow.StatusParked) } if got.ParkReason != "taskrun run failed: registry build failed" { t.Errorf("ParkReason = %q, want worker run failure", got.ParkReason) } } func TestRunViaAgentSendsExpectedRequest(t *testing.T) { d, s, busClient := daemonWithNATS(t) prev := d.Cfg.Get() prev.DiffCapLines = new(999) d.Cfg.Set(prev) ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 3, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } received := make(chan taskrun.Request, 1) var requestCount atomic.Int32 sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { requestCount.Add(1) var req taskrun.Request _ = json.Unmarshal(msg.Data, &req) received <- req data, _ := json.Marshal(taskrun.Response{Status: workflow.StatusPROpen}) _ = msg.Respond(data) }) if err != nil { t.Fatalf("subscribe: %v", err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) repo := config.Repo{Owner: "acme", Name: "widget", Base: "main"} runPinnedViaAgent(ctx, d, task, repo) select { case req := <-received: if req.Task == nil || req.Task.ID != task.ID { t.Fatalf("request Task = %+v, want ID %d", req.Task, task.ID) } if req.Repo.FullName() != "acme/widget" { t.Fatalf("request Repo = %+v", req.Repo) } if req.Cfg.DiffCapLines != 999 { t.Fatalf("request Cfg.DiffCapLines = %d, want 999", req.Cfg.DiffCapLines) } if req.WorktreeGrant != "test-worktree-grant" { t.Fatalf("request WorktreeGrant = %q, want dispatch grant", req.WorktreeGrant) } case <-time.After(time.Second): t.Fatal("archied did not publish a taskrun request") } if got := requestCount.Load(); got != 1 { t.Errorf("taskrun request count = %d, want exactly 1", got) } // Success path: runViaAgent must not park a task that archie-agent // already reported as complete -- the daemon just logs it. got, err := s.TaskByIssue(ctx, "acme", "widget", 3) if err != nil || got == nil { t.Fatalf("TaskByIssue: (%+v, %v)", got, err) } if got.Status == workflow.StatusParked { t.Fatal("runViaAgent parked a task that succeeded") } } // TestRunViaAgentSendsMCPServers is the daemon half of persistence-2: the // configured daemon-wide MCP servers must be carried in the taskrun request // so the archie-agent worker can construct transports and register their // tools (task_execution.go startMCPProviders). A request missing them leaves // the whole agent-side MCP tool path dead. func TestRunViaAgentSendsMCPServers(t *testing.T) { d, s, busClient := daemonWithNATS(t) cfg := d.Cfg.Get() cfg.Tools.MCPServers = []config.MCPServer{ {Name: "mcp-npm", Transport: "stdio", Command: "npx", Args: []string{"-y", "@modelcontextprotocol/server-filesystem"}}, } d.Cfg.Set(cfg) ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 10, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } received := make(chan taskrun.Request, 1) sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { var req taskrun.Request _ = json.Unmarshal(msg.Data, &req) received <- req data, _ := json.Marshal(taskrun.Response{Status: workflow.StatusPROpen}) _ = msg.Respond(data) }) if err != nil { t.Fatalf("subscribe: %v", err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) runPinnedViaAgent(ctx, d, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}) select { case req := <-received: if len(req.MCPServers) != 1 || req.MCPServers[0].Name != "mcp-npm" { t.Fatalf("request MCPServers = %+v, want the configured mcp-npm server", req.MCPServers) } case <-time.After(time.Second): t.Fatal("archied did not publish a taskrun request") } } // TestRunViaAgentSendsRoutingBindings is the daemon half of // workflow-engine-1: the kind/label -> workflow bindings the daemon loaded // must be carried in the taskrun request, because workflow.Route runs in the // separate archie-agent process. A request missing them would silently route // with the built-in defaults. func TestRunViaAgentSendsRoutingBindings(t *testing.T) { d, s, busClient := daemonWithNATS(t) d.KindWorkflows = workflow.KindWorkflows{workintake.KindBug: "custom-bug"} d.LabelWorkflows = workflow.LabelWorkflows{"security": "security-review"} ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 9, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } // The published request is checked for its routing bindings, so a decode // failure is carried out rather than discarded: otherwise a payload-shape // break would surface as an empty binding and read as a routing bug. type published struct { req taskrun.Request err error } received := make(chan published, 1) sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { var p published p.err = json.Unmarshal(msg.Data, &p.req) received <- p data, _ := json.Marshal(taskrun.Response{Status: workflow.StatusPROpen}) _ = msg.Respond(data) }) if err != nil { t.Fatalf("subscribe: %v", err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) runPinnedViaAgent(ctx, d, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}) select { case p := <-received: if p.err != nil { t.Fatalf("decode published taskrun request: %v", p.err) } if got := p.req.KindWorkflows[workintake.KindBug]; got != "custom-bug" { t.Errorf("request KindWorkflows[bug] = %q, want %q", got, "custom-bug") } if got := p.req.LabelWorkflows["security"]; got != "security-review" { t.Errorf("request LabelWorkflows[security] = %q, want %q", got, "security-review") } case <-time.After(time.Second): t.Fatal("archied did not publish a taskrun request") } } // TestRunViaAgentObservesReportedAgentVersion is the regression proof for // daemon.AgentStatus actually being fed: every archie-agent process is // task-scoped and ephemeral, so a completed taskrun.Response is the only // channel the daemon has to learn what build is really running (see // cmd/archied's daemonRunningVersions, which reads this back out via // Snapshot for releaseupdate.Report.Verify). func TestRunViaAgentObservesReportedAgentVersion(t *testing.T) { d, s, busClient := daemonWithNATS(t) var status AgentStatus d.AgentStatus = &status ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 4, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { data, _ := json.Marshal(taskrun.Response{ Status: workflow.StatusPROpen, AgentVersion: "1.9.11", AgentInstallType: "container", }) _ = msg.Respond(data) }) if err != nil { t.Fatalf("subscribe: %v", err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) runPinnedViaAgent(ctx, d, task, config.Repo{Owner: "acme", Name: "widget"}) version, installType, ok := status.Snapshot() if !ok || version != "1.9.11" || installType != "container" { t.Fatalf("Snapshot() = (%q, %q, %v), want (\"1.9.11\", \"container\", true)", version, installType, ok) } } // TestRunViaAgentIgnoresEmptyAgentVersion: a worker older than this field, // or one whose build was never stamped, reports an empty AgentVersion. // Recording that as observed would read as a confirmed "unknown" version // rather than as nothing having been reported yet. func TestRunViaAgentIgnoresEmptyAgentVersion(t *testing.T) { d, s, busClient := daemonWithNATS(t) var status AgentStatus d.AgentStatus = &status ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 5, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { data, _ := json.Marshal(taskrun.Response{Status: workflow.StatusPROpen}) _ = msg.Respond(data) }) if err != nil { t.Fatalf("subscribe: %v", err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) runPinnedViaAgent(ctx, d, task, config.Repo{Owner: "acme", Name: "widget"}) if _, _, ok := status.Snapshot(); ok { t.Fatal("Snapshot() reports observed after an empty AgentVersion response") } } // testForge implements forge.Forge for daemon tests. type testForge struct { comments []commentCall stateLabels []stateLabelCall closedIssues []closeIssueCall prStates map[int]string } // pollingForge records every repo the daemon asks about, so a test can // observe which repos are being polled -- including that a repo removed // by a reload stops new intake. type pollingForge struct { testForge asked []string } func (f *pollingForge) AssignedIssues(_ context.Context, owner, repo, _ string) ([]forge.Issue, error) { f.asked = append(f.asked, owner+"/"+repo) return nil, nil } // TestRemovingRepoStopsNewIntake pins the reload behaviour: when a repo // vanishes from the config, the next poll cycle no longer asks the forge // about it. In-flight tasks are unaffected because their TaskContext.Cfg // is a value snapshot taken at dispatch -- the type system guarantees // that half; this test covers the intake half. func TestRemovingRepoStopsNewIntake(t *testing.T) { fg := &pollingForge{} d := &Daemon{ Cfg: config.NewHolder(config.Config{ BotUser: "archie", Repos: []config.Repo{{Owner: "acme", Name: "widget"}}, Dispatch: config.Dispatch{Trigger: "assignee"}, }), Forge: fg, Log: slog.New(slog.DiscardHandler), } ctx := context.Background() d.poll(ctx) if len(fg.asked) != 1 || fg.asked[0] != "acme/widget" { t.Fatalf("after first poll, asked = %v, want [acme/widget]", fg.asked) } // Simulate a reload that drops the repo: the polling loop re-reads // d.Cfg.Get().Repos fresh, so the next poll asks about nothing. d.Cfg.Set(config.Config{ BotUser: "archie", Dispatch: config.Dispatch{Trigger: "assignee"}, }) d.poll(ctx) if len(fg.asked) != 1 { t.Fatalf("after reload, asked = %v, want unchanged [acme/widget] (no new intake)", fg.asked) } } type closeIssueCall struct { owner, repo string number int comment string } type commentCall struct { owner, repo string number int body string } type stateLabelCall struct { owner, repo string number int label string knownLabels []string } func (f *testForge) Comment(_ context.Context, owner, repo string, number int, body string) (int64, error) { f.comments = append(f.comments, commentCall{owner, repo, number, body}) return 1, nil } func (f *testForge) SetStateLabel(_ context.Context, owner, repo string, number int, label string, knownLabels []string) { f.stateLabels = append(f.stateLabels, stateLabelCall{owner, repo, number, label, knownLabels}) } func (f *testForge) AcceptInvitations(context.Context) error { return nil } func (f *testForge) AssignedIssues(context.Context, string, string, string) ([]forge.Issue, error) { panic("unexpected call") } func (f *testForge) IssuesWithLabel(context.Context, string, string, string) ([]forge.Issue, error) { panic("unexpected call") } func (f *testForge) CreatePR(context.Context, string, string, string, string, string, string) (int, error) { panic("unexpected call") } func (f *testForge) PRState(_ context.Context, _, _ string, number int) (string, error) { if state, ok := f.prStates[number]; ok { return state, nil } panic("unexpected call") } func (f *testForge) CloseIssue(_ context.Context, owner, repo string, number int, comment string) error { f.closedIssues = append(f.closedIssues, closeIssueCall{owner, repo, number, comment}) return nil } func (f *testForge) CreateIssue(context.Context, string, string, string, string, []string) (int, error) { return 0, nil } func (f *testForge) React(context.Context, string, string, int, string) error { return nil } func (f *testForge) VerifyPush(context.Context, string, string) error { return nil } func (f *testForge) LinkBranch(context.Context, string, string, int, string) error { return nil } func testDaemon(t *testing.T, maxRetries, _ int) (*Daemon, *pgstore.TaskDB, *testForge) { t.Helper() s := pgstore.Open(t) fg := &testForge{} cfg := config.Config{ MaxRetries: maxRetries, Dispatch: config.Dispatch{ Labels: map[string]string{ "parked": "archie:parked", "queued": "archie:queued", "dead": "archie:dead", }, }, } log := slog.New(slog.NewTextHandler(os.Stderr, nil)) return &Daemon{ Cfg: config.NewHolder(cfg), Store: s, Forge: fg, // Reconcile paths call Trees.Cleanup, which dereferences the // manager. A zero Manager is enough: cleanup of a worktree that was // never created is a no-op. Trees: &worktree.Manager{}, Log: log, }, s, fg } func TestNewIdentityRunnerPopulatesFromConfig(t *testing.T) { // Identity names come from config, not hardcoded by the daemon. tests := []struct { name string idCfg config.IdentityConfig wantErr bool }{ { name: "valid identity", idCfg: config.IdentityConfig{ Name: "my-bot", BotUser: "my-bot", Forge: config.Forge{Type: "gitea", Token: secret.SecretRef{Engine: "env", Key: "X"}}, Repos: []config.Repo{{Owner: "o", Name: "r"}}, }, }, { name: "another identity", idCfg: config.IdentityConfig{ Name: "other-bot", BotUser: "other-bot", Forge: config.Forge{Type: "github", Token: secret.SecretRef{Engine: "env", Key: "Y"}}, Repos: []config.Repo{{Owner: "a", Name: "b"}, {Owner: "c", Name: "d"}}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctx := context.Background() fg := &testForge{} trees := &worktree.Manager{WorkDir: t.TempDir(), Token: "t", BotUser: tt.idCfg.BotUser, BotEmail: "bot@test"} ir, err := NewIdentityRunner(ctx, tt.idCfg, fg, trees, slog.New(slog.DiscardHandler)) if (err != nil) != tt.wantErr { t.Fatalf("NewIdentityRunner error = %v, wantErr = %v", err, tt.wantErr) } if err != nil { return } if ir.Name != tt.idCfg.Name { t.Errorf("Name = %q, want %q", ir.Name, tt.idCfg.Name) } if ir.Forge != fg { t.Error("Forge not wired") } if len(ir.Repos) != len(tt.idCfg.Repos) { t.Errorf("len(Repos) = %d, want %d", len(ir.Repos), len(tt.idCfg.Repos)) } }) } } func TestNewIdentityRunnerRejectsEmptyName(t *testing.T) { ctx := context.Background() _, err := NewIdentityRunner(ctx, config.IdentityConfig{Name: ""}, &testForge{}, nil, slog.New(slog.DiscardHandler)) if err == nil { t.Error("expected error for empty name") } } func TestConfigForUsesIdentityConfigAndProvider(t *testing.T) { d := &Daemon{ Cfg: config.NewHolder(config.Config{ Models: map[string]string{"builder": "root/model"}, Providers: map[string]config.Provider{ "root": {Class: "openai", APIKeyEnv: "ROOT_KEY"}, }, }), Identities: []*IdentityRunner{{ Name: "worker", Cfg: config.IdentityConfig{ Name: "worker", Models: map[string]string{"builder": "worker/model"}, Providers: map[string]config.Provider{ "worker": {Class: "anthropic", APIKeyEnv: "WORKER_KEY"}, }, }, }}, } cfg := d.configFor(&workflow.Task{Identity: "worker"}) if cfg.Models["builder"] != "worker/model" { t.Fatalf("builder model = %q", cfg.Models["builder"]) } if _, ok := cfg.Providers["worker"]; !ok { t.Fatal("identity provider missing from execution config") } if _, ok := cfg.Providers["root"]; ok { t.Fatal("root provider leaked into identity execution config") } } func TestDaemonStoresIdentityRunners(t *testing.T) { d := &Daemon{ Identities: []*IdentityRunner{ {Name: "one"}, {Name: "two"}, }, } if len(d.Identities) != 2 { t.Errorf("len(Identities) = %d, want 2", len(d.Identities)) } } // ── Multi-identity isolation (archie-core-abg.37) ────────────────────── // // Two identities ("archie" and "winter") are both configured to work the // same owner/repo -- the exact overlapping-assignment scenario the bead // exists to make safe. A task enqueued under one identity's name must // only ever be acted on through that identity's own forge client and // worktree manager; it must never fall back to the root d.Forge/d.Trees // or leak into the other identity's client. func twoIdentityDaemon(t *testing.T) (d *Daemon, s *pgstore.TaskDB, rootFg, archieFg, winterFg *testForge) { t.Helper() s = pgstore.Open(t) rootFg = &testForge{} archieFg = &testForge{} winterFg = &testForge{} log := slog.New(slog.NewTextHandler(os.Stderr, nil)) repo := config.Repo{Owner: "acme", Name: "shared"} d = &Daemon{ Cfg: config.NewHolder(config.Config{ Dispatch: config.Dispatch{ Labels: map[string]string{ "queued": "archie:queued", "dead": "archie:dead", }, }, Repos: []config.Repo{repo}, // root/legacy repo list -- must NOT be consulted for identity tasks }), Store: s, Forge: rootFg, // must NOT be called for identity-owned tasks Log: log, Identities: []*IdentityRunner{ {Name: "archie", Forge: archieFg, Trees: &worktree.Manager{}, Repos: []config.Repo{repo}, Cfg: config.IdentityConfig{Name: "archie"}, Log: log}, {Name: "winter", Forge: winterFg, Trees: &worktree.Manager{}, Repos: []config.Repo{repo}, Cfg: config.IdentityConfig{Name: "winter"}, Log: log}, }, } return d, s, rootFg, archieFg, winterFg } func TestIdentityForResolvesOwningRunner(t *testing.T) { d, _, _, _, _ := twoIdentityDaemon(t) if id := d.identityFor(&workflow.Task{Identity: "archie"}); id == nil || id.Name != "archie" { t.Fatalf("identityFor(archie) = %v, want archie runner", id) } if id := d.identityFor(&workflow.Task{Identity: "winter"}); id == nil || id.Name != "winter" { t.Fatalf("identityFor(winter) = %v, want winter runner", id) } if id := d.identityFor(&workflow.Task{Identity: ""}); id != nil { t.Fatalf("identityFor(\"\") = %v, want nil (legacy single-identity path)", id) } if id := d.identityFor(&workflow.Task{Identity: "nonexistent"}); id != nil { t.Fatalf("identityFor(nonexistent) = %v, want nil", id) } } func TestAcknowledgeUsesOwningIdentityForgeNotRoot(t *testing.T) { d, _, rootFg, archieFg, winterFg := twoIdentityDaemon(t) ctx := context.Background() repo := config.Repo{Owner: "acme", Name: "shared"} is := forge.Issue{Number: 1, Title: "shared issue"} // archie's poll cycle acknowledges via its own forge client. d.acknowledge(ctx, archieFg, d.Cfg.Get(), repo, is) if len(archieFg.stateLabels) != 0 { t.Fatalf("archie forge got %d state label calls, want 0", len(archieFg.stateLabels)) } if len(winterFg.stateLabels) != 0 || len(rootFg.stateLabels) != 0 { t.Fatal("acknowledge leaked a state label call to winter's or root's forge client") } } func TestRepoForPrefersOwningIdentityRepoList(t *testing.T) { d, _, _, _, _ := twoIdentityDaemon(t) // Identity-only repo, absent from the root Cfg.Repos list. identityOnlyRepo := config.Repo{Owner: "acme", Name: "identity-only"} d.Identities[0].Repos = append(d.Identities[0].Repos, identityOnlyRepo) got, ok := d.repoFor(&workflow.Task{Owner: "acme", Repo: "identity-only", Identity: "archie"}) if !ok { t.Fatal("repoFor did not find an identity-only repo via the owning identity's repo list") } if got.Owner != "acme" || got.Name != "identity-only" { t.Fatalf("repoFor = %+v, want sam/identity-only", got) } // The same repo name must NOT resolve for an unrelated identity. if _, ok := d.repoFor(&workflow.Task{Owner: "acme", Repo: "identity-only", Identity: "winter"}); ok { t.Fatal("repoFor leaked archie's identity-only repo to winter") } } // sweepForge records invitation sweeps and push verifications so a test can // assert which forges Startup swept. It embeds testForge for the rest of the // forge.Forge surface; those methods are unreachable from Startup. Startup // sweeps every forge concurrently, so the recorders are mutex-guarded. type sweepForge struct { testForge mu sync.Mutex invitations int verifiedRepos []string } func (f *sweepForge) AcceptInvitations(context.Context) error { f.mu.Lock() defer f.mu.Unlock() f.invitations++ return nil } func (f *sweepForge) VerifyPush(_ context.Context, owner, repo string) error { f.mu.Lock() defer f.mu.Unlock() f.verifiedRepos = append(f.verifiedRepos, owner+"/"+repo) return nil } func (f *sweepForge) snapshot() (int, []string) { f.mu.Lock() defer f.mu.Unlock() return f.invitations, slices.Clone(f.verifiedRepos) } // TestStartupSweepsEveryConfiguredForge pins the boot-time invitation sweep // and push verification. Startup used to sweep only d.Forge and // d.Cfg.Get().Repos, so in multi-identity mode per-identity invites were // never accepted and per-identity repos never push-verified: a task from one // of those repos failed at dispatch instead of warning at boot. // // The root forge is swept in both modes. Run never polls the root repo list // in multi-identity mode, but dispatchBinding enqueues with an empty identity // against resolveBindingRepo's root-repo fallback, and forgeFor/repoFor // resolve an empty identity to the root forge and root repos, so root targets // stay reachable there. func TestStartupSweepsEveryConfiguredForge(t *testing.T) { type want struct { invitations int repos []string } tests := []struct { name string rootRepos []config.Repo identities []config.Repo // one identity per entry, keyed by owner wantRoot want wantIDs []want }{ { name: "single identity sweeps the root forge", rootRepos: []config.Repo{{Owner: "root", Name: "repo"}}, wantRoot: want{invitations: 1, repos: []string{"root/repo"}}, }, { name: "multi identity sweeps the root forge and every identity", rootRepos: []config.Repo{{Owner: "root", Name: "repo"}}, identities: []config.Repo{ {Owner: "one", Name: "a"}, {Owner: "two", Name: "b"}, }, wantRoot: want{invitations: 1, repos: []string{"root/repo"}}, wantIDs: []want{ {invitations: 1, repos: []string{"one/a"}}, {invitations: 1, repos: []string{"two/b"}}, }, }, { name: "identity with no repos still gets its invitations swept", rootRepos: nil, identities: []config.Repo{ {Owner: "one", Name: ""}, }, wantRoot: want{invitations: 1}, wantIDs: []want{{invitations: 1}}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { rootFg := &sweepForge{} idForges := make([]*sweepForge, len(tc.identities)) runners := make([]*IdentityRunner, len(tc.identities)) for i, r := range tc.identities { idForges[i] = &sweepForge{} runner := &IdentityRunner{Name: r.Owner, Forge: idForges[i]} if r.Name != "" { runner.Repos = []config.Repo{r} } runners[i] = runner } d := &Daemon{ Cfg: config.NewHolder(config.Config{Repos: tc.rootRepos}), Store: pgstore.Open(t), Forge: rootFg, Log: slog.New(slog.DiscardHandler), Identities: runners, } if err := d.Startup(context.Background()); err != nil { t.Fatalf("Startup: %v", err) } assertSwept(t, "root", rootFg, tc.wantRoot.invitations, tc.wantRoot.repos) for i, w := range tc.wantIDs { assertSwept(t, runners[i].Name, idForges[i], w.invitations, w.repos) } }) } } func assertSwept(t *testing.T, name string, fg *sweepForge, wantInvitations int, wantRepos []string) { t.Helper() invitations, repos := fg.snapshot() if invitations != wantInvitations { t.Errorf("%s AcceptInvitations = %d, want %d", name, invitations, wantInvitations) } if !slices.Equal(repos, wantRepos) { t.Errorf("%s verified repos = %v, want %v", name, repos, wantRepos) } } // stallForge stands in for a forge client whose request never returns: // AcceptInvitations signals that its sweep has started, then blocks until the // test releases it or the sweep's own deadline fires. type stallForge struct { testForge entered chan struct{} release chan struct{} } func (f *stallForge) AcceptInvitations(ctx context.Context) error { close(f.entered) select { case <-f.release: return nil case <-ctx.Done(): return ctx.Err() } } // TestStartupSweepIsolatesAStalledForge pins the concurrency half of the // stalled-forge fix. Sweeping forges one at a time on the startup goroutine // left every later identity unswept when one forge request never returned, // and Startup gates the gateways and the run loop, so it held boot open too. // Each forge now gets its own goroutine; this asserts the ordering that // proves it: the sibling identity's sweep completes while the root forge is // still blocked. func TestStartupSweepIsolatesAStalledForge(t *testing.T) { rootFg := &stallForge{entered: make(chan struct{}), release: make(chan struct{})} siblingFg := &sweepForge{} d := &Daemon{ Cfg: config.NewHolder(config.Config{}), Store: pgstore.Open(t), Forge: rootFg, Log: slog.New(slog.DiscardHandler), Identities: []*IdentityRunner{{Name: "sibling", Forge: siblingFg}}, } errs := make(chan error, 1) done := make(chan struct{}) go func() { defer close(done) errs <- d.Startup(context.Background()) }() waitFor(t, 5*time.Second, "the stalled root forge to start its sweep", func() bool { select { case <-rootFg.entered: return true default: return false } }) waitFor(t, 5*time.Second, "the sibling identity's sweep to finish", func() bool { invitations, _ := siblingFg.snapshot() return invitations == 1 }) close(rootFg.release) waitFor(t, 5*time.Second, "Startup to return once the stall is released", func() bool { select { case <-done: return true default: return false } }) if err := <-errs; err != nil { t.Errorf("Startup: %v", err) } } // waitFor polls cond until it holds or the timeout expires. func waitFor(t *testing.T, timeout time.Duration, what string, cond func() bool) { t.Helper() deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { if cond() { return } time.Sleep(time.Millisecond) } t.Fatalf("timed out waiting for %s", what) } // verifyForge returns a fixed error from VerifyPush, or blocks until the // sweep's own deadline expires first, standing in for an unreachable host. type verifyForge struct { testForge mu sync.Mutex calls int block bool } var errVerifyDenied = errors.New("push access denied") func (f *verifyForge) VerifyPush(ctx context.Context, _, _ string) error { f.mu.Lock() f.calls++ f.mu.Unlock() if f.block { <-ctx.Done() return ctx.Err() } return errVerifyDenied } func (f *verifyForge) callCount() int { f.mu.Lock() defer f.mu.Unlock() return f.calls } // TestSweepNamesTheCauseOfAnAbandonedPushCheck pins what the boot sweep // reports when a push check does not come back. One budget covers a forge's // whole sweep, so a repo cut short by the deadline used to warn that it was // "not pushable" -- asserting access had been denied when nothing about // access had been learned, once per remaining repo. An operator shutdown // cancels the same context, and that is neither a spent budget nor a denial. func TestSweepNamesTheCauseOfAnAbandonedPushCheck(t *testing.T) { repos := []config.Repo{ {Owner: "acme", Name: "one"}, {Owner: "acme", Name: "two"}, {Owner: "acme", Name: "three"}, } tests := []struct { name string mode string wantDenied int wantBudget int wantCalls int }{ { name: "a denied push is reported per repo", mode: "deny", wantDenied: len(repos), wantCalls: len(repos), }, { name: "an exhausted budget is reported once, as the cause", mode: "deadline", wantBudget: 1, wantCalls: 1, }, { name: "a cancelled startup is reported as neither", mode: "cancel", wantCalls: 1, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { fg := &verifyForge{block: tc.mode != "deny"} var logs logBuffer d := &Daemon{ Cfg: config.NewHolder(config.Config{Repos: repos}), Forge: fg, Log: slog.New(slog.NewTextHandler(&logs, nil)), } ctx := context.Background() switch tc.mode { case "deadline": // Expire the sweep's budget without waiting out sweepTimeout. deadline, cancel := context.WithTimeout(ctx, 50*time.Millisecond) defer cancel() ctx = deadline case "cancel": cancelled, cancel := context.WithCancel(ctx) cancel() ctx = cancelled } d.sweepAccess(ctx) if got := fg.callCount(); got != tc.wantCalls { t.Errorf("VerifyPush calls = %d, want %d", got, tc.wantCalls) } if got := strings.Count(logs.String(), "repo not pushable"); got != tc.wantDenied { t.Errorf("denied-push warnings = %d, want %d\nlogs:\n%s", got, tc.wantDenied, logs.String()) } if got := strings.Count(logs.String(), "sweep budget exhausted"); got != tc.wantBudget { t.Errorf("budget warnings = %d, want %d\nlogs:\n%s", got, tc.wantBudget, logs.String()) } }) } } // mustCoreConn returns the raw NATS connection for tests that drive core-NATS // subscriptions directly, failing the test if the client is not connected. func mustCoreConn(t *testing.T, c *arnats.Client) *natsio.Conn { t.Helper() conn, err := c.CoreConn() if err != nil { t.Fatalf("CoreConn: %v", err) } return conn } // Multi-identity mode must still run the store-wide maintenance passes: // storage cleanup and PR reconciliation. Before this loop was split into // per-identity polling plus a single shared maintainAndDrain loop, neither // ran in multi-identity mode -- merged PRs were never reconciled, tasks // stayed pr_open forever, and worktrees were never cleaned up. func TestMaintainAndDrainReconcilesPRsInMultiIdentityMode(t *testing.T) { d, s, _, archieFg, _ := twoIdentityDaemon(t) ctx := context.Background() // A forge-sourced task owned by the "archie" identity whose PR merged. if _, err := s.EnqueueIssue(ctx, "acme", "shared", 7, "t", "b", "", "archie"); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("ClaimNext = (%+v, %v)", task, err) } task.PRNumber = 42 if err := s.Update(ctx, task); err != nil { t.Fatal(err) } if err := s.Transition(ctx, task.ID, workflow.StatusRunning, workflow.StatusPROpen, ""); err != nil { t.Fatal(err) } archieFg.prStates = map[int]string{42: "merged"} d.maintainAndDrain(ctx) got, err := s.TaskByID(ctx, task.ID) if err != nil || got == nil { t.Fatalf("TaskByID = (%+v, %v)", got, err) } if got.Status != workflow.StatusMerged { t.Fatalf("status = %q, want %q (multi-identity reconcile never ran)", got.Status, workflow.StatusMerged) } if len(archieFg.closedIssues) != 1 { t.Fatalf("CloseIssue calls = %d, want 1 (issue must be closed on merge)", len(archieFg.closedIssues)) } } // The identity poll path must use the identity's own dispatch config -- // its bot user, label, and trigger -- not the root daemon's. An identity // with a different bot user polling "assignee" would otherwise query for // issues assigned to the root bot user. func TestPollIssuesWithConfigUsesIdentityScopedDispatch(t *testing.T) { fg := &recordingForge{} d := &Daemon{Cfg: config.NewHolder(config.Config{}), Log: slog.New(slog.DiscardHandler)} // Root uses bot_user "root"; the identity uses "winter". Both default // to the assignee trigger. rootCfg := config.Config{BotUser: "root", Dispatch: config.Dispatch{Trigger: "assignee"}} idCfg := config.IdentityConfig{BotUser: "winter", Dispatch: config.Dispatch{Trigger: "assignee"}} // Root poll queries root's bot user. _ = d.pollIssuesWithConfig(context.Background(), fg, rootCfg, config.Repo{Owner: "acme", Name: "shared"}) _ = d.pollIssuesWithConfig(context.Background(), fg, configForIdentity(rootCfg, idCfg), config.Repo{Owner: "acme", Name: "shared"}) if got := fg.assigned; len(got) != 2 || got[0] != "root" || got[1] != "winter" { t.Fatalf("AssignedIssues bot users = %v, want [root winter] (identity dispatch ignored)", got) } } // An empty cfg.Label with trigger "label" or "either" must never reach // IssuesWithLabel: GitHub's issues-list API treats an empty label filter as // "no filter" and returns every open issue in the repo. A live incident // (GH#445) queued 124 unrelated issues in one poll cycle this way, after // [dispatch.labels] (a different, unrelated field for task-state labels) was // configured while the actual trigger-match label was left unset. func TestPollIssuesWithConfigRefusesEmptyLabel(t *testing.T) { tests := []struct { name string trigger string }{ {name: "label trigger", trigger: "label"}, {name: "either trigger", trigger: "either"}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { fg := &recordingForge{} d := &Daemon{Cfg: config.NewHolder(config.Config{}), Log: slog.New(slog.DiscardHandler)} cfg := config.Config{Dispatch: config.Dispatch{Trigger: tc.trigger}} // Label left blank. issues := d.pollIssuesWithConfig(context.Background(), fg, cfg, config.Repo{Owner: "acme", Name: "shared"}) if issues != nil { t.Fatalf("issues = %v, want nil when label is unset", issues) } if len(fg.labelled) != 0 { t.Fatalf("IssuesWithLabel called with %v, want no call at all (empty label matches every open issue)", fg.labelled) } }) } } // The "either" trigger must keep polling by assignee when the label is // empty -- only the label side is unsafe, not the assignee side. A prior // version of this guard passed even when the empty-label branch dropped // assignee matches too (return nil instead of return out), because the // fixture always returned nil from AssignedIssues regardless. func TestPollIssuesWithConfigEitherTriggerKeepsAssigneeMatchesWhenLabelEmpty(t *testing.T) { fg := &recordingForge{assignedIssues: []forge.Issue{{Number: 7}}} d := &Daemon{Cfg: config.NewHolder(config.Config{}), Log: slog.New(slog.DiscardHandler)} cfg := config.Config{BotUser: "archie", Dispatch: config.Dispatch{Trigger: "either"}} // Label left blank. issues := d.pollIssuesWithConfig(context.Background(), fg, cfg, config.Repo{Owner: "acme", Name: "shared"}) if len(issues) != 1 || issues[0].Number != 7 { t.Fatalf("issues = %v, want the one assignee-matched issue preserved", issues) } if len(fg.labelled) != 0 { t.Fatalf("IssuesWithLabel called with %v, want no call at all (empty label matches every open issue)", fg.labelled) } } // recordingForge records the bot user/label passed to the discovery calls // so tests can assert identity-scoped polling. Everything else panics. type recordingForge struct { assigned []string labelled []string // assignedIssues is returned by AssignedIssues, letting tests confirm // assignee matches survive alongside (or despite) the label poll. assignedIssues []forge.Issue } func (f *recordingForge) AssignedIssues(_ context.Context, _, _, botUser string) ([]forge.Issue, error) { f.assigned = append(f.assigned, botUser) return f.assignedIssues, nil } func (f *recordingForge) IssuesWithLabel(_ context.Context, _, _, label string) ([]forge.Issue, error) { f.labelled = append(f.labelled, label) return nil, nil } func (f *recordingForge) Comment(context.Context, string, string, int, string) (int64, error) { panic("unexpected call") } func (f *recordingForge) SetStateLabel(context.Context, string, string, int, string, []string) { panic("unexpected call") } func (f *recordingForge) AcceptInvitations(context.Context) error { panic("unexpected call") } func (f *recordingForge) CreatePR(context.Context, string, string, string, string, string, string) (int, error) { panic("unexpected call") } func (f *recordingForge) PRState(context.Context, string, string, int) (string, error) { panic("unexpected call") } func (f *recordingForge) CloseIssue(context.Context, string, string, int, string) error { panic("unexpected call") } func (f *recordingForge) CreateIssue(context.Context, string, string, string, string, []string) (int, error) { panic("unexpected call") } func (f *recordingForge) React(context.Context, string, string, int, string) error { panic("unexpected call") } func (f *recordingForge) VerifyPush(context.Context, string, string) error { panic("unexpected call") } func (f *recordingForge) LinkBranch(context.Context, string, string, int, string) error { panic("unexpected call") } // A merged PR must close the forge issue. // // "Closes #N" was removed from PR bodies in favour of LinkBranch, which is // sidebar linkage on Gitea and a no-op stub on GitHub -- neither closes // anything. So the issue stayed open, labelled and assigned, and the next // poll re-enqueued it. Once the operator uses the dashboard's Clear, which // deletes the task row, that is a second implementation and a second PR for // work already merged. func TestReconcilePRsClosesTheIssueOnMerge(t *testing.T) { tests := []struct { name string prState string wantClosed bool }{ {name: "merged closes the issue", prState: "merged", wantClosed: true}, // A PR closed without merging usually means "try again", so the // issue stays open deliberately. {name: "closed without merge leaves it open", prState: "closed"}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { d, s, fg := testDaemon(t, 3, 0) ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 42, "a task", "", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("ClaimNext = (%+v, %v)", task, err) } task.PRNumber = 7 if err := s.Update(ctx, task); err != nil { t.Fatal(err) } if err := s.Transition(ctx, task.ID, workflow.StatusRunning, workflow.StatusPROpen, ""); err != nil { t.Fatal(err) } fg.prStates = map[int]string{7: tc.prState} d.reconcilePRs(ctx) if tc.wantClosed { if len(fg.closedIssues) != 1 { t.Fatalf("CloseIssue calls = %d, want 1: the issue stays open "+ "and is re-polled, so the work is done twice", len(fg.closedIssues)) } got := fg.closedIssues[0] if got.owner != "acme" || got.repo != "widget" || got.number != 42 { t.Errorf("closed %s/%s#%d, want acme/widget#42", got.owner, got.repo, got.number) } if got.comment == "" { t.Error("closed with no comment") } return } if len(fg.closedIssues) != 0 { t.Errorf("CloseIssue called %d times for a PR closed without merge", len(fg.closedIssues)) } }) } } // A chat task's issue number is synthetic, so closing it would close an // unrelated issue or error. func TestReconcilePRsSkipsChatTasks(t *testing.T) { d, s, fg := testDaemon(t, 3, 0) ctx := context.Background() task, err := s.EnqueueChatTask(ctx, "acme", "widget", "chat task", "", "", "") if err != nil { t.Fatal(err) } if _, err := s.ClaimNext(ctx); err != nil { t.Fatal(err) } task.PRNumber = 8 if err := s.Update(ctx, task); err != nil { t.Fatal(err) } if err := s.Transition(ctx, task.ID, workflow.StatusRunning, workflow.StatusPROpen, ""); err != nil { t.Fatal(err) } fg.prStates = map[int]string{8: "merged"} d.reconcilePRs(ctx) if len(fg.closedIssues) != 0 { t.Errorf("CloseIssue called for a chat task: issue number is synthetic") } } // daemonCanarySecrets are recognisable secret references and values that must // never appear in a captured configuration document. The event this capture // writes is readable by anyone who can read the task's timeline, so the // document is a disclosure boundary, not just a convenience. const ( canaryForgeToken = "ghp_DAEMONCANARYFORGETOKEN01" canaryWebhookSecret = "CANARY-WEBHOOK-SECRET-02" canaryIdentityToken = "ghp_DAEMONCANARYIDENTITY03" canaryProviderKey = "sk-DAEMONCANARYPROVIDERKEY04" canaryBWSKeyName = "DAEMON-CANARY-BWS-KEY-NAME-05" canaryTelegramToken = "tg-DAEMONCANARYTELEGRAMTOKEN06" canaryNATSTokenEnv = "DAEMON_CANARY_NATS_TOKEN_ENV_07" canaryStateStoreToken = "DAEMON-CANARY-STATE-STORE-TOKEN-08" ) // configWithCanarySecrets is a fully populated configuration carrying // recognisable secrets in every place a deployment keeps one: the root forge, // an identity's forge, the model providers, the chat channel, the state store // and the NATS credential. A capture that serialized the wrong object -- the // whole config, or a projection that forgot to drop a secret -- would carry // one of them. func configWithCanarySecrets() config.Config { return config.Config{ WorkDir: "/work/archie", BotUser: "archie-bot", BotEmail: "archie@example.com", Label: "archie", Forge: config.Forge{ Type: "gitea", Host: "gitea.example.com", Token: secret.SecretRef{Engine: "bws", Key: canaryForgeToken}, WebhookSecret: secret.SecretRef{Engine: "bws", Key: canaryWebhookSecret}, Intake: "webhook", }, Models: map[string]string{"builder": "openai/gpt-4", "planner": "openai/gpt-4"}, Providers: map[string]config.Provider{ "openai": { Class: "openai", APIKeyEnv: "OPENAI_API_KEY", BaseURL: "https://api.openai.com/v1", APIKey: secret.SecretRef{Engine: "bws", Key: canaryBWSKeyName}, }, "custom": { Class: "custom", APIKey: secret.SecretRef{Engine: "literal", Key: canaryProviderKey}, }, }, Budgets: config.Budgets{MaxSteps: 40, GateMaxFailures: 2}, DiffCapLines: new(800), Dispatch: config.Dispatch{ Trigger: "either", AckReaction: "eyes", Labels: map[string]string{"queued": "archie:queued"}, }, Notify: config.Notify{Webhook: "https://hooks.example.com/notify"}, Tools: config.ToolsConfig{Policy: config.ToolPolicy{MaxResultChars: 12000, SpillDir: "/tmp/spill"}}, NATS: config.NATSConfig{Mode: "external", URL: "nats://127.0.0.1:4222", TokenEnv: canaryNATSTokenEnv}, Chat: config.ChatConfig{Operator: "sam", Telegram: config.TelegramConfig{Token: secret.SecretRef{Engine: "bws", Key: canaryTelegramToken}}}, Services: config.Services{config.ServiceNameState: {Target: "127.0.0.1:50051", TargetToken: canaryStateStoreToken}}, ModelLimits: map[string]config.ModelLimits{ "openai/gpt-4": {ContextWindow: 128000, MaxOutputTokens: 4096}, }, Identities: []config.IdentityConfig{ { Name: "gitea-bot", BotUser: "gitea-bot", BotEmail: "gitea-bot@example.com", Forge: config.Forge{Type: "gitea", Host: "gitea.example.com", Token: secret.SecretRef{Engine: "bws", Key: canaryIdentityToken}}, Models: map[string]string{"builder": "openai/gpt-4"}, Budgets: config.Budgets{MaxSteps: 40}, }, }, } } // TestDispatchCapturesTheAttemptsEffectiveConfig is R4's producer half: the // configuration an attempt ran under is retrievable for that attempt. // // It asserts three separate claims rather than one: the event is durable and // attributed to the attempt, the document is the dispatch's own TaskConfig, // and no secret value anywhere in the configuration reaches it. func TestDispatchCapturesTheAttemptsEffectiveConfig(t *testing.T) { d, s, busClient := daemonWithNATS(t) d.Cfg.Set(configWithCanarySecrets()) ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 5, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } if task.Attempt == 0 { t.Fatal("claimed task has attempt 0; the event below could not be attributed to a run") } sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { data, _ := json.Marshal(taskrun.Response{Status: workflow.StatusPROpen}) _ = msg.Respond(data) }) if err != nil { t.Fatalf("subscribe: %v", err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) runPinnedViaAgent(ctx, d, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}) timeline, err := s.TaskEvents(ctx, task.ID) if err != nil { t.Fatal(err) } var captures []events.Event for _, e := range timeline { if e.Kind == events.KindConfigCaptured { captures = append(captures, e) } } if len(captures) != 1 { t.Fatalf("config_captured events = %d, want exactly 1 for the attempt", len(captures)) } captured := captures[0] if captured.Attempt != task.Attempt { t.Errorf("capture attempt = %d, want the dispatched attempt %d", captured.Attempt, task.Attempt) } encoded, err := json.Marshal(captured) if err != nil { t.Fatal(err) } document := string(encoded) for _, leak := range []string{ canaryForgeToken, canaryWebhookSecret, canaryIdentityToken, canaryProviderKey, canaryBWSKeyName, canaryTelegramToken, canaryNATSTokenEnv, canaryStateStoreToken, } { if strings.Contains(document, leak) { t.Errorf("captured configuration document leaked %q:\n%s", leak, document) } } var data struct { Schema string `json:"schema"` Document json.RawMessage `json:"document"` } if err := json.Unmarshal(encoded, &data); err != nil { t.Fatal(err) } // The event wrapper carries kind/stage/attempt; the document itself is // what the Config tab renders. var wrapper struct { Data struct { Schema string `json:"schema"` Document json.RawMessage `json:"document"` } `json:"data"` } if err := json.Unmarshal(encoded, &wrapper); err != nil { t.Fatal(err) } if wrapper.Data.Schema != events.ConfigCapturedSchema { t.Errorf("schema = %q, want %q", wrapper.Data.Schema, events.ConfigCapturedSchema) } var doc map[string]any if err := json.Unmarshal(wrapper.Data.Document, &doc); err != nil { t.Fatalf("document is not a JSON object: %v", err) } // The document is exactly TaskConfig's field set, not the larger dashboard // projection: a reader must be able to trust that it carries no providers, // repositories, identities or lock state. wantKeys := []string{ "bot_email", "bot_user", "budgets", "diff_cap_lines", "dispatch", "forge", "model_limits", "models", "notify", "tool_policy", } gotKeys := make([]string, 0, len(doc)) for key := range doc { gotKeys = append(gotKeys, key) } sort.Strings(gotKeys) if !slices.Equal(gotKeys, wantKeys) { t.Errorf("document keys = %v, want exactly %v", gotKeys, wantKeys) } if doc["forge"] == nil || !strings.Contains(document, "gitea.example.com") { t.Error("document is missing the non-secret forge host the Config tab renders") } // The size decides whether a per-attempt document belongs in an event at // all; recorded so the number is reviewable, not assumed. t.Logf("serialized config document: %d bytes for a fully populated deployment", len(wrapper.Data.Document)) if len(wrapper.Data.Document) == 0 { t.Error("document is empty; the Config tab would render nothing") } // Models and ModelLimits are the only two fields whose size is set by the // deployment rather than by the struct, so the growth factor gets measured // too instead of assumed. scaled := configWithCanarySecrets() scaled.Models = map[string]string{} scaled.ModelLimits = map[string]config.ModelLimits{} for i := range 20 { ref := fmt.Sprintf("openai/model-%d", i) scaled.Models[fmt.Sprintf("role-%d", i)] = ref scaled.ModelLimits[ref] = config.ModelLimits{ContextWindow: 128000, MaxOutputTokens: 4096} } scaledDoc, err := json.Marshal(scaled.ForTask()) if err != nil { t.Fatal(err) } t.Logf("serialized config document with 20 named models: %d bytes", len(scaledDoc)) } // failingEventStore is the daemon's store surface with an event insert that // always fails, so the fail-open policy of the capture can be observed without // breaking everything else the dispatch needs. type failingEventStore struct { *pgstore.TaskDB } func (f failingEventStore) InsertEvent(context.Context, events.Event) (int64, error) { return 0, fmt.Errorf("events table unavailable") } // TestCaptureAttemptConfigFailureDoesNotStopTheDispatch: a provenance record // that could not be written must not cost the run. The taskrun request has to // go out anyway, and nothing may park. func TestCaptureAttemptConfigFailureDoesNotStopTheDispatch(t *testing.T) { d, s, busClient := daemonWithNATS(t) d.Store = failingEventStore{TaskDB: s} ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 6, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } dispatched := make(chan struct{}, 1) sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { dispatched <- struct{}{} data, _ := json.Marshal(taskrun.Response{Status: workflow.StatusPROpen}) _ = msg.Respond(data) }) if err != nil { t.Fatalf("subscribe: %v", err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) runPinnedViaAgent(ctx, d, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}) select { case <-dispatched: case <-time.After(time.Second): t.Fatal("the taskrun request was never published; a failed config capture stopped the dispatch") } got, err := s.TaskByIssue(ctx, "acme", "widget", 6) if err != nil || got == nil { t.Fatalf("TaskByIssue: (%+v, %v)", got, err) } if got.Status == workflow.StatusParked { t.Errorf("task parked with reason %q; a failed config capture must not park a run", got.ParkReason) } } // TestTaskAgnosticDaemonEventsCarryNoAttempt pins the other half of the // attribution rule: an event produced before any task row exists has no // attempt to name, and inventing one -- the deployment's "current" attempt, a // zero-that-means-first -- would be a fabricated claim. It stays 0, which the // reader reports as unattributed. func TestTaskAgnosticDaemonEventsCarryNoAttempt(t *testing.T) { d := &Daemon{Bus: events.NewBus(), Log: slog.New(slog.DiscardHandler)} sub := d.Bus.Subscribe(4) t.Cleanup(sub.Close) d.acknowledge(context.Background(), &recordingForge{}, config.Config{}, config.Repo{Owner: "acme", Name: "widget"}, forge.Issue{Number: 3, Title: "a task"}) select { case e := <-sub.C: if e.Kind != events.KindTaskQueued { t.Fatalf("kind = %q, want %q", e.Kind, events.KindTaskQueued) } if e.Attempt != 0 { t.Errorf("queued event attempt = %d, want 0 (no task exists yet to attribute it to)", e.Attempt) } case <-time.After(time.Second): t.Fatal("no queued event was published") } } // TestReconcilePRsAttributesTheOutcomeToTheTasksAttempt: a PR is merged or // rejected against the attempt whose task recorded that PR number, so the // timeline can attribute the outcome to the run that produced it instead of // leaving it unattributed. func TestReconcilePRsAttributesTheOutcomeToTheTasksAttempt(t *testing.T) { d, s, fg := testDaemon(t, 3, 0) ctx := context.Background() d.Bus = events.NewBus() sub := d.Bus.Subscribe(8) t.Cleanup(sub.Close) if _, err := s.EnqueueIssue(ctx, "acme", "widget", 42, "a task", "", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("ClaimNext = (%+v, %v)", task, err) } if task.Attempt == 0 { t.Fatal("claimed task has attempt 0; this test could not tell a stamp from a coincidence") } task.PRNumber = 7 if err := s.Update(ctx, task); err != nil { t.Fatal(err) } if err := s.Transition(ctx, task.ID, workflow.StatusRunning, workflow.StatusPROpen, ""); err != nil { t.Fatal(err) } fg.prStates = map[int]string{7: "merged"} d.reconcilePRs(ctx) select { case e := <-sub.C: if e.Kind != events.KindPRMerged { t.Fatalf("kind = %q, want %q", e.Kind, events.KindPRMerged) } if e.Attempt != task.Attempt { t.Errorf("pr_merged attempt = %d, want the task's attempt %d", e.Attempt, task.Attempt) } case <-time.After(time.Second): t.Fatal("reconcilePRs published no merge event") } } // TestConfigForIdentityInheritsAnUnsetDiffCap is the regression case for an // identity silently disabling the diff cap. configForIdentity copied every // identity field over the root unconditionally, so an identity that simply did // not mention diff_cap_lines overwrote the shared cap with the zero value -- // which StageDiffCap reads as "no cap". The safety rail vanished for exactly // the deployments that never configured it per identity. func TestConfigForIdentityInheritsAnUnsetDiffCap(t *testing.T) { root := config.Config{DiffCapLines: new(400)} inherited := configForIdentity(root, config.IdentityConfig{BotUser: "one"}) if inherited.DiffCap() != 400 { t.Fatalf("DiffCap() = %d, want the shared 400 when the identity sets none", inherited.DiffCap()) } overridden := configForIdentity(root, config.IdentityConfig{BotUser: "two", DiffCapLines: new(50)}) if overridden.DiffCap() != 50 { t.Fatalf("DiffCap() = %d, want the identity's 50", overridden.DiffCap()) } off := configForIdentity(root, config.IdentityConfig{BotUser: "three", DiffCapLines: new(0)}) if off.DiffCap() != 0 { t.Fatalf("DiffCap() = %d, want 0: an identity may switch its own cap off explicitly", off.DiffCap()) } } // A workflow's profile decides the tools the agent gets: its allowlist is // carried in the taskrun request, and a profile that is not configured parks // the task for an operator instead of running it under the default. func TestRunViaAgentCarriesTheWorkflowProfile(t *testing.T) { d, s, busClient := daemonWithNATS(t) cfg := d.Cfg.Get() cfg.Containers.Profiles = map[string]config.AgentProfile{"net": {Image: "agent-net:1", Tools: []string{"whois"}}} d.Cfg.Set(cfg) d.WorkflowDefinitions = &workflowDefinitionsStub{collection: workflow.WorkflowDefinitionCollection{Definitions: []workflow.WorkflowDefinitionEntry{ {ID: "netflow", YAML: "id: netflow\nprofile: net\nsteps:\n - type: implement.prepare\n"}, {ID: "missing", YAML: "id: missing\nprofile: gone\nsteps:\n - type: implement.prepare\n"}, }}, version: 1} ctx := context.Background() claim := func(number int, wf string) *workflow.Task { t.Helper() if _, err := s.EnqueueIssue(ctx, "acme", "widget", number, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } task.Workflow = wf return task } task := claim(11, "netflow") received := make(chan taskrun.Request, 1) sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { var req taskrun.Request _ = json.Unmarshal(msg.Data, &req) received <- req data, _ := json.Marshal(taskrun.Response{Status: workflow.StatusPROpen}) _ = msg.Respond(data) }) if err != nil { t.Fatalf("subscribe: %v", err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) profile, ok := d.pinTaskProfile(ctx, task) if !ok || profile.Image != "agent-net:1" { t.Fatalf("pinTaskProfile() = %+v, %v; want the net profile", profile, ok) } d.runViaAgent(ctx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, profile, nil) select { case req := <-received: if len(req.Tools) != 1 || req.Tools[0] != "whois" { t.Fatalf("request Tools = %v, want the profile's allowlist", req.Tools) } case <-time.After(time.Second): t.Fatal("archied did not publish a taskrun request") } parked := claim(12, "missing") if _, ok := d.pinTaskProfile(ctx, parked); ok { t.Fatal("a workflow naming an unconfigured profile was not refused") } got, err := s.TaskByID(ctx, parked.ID) if err != nil || got.Status != workflow.StatusParked || !strings.Contains(got.ParkReason, `"gone" is not configured`) { t.Fatalf("task = %+v, %v; want parked naming the profile", got, err) } } // TestPinTaskProfileRefusesCapturesTheHarnessCannotServe is the harness PRD's // (Contract) "workflow definition validation rejects... a stage whose needs // the selected harness cannot meet: capture tools without an adapter". A Kit // profile's own adapter -- not the workflow -- decides whether captures are // servable, so a workflow that needs them but names a profile whose CLI // serves no capture tools (empty MCPConfig, e.g. Codex/Pi/OMP) must be // refused rather than silently running with no way to return its result. func TestPinTaskProfileRefusesCapturesTheHarnessCannotServe(t *testing.T) { d, s, _ := daemonWithNATS(t) cfg := d.Cfg.Get() cfg.Containers.Profiles = map[string]config.AgentProfile{ "serves-captures": {Kit: "kit@sha256:ab", Adapter: agentexec.AdapterClaudeCode}, "no-captures": {Kit: "kit@sha256:ab", Adapter: agentexec.AdapterCodex}, "no-adapter": {Kit: "kit@sha256:ab"}, } d.Cfg.Set(cfg) d.WorkflowDefinitions = &workflowDefinitionsStub{collection: workflow.WorkflowDefinitionCollection{Definitions: []workflow.WorkflowDefinitionEntry{ {ID: "needs-captures-served", YAML: "id: needs-captures-served\nrepository: none\nprofile: serves-captures\nneeds: {captures: true}\nsteps:\n - type: agent.run\n settings: {mission: go}\n"}, {ID: "needs-captures-codex", YAML: "id: needs-captures-codex\nrepository: none\nprofile: no-captures\nneeds: {captures: true}\nsteps:\n - type: agent.run\n settings: {mission: go}\n"}, {ID: "needs-captures-bare", YAML: "id: needs-captures-bare\nrepository: none\nprofile: no-adapter\nneeds: {captures: true}\nsteps:\n - type: agent.run\n settings: {mission: go}\n"}, }}, version: 1} ctx := context.Background() claim := func(number int, wf string) *workflow.Task { t.Helper() if _, err := s.EnqueueIssue(ctx, "acme", "widget", number, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } task.Workflow = wf return task } served := claim(51, "needs-captures-served") if _, ok := d.pinTaskProfile(ctx, served); !ok { t.Fatal("a Kit profile whose adapter serves captures was refused") } for _, tt := range []struct { number int workflow string }{ {52, "needs-captures-codex"}, {53, "needs-captures-bare"}, } { task := claim(tt.number, tt.workflow) if _, ok := d.pinTaskProfile(ctx, task); ok { t.Fatalf("%s: a workflow needing captures was not refused by a profile that cannot serve them", tt.workflow) } got, err := s.TaskByID(ctx, task.ID) if err != nil || got.Status != workflow.StatusParked || !strings.Contains(got.ParkReason, "captures") { t.Fatalf("%s: task = %+v, %v; want parked naming the unmet captures need", tt.workflow, got, err) } } } // TestWorkflowCallGivesACalleeItsOwnProfile is the harness PRD's "a stage may // name a different profile" (docs/prds/external-agent-harness.md, Selection): // there is no mid-container runner switch, and none is built here. A stage // wanting a different profile -- a Kit harness where the rest of the workflow // runs on the built-in loop -- is its own callee workflow, started the way // workflow.call already starts one (StartCall), with its own agent profile // resolved independently by pinTaskProfile off the callee's own task row. // TestWorkflowCallerConformance (staterpc) already proves the callee's // Workflow field is independent of the caller's; this proves that // independence is what makes Kit-vs-image runner selection independent too. func TestWorkflowCallGivesACalleeItsOwnProfile(t *testing.T) { d, s, _ := daemonWithNATS(t) cfg := d.Cfg.Get() cfg.Containers.Profiles = map[string]config.AgentProfile{ "net": {Image: "agent-net:1"}, "contain": {Kit: "contain-kit@sha256:ab", Adapter: agentexec.AdapterClaudeCode}, } d.Cfg.Set(cfg) d.WorkflowDefinitions = &workflowDefinitionsStub{collection: workflow.WorkflowDefinitionCollection{Definitions: []workflow.WorkflowDefinitionEntry{ {ID: "investigate", YAML: "id: investigate\nprofile: net\nsteps:\n - type: implement.prepare\n"}, {ID: "contain", YAML: "id: contain\nrepository: none\nprofile: contain\nsteps:\n - type: agent.run\n settings: {mission: contain it}\n"}, }}, version: 1} ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 41, "t", "b", "", ""); err != nil { t.Fatal(err) } caller, err := s.ClaimNext(ctx) if err != nil || caller == nil { t.Fatalf("claim caller: (%v, %v)", caller, err) } caller.Workflow = "investigate" // Exactly the call a workflow.call step makes (workflow_call.go's // runWorkflowCall -> tc.Calls.StartCall), not a hand-rolled shortcut. callee, err := s.StartCall(ctx, caller.ID, "contain", nil) if err != nil || callee == nil { t.Fatalf("StartCall: (%+v, %v)", callee, err) } callerProfile, ok := d.pinTaskProfile(ctx, caller) if !ok || callerProfile.IsKit() { t.Fatalf("caller profile = %+v, %v; want a non-Kit profile", callerProfile, ok) } calleeProfile, ok := d.pinTaskProfile(ctx, callee) if !ok || !calleeProfile.IsKit() || calleeProfile.Adapter != agentexec.AdapterClaudeCode { t.Fatalf("callee profile = %+v, %v; want the Kit profile its own workflow names", calleeProfile, ok) } } // A container that dies mid-run (max-uptime reaper, OOM, crash) never answers // the taskrun request. The run must end with the task parked, not left // running with the worker blocked on a reply that cannot come. func TestRunViaAgentParksWhenTheContainerExits(t *testing.T) { d, s, busClient := daemonWithNATS(t) ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 21, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } received := make(chan struct{}, 1) sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(*natsio.Msg) { received <- struct{}{} }) if err != nil { t.Fatalf("subscribe: %v", err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) exited := make(chan struct{}) runCtx, stop := withContainerExit(ctx, exited) defer stop() done := make(chan struct{}) go func() { d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{}, nil) close(done) }() select { case <-received: case <-time.After(5 * time.Second): t.Fatal("archied did not publish a taskrun request") } close(exited) select { case <-done: case <-time.After(5 * time.Second): t.Fatal("runViaAgent still blocked after its container exited") } got, err := s.TaskByID(ctx, task.ID) if err != nil || got.Status != workflow.StatusParked || !strings.Contains(got.ParkReason, "agent container exited") { t.Fatalf("task = %+v, %v; want parked naming the container exit", got, err) } } // A task that outruns its overall time limit is parked with the limit named, // even while its agent is still alive and silent. func TestRunViaAgentParksWhenTheTaskTimeLimitPasses(t *testing.T) { d, s, busClient := daemonWithNATS(t) ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 22, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(*natsio.Msg) {}) if err != nil { t.Fatalf("subscribe: %v", err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) runCtx, stop := withTaskTimeLimit(ctx, 100*time.Millisecond) defer stop() done := make(chan struct{}) go func() { d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{}, nil) close(done) }() select { case <-done: case <-time.After(5 * time.Second): t.Fatal("runViaAgent still blocked after the task time limit passed") } got, err := s.TaskByID(ctx, task.ID) if err != nil || got.Status != workflow.StatusParked || !strings.Contains(got.ParkReason, "task exceeded its time limit (100ms)") { t.Fatalf("task = %+v, %v; want parked naming the time limit", got, err) } } type fakeKitLauncher struct { got kitrun.Request released bool removed []kit.Volume } func (f *fakeKitLauncher) Launch(_ context.Context, req kitrun.Request) (*kitrun.Run, error) { f.got = req return &kitrun.Run{ Container: &container.Container{}, Harness: agentexec.HarnessSpec{Adapter: agentexec.AdapterClaudeCode, Launch: []string{"claude"}}, Volumes: []kit.Volume{{Name: "v", Path: "/home/agent/.claude"}}, }, nil } func (f *fakeKitLauncher) Release(context.Context, *kitrun.Run) error { f.released = true return nil } func (f *fakeKitLauncher) RemoveVolumes(_ context.Context, v []kit.Volume) error { f.removed = append(f.removed, v...) return nil } // A Kit profile starts the task through the Kit launcher, hands the worker // the Kit's harness, keeps model provider keys out of the container, and // removes the execution's volumes once the task is no longer retryable. func TestKitProfileRunsTheTaskOnItsHarness(t *testing.T) { d, s, busClient := daemonWithNATS(t) launcher := &fakeKitLauncher{} d.KitLauncher = launcher d.ConnectedNATS.URL = "nats://172.17.0.1:4222" t.Setenv("KIT_TEST_PROVIDER_KEY", "sk-real") cfg := d.Cfg.Get() cfg.Providers = map[string]config.Provider{"p": {APIKeyEnv: "KIT_TEST_PROVIDER_KEY"}} d.Cfg.Set(cfg) ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 31, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } received := make(chan taskrun.Request, 1) sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { var req taskrun.Request _ = json.Unmarshal(msg.Data, &req) received <- req data, _ := json.Marshal(taskrun.Response{Status: workflow.StatusPROpen}) _ = msg.Respond(data) }) if err != nil { t.Fatalf("subscribe: %v", err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) profile := config.AgentProfile{Kit: "kit@sha256:ab", Adapter: agentexec.AdapterClaudeCode} d.runKitTask(ctx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, t.TempDir(), profile) select { case req := <-received: if req.Harness == nil || req.Harness.Adapter != agentexec.AdapterClaudeCode { t.Fatalf("request harness = %+v, want the Kit's", req.Harness) } case <-time.After(time.Second): t.Fatal("archied did not publish a taskrun request") } if launcher.got.Kit != profile.Kit || launcher.got.Adapter != profile.Adapter { t.Fatalf("launch request %+v, want the profile's kit and adapter", launcher.got) } for _, e := range launcher.got.WorkerEnv { if strings.Contains(e, "sk-real") { t.Fatalf("worker env carries a model provider key: %q", e) } } if !launcher.released || len(launcher.removed) != 1 { t.Fatalf("released %v, removed %v; want the run released and its volumes removed", launcher.released, launcher.removed) } } // TestRunKitTaskThreadsGateRetriesToTheLauncher: KitLauncher.Launch is where // the Kit's actual agent-sessions capability is known (it composes the Kit), // so that is where docs/prds/external-agent-harness.md's "a gate retry // without a resume verb" refusal has to run (kit.ValidateNeeds, wired in // kitrun.Launcher.Launch). This proves runKitTask carries the workflow's // declared retry budget there rather than dropping it. func TestRunKitTaskThreadsGateRetriesToTheLauncher(t *testing.T) { d, s, busClient := daemonWithNATS(t) launcher := &fakeKitLauncher{} d.KitLauncher = launcher d.ConnectedNATS.URL = "nats://172.17.0.1:4222" ctx := context.Background() if _, err := s.EnqueueIssue(ctx, "acme", "widget", 32, "t", "b", "", ""); err != nil { t.Fatal(err) } task, err := s.ClaimNext(ctx) if err != nil || task == nil { t.Fatalf("claim: (%v, %v)", task, err) } task.WorkflowDefinitionYAML = "id: contain\nrepository: none\nneeds: {gate_retries: 4}\nsteps:\n - type: agent.run\n settings: {mission: go}\n" // runKitTask calls through to runViaAgent after Launch, which blocks on a // NATS reply bounded by the task wall-clock budget (zero here, i.e. no // bound at all) -- a handler that never responds hangs the daemon // goroutine forever rather than timing out, so this must answer even // though the assertion below only cares about what Launch already saw. sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { data, _ := json.Marshal(taskrun.Response{Status: workflow.StatusPROpen}) _ = msg.Respond(data) }) if err != nil { t.Fatalf("subscribe: %v", err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) d.runKitTask(ctx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, t.TempDir(), config.AgentProfile{Kit: "kit@sha256:ab", Adapter: agentexec.AdapterClaudeCode}) if launcher.got.GateRetries != 4 { t.Fatalf("launch request GateRetries = %d, want the workflow's declared 4", launcher.got.GateRetries) } }