fix(statestore): skip the legacy cron jobs the schedules resource cannot hold

migrateLegacySchedules replaced the whole legacy cron/jobs.json list into
the schedules resource, whose validator refuses every job whose kind is
not 'workflow' -- and chat is the legacy default (an empty kind on
schema-v1 jobs means chat). One chat job therefore failed the replace,
the migration returned the error and RunStateStore aborted before the
server ever listened. The failure was permanent: the seed had already
landed at version 1, so every restart re-ran the migration and refused
again, until an operator renamed cron/jobs.json by hand -- a remedy no
recovery doc mentions (archie-core-q6vw).

The migration now moves only the jobs the schedules resource can hold
and reports the rest instead of refusing them: chat and unrecognised
kinds were already dead deliveries in the old engine (the delivery
router maps KindWorkflow alone). An unreadable or corrupt sidecar is
logged and left in place for the next start, the way ImportConfig skips
a seed the resource validator refuses -- a value this process cannot
use must not stop the one process every other process dials. A
successful replace is marked by renaming the file to jobs.json.migrated
(the skipped jobs stay readable in it); until the rename lands, the
replaying request ID keeps the replace idempotent across restarts.
This commit is contained in:
sam committed 2026-09-22 00:14:08 +10:00
1 parent 51ed7178e6
commit 553330df41
2 files changed
+267 -20

No files matched your search

@@ -0,0 +1,206 @@
package archied
import (
"encoding/json"
"log/slog"
"os"
"path/filepath"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/app/controlplane"
controlpb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
"github.com/samcharles93/archie-core/internal/infrastructure/cronstore"
"github.com/samcharles93/archie-core/internal/store"
)
// migrationServer builds the validating side exactly as RunStateStore composes
// it: a store over a throwaway file and openStateStoreControlPlane, the root
// helper that registers the shared step vocabulary.
func migrationServer(t *testing.T) (*controlplane.Server, *store.Store) {
t.Helper()
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "archie.db-tasks.sqlite"))
if err != nil {
t.Fatalf("open store: %v", err)
}
t.Cleanup(func() { _ = st.Close() })
server, err := openStateStoreControlPlane(st)
if err != nil {
t.Fatalf("open control plane: %v", err)
}
return server, st
}
// seedSchedules mirrors RunStateStore's ImportConfig: the schedules resource
// starts as the empty seed at version 1, which is the state the legacy
// migration runs against.
func seedSchedules(t *testing.T, control *controlplane.Server) int64 {
t.Helper()
seed, err := json.Marshal([]cronstore.JobSpec{})
if err != nil {
t.Fatalf("encode empty schedules seed: %v", err)
}
response, err := control.Command(t.Context(), &controlpb.CommandRequest{
Kind: controlplane.SchedulesKind, Command: "replace", ValueJson: seed,
ExpectedVersion: 0, Actor: "system:migration", Source: "legacy-config", RequestId: "import:schedules",
})
if err != nil {
t.Fatalf("seed schedules: %v", err)
}
return response.GetResource().GetVersion()
}
// writeLegacyJobs writes a legacy cronstore file at the layout
// migrateLegacySchedules reads: <dir of dbPath>/cron/jobs.json, in the
// store's own on-disk envelope shape (schema v2).
func writeLegacyJobs(t *testing.T, dir string, jobs []cronstore.JobSpec) string {
t.Helper()
data, err := json.Marshal(struct {
SchemaVersion int `json:"schema_version"`
Jobs []cronstore.JobSpec `json:"jobs"`
}{SchemaVersion: 2, Jobs: jobs})
if err != nil {
t.Fatalf("encode fixture: %v", err)
}
return writeLegacyFile(t, dir, string(data))
}
func writeLegacyFile(t *testing.T, dir, content string) string {
t.Helper()
path := filepath.Join(dir, "cron", "jobs.json")
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("create legacy cron directory: %v", err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatalf("write legacy jobs file: %v", err)
}
return path
}
// legacyJob is a schema-v1-shaped job the old daemon's cron engine wrote.
// An empty Kind means chat, the pre-field default (cronstore.KindChat).
func legacyJob(id, kind string, interval time.Duration) cronstore.JobSpec {
stamped := time.Date(2026, 9, 1, 9, 0, 0, 0, time.UTC)
return cronstore.JobSpec{
ID: id, Kind: kind, Detail: "legacy " + id,
Schedule: cronstore.Schedule{Kind: cronstore.ScheduleInterval, Interval: interval},
NextRun: stamped, Created: stamped, Updated: stamped,
}
}
// storedScheduleIDs decodes the schedules resource into the job IDs it holds.
func storedScheduleIDs(t *testing.T, st *store.Store) []string {
t.Helper()
resource, err := st.Resource(t.Context(), controlplane.SchedulesKind)
if err != nil {
t.Fatalf("read schedules resource: %v", err)
}
var jobs []cronstore.JobSpec
if err := json.Unmarshal(resource.Value, &jobs); err != nil {
t.Fatalf("decode schedules resource: %v", err)
}
ids := make([]string, 0, len(jobs))
for _, job := range jobs {
ids = append(ids, job.ID)
}
return ids
}
// TestMigrateLegacySchedulesMovesOnlyWorkflowJobs: a legacy file holding a
// chat job (the pre-field default, kind omitted) and a workflow job migrates
// the workflow job and skips -- never refuses -- the chat one. The current
// behaviour replaces the whole list and lets the schedules validator fail the
// State Store on the chat job, taking every process that dials it down with
// it (archie-core-q6vw).
func TestMigrateLegacySchedulesMovesOnlyWorkflowJobs(t *testing.T) {
dir := t.TempDir()
dbPath := filepath.Join(dir, "archie.db-tasks.sqlite")
writeLegacyJobs(t, dir, []cronstore.JobSpec{
legacyJob("chat-daily", "", 24*time.Hour),
legacyJob("tdd-hourly", cronstore.KindWorkflow, time.Hour),
})
server, st := migrationServer(t)
version := seedSchedules(t, server)
migrateLegacySchedules(t.Context(), server, dbPath, version, discardLog())
got := storedScheduleIDs(t, st)
if len(got) != 1 || got[0] != "tdd-hourly" {
t.Fatalf("schedules resource holds %v, want only the migratable workflow job", got)
}
if _, err := os.Stat(path_migrated(dir)); err != nil {
t.Fatalf("migrated legacy file not marked (jobs.json.migrated): %v", err)
}
if _, err := os.Stat(filepath.Join(dir, "cron", "jobs.json")); !os.IsNotExist(err) {
t.Fatalf("legacy file still at its original path after a successful migration")
}
}
// path_migrated is where a completed migration leaves the legacy file.
func path_migrated(dir string) string {
return filepath.Join(dir, "cron", "jobs.json.migrated")
}
// TestMigrateLegacySchedulesKeepsStoreRunningWhenNothingIsMigratable: a
// legacy file holding only chat jobs leaves the schedules resource untouched
// and reports -- never fails -- the skip. The store's whole job is to come up.
func TestMigrateLegacySchedulesKeepsStoreRunningWhenNothingIsMigratable(t *testing.T) {
dir := t.TempDir()
dbPath := filepath.Join(dir, "archie.db-tasks.sqlite")
path := writeLegacyJobs(t, dir, []cronstore.JobSpec{
legacyJob("chat-daily", "", 24*time.Hour),
legacyJob("chat-weekly", "chat", 7*24*time.Hour),
})
server, st := migrationServer(t)
version := seedSchedules(t, server)
migrateLegacySchedules(t.Context(), server, dbPath, version, discardLog())
if _, err := os.Stat(path); err != nil {
t.Fatalf("legacy file removed though nothing was migrated: %v", err)
}
if got := storedScheduleIDs(t, st); len(got) != 0 {
t.Fatalf("schedules resource holds %v, want the untouched empty seed", got)
}
}
// TestMigrateLegacySchedulesKeepsStoreRunningOnUnreadableLegacyStore: a
// corrupt legacy file is a diagnosis, not a boot blocker. The store must come
// up on the empty seed and the file must stay in place for the operator.
func TestMigrateLegacySchedulesKeepsStoreRunningOnUnreadableLegacyStore(t *testing.T) {
dir := t.TempDir()
dbPath := filepath.Join(dir, "archie.db-tasks.sqlite")
path := writeLegacyFile(t, dir, "{not json")
server, _ := migrationServer(t)
version := seedSchedules(t, server)
migrateLegacySchedules(t.Context(), server, dbPath, version, discardLog())
if _, err := os.Stat(path); err != nil {
t.Fatalf("legacy file removed though it was never read: %v", err)
}
}
// TestMigrateLegacySchedulesLeavesTheFileWhenTheReplaceIsRefused: even a
// workflow job the schedules validator refuses is skipped and retried on the
// next start, the way ImportConfig skips a refused seed -- never fatal.
func TestMigrateLegacySchedulesLeavesTheFileWhenTheReplaceIsRefused(t *testing.T) {
dir := t.TempDir()
dbPath := filepath.Join(dir, "archie.db-tasks.sqlite")
path := writeLegacyJobs(t, dir, []cronstore.JobSpec{
legacyJob("broken-interval", cronstore.KindWorkflow, 0), // interval schedule with no interval
})
server, st := migrationServer(t)
version := seedSchedules(t, server)
migrateLegacySchedules(t.Context(), server, dbPath, version, discardLog())
if _, err := os.Stat(path); err != nil {
t.Fatalf("legacy file removed though the replace never landed: %v", err)
}
if got := storedScheduleIDs(t, st); len(got) != 0 {
t.Fatalf("schedules resource holds %v, want the untouched empty seed", got)
}
}
// discardLog returns a logger that drops every record: these tests pin the
// migration's effects on the resource and the file, not its reporting.
func discardLog() *slog.Logger {
return slog.New(slog.DiscardHandler)
}
+61 -20
View File
@@ -13,6 +13,7 @@ import (
"context"
"encoding/json"
"fmt"
"log/slog"
"net"
"net/http"
"os"
@@ -111,9 +112,7 @@ func RunStateStore(ctx context.Context, options StateStoreOptions) error {
return fmt.Errorf("import control-plane resources: %w", err)
}
b.reportUnseededResources(skipped)
if err := migrateLegacySchedules(ctx, control, b.cfg.DBPath, versions[controlplane.SchedulesKind]); err != nil {
return fmt.Errorf("migrate schedules: %w", err)
}
migrateLegacySchedules(ctx, control, b.cfg.DBPath, versions[controlplane.SchedulesKind], b.log)
token := options.Token
if token == "" {
@@ -189,37 +188,79 @@ func openStateStoreControlPlane(resources controlplane.ResourceStore) (*controlp
return server, nil
}
func migrateLegacySchedules(ctx context.Context, control *controlplane.Server, dbPath string, version int64) error {
// migrateLegacySchedules carries a legacy cronstore file -- the old daemon's
// <state dir>/cron/jobs.json -- into the schedules resource, once, on a fresh
// seed. It moves only the jobs the schedules resource can hold: workflow
// jobs, the one delivery the router maps. Chat and unrecognised kinds were
// already dead deliveries in the old engine (the delivery router maps
// KindWorkflow alone), so they are reported and left in the legacy file
// rather than being allowed to refuse the replace the whole store depends on.
//
// Nothing here is fatal. The migration is a legacy-data courtesy: an
// unreadable or refused sidecar is logged and left in place for the next
// start, the way ImportConfig skips a seed the resource validator refuses --
// a value this process cannot use must not stop the one process every other
// process dials (docs/prds/runtime-control-plane.md, "Bootstrap, migration,
// and recovery"). A successful replace is marked by renaming the file to
// jobs.json.migrated, so the whole legacy list -- the skipped jobs included
// -- stays on disk for the operator; until the rename lands, the replaying
// request ID keeps the replace idempotent across restarts.
func migrateLegacySchedules(ctx context.Context, control *controlplane.Server, dbPath string, version int64, log *slog.Logger) {
if version != 1 {
return nil
return
}
path := filepath.Join(filepath.Dir(dbPath), "cron", "jobs.json")
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
return nil
return
}
return err
log.Warn("legacy cron store not readable; schedules migration skipped and retried on the next start", "path", path, "err", err)
return
}
jobs, err := readLegacyJobs(ctx, path)
if err != nil {
log.Error("legacy cron store unreadable; schedules migration skipped and retried on the next start", "path", path, "err", err)
return
}
migratable := make([]cronstore.JobSpec, 0, len(jobs))
for _, job := range jobs {
if job.Kind == cronstore.KindWorkflow {
migratable = append(migratable, job)
continue
}
log.Warn("legacy cron job is not a workflow job; the schedules resource does not carry it and it stays in the legacy file", "id", job.ID, "kind", job.Kind)
}
if len(migratable) == 0 {
log.Info("legacy cron store holds no workflow jobs; nothing to migrate", "path", path, "jobs", len(jobs))
return
}
value, err := json.Marshal(migratable)
if err != nil {
log.Error("encode legacy cron jobs; schedules migration skipped and retried on the next start", "path", path, "err", err)
return
}
if _, err := control.Command(ctx, &controlpb.CommandRequest{Kind: controlplane.SchedulesKind, Command: "replace", ValueJson: value, ExpectedVersion: 1, Actor: "system:migration", Source: "legacy-cronstore", RequestId: "import:legacy-schedules"}); err != nil {
log.Error("legacy cron jobs do not satisfy the schedules resource; migration skipped and retried on the next start", "path", path, "err", err)
return
}
if err := os.Rename(path, path+".migrated"); err != nil {
log.Error("mark the migrated legacy cron file; migration retried on the next start", "path", path, "err", err)
}
}
// readLegacyJobs reads the legacy cronstore file and closes its handle,
// reporting whichever failure came first.
func readLegacyJobs(ctx context.Context, path string) ([]cronstore.JobSpec, error) {
legacy, err := cronstore.Open(path)
if err != nil {
return err
return nil, err
}
jobs, listErr := legacy.List(ctx)
closeErr := legacy.Close()
if listErr != nil {
return listErr
return nil, listErr
}
if closeErr != nil {
return closeErr
}
value, err := json.Marshal(jobs)
if err != nil {
return err
}
if _, err := control.Command(ctx, &controlpb.CommandRequest{Kind: controlplane.SchedulesKind, Command: "replace", ValueJson: value, ExpectedVersion: 1, Actor: "system:migration", Source: "legacy-cronstore", RequestId: "import:legacy-schedules"}); err != nil {
return err
}
return os.Rename(path, path+".migrated")
return jobs, closeErr
}
// openStateStore opens the single task-store SQLite file exactly once for