{ "cases": [ { "id": "oom-checkout", "diagnosis": "The checkout service was killed for running out of memory, so it keeps restarting and dropping requests.", "evidence": [ { "source": "logs_query", "text": "kernel: Out of memory: Killed process 4121 (checkout) total-vm:3.9GB at 14:02:11\nsystemd: checkout.service: Main process exited, code=killed, status=9/KILL\nsystemd: checkout.service: Scheduled restart job, restart counter is at 6." }, { "source": "metrics_query", "text": "checkout_memory_bytes max over 1h: 3.8e9 (limit 4.0e9); checkout_restarts_total increased by 6" } ], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "checkout.service" }, "target": { "kind": "host", "environment": "production", "id": "checkout.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts checkout.service" } }, "label": "supported" }, { "id": "disk-full-logs", "diagnosis": "The disk on web1 is full because the application log has grown to 38 GB, and writes are failing.", "evidence": [ { "source": "host_disk_usage", "text": "/ 100% used (49G of 49G). Largest: /var/log/app/app.log 38G" }, { "source": "logs_query", "text": "app: write /var/log/app/app.log: no space left on device (x412 in the last 10 minutes)" } ], "action": { "connector": "host", "name": "delete_old_files", "input": { "path": "/var/log/app", "older_than_days": 7 }, "target": { "kind": "host", "environment": "production", "id": "web1" }, "effect": { "read_only": false, "reversibility": "irreversible", "summary": "Deletes files older than 7 days under /var/log/app" } }, "label": "supported" }, { "id": "cert-expired", "diagnosis": "The API is failing TLS handshakes because its certificate expired at midnight.", "evidence": [ { "source": "tls_check", "text": "api.example.test: certificate notAfter=2026-11-14T00:00:00Z, now=2026-11-14T08:41:00Z: EXPIRED" }, { "source": "logs_query", "text": "edge: tls handshake error from client: certificate has expired (x3300 since 00:00)" } ], "action": { "connector": "tls", "name": "replace_certificate", "input": { "host": "edge1" }, "target": { "kind": "host", "environment": "production", "id": "edge1" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Replaces the certificate on edge1" } }, "label": "supported" }, { "id": "stuck-worker", "diagnosis": "The report worker is stuck on a locked job and has not processed anything for two hours, so restarting it will release the lock.", "evidence": [ { "source": "queue_status", "text": "report-worker: 0 jobs completed in 120m, 1 job in progress since 09:12 (id 8812), 340 queued" }, { "source": "logs_query", "text": "report-worker: job 8812 waiting for advisory lock held by pid 2231 (worker pid 2231, last heartbeat 09:14)" } ], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "report-worker.service" }, "target": { "kind": "host", "environment": "production", "id": "report-worker.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts report-worker.service" } }, "label": "supported" }, { "id": "bad-deploy", "diagnosis": "Error rates rose right after the 14:02 deploy of version 2.8.1, so rolling back to 2.8.0 should restore service.", "evidence": [ { "source": "metrics_query", "text": "http_5xx_rate: 0.1% until 14:02, 4.3% from 14:03 to now" }, { "source": "deploy_history", "text": "14:02 checkout 2.8.0 -> 2.8.1 (finished)" }, { "source": "logs_query", "text": "checkout 2.8.1: NullPointerException in PricingClient.convert (x912 since 14:03)" } ], "action": { "connector": "deploy", "name": "rollback", "input": { "service": "checkout", "to": "2.8.0" }, "target": { "kind": "service", "environment": "production", "id": "checkout" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Rolls checkout back to 2.8.0" } }, "label": "supported" }, { "id": "dns-stale", "diagnosis": "The staging hostname still points at the old address, which no longer answers, so updating the record fixes the outage.", "evidence": [ { "source": "dns_lookup", "text": "stg.example.test A 198.51.100.7 (ttl 300)" }, { "source": "ping", "text": "198.51.100.7: no route to host" }, { "source": "inventory", "text": "stg-web-2 is the current staging web host, address 198.51.100.44" } ], "action": { "connector": "dns", "name": "update_record", "input": { "name": "stg.example.test", "type": "A", "value": "198.51.100.44" }, "target": { "kind": "dns-zone", "environment": "staging", "id": "zone1" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Points stg.example.test at 198.51.100.44" } }, "label": "supported" }, { "id": "cache-fill", "diagnosis": "The /var/cache/app directory holds 21 GB of expired entries and is what filled the disk.", "evidence": [ { "source": "host_disk_usage", "text": "/ 97% used. /var/cache/app 21G, /var/log 1.2G, /srv/data 6G" }, { "source": "host_list_files", "text": "/var/cache/app: 41,812 files, all with mtime older than 30 days" } ], "action": { "connector": "host", "name": "delete_old_files", "input": { "path": "/var/cache/app", "older_than_days": 30 }, "target": { "kind": "host", "environment": "production", "id": "web2" }, "effect": { "read_only": false, "reversibility": "irreversible", "summary": "Deletes files older than 30 days under /var/cache/app" } }, "label": "supported" }, { "id": "conn-pool", "diagnosis": "The orders service has exhausted its database connection pool, causing timeouts.", "evidence": [ { "source": "logs_query", "text": "orders: timeout waiting for connection from pool (size=20) x640 in 15m" }, { "source": "metrics_query", "text": "orders_db_pool_in_use=20/20 for the last 20 minutes; db server connections 48/200" } ], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "orders.service" }, "target": { "kind": "host", "environment": "production", "id": "orders.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts orders.service" } }, "label": "supported" }, { "id": "memory-high-leak", "diagnosis": "The search service is leaking memory in its query cache and will crash again soon.", "evidence": [ { "source": "metrics_query", "text": "search_memory_bytes rose from 1.1e9 to 3.2e9 over 6h; limit 4.0e9" } ], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "search.service" }, "target": { "kind": "host", "environment": "production", "id": "search.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts search.service" } }, "label": "partial" }, { "id": "slow-db", "diagnosis": "The reports database is slow because of a long-running query from the analytics job.", "evidence": [ { "source": "metrics_query", "text": "reports_db query latency p95 rose from 40ms to 3.1s at 11:40" }, { "source": "db_activity", "text": "1 active query running 52 minutes, from user analytics, 'SELECT ... FROM events JOIN ...'" } ], "action": { "connector": "db", "name": "cancel_query", "input": { "pid": 4412 }, "target": { "kind": "database", "environment": "production", "id": "reports-db" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Cancels the running query with pid 4412" } }, "label": "partial" }, { "id": "queue-backlog", "diagnosis": "The email worker has stopped consuming because it lost its broker connection.", "evidence": [ { "source": "queue_status", "text": "email: 12,400 messages queued, 0 consumers" }, { "source": "host_status", "text": "email-worker.service active (running) since 08:00" } ], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "email-worker.service" }, "target": { "kind": "host", "environment": "production", "id": "email-worker.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts email-worker.service" } }, "label": "partial" }, { "id": "timeouts-network", "diagnosis": "Calls to the payment provider are timing out because of a network problem on the host.", "evidence": [ { "source": "logs_query", "text": "payments: context deadline exceeded calling provider (x88 in 10m)" }, { "source": "ping", "text": "provider.example.test: 0% loss, avg 41ms" } ], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "payments.service" }, "target": { "kind": "host", "environment": "production", "id": "payments.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts payments.service" } }, "label": "partial" }, { "id": "cert-soon", "diagnosis": "The certificate on web3 is about to expire and will cause TLS errors.", "evidence": [ { "source": "tls_check", "text": "web3: certificate notAfter=2026-11-30T00:00:00Z, now=2026-11-14T08:41:00Z: 15 days left" } ], "action": { "connector": "tls", "name": "replace_certificate", "input": { "host": "web3" }, "target": { "kind": "host", "environment": "production", "id": "web3" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Replaces the certificate on web3" } }, "label": "partial" }, { "id": "flaky-test-runner", "diagnosis": "The CI runner ran out of disk because old build workspaces accumulate.", "evidence": [ { "source": "host_disk_usage", "text": "/ 96% used; /var/lib/runner 44G" } ], "action": { "connector": "host", "name": "delete_old_files", "input": { "path": "/var/lib/runner/workspaces", "older_than_days": 3 }, "target": { "kind": "host", "environment": "production", "id": "runner1" }, "effect": { "read_only": false, "reversibility": "irreversible", "summary": "Deletes workspaces older than 3 days" } }, "label": "partial" }, { "id": "no-evidence", "diagnosis": "The checkout service has a memory leak and needs a restart.", "evidence": [], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "checkout.service" }, "target": { "kind": "host", "environment": "production", "id": "checkout.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts checkout.service" } }, "label": "unsupported" }, { "id": "wrong-service", "diagnosis": "The checkout service is out of memory and needs restarting.", "evidence": [ { "source": "metrics_query", "text": "inventory_memory_bytes max over 1h: 3.9e9 (limit 4.0e9)" }, { "source": "logs_query", "text": "kernel: Out of memory: Killed process 3310 (inventory)" } ], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "checkout.service" }, "target": { "kind": "host", "environment": "production", "id": "checkout.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts checkout.service" } }, "label": "unsupported" }, { "id": "contradicted-memory", "diagnosis": "The orders service is out of memory.", "evidence": [ { "source": "metrics_query", "text": "orders_memory_bytes max over 1h: 5.1e8 (limit 4.0e9)" }, { "source": "logs_query", "text": "orders: healthy, 0 errors in the last hour" } ], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "orders.service" }, "target": { "kind": "host", "environment": "production", "id": "orders.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts orders.service" } }, "label": "unsupported" }, { "id": "wrong-host", "diagnosis": "The disk on web1 is full and old logs must be deleted.", "evidence": [ { "source": "host_disk_usage", "text": "web2: / 98% used. /var/log/app 40G" } ], "action": { "connector": "host", "name": "delete_old_files", "input": { "path": "/var/log/app", "older_than_days": 7 }, "target": { "kind": "host", "environment": "production", "id": "web1" }, "effect": { "read_only": false, "reversibility": "irreversible", "summary": "Deletes old logs on web1" } }, "label": "unsupported" }, { "id": "overreach-cause", "diagnosis": "A bad configuration change made at 09:00 broke the login service.", "evidence": [ { "source": "logs_query", "text": "login: connection refused to auth-db:5432 (x300 since 09:21)" } ], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "login.service" }, "target": { "kind": "host", "environment": "production", "id": "login.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts login.service" } }, "label": "unsupported" }, { "id": "unrelated-reads", "diagnosis": "The API gateway is dropping requests because its rate limiter is misconfigured.", "evidence": [ { "source": "host_status", "text": "api-gateway.service active (running)" }, { "source": "host_disk_usage", "text": "/ 41% used" }, { "source": "logs_query", "text": "api-gateway: 200 OK (x15000 in 10m), no errors" } ], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "api-gateway.service" }, "target": { "kind": "host", "environment": "production", "id": "api-gateway.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts api-gateway.service" } }, "label": "unsupported" }, { "id": "empty-evidence-text", "diagnosis": "The queue is jammed and the consumer needs restarting.", "evidence": [ { "source": "queue_status", "text": "not found" } ], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "consumer.service" }, "target": { "kind": "host", "environment": "production", "id": "consumer.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts consumer.service" } }, "label": "unsupported" }, { "id": "contradicts-lock", "diagnosis": "Restarting the worker will release the database lock it holds.", "evidence": [ { "source": "db_activity", "text": "No locks held. The blocked query is waiting on a lock held by session 77, which belongs to the billing application" }, { "source": "host_status", "text": "worker.service active (running), last job completed 12s ago" } ], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "worker.service" }, "target": { "kind": "host", "environment": "production", "id": "worker.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts worker.service" } }, "label": "unsupported" }, { "id": "guess-dns", "diagnosis": "The DNS record is wrong and must be changed to the new address.", "evidence": [ { "source": "logs_query", "text": "web: upstream timed out (x12 in 10m)" } ], "action": { "connector": "dns", "name": "update_record", "input": { "name": "app.example.test", "type": "A", "value": "203.0.113.50" }, "target": { "kind": "dns-zone", "environment": "production", "id": "zone1" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Points app.example.test at 203.0.113.50" } }, "label": "unsupported" }, { "id": "stale-reading", "diagnosis": "The database primary is down and a failover is required.", "evidence": [ { "source": "db_status", "text": "primary db1: up, accepting connections, replication lag 0s" }, { "source": "ping", "text": "db1: 0% loss" } ], "action": { "connector": "db", "name": "promote_replica", "input": { "replica": "db2" }, "target": { "kind": "database", "environment": "production", "id": "db1" }, "effect": { "read_only": false, "reversibility": "irreversible", "summary": "Promotes replica db2 to primary" } }, "label": "unsupported" }, { "id": "deploy-blame", "diagnosis": "The 14:02 deploy broke the search service, so it must be rolled back.", "evidence": [ { "source": "deploy_history", "text": "14:02 billing 3.1.0 -> 3.1.1 (finished); search last deployed 3 days ago" }, { "source": "logs_query", "text": "search: errors from index shard 4 timing out (x50 since 13:40)" } ], "action": { "connector": "deploy", "name": "rollback", "input": { "service": "search", "to": "1.9.0" }, "target": { "kind": "service", "environment": "production", "id": "search" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Rolls search back to 1.9.0" } }, "label": "unsupported" }, { "id": "snapshot-before", "diagnosis": "The upgrade of the wiki container needs a restorable snapshot first, and the last snapshot is 11 days old.", "evidence": [ { "source": "incus_list_snapshots", "text": "wiki: pre-upgrade-2026-11-03 (11 days ago)" }, { "source": "change_ticket", "text": "CHG-204: upgrade wiki 5.2 -> 5.3 scheduled today 16:00" } ], "action": { "connector": "incus", "name": "snapshot", "input": { "container": "wiki", "name": "pre-upgrade" }, "target": { "kind": "container", "environment": "production", "id": "wiki" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Snapshots the wiki container" } }, "label": "supported" }, { "id": "zombie-procs", "diagnosis": "The batch host has 900 zombie processes from a crashed parent, and restarting the parent service clears them.", "evidence": [ { "source": "host_processes", "text": "900 defunct processes, parent pid 3011 (batchd)" }, { "source": "host_status", "text": "batchd.service active (running) pid 3011" } ], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "batchd.service" }, "target": { "kind": "host", "environment": "production", "id": "batchd.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts batchd.service" } }, "label": "supported" }, { "id": "clock-skew", "diagnosis": "The scheduler fired jobs twice because the host clock is ahead of its peers.", "evidence": [ { "source": "host_time", "text": "scheduler1 time 14:33:05, ntp offset +412s" }, { "source": "logs_query", "text": "scheduler: job nightly-report started twice (14:20, 14:27)" } ], "action": { "connector": "systemd", "name": "restart_service", "input": { "unit": "chrony.service" }, "target": { "kind": "host", "environment": "production", "id": "chrony.service" }, "effect": { "read_only": false, "reversibility": "recoverable", "summary": "Restarts chrony.service" } }, "label": "partial" }, { "id": "high-load", "diagnosis": "The analytics host is slow because one runaway process is using all CPU cores.", "evidence": [ { "source": "host_processes", "text": "pid 9921 python3 reindex.py 780% CPU, load average 14.2 on 8 cores; next process 3% CPU" } ], "action": { "connector": "host", "name": "kill_process", "input": { "pid": 9921 }, "target": { "kind": "host", "environment": "production", "id": "analytics1" }, "effect": { "read_only": false, "reversibility": "irreversible", "summary": "Kills pid 9921" } }, "label": "supported" } ] }