Files

914 lines
22 KiB
JSON

{
"cases": [
{
"id": "oom-checkout",
"diagnosis": "The checkout service was killed for running out of memory, so it keeps restarting and dropping requests.",
"evidence": [
{
"source": "logs_query",
"text": "kernel: Out of memory: Killed process 4121 (checkout) total-vm:3.9GB at 14:02:11\nsystemd: checkout.service: Main process exited, code=killed, status=9/KILL\nsystemd: checkout.service: Scheduled restart job, restart counter is at 6."
},
{
"source": "metrics_query",
"text": "checkout_memory_bytes max over 1h: 3.8e9 (limit 4.0e9); checkout_restarts_total increased by 6"
}
],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "checkout.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "checkout.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts checkout.service"
}
},
"label": "supported"
},
{
"id": "disk-full-logs",
"diagnosis": "The disk on web1 is full because the application log has grown to 38 GB, and writes are failing.",
"evidence": [
{
"source": "host_disk_usage",
"text": "/ 100% used (49G of 49G). Largest: /var/log/app/app.log 38G"
},
{
"source": "logs_query",
"text": "app: write /var/log/app/app.log: no space left on device (x412 in the last 10 minutes)"
}
],
"action": {
"connector": "host",
"name": "delete_old_files",
"input": {
"path": "/var/log/app",
"older_than_days": 7
},
"target": {
"kind": "host",
"environment": "production",
"id": "web1"
},
"effect": {
"read_only": false,
"reversibility": "irreversible",
"summary": "Deletes files older than 7 days under /var/log/app"
}
},
"label": "supported"
},
{
"id": "cert-expired",
"diagnosis": "The API is failing TLS handshakes because its certificate expired at midnight.",
"evidence": [
{
"source": "tls_check",
"text": "api.example.test: certificate notAfter=2026-11-14T00:00:00Z, now=2026-11-14T08:41:00Z: EXPIRED"
},
{
"source": "logs_query",
"text": "edge: tls handshake error from client: certificate has expired (x3300 since 00:00)"
}
],
"action": {
"connector": "tls",
"name": "replace_certificate",
"input": {
"host": "edge1"
},
"target": {
"kind": "host",
"environment": "production",
"id": "edge1"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Replaces the certificate on edge1"
}
},
"label": "supported"
},
{
"id": "stuck-worker",
"diagnosis": "The report worker is stuck on a locked job and has not processed anything for two hours, so restarting it will release the lock.",
"evidence": [
{
"source": "queue_status",
"text": "report-worker: 0 jobs completed in 120m, 1 job in progress since 09:12 (id 8812), 340 queued"
},
{
"source": "logs_query",
"text": "report-worker: job 8812 waiting for advisory lock held by pid 2231 (worker pid 2231, last heartbeat 09:14)"
}
],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "report-worker.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "report-worker.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts report-worker.service"
}
},
"label": "supported"
},
{
"id": "bad-deploy",
"diagnosis": "Error rates rose right after the 14:02 deploy of version 2.8.1, so rolling back to 2.8.0 should restore service.",
"evidence": [
{
"source": "metrics_query",
"text": "http_5xx_rate: 0.1% until 14:02, 4.3% from 14:03 to now"
},
{
"source": "deploy_history",
"text": "14:02 checkout 2.8.0 -> 2.8.1 (finished)"
},
{
"source": "logs_query",
"text": "checkout 2.8.1: NullPointerException in PricingClient.convert (x912 since 14:03)"
}
],
"action": {
"connector": "deploy",
"name": "rollback",
"input": {
"service": "checkout",
"to": "2.8.0"
},
"target": {
"kind": "service",
"environment": "production",
"id": "checkout"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Rolls checkout back to 2.8.0"
}
},
"label": "supported"
},
{
"id": "dns-stale",
"diagnosis": "The staging hostname still points at the old address, which no longer answers, so updating the record fixes the outage.",
"evidence": [
{
"source": "dns_lookup",
"text": "stg.example.test A 198.51.100.7 (ttl 300)"
},
{
"source": "ping",
"text": "198.51.100.7: no route to host"
},
{
"source": "inventory",
"text": "stg-web-2 is the current staging web host, address 198.51.100.44"
}
],
"action": {
"connector": "dns",
"name": "update_record",
"input": {
"name": "stg.example.test",
"type": "A",
"value": "198.51.100.44"
},
"target": {
"kind": "dns-zone",
"environment": "staging",
"id": "zone1"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Points stg.example.test at 198.51.100.44"
}
},
"label": "supported"
},
{
"id": "cache-fill",
"diagnosis": "The /var/cache/app directory holds 21 GB of expired entries and is what filled the disk.",
"evidence": [
{
"source": "host_disk_usage",
"text": "/ 97% used. /var/cache/app 21G, /var/log 1.2G, /srv/data 6G"
},
{
"source": "host_list_files",
"text": "/var/cache/app: 41,812 files, all with mtime older than 30 days"
}
],
"action": {
"connector": "host",
"name": "delete_old_files",
"input": {
"path": "/var/cache/app",
"older_than_days": 30
},
"target": {
"kind": "host",
"environment": "production",
"id": "web2"
},
"effect": {
"read_only": false,
"reversibility": "irreversible",
"summary": "Deletes files older than 30 days under /var/cache/app"
}
},
"label": "supported"
},
{
"id": "conn-pool",
"diagnosis": "The orders service has exhausted its database connection pool, causing timeouts.",
"evidence": [
{
"source": "logs_query",
"text": "orders: timeout waiting for connection from pool (size=20) x640 in 15m"
},
{
"source": "metrics_query",
"text": "orders_db_pool_in_use=20/20 for the last 20 minutes; db server connections 48/200"
}
],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "orders.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "orders.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts orders.service"
}
},
"label": "supported"
},
{
"id": "memory-high-leak",
"diagnosis": "The search service is leaking memory in its query cache and will crash again soon.",
"evidence": [
{
"source": "metrics_query",
"text": "search_memory_bytes rose from 1.1e9 to 3.2e9 over 6h; limit 4.0e9"
}
],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "search.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "search.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts search.service"
}
},
"label": "partial"
},
{
"id": "slow-db",
"diagnosis": "The reports database is slow because of a long-running query from the analytics job.",
"evidence": [
{
"source": "metrics_query",
"text": "reports_db query latency p95 rose from 40ms to 3.1s at 11:40"
},
{
"source": "db_activity",
"text": "1 active query running 52 minutes, from user analytics, 'SELECT ... FROM events JOIN ...'"
}
],
"action": {
"connector": "db",
"name": "cancel_query",
"input": {
"pid": 4412
},
"target": {
"kind": "database",
"environment": "production",
"id": "reports-db"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Cancels the running query with pid 4412"
}
},
"label": "partial"
},
{
"id": "queue-backlog",
"diagnosis": "The email worker has stopped consuming because it lost its broker connection.",
"evidence": [
{
"source": "queue_status",
"text": "email: 12,400 messages queued, 0 consumers"
},
{
"source": "host_status",
"text": "email-worker.service active (running) since 08:00"
}
],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "email-worker.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "email-worker.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts email-worker.service"
}
},
"label": "partial"
},
{
"id": "timeouts-network",
"diagnosis": "Calls to the payment provider are timing out because of a network problem on the host.",
"evidence": [
{
"source": "logs_query",
"text": "payments: context deadline exceeded calling provider (x88 in 10m)"
},
{
"source": "ping",
"text": "provider.example.test: 0% loss, avg 41ms"
}
],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "payments.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "payments.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts payments.service"
}
},
"label": "partial"
},
{
"id": "cert-soon",
"diagnosis": "The certificate on web3 is about to expire and will cause TLS errors.",
"evidence": [
{
"source": "tls_check",
"text": "web3: certificate notAfter=2026-11-30T00:00:00Z, now=2026-11-14T08:41:00Z: 15 days left"
}
],
"action": {
"connector": "tls",
"name": "replace_certificate",
"input": {
"host": "web3"
},
"target": {
"kind": "host",
"environment": "production",
"id": "web3"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Replaces the certificate on web3"
}
},
"label": "partial"
},
{
"id": "flaky-test-runner",
"diagnosis": "The CI runner ran out of disk because old build workspaces accumulate.",
"evidence": [
{
"source": "host_disk_usage",
"text": "/ 96% used; /var/lib/runner 44G"
}
],
"action": {
"connector": "host",
"name": "delete_old_files",
"input": {
"path": "/var/lib/runner/workspaces",
"older_than_days": 3
},
"target": {
"kind": "host",
"environment": "production",
"id": "runner1"
},
"effect": {
"read_only": false,
"reversibility": "irreversible",
"summary": "Deletes workspaces older than 3 days"
}
},
"label": "partial"
},
{
"id": "no-evidence",
"diagnosis": "The checkout service has a memory leak and needs a restart.",
"evidence": [],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "checkout.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "checkout.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts checkout.service"
}
},
"label": "unsupported"
},
{
"id": "wrong-service",
"diagnosis": "The checkout service is out of memory and needs restarting.",
"evidence": [
{
"source": "metrics_query",
"text": "inventory_memory_bytes max over 1h: 3.9e9 (limit 4.0e9)"
},
{
"source": "logs_query",
"text": "kernel: Out of memory: Killed process 3310 (inventory)"
}
],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "checkout.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "checkout.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts checkout.service"
}
},
"label": "unsupported"
},
{
"id": "contradicted-memory",
"diagnosis": "The orders service is out of memory.",
"evidence": [
{
"source": "metrics_query",
"text": "orders_memory_bytes max over 1h: 5.1e8 (limit 4.0e9)"
},
{
"source": "logs_query",
"text": "orders: healthy, 0 errors in the last hour"
}
],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "orders.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "orders.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts orders.service"
}
},
"label": "unsupported"
},
{
"id": "wrong-host",
"diagnosis": "The disk on web1 is full and old logs must be deleted.",
"evidence": [
{
"source": "host_disk_usage",
"text": "web2: / 98% used. /var/log/app 40G"
}
],
"action": {
"connector": "host",
"name": "delete_old_files",
"input": {
"path": "/var/log/app",
"older_than_days": 7
},
"target": {
"kind": "host",
"environment": "production",
"id": "web1"
},
"effect": {
"read_only": false,
"reversibility": "irreversible",
"summary": "Deletes old logs on web1"
}
},
"label": "unsupported"
},
{
"id": "overreach-cause",
"diagnosis": "A bad configuration change made at 09:00 broke the login service.",
"evidence": [
{
"source": "logs_query",
"text": "login: connection refused to auth-db:5432 (x300 since 09:21)"
}
],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "login.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "login.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts login.service"
}
},
"label": "unsupported"
},
{
"id": "unrelated-reads",
"diagnosis": "The API gateway is dropping requests because its rate limiter is misconfigured.",
"evidence": [
{
"source": "host_status",
"text": "api-gateway.service active (running)"
},
{
"source": "host_disk_usage",
"text": "/ 41% used"
},
{
"source": "logs_query",
"text": "api-gateway: 200 OK (x15000 in 10m), no errors"
}
],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "api-gateway.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "api-gateway.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts api-gateway.service"
}
},
"label": "unsupported"
},
{
"id": "empty-evidence-text",
"diagnosis": "The queue is jammed and the consumer needs restarting.",
"evidence": [
{
"source": "queue_status",
"text": "not found"
}
],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "consumer.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "consumer.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts consumer.service"
}
},
"label": "unsupported"
},
{
"id": "contradicts-lock",
"diagnosis": "Restarting the worker will release the database lock it holds.",
"evidence": [
{
"source": "db_activity",
"text": "No locks held. The blocked query is waiting on a lock held by session 77, which belongs to the billing application"
},
{
"source": "host_status",
"text": "worker.service active (running), last job completed 12s ago"
}
],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "worker.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "worker.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts worker.service"
}
},
"label": "unsupported"
},
{
"id": "guess-dns",
"diagnosis": "The DNS record is wrong and must be changed to the new address.",
"evidence": [
{
"source": "logs_query",
"text": "web: upstream timed out (x12 in 10m)"
}
],
"action": {
"connector": "dns",
"name": "update_record",
"input": {
"name": "app.example.test",
"type": "A",
"value": "203.0.113.50"
},
"target": {
"kind": "dns-zone",
"environment": "production",
"id": "zone1"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Points app.example.test at 203.0.113.50"
}
},
"label": "unsupported"
},
{
"id": "stale-reading",
"diagnosis": "The database primary is down and a failover is required.",
"evidence": [
{
"source": "db_status",
"text": "primary db1: up, accepting connections, replication lag 0s"
},
{
"source": "ping",
"text": "db1: 0% loss"
}
],
"action": {
"connector": "db",
"name": "promote_replica",
"input": {
"replica": "db2"
},
"target": {
"kind": "database",
"environment": "production",
"id": "db1"
},
"effect": {
"read_only": false,
"reversibility": "irreversible",
"summary": "Promotes replica db2 to primary"
}
},
"label": "unsupported"
},
{
"id": "deploy-blame",
"diagnosis": "The 14:02 deploy broke the search service, so it must be rolled back.",
"evidence": [
{
"source": "deploy_history",
"text": "14:02 billing 3.1.0 -> 3.1.1 (finished); search last deployed 3 days ago"
},
{
"source": "logs_query",
"text": "search: errors from index shard 4 timing out (x50 since 13:40)"
}
],
"action": {
"connector": "deploy",
"name": "rollback",
"input": {
"service": "search",
"to": "1.9.0"
},
"target": {
"kind": "service",
"environment": "production",
"id": "search"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Rolls search back to 1.9.0"
}
},
"label": "unsupported"
},
{
"id": "snapshot-before",
"diagnosis": "The upgrade of the wiki container needs a restorable snapshot first, and the last snapshot is 11 days old.",
"evidence": [
{
"source": "incus_list_snapshots",
"text": "wiki: pre-upgrade-2026-11-03 (11 days ago)"
},
{
"source": "change_ticket",
"text": "CHG-204: upgrade wiki 5.2 -> 5.3 scheduled today 16:00"
}
],
"action": {
"connector": "incus",
"name": "snapshot",
"input": {
"container": "wiki",
"name": "pre-upgrade"
},
"target": {
"kind": "container",
"environment": "production",
"id": "wiki"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Snapshots the wiki container"
}
},
"label": "supported"
},
{
"id": "zombie-procs",
"diagnosis": "The batch host has 900 zombie processes from a crashed parent, and restarting the parent service clears them.",
"evidence": [
{
"source": "host_processes",
"text": "900 defunct processes, parent pid 3011 (batchd)"
},
{
"source": "host_status",
"text": "batchd.service active (running) pid 3011"
}
],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "batchd.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "batchd.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts batchd.service"
}
},
"label": "supported"
},
{
"id": "clock-skew",
"diagnosis": "The scheduler fired jobs twice because the host clock is ahead of its peers.",
"evidence": [
{
"source": "host_time",
"text": "scheduler1 time 14:33:05, ntp offset +412s"
},
{
"source": "logs_query",
"text": "scheduler: job nightly-report started twice (14:20, 14:27)"
}
],
"action": {
"connector": "systemd",
"name": "restart_service",
"input": {
"unit": "chrony.service"
},
"target": {
"kind": "host",
"environment": "production",
"id": "chrony.service"
},
"effect": {
"read_only": false,
"reversibility": "recoverable",
"summary": "Restarts chrony.service"
}
},
"label": "partial"
},
{
"id": "high-load",
"diagnosis": "The analytics host is slow because one runaway process is using all CPU cores.",
"evidence": [
{
"source": "host_processes",
"text": "pid 9921 python3 reindex.py 780% CPU, load average 14.2 on 8 cores; next process 3% CPU"
}
],
"action": {
"connector": "host",
"name": "kill_process",
"input": {
"pid": 9921
},
"target": {
"kind": "host",
"environment": "production",
"id": "analytics1"
},
"effect": {
"read_only": false,
"reversibility": "irreversible",
"summary": "Kills pid 9921"
}
},
"label": "supported"
}
]
}