914 lines
22 KiB
JSON
914 lines
22 KiB
JSON
{
|
|
"cases": [
|
|
{
|
|
"id": "oom-checkout",
|
|
"diagnosis": "The checkout service was killed for running out of memory, so it keeps restarting and dropping requests.",
|
|
"evidence": [
|
|
{
|
|
"source": "logs_query",
|
|
"text": "kernel: Out of memory: Killed process 4121 (checkout) total-vm:3.9GB at 14:02:11\nsystemd: checkout.service: Main process exited, code=killed, status=9/KILL\nsystemd: checkout.service: Scheduled restart job, restart counter is at 6."
|
|
},
|
|
{
|
|
"source": "metrics_query",
|
|
"text": "checkout_memory_bytes max over 1h: 3.8e9 (limit 4.0e9); checkout_restarts_total increased by 6"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "checkout.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "checkout.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts checkout.service"
|
|
}
|
|
},
|
|
"label": "supported"
|
|
},
|
|
{
|
|
"id": "disk-full-logs",
|
|
"diagnosis": "The disk on web1 is full because the application log has grown to 38 GB, and writes are failing.",
|
|
"evidence": [
|
|
{
|
|
"source": "host_disk_usage",
|
|
"text": "/ 100% used (49G of 49G). Largest: /var/log/app/app.log 38G"
|
|
},
|
|
{
|
|
"source": "logs_query",
|
|
"text": "app: write /var/log/app/app.log: no space left on device (x412 in the last 10 minutes)"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "host",
|
|
"name": "delete_old_files",
|
|
"input": {
|
|
"path": "/var/log/app",
|
|
"older_than_days": 7
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "web1"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "irreversible",
|
|
"summary": "Deletes files older than 7 days under /var/log/app"
|
|
}
|
|
},
|
|
"label": "supported"
|
|
},
|
|
{
|
|
"id": "cert-expired",
|
|
"diagnosis": "The API is failing TLS handshakes because its certificate expired at midnight.",
|
|
"evidence": [
|
|
{
|
|
"source": "tls_check",
|
|
"text": "api.example.test: certificate notAfter=2026-11-14T00:00:00Z, now=2026-11-14T08:41:00Z: EXPIRED"
|
|
},
|
|
{
|
|
"source": "logs_query",
|
|
"text": "edge: tls handshake error from client: certificate has expired (x3300 since 00:00)"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "tls",
|
|
"name": "replace_certificate",
|
|
"input": {
|
|
"host": "edge1"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "edge1"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Replaces the certificate on edge1"
|
|
}
|
|
},
|
|
"label": "supported"
|
|
},
|
|
{
|
|
"id": "stuck-worker",
|
|
"diagnosis": "The report worker is stuck on a locked job and has not processed anything for two hours, so restarting it will release the lock.",
|
|
"evidence": [
|
|
{
|
|
"source": "queue_status",
|
|
"text": "report-worker: 0 jobs completed in 120m, 1 job in progress since 09:12 (id 8812), 340 queued"
|
|
},
|
|
{
|
|
"source": "logs_query",
|
|
"text": "report-worker: job 8812 waiting for advisory lock held by pid 2231 (worker pid 2231, last heartbeat 09:14)"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "report-worker.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "report-worker.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts report-worker.service"
|
|
}
|
|
},
|
|
"label": "supported"
|
|
},
|
|
{
|
|
"id": "bad-deploy",
|
|
"diagnosis": "Error rates rose right after the 14:02 deploy of version 2.8.1, so rolling back to 2.8.0 should restore service.",
|
|
"evidence": [
|
|
{
|
|
"source": "metrics_query",
|
|
"text": "http_5xx_rate: 0.1% until 14:02, 4.3% from 14:03 to now"
|
|
},
|
|
{
|
|
"source": "deploy_history",
|
|
"text": "14:02 checkout 2.8.0 -> 2.8.1 (finished)"
|
|
},
|
|
{
|
|
"source": "logs_query",
|
|
"text": "checkout 2.8.1: NullPointerException in PricingClient.convert (x912 since 14:03)"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "deploy",
|
|
"name": "rollback",
|
|
"input": {
|
|
"service": "checkout",
|
|
"to": "2.8.0"
|
|
},
|
|
"target": {
|
|
"kind": "service",
|
|
"environment": "production",
|
|
"id": "checkout"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Rolls checkout back to 2.8.0"
|
|
}
|
|
},
|
|
"label": "supported"
|
|
},
|
|
{
|
|
"id": "dns-stale",
|
|
"diagnosis": "The staging hostname still points at the old address, which no longer answers, so updating the record fixes the outage.",
|
|
"evidence": [
|
|
{
|
|
"source": "dns_lookup",
|
|
"text": "stg.example.test A 198.51.100.7 (ttl 300)"
|
|
},
|
|
{
|
|
"source": "ping",
|
|
"text": "198.51.100.7: no route to host"
|
|
},
|
|
{
|
|
"source": "inventory",
|
|
"text": "stg-web-2 is the current staging web host, address 198.51.100.44"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "dns",
|
|
"name": "update_record",
|
|
"input": {
|
|
"name": "stg.example.test",
|
|
"type": "A",
|
|
"value": "198.51.100.44"
|
|
},
|
|
"target": {
|
|
"kind": "dns-zone",
|
|
"environment": "staging",
|
|
"id": "zone1"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Points stg.example.test at 198.51.100.44"
|
|
}
|
|
},
|
|
"label": "supported"
|
|
},
|
|
{
|
|
"id": "cache-fill",
|
|
"diagnosis": "The /var/cache/app directory holds 21 GB of expired entries and is what filled the disk.",
|
|
"evidence": [
|
|
{
|
|
"source": "host_disk_usage",
|
|
"text": "/ 97% used. /var/cache/app 21G, /var/log 1.2G, /srv/data 6G"
|
|
},
|
|
{
|
|
"source": "host_list_files",
|
|
"text": "/var/cache/app: 41,812 files, all with mtime older than 30 days"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "host",
|
|
"name": "delete_old_files",
|
|
"input": {
|
|
"path": "/var/cache/app",
|
|
"older_than_days": 30
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "web2"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "irreversible",
|
|
"summary": "Deletes files older than 30 days under /var/cache/app"
|
|
}
|
|
},
|
|
"label": "supported"
|
|
},
|
|
{
|
|
"id": "conn-pool",
|
|
"diagnosis": "The orders service has exhausted its database connection pool, causing timeouts.",
|
|
"evidence": [
|
|
{
|
|
"source": "logs_query",
|
|
"text": "orders: timeout waiting for connection from pool (size=20) x640 in 15m"
|
|
},
|
|
{
|
|
"source": "metrics_query",
|
|
"text": "orders_db_pool_in_use=20/20 for the last 20 minutes; db server connections 48/200"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "orders.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "orders.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts orders.service"
|
|
}
|
|
},
|
|
"label": "supported"
|
|
},
|
|
{
|
|
"id": "memory-high-leak",
|
|
"diagnosis": "The search service is leaking memory in its query cache and will crash again soon.",
|
|
"evidence": [
|
|
{
|
|
"source": "metrics_query",
|
|
"text": "search_memory_bytes rose from 1.1e9 to 3.2e9 over 6h; limit 4.0e9"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "search.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "search.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts search.service"
|
|
}
|
|
},
|
|
"label": "partial"
|
|
},
|
|
{
|
|
"id": "slow-db",
|
|
"diagnosis": "The reports database is slow because of a long-running query from the analytics job.",
|
|
"evidence": [
|
|
{
|
|
"source": "metrics_query",
|
|
"text": "reports_db query latency p95 rose from 40ms to 3.1s at 11:40"
|
|
},
|
|
{
|
|
"source": "db_activity",
|
|
"text": "1 active query running 52 minutes, from user analytics, 'SELECT ... FROM events JOIN ...'"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "db",
|
|
"name": "cancel_query",
|
|
"input": {
|
|
"pid": 4412
|
|
},
|
|
"target": {
|
|
"kind": "database",
|
|
"environment": "production",
|
|
"id": "reports-db"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Cancels the running query with pid 4412"
|
|
}
|
|
},
|
|
"label": "partial"
|
|
},
|
|
{
|
|
"id": "queue-backlog",
|
|
"diagnosis": "The email worker has stopped consuming because it lost its broker connection.",
|
|
"evidence": [
|
|
{
|
|
"source": "queue_status",
|
|
"text": "email: 12,400 messages queued, 0 consumers"
|
|
},
|
|
{
|
|
"source": "host_status",
|
|
"text": "email-worker.service active (running) since 08:00"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "email-worker.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "email-worker.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts email-worker.service"
|
|
}
|
|
},
|
|
"label": "partial"
|
|
},
|
|
{
|
|
"id": "timeouts-network",
|
|
"diagnosis": "Calls to the payment provider are timing out because of a network problem on the host.",
|
|
"evidence": [
|
|
{
|
|
"source": "logs_query",
|
|
"text": "payments: context deadline exceeded calling provider (x88 in 10m)"
|
|
},
|
|
{
|
|
"source": "ping",
|
|
"text": "provider.example.test: 0% loss, avg 41ms"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "payments.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "payments.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts payments.service"
|
|
}
|
|
},
|
|
"label": "partial"
|
|
},
|
|
{
|
|
"id": "cert-soon",
|
|
"diagnosis": "The certificate on web3 is about to expire and will cause TLS errors.",
|
|
"evidence": [
|
|
{
|
|
"source": "tls_check",
|
|
"text": "web3: certificate notAfter=2026-11-30T00:00:00Z, now=2026-11-14T08:41:00Z: 15 days left"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "tls",
|
|
"name": "replace_certificate",
|
|
"input": {
|
|
"host": "web3"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "web3"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Replaces the certificate on web3"
|
|
}
|
|
},
|
|
"label": "partial"
|
|
},
|
|
{
|
|
"id": "flaky-test-runner",
|
|
"diagnosis": "The CI runner ran out of disk because old build workspaces accumulate.",
|
|
"evidence": [
|
|
{
|
|
"source": "host_disk_usage",
|
|
"text": "/ 96% used; /var/lib/runner 44G"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "host",
|
|
"name": "delete_old_files",
|
|
"input": {
|
|
"path": "/var/lib/runner/workspaces",
|
|
"older_than_days": 3
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "runner1"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "irreversible",
|
|
"summary": "Deletes workspaces older than 3 days"
|
|
}
|
|
},
|
|
"label": "partial"
|
|
},
|
|
{
|
|
"id": "no-evidence",
|
|
"diagnosis": "The checkout service has a memory leak and needs a restart.",
|
|
"evidence": [],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "checkout.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "checkout.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts checkout.service"
|
|
}
|
|
},
|
|
"label": "unsupported"
|
|
},
|
|
{
|
|
"id": "wrong-service",
|
|
"diagnosis": "The checkout service is out of memory and needs restarting.",
|
|
"evidence": [
|
|
{
|
|
"source": "metrics_query",
|
|
"text": "inventory_memory_bytes max over 1h: 3.9e9 (limit 4.0e9)"
|
|
},
|
|
{
|
|
"source": "logs_query",
|
|
"text": "kernel: Out of memory: Killed process 3310 (inventory)"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "checkout.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "checkout.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts checkout.service"
|
|
}
|
|
},
|
|
"label": "unsupported"
|
|
},
|
|
{
|
|
"id": "contradicted-memory",
|
|
"diagnosis": "The orders service is out of memory.",
|
|
"evidence": [
|
|
{
|
|
"source": "metrics_query",
|
|
"text": "orders_memory_bytes max over 1h: 5.1e8 (limit 4.0e9)"
|
|
},
|
|
{
|
|
"source": "logs_query",
|
|
"text": "orders: healthy, 0 errors in the last hour"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "orders.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "orders.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts orders.service"
|
|
}
|
|
},
|
|
"label": "unsupported"
|
|
},
|
|
{
|
|
"id": "wrong-host",
|
|
"diagnosis": "The disk on web1 is full and old logs must be deleted.",
|
|
"evidence": [
|
|
{
|
|
"source": "host_disk_usage",
|
|
"text": "web2: / 98% used. /var/log/app 40G"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "host",
|
|
"name": "delete_old_files",
|
|
"input": {
|
|
"path": "/var/log/app",
|
|
"older_than_days": 7
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "web1"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "irreversible",
|
|
"summary": "Deletes old logs on web1"
|
|
}
|
|
},
|
|
"label": "unsupported"
|
|
},
|
|
{
|
|
"id": "overreach-cause",
|
|
"diagnosis": "A bad configuration change made at 09:00 broke the login service.",
|
|
"evidence": [
|
|
{
|
|
"source": "logs_query",
|
|
"text": "login: connection refused to auth-db:5432 (x300 since 09:21)"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "login.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "login.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts login.service"
|
|
}
|
|
},
|
|
"label": "unsupported"
|
|
},
|
|
{
|
|
"id": "unrelated-reads",
|
|
"diagnosis": "The API gateway is dropping requests because its rate limiter is misconfigured.",
|
|
"evidence": [
|
|
{
|
|
"source": "host_status",
|
|
"text": "api-gateway.service active (running)"
|
|
},
|
|
{
|
|
"source": "host_disk_usage",
|
|
"text": "/ 41% used"
|
|
},
|
|
{
|
|
"source": "logs_query",
|
|
"text": "api-gateway: 200 OK (x15000 in 10m), no errors"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "api-gateway.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "api-gateway.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts api-gateway.service"
|
|
}
|
|
},
|
|
"label": "unsupported"
|
|
},
|
|
{
|
|
"id": "empty-evidence-text",
|
|
"diagnosis": "The queue is jammed and the consumer needs restarting.",
|
|
"evidence": [
|
|
{
|
|
"source": "queue_status",
|
|
"text": "not found"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "consumer.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "consumer.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts consumer.service"
|
|
}
|
|
},
|
|
"label": "unsupported"
|
|
},
|
|
{
|
|
"id": "contradicts-lock",
|
|
"diagnosis": "Restarting the worker will release the database lock it holds.",
|
|
"evidence": [
|
|
{
|
|
"source": "db_activity",
|
|
"text": "No locks held. The blocked query is waiting on a lock held by session 77, which belongs to the billing application"
|
|
},
|
|
{
|
|
"source": "host_status",
|
|
"text": "worker.service active (running), last job completed 12s ago"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "worker.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "worker.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts worker.service"
|
|
}
|
|
},
|
|
"label": "unsupported"
|
|
},
|
|
{
|
|
"id": "guess-dns",
|
|
"diagnosis": "The DNS record is wrong and must be changed to the new address.",
|
|
"evidence": [
|
|
{
|
|
"source": "logs_query",
|
|
"text": "web: upstream timed out (x12 in 10m)"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "dns",
|
|
"name": "update_record",
|
|
"input": {
|
|
"name": "app.example.test",
|
|
"type": "A",
|
|
"value": "203.0.113.50"
|
|
},
|
|
"target": {
|
|
"kind": "dns-zone",
|
|
"environment": "production",
|
|
"id": "zone1"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Points app.example.test at 203.0.113.50"
|
|
}
|
|
},
|
|
"label": "unsupported"
|
|
},
|
|
{
|
|
"id": "stale-reading",
|
|
"diagnosis": "The database primary is down and a failover is required.",
|
|
"evidence": [
|
|
{
|
|
"source": "db_status",
|
|
"text": "primary db1: up, accepting connections, replication lag 0s"
|
|
},
|
|
{
|
|
"source": "ping",
|
|
"text": "db1: 0% loss"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "db",
|
|
"name": "promote_replica",
|
|
"input": {
|
|
"replica": "db2"
|
|
},
|
|
"target": {
|
|
"kind": "database",
|
|
"environment": "production",
|
|
"id": "db1"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "irreversible",
|
|
"summary": "Promotes replica db2 to primary"
|
|
}
|
|
},
|
|
"label": "unsupported"
|
|
},
|
|
{
|
|
"id": "deploy-blame",
|
|
"diagnosis": "The 14:02 deploy broke the search service, so it must be rolled back.",
|
|
"evidence": [
|
|
{
|
|
"source": "deploy_history",
|
|
"text": "14:02 billing 3.1.0 -> 3.1.1 (finished); search last deployed 3 days ago"
|
|
},
|
|
{
|
|
"source": "logs_query",
|
|
"text": "search: errors from index shard 4 timing out (x50 since 13:40)"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "deploy",
|
|
"name": "rollback",
|
|
"input": {
|
|
"service": "search",
|
|
"to": "1.9.0"
|
|
},
|
|
"target": {
|
|
"kind": "service",
|
|
"environment": "production",
|
|
"id": "search"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Rolls search back to 1.9.0"
|
|
}
|
|
},
|
|
"label": "unsupported"
|
|
},
|
|
{
|
|
"id": "snapshot-before",
|
|
"diagnosis": "The upgrade of the wiki container needs a restorable snapshot first, and the last snapshot is 11 days old.",
|
|
"evidence": [
|
|
{
|
|
"source": "incus_list_snapshots",
|
|
"text": "wiki: pre-upgrade-2026-11-03 (11 days ago)"
|
|
},
|
|
{
|
|
"source": "change_ticket",
|
|
"text": "CHG-204: upgrade wiki 5.2 -> 5.3 scheduled today 16:00"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "incus",
|
|
"name": "snapshot",
|
|
"input": {
|
|
"container": "wiki",
|
|
"name": "pre-upgrade"
|
|
},
|
|
"target": {
|
|
"kind": "container",
|
|
"environment": "production",
|
|
"id": "wiki"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Snapshots the wiki container"
|
|
}
|
|
},
|
|
"label": "supported"
|
|
},
|
|
{
|
|
"id": "zombie-procs",
|
|
"diagnosis": "The batch host has 900 zombie processes from a crashed parent, and restarting the parent service clears them.",
|
|
"evidence": [
|
|
{
|
|
"source": "host_processes",
|
|
"text": "900 defunct processes, parent pid 3011 (batchd)"
|
|
},
|
|
{
|
|
"source": "host_status",
|
|
"text": "batchd.service active (running) pid 3011"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "batchd.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "batchd.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts batchd.service"
|
|
}
|
|
},
|
|
"label": "supported"
|
|
},
|
|
{
|
|
"id": "clock-skew",
|
|
"diagnosis": "The scheduler fired jobs twice because the host clock is ahead of its peers.",
|
|
"evidence": [
|
|
{
|
|
"source": "host_time",
|
|
"text": "scheduler1 time 14:33:05, ntp offset +412s"
|
|
},
|
|
{
|
|
"source": "logs_query",
|
|
"text": "scheduler: job nightly-report started twice (14:20, 14:27)"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "systemd",
|
|
"name": "restart_service",
|
|
"input": {
|
|
"unit": "chrony.service"
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "chrony.service"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "recoverable",
|
|
"summary": "Restarts chrony.service"
|
|
}
|
|
},
|
|
"label": "partial"
|
|
},
|
|
{
|
|
"id": "high-load",
|
|
"diagnosis": "The analytics host is slow because one runaway process is using all CPU cores.",
|
|
"evidence": [
|
|
{
|
|
"source": "host_processes",
|
|
"text": "pid 9921 python3 reindex.py 780% CPU, load average 14.2 on 8 cores; next process 3% CPU"
|
|
}
|
|
],
|
|
"action": {
|
|
"connector": "host",
|
|
"name": "kill_process",
|
|
"input": {
|
|
"pid": 9921
|
|
},
|
|
"target": {
|
|
"kind": "host",
|
|
"environment": "production",
|
|
"id": "analytics1"
|
|
},
|
|
"effect": {
|
|
"read_only": false,
|
|
"reversibility": "irreversible",
|
|
"summary": "Kills pid 9921"
|
|
}
|
|
},
|
|
"label": "supported"
|
|
}
|
|
]
|
|
} |