{"overall": "degraded", "checks": {"jobs": {"status": "healthy", "message": "All systems operational", "last_check": "2026-10-15T23:33:08.137388+00:00", "last_change": "2026-10-15T16:02:36.003516+00:00", "details": {"prod": {"status": "healthy", "message": "prod: operational (31 completed in last 5min)", "details": {"abandon_rate": "0.0%", "recent_created": 31, "recent_completed": 31, "recent_abandoned": 0, "last_completed_at": null}}, "canary": {"status": "healthy", "message": "canary: operational (31 completed in last 5min)", "details": {"abandon_rate": "0.0%", "recent_created": 31, "recent_completed": 31, "recent_abandoned": 0, "last_completed_at": null}}}, "consecutive_failures": 0, "required_confirmations": 2}, "pods": {"status": "degraded", "message": "canary/sites-service-7876df949f-fz7v8 pending", "last_check": "2026-10-15T23:33:08.496564+00:00", "last_change": "2026-10-14T21:13:50.341134+00:00", "details": {"prod": {"pods": [{"name": "jobs-api-cbdb9b568-v8r72", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "merciful-frontend-6c894f7d49-qbgcm", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "merciful-pages-6f6cf95c89-qsqtj", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "openai-text-runner-77f56459c6-5qfxw", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "openai-text-runner-77f56459c6-fz9jw", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "openai-text-runner-77f56459c6-n98rx", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "openai-text-runner-77f56459c6-r7259", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "openai-text-runner-77f56459c6-tjgb9", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "openai-text-runner-77f56459c6-xmzx5", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "text-runner-847fb7b6b4-nlx6x", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "text-runner-847fb7b6b4-pkqfm", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "text-runner-847fb7b6b4-r62j6", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}], "total": 12, "healthy": 12}, "canary": {"pods": [{"name": "jobs-api-564b4cdb8f-rdsc7", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "merciful-frontend-5bbdc4745f-hprt2", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "merciful-pages-6ffcbf78ff-hghcc", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "moderation-service-5f889c6bcb-wtxkf", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "openai-text-runner-d786f97bd-7xz8x", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "orders-up-659d89f94d-9vptq", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}, {"name": "sites-service-7876df949f-fz7v8", "phase": "Pending", "ready": false, "restarts": 0, "image_error": null}, {"name": "text-runner-86448d86b8-xjdkd", "phase": "Running", "ready": true, "restarts": 0, "image_error": null}], "total": 8, "healthy": 7}}, "consecutive_failures": 1435, "required_confirmations": 2}, "disk": {"status": "degraded", "message": "Disk warning: /: 83% full (24.6GB free); /shared: 83% full (24.6GB free); /var/lib: 83% full (24.6GB free)", "last_check": "2026-10-15T23:33:08.496674+00:00", "last_change": "2026-10-14T21:09:02.008968+00:00", "details": {"/": {"used_pct": 83.1, "free_gb": 24.63, "total_gb": 154.88}, "/shared": {"used_pct": 83.1, "free_gb": 24.63, "total_gb": 154.88}, "/var/lib": {"used_pct": 83.1, "free_gb": 24.63, "total_gb": 154.88}}, "consecutive_failures": 1439, "required_confirmations": 2}, "runners": {"status": "healthy", "message": "11/11 healthy", "last_check": "2026-10-15T23:33:12.130317+00:00", "last_change": "2026-10-15T22:41:18.109824+00:00", "details": {"prod": {"runners": [{"runner_id": "text-runner-prod-01", "runner_type": "ollama", "backend": "ollama", "environment": "prod", "started_at": "2026-10-13T23:20:58.121407+00:00", "draining": false, "shutdown_reason": null, "backend_health": {"ollama": {"status": "healthy", "consecutive_failures": 0, "in_deep_sleep": false, "is_probing": true}}, "supported_tiers": ["anime", "advanced", "standard", "economy"], "tier_model_map": {"anime": "glm-5.3-flash:cloud", "advanced": "glm-5.3:cloud", "standard": "glm-5.3-flash:cloud", "economy": "glm-5.3-flash:cloud"}, "current_job": null, "last_completion_at": "2026-10-15T23:32:38.752659+00:00", "last_completion_duration": 4.02, "avg_completion_time_10min": 4.24, "completions_last_10min": 14, "total_completed": 2493, "total_failed": 0, "status_port": 9091, "status": "healthy", "pod": "text-runner-847fb7b6b4-nlx6x", "namespace": "prod", "host_network": false}, {"runner_id": "text-runner-prod-01", "runner_type": "ollama", "backend": "ollama", "environment": "prod", "started_at": "2026-10-13T23:21:21.989073+00:00", "draining": false, "shutdown_reason": null, "backend_health": {"ollama": {"status": "healthy", "consecutive_failures": 0, "in_deep_sleep": false, "is_probing": true}}, "supported_tiers": ["anime", "advanced", "standard", "economy"], "tier_model_map": {"anime": "glm-5.3-flash:cloud", "advanced": "glm-5.3:cloud", "standard": "glm-5.3-flash:cloud", "economy": "glm-5.3-flash:cloud"}, "current_job": null, "last_completion_at": "2026-10-15T23:32:40.894803+00:00", "last_completion_duration": 18.73, "avg_completion_time_10min": 4.82, "completions_last_10min": 21, "total_completed": 2565, "total_failed": 0, "status_port": 9091, "status": "healthy", "pod": "text-runner-847fb7b6b4-pkqfm", "namespace": "prod", "host_network": false}, {"runner_id": "text-runner-prod-01", "runner_type": "ollama", "backend": "ollama", "environment": "prod", "started_at": "2026-10-13T23:21:09.797734+00:00", "draining": false, "shutdown_reason": null, "backend_health": {"ollama": {"status": "healthy", "consecutive_failures": 0, "in_deep_sleep": false, "is_probing": true}}, "supported_tiers": ["anime", "advanced", "standard", "economy"], "tier_model_map": {"anime": "glm-5.3-flash:cloud", "advanced": "glm-5.3:cloud", "standard": "glm-5.3-flash:cloud", "economy": "glm-5.3-flash:cloud"}, "current_job": null, "last_completion_at": "2026-10-15T23:32:51.367027+00:00", "last_completion_duration": 4.59, "avg_completion_time_10min": 5.57, "completions_last_10min": 15, "total_completed": 2536, "total_failed": 0, "status_port": 9091, "status": "healthy", "pod": "text-runner-847fb7b6b4-r62j6", "namespace": "prod", "host_network": false}, {"runner_id": "openai-text-runner-prod-01", "runner_type": "openai", "backend": "openai", "environment": "prod", "started_at": "2026-10-13T23:21:59.034571+00:00", "draining": false, "shutdown_reason": null, "backend_health": {"openai": {"status": "healthy", "consecutive_failures": 0, "in_deep_sleep": false, "is_probing": true}}, "supported_tiers": ["anime", "advanced", "standard", "economy"], "tier_model_map": {"anime": "glm-5.3-flash", "advanced": "glm-5.3", "standard": "glm-5.3-flash", "economy": "glm-5.3-flash"}, "current_job": null, "last_completion_at": "2026-10-15T23:32:44.231458+00:00", "last_completion_duration": 6.81, "avg_completion_time_10min": 3.57, "completions_last_10min": 6, "total_completed": 528, "total_failed": 0, "status_port": 9091, "status": "healthy", "pod": "openai-text-runner-77f56459c6-5qfxw", "namespace": "prod", "host_network": false}, {"runner_id": "openai-text-runner-prod-01", "runner_type": "openai", "backend": "openai", "environment": "prod", "started_at": "2026-10-14T05:00:49.362384+00:00", "draining": false, "shutdown_reason": null, "backend_health": {"openai": {"status": "healthy", "consecutive_failures": 0, "in_deep_sleep": false, "is_probing": true}}, "supported_tiers": ["anime", "advanced", "standard", "economy"], "tier_model_map": {"anime": "glm-5.3-flash", "advanced": "glm-5.3", "standard": "glm-5.3-flash", "economy": "glm-5.3-flash"}, "current_job": null, "last_completion_at": "2026-10-15T23:31:54.150382+00:00", "last_completion_duration": 3.42, "avg_completion_time_10min": 2.62, "completions_last_10min": 3, "total_completed": 420, "total_failed": 0, "status_port": 9091, "status": "healthy", "pod": "openai-text-runner-77f56459c6-fz9jw", "namespace": "prod", "host_network": false}, {"runner_id": "openai-text-runner-prod-01", "runner_type": "openai", "backend": "openai", "environment": "prod", "started_at": "2026-10-13T23:21:46.334250+00:00", "draining": false, "shutdown_reason": null, "backend_health": {"openai": {"status": "healthy", "consecutive_failures": 0, "in_deep_sleep": false, "is_probing": true}}, "supported_tiers": ["anime", "advanced", "standard", "economy"], "tier_model_map": {"anime": "glm-5.3-flash", "advanced": "glm-5.3", "standard": "glm-5.3-flash", "economy": "glm-5.3-flash"}, "current_job": null, "last_completion_at": "2026-10-15T23:23:17.702810+00:00", "last_completion_duration": 1.66, "avg_completion_time_10min": 1.66, "completions_last_10min": 1, "total_completed": 505, "total_failed": 0, "status_port": 9091, "status": "healthy", "pod": "openai-text-runner-77f56459c6-n98rx", "namespace": "prod", "host_network": false}, {"runner_id": "openai-text-runner-prod-01", "runner_type": "openai", "backend": "openai", "environment": "prod", "started_at": "2026-10-14T05:00:49.362138+00:00", "draining": false, "shutdown_reason": null, "backend_health": {"openai": {"status": "healthy", "consecutive_failures": 0, "in_deep_sleep": false, "is_probing": true}}, "supported_tiers": ["anime", "advanced", "standard", "economy"], "tier_model_map": {"anime": "glm-5.3-flash", "advanced": "glm-5.3", "standard": "glm-5.3-flash", "economy": "glm-5.3-flash"}, "current_job": null, "last_completion_at": "2026-10-15T23:30:42.623286+00:00", "last_completion_duration": 1.98, "avg_completion_time_10min": 5.91, "completions_last_10min": 4, "total_completed": 493, "total_failed": 0, "status_port": 9091, "status": "healthy", "pod": "openai-text-runner-77f56459c6-r7259", "namespace": "prod", "host_network": false}, {"runner_id": "openai-text-runner-prod-01", "runner_type": "openai", "backend": "openai", "environment": "prod", "started_at": "2026-10-14T05:00:49.362385+00:00", "draining": false, "shutdown_reason": null, "backend_health": {"openai": {"status": "healthy", "consecutive_failures": 0, "in_deep_sleep": false, "is_probing": true}}, "supported_tiers": ["anime", "advanced", "standard", "economy"], "tier_model_map": {"anime": "glm-5.3-flash", "advanced": "glm-5.3", "standard": "glm-5.3-flash", "economy": "glm-5.3-flash"}, "current_job": null, "last_completion_at": "2026-10-15T23:32:32.298273+00:00", "last_completion_duration": 2.36, "avg_completion_time_10min": 2.86, "completions_last_10min": 5, "total_completed": 413, "total_failed": 0, "status_port": 9091, "status": "healthy", "pod": "openai-text-runner-77f56459c6-tjgb9", "namespace": "prod", "host_network": false}, {"runner_id": "openai-text-runner-prod-01", "runner_type": "openai", "backend": "openai", "environment": "prod", "started_at": "2026-10-13T23:21:34.136986+00:00", "draining": false, "shutdown_reason": null, "backend_health": {"openai": {"status": "healthy", "consecutive_failures": 0, "in_deep_sleep": false, "is_probing": true}}, "supported_tiers": ["anime", "advanced", "standard", "economy"], "tier_model_map": {"anime": "glm-5.3-flash", "advanced": "glm-5.3", "standard": "glm-5.3-flash", "economy": "glm-5.3-flash"}, "current_job": null, "last_completion_at": "2026-10-15T23:26:56.778145+00:00", "last_completion_duration": 2.36, "avg_completion_time_10min": 3.77, "completions_last_10min": 2, "total_completed": 500, "total_failed": 0, "status_port": 9091, "status": "healthy", "pod": "openai-text-runner-77f56459c6-xmzx5", "namespace": "prod", "host_network": false}], "total": 9, "reachable": 9}, "canary": {"runners": [{"runner_id": "text-runner-canary-01", "runner_type": "ollama", "backend": "ollama", "environment": "canary", "started_at": "2026-10-14T21:09:03.730173+00:00", "draining": false, "shutdown_reason": null, "backend_health": {"ollama": {"status": "healthy", "consecutive_failures": 0, "in_deep_sleep": false, "is_probing": true}}, "supported_tiers": ["anime", "advanced", "standard", "economy"], "tier_model_map": {"anime": "glm-5.3-flash:cloud", "advanced": "glm-5.3-flash:cloud", "standard": "glm-5.3-flash:cloud", "economy": "glm-5.3-flash:cloud"}, "current_job": null, "last_completion_at": "2026-10-15T20:19:24.309527+00:00", "last_completion_duration": 26.39, "avg_completion_time_10min": null, "completions_last_10min": 0, "total_completed": 31, "total_failed": 0, "status_port": 9091, "status": "healthy", "pod": "text-runner-86448d86b8-xjdkd", "namespace": "canary", "host_network": false}, {"runner_id": "openai-text-runner-canary-01", "runner_type": "openai", "backend": "openai", "environment": "canary", "started_at": "2026-10-14T21:09:16.306912+00:00", "draining": false, "shutdown_reason": null, "backend_health": {"openai": {"status": "healthy", "consecutive_failures": 0, "in_deep_sleep": false, "is_probing": true}}, "supported_tiers": ["anime", "advanced", "standard", "economy"], "tier_model_map": {"anime": "glm-5.3-flash", "advanced": "glm-5.3-flash", "standard": "glm-5.3-flash", "economy": "glm-5.3-flash"}, "current_job": null, "last_completion_at": "2026-10-15T20:25:14.559032+00:00", "last_completion_duration": 11.38, "avg_completion_time_10min": null, "completions_last_10min": 0, "total_completed": 19, "total_failed": 0, "status_port": 9091, "status": "healthy", "pod": "openai-text-runner-d786f97bd-7xz8x", "namespace": "canary", "host_network": false}], "total": 2, "reachable": 2}}, "consecutive_failures": 0, "required_confirmations": 2}, "email": {"status": "healthy", "message": "Email service operational", "last_check": "2026-10-15T23:33:12.652099+00:00", "last_change": "2026-10-15T13:31:25.255808+00:00", "details": {"prod": {"status": "healthy", "message": "prod: Brevo connected, last email: 2026-10-15 23:07:47.84996", "details": {"brevo": "ok", "service": "brevo", "last_email_at": "2026-10-15 23:07:47.84996"}}, "canary": {"status": "healthy", "message": "canary: Brevo connected, last email: 2026-10-15 23:07:47.84996", "details": {"brevo": "ok", "service": "brevo", "last_email_at": "2026-10-15 23:07:47.84996"}}}, "consecutive_failures": 0, "required_confirmations": 1}, "billing": {"status": "healthy", "message": "Billing service operational", "last_check": "2026-10-15T23:33:12.989946+00:00", "last_change": "2026-10-02T17:04:39.971726+00:00", "details": {"prod": {"status": "healthy", "message": "prod: Stripe connected (LIVE mode)", "details": {"stripe": "ok", "livemode": true, "service": "stripe"}}, "canary": {"status": "healthy", "message": "canary: Stripe connected (TEST mode)", "details": {"stripe": "ok", "livemode": false, "service": "stripe"}}}, "consecutive_failures": 0, "required_confirmations": 1}}, "incidents": [{"id": "email-20261010180005", "check": "email", "status": "down", "message": "prod: prod: orders-up unreachable: Get \"http://orders-up.canary.svc.cluster.local:3032/health/email\": context deadline exceeded (Client.Timeout exceeded while awaiting headers); canary: canary: unknown", "started_at": "2026-10-10T18:00:05.623235+00:00", "resolved_at": "2026-10-10T18:05:36.337151+00:00", "resolution": "Email service operational"}, {"id": "jobs-20261011000237", "check": "jobs", "status": "down", "message": "prod: prod: jobs-api timeout", "started_at": "2026-10-11T00:02:37.483353+00:00", "resolved_at": "2026-10-11T00:03:54.370516+00:00", "resolution": "All systems operational"}, {"id": "disk-20261011000241", "check": "disk", "status": "down", "message": "Disk critical: /: 93% full (8.8GB free); /shared: 93% full (8.8GB free); /var/lib: 93% full (8.8GB free)", "started_at": "2026-10-11T00:02:41.440635+00:00", "resolved_at": "2026-10-11T00:03:54.798992+00:00", "resolution": "Disk OK (worst: 74.8%)"}, {"id": "email-20261012181859", "check": "email", "status": "down", "message": "prod: prod: unknown; canary: canary: orders-up unreachable: Get \"http://orders-up.canary.svc.cluster.local:3032/health/email\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)", "started_at": "2026-10-12T18:18:59.878699+00:00", "resolved_at": "2026-10-12T18:24:29.409163+00:00", "resolution": "Email service operational"}, {"id": "email-20261013120026", "check": "email", "status": "down", "message": "prod: prod: orders-up unreachable: Get \"http://orders-up.canary.svc.cluster.local:3032/health/email\": context deadline exceeded (Client.Timeout exceeded while awaiting headers); canary: canary: orders-up unreachable: Get \"http://orders-up.canary.svc.cluster.local:3032/health/email\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)", "started_at": "2026-10-13T12:00:26.547292+00:00", "resolved_at": "2026-10-13T12:05:55.385022+00:00", "resolution": "Email service operational"}, {"id": "email-20261013150014", "check": "email", "status": "down", "message": "prod: prod: orders-up unreachable: Get \"http://orders-up.canary.svc.cluster.local:3032/health/email\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)", "started_at": "2026-10-13T15:00:14.997360+00:00", "resolved_at": "2026-10-13T15:05:41.267723+00:00", "resolution": "Email service operational"}, {"id": "email-20261013160015", "check": "email", "status": "down", "message": "prod: prod: orders-up unreachable: Get \"http://orders-up.canary.svc.cluster.local:3032/health/email\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)", "started_at": "2026-10-13T16:00:15.308283+00:00", "resolved_at": "2026-10-13T16:05:42.874370+00:00", "resolution": "Email service operational"}, {"id": "email-20261013170018", "check": "email", "status": "down", "message": "prod: prod: orders-up unreachable: Get \"http://orders-up.canary.svc.cluster.local:3032/health/email\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)", "started_at": "2026-10-13T17:00:18.101742+00:00", "resolved_at": "2026-10-13T17:05:47.669169+00:00", "resolution": "Email service operational"}, {"id": "email-20261013180030", "check": "email", "status": "down", "message": "prod: prod: orders-up unreachable: Get \"http://orders-up.canary.svc.cluster.local:3032/health/email\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)", "started_at": "2026-10-13T18:00:30.608914+00:00", "resolved_at": "2026-10-13T18:05:57.777459+00:00", "resolution": "Email service operational"}, {"id": "disk-20261013202009", "check": "disk", "status": "degraded", "message": "Disk warning: /: 80% full (29.4GB free); /shared: 80% full (29.4GB free); /var/lib: 80% full (29.4GB free)", "started_at": "2026-10-13T20:20:09.040072+00:00", "resolved_at": "2026-10-14T21:05:32.602062+00:00", "resolution": "Disk OK (worst: 80.0%)"}, {"id": "email-20261014090018", "check": "email", "status": "down", "message": "prod: prod: orders-up unreachable: Get \"http://orders-up.canary.svc.cluster.local:3032/health/email\": context deadline exceeded (Client.Timeout exceeded while awaiting headers); canary: canary: orders-up unreachable: Get \"http://orders-up.canary.svc.cluster.local:3032/health/email\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)", "started_at": "2026-10-14T09:00:18.162028+00:00", "resolved_at": "2026-10-14T09:05:47.202980+00:00", "resolution": "Email service operational"}, {"id": "runners-20261014120734", "check": "runners", "status": "degraded", "message": "no completions for 7 min", "started_at": "2026-10-14T12:07:34.333717+00:00", "resolved_at": "2026-10-14T12:08:41.115381+00:00", "resolution": "11/11 healthy"}, {"id": "email-20261014210755", "check": "email", "status": "down", "message": "prod: prod: orders-up unreachable: Get \"http://orders-up.canary.svc.cluster.local:3032/health/email\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)", "started_at": "2026-10-14T21:07:55.588166+00:00", "resolved_at": "2026-10-14T21:13:56.358935+00:00", "resolution": "Email service operational"}, {"id": "disk-20261014211014", "check": "disk", "status": "degraded", "message": "Disk warning: /: 80% full (28.6GB free); /shared: 80% full (28.6GB free); /var/lib: 80% full (28.6GB free)", "started_at": "2026-10-14T21:10:14.744286+00:00", "resolved_at": null}, {"id": "pods-20261014211502", "check": "pods", "status": "degraded", "message": "canary/sites-service-7876df949f-fz7v8 pending", "started_at": "2026-10-14T21:15:02.635934+00:00", "resolved_at": null}, {"id": "jobs-20261014212100", "check": "jobs", "status": "down", "message": "prod: prod: jobs-api timeout", "started_at": "2026-10-14T21:21:00.056031+00:00", "resolved_at": "2026-10-14T21:22:15.555694+00:00", "resolution": "All systems operational"}, {"id": "email-20261014213014", "check": "email", "status": "down", "message": "prod: prod: orders-up unreachable: Get \"http://orders-up.canary.svc.cluster.local:3032/health/email\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)", "started_at": "2026-10-14T21:30:14.371133+00:00", "resolved_at": "2026-10-14T21:35:43.798315+00:00", "resolution": "Email service operational"}, {"id": "email-20261015030026", "check": "email", "status": "down", "message": "prod: prod: orders-up unreachable: Get \"http://orders-up.canary.svc.cluster.local:3032/health/email\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)", "started_at": "2026-10-15T03:00:26.416056+00:00", "resolved_at": "2026-10-15T03:05:56.173018+00:00", "resolution": "Email service operational"}, {"id": "email-20261015132552", "check": "email", "status": "down", "message": "prod: prod: unknown; canary: canary: unknown", "started_at": "2026-10-15T13:25:52.735060+00:00", "resolved_at": "2026-10-15T13:31:25.255816+00:00", "resolution": "Email service operational"}, {"id": "jobs-20261015160129", "check": "jobs", "status": "degraded", "message": "prod: prod: 22% abandon rate (2/9)", "started_at": "2026-10-15T16:01:29.562172+00:00", "resolved_at": "2026-10-15T16:02:36.003524+00:00", "resolution": "All systems operational"}], "started_at": "2026-09-20T22:55:20.889337+00:00"}