Observability triage

# 5xx rate / total rate
sum(rate(http_requests_total{status=~"5.."}[5m]))
/
sum(rate(http_requests_total[5m]))

# by ingress
sum by (ingress) (rate(nginx_ingress_controller_requests{status=~"5.."}[5m]))

sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total[1h])) > 3

# top restarters
 topk(10, sum by (pod, namespace) (kube_pod_container_status_restarts_total))

histogram_quantile(0.99,
  sum by (le) (rate(http_request_duration_seconds_bucket[5m]))
)

# ingress latency
histogram_quantile(0.99, sum by (le) (rate(nginx_ingress_controller_request_duration_seconds_bucket[5m])))

100 - (
  avg by (instance) (node_filesystem_avail_bytes{mountpoint="/"}) /
  avg by (instance) (node_filesystem_size_bytes{mountpoint="/"}) * 100
)

(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14

# alert when < 7 days
(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400

{namespace="apps"} |= "error" or |= "ERROR" or |= "panic"
| json
| line_format "{{.pod}} {{.message}}"

# rate of error log lines
sum(rate({namespace="apps"} |= "error" [5m])) by (pod)

fields @timestamp, @message
| filter @message like /(?i)(error|5\d\d|exception)/
| sort @timestamp desc
| limit 100

# EKS container logs
fields @timestamp, kubernetes.pod_name, @message
| filter @message like /ERROR/
| stats count() by kubernetes.pod_name

sum by (pod, namespace) (rate(container_cpu_cfs_throttled_seconds_total[5m]))

# top throttled
 topk(10, sum by (pod) (rate(container_cpu_cfs_throttled_seconds_total[5m])))

sum by (pod, namespace) (container_memory_working_set_bytes)
/
sum by (pod, namespace) (kube_pod_container_resource_limits{resource="memory"}) * 100

# OOM risk > 90%

curl -s http://localhost:9200/_cluster/health?pretty
curl -s http://localhost:9200/_cat/indices?v
curl -s http://localhost:9200/_cat/shards?v | grep UNASSIGNED