Verify AWS identity
aws · debug
aws sts get-caller-identity aws configure list
On-call mode
Error search, 23 runbooks, 45 debug snippets — no install guides.
Fix 502 behind Nginx + K8s Ingress
Upstream unreachable — trace from browser to pod through ingress and service.
🔒Terraform state lock stuck
plan/apply blocked by DynamoDB or remote backend lock — release safely.
⎈Helm release failed or pending
Upgrade stuck in pending-install or pending-upgrade — diagnose and recover.
🔄CI pipeline failed at deploy step
Build passed but deploy to K8s/registry failed — common fixes across CI platforms.
💥Pod CrashLoopBackOff
Container starts then exits — check logs and exit code.
📦ImagePullBackOff
Cluster cannot pull container image.
🔐Ingress TLS cert expired or missing
Browser SSL error or cert-manager not ready.
🖥️Node NotReady / disk pressure
Scheduler cannot place pods; node conditions unhealthy.
🔑CI pipeline auth failed (registry / cloud)
401 on docker push or AWS/kubectl denied in CI job.
📐Terraform drift / unexpected plan changes
Plan wants to recreate resources or state differs from reality.
🔒Vault is sealed
Secrets unavailable after restart until unseal.
⏳Pod running but not Ready
readinessProbe failing — service has no endpoints.
🔌Connection refused upstream
Nginx/ingress cannot reach backend service.
☁️AWS credentials expired or invalid
CLI or CI fails STS or API calls.
⎈Helm release stuck pending
Upgrade cannot complete; revision hung.
🔑Rotate secrets without downtime
DB creds, IRSA, TLS, API keys — dual-credential pattern and rolling restarts.
💾Database restore drill (quarterly)
Prove backups work — restore to staging before prod depends on it.
☸️Velero restore failed or incomplete
Backup exists but restore hangs, missing PVCs, or wrong namespace.
💥Pod OOMKilled — out of memory
Container killed by kernel OOM — fix limits, leaks, or node pressure.
🔄Argo CD sync failed or OutOfSync
GitOps app stuck — compare diff, fix RBAC, hooks, or resource conflicts.
🔍DNS not resolving in cluster or from pod
Service name fails, external DNS wrong, or CoreDNS unhealthy.
🔴Redis connection refused or timeout
App cannot reach Redis — network, auth, maxclients, or wrong host.
📈High CPU throttling or latency spike
Pods hit CPU limits, HPA lagging, or node CPU saturated.
aws · debug
aws sts get-caller-identity aws configure list
k8s · debug
kubectl run netshoot --rm -it --image=nicolaka/netshoot --restart=Never -n <ns> -- /bin/bash # inside: nslookup kubernetes.default, curl -v svc:port
k8s · debug
kubectl port-forward svc/<service> 8080:80 -n <namespace> curl -I http://localhost:8080
k8s · debug
helm history myrelease -n apps helm rollback myrelease <revision> -n apps helm status myrelease -n apps
terraform · debug
terraform force-unlock <LOCK_ID> # LOCK_ID from error message — confirm no other apply running first
nginx · debug
location / {
proxy_pass http://backend:8080;
proxy_connect_timeout 60s;
proxy_send_timeout 60s;
proxy_read_timeout 60s;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
}
sudo nginx -t && sudo systemctl reload nginxnginx · debug
sudo tail -f /var/log/nginx/error.log sudo nginx -t
docker · debug
docker system df docker system prune -a --volumes # careful: removes unused images
docker · debug
docker logs -f <container>
docker inspect <container> --format='{{.State.ExitCode}}'k8s · debug
kubectl logs <pod> -n <ns> --previous kubectl describe pod <pod> -n <ns> kubectl get events -n <ns> --sort-by='.lastTimestamp'
k8s · debug
kubectl describe pod <pod> -n <ns> | grep -A5 Events kubectl get secret -n <ns> # ensure imagePullSecrets if private registry
k8s · debug
kubectl top nodes kubectl top pods -A --sort-by=memory
debug
vault status vault operator unseal # repeat until Sealed: false
debug
sudo journalctl -u <service> -f --since "1 hour ago" sudo systemctl status <service>
debug
df -h du -sh /* 2>/dev/null | sort -hr | head -20 sudo journalctl --vacuum-time=7d
k8s · nginx · debug
kubectl get certificate -A kubectl describe certificate <name> -n <ns> echo | openssl s_client -connect app.example.com:443 2>/dev/null | openssl x509 -noout -dates
k8s · debug
kubectl describe ingress <name> -n <ns> kubectl get endpoints -n <ns> kubectl logs -n ingress-nginx -l app.kubernetes.io/name=ingress-nginx --tail=50
terraform · debug
terraform state list terraform state show aws_instance.web terraform refresh
ci · docker · debug
# Verify registry login in CI before push docker login registry.example.com -u $CI_USER -p $CI_TOKEN # K8s: create docker-registry secret kubectl create secret docker-registry regcred --docker-server=... --docker-username=... --docker-password=... -n <ns>
k8s · debug
kubectl exec -it <pod> -n <ns> -- /bin/sh # or -c <container> if multi-container
k8s · debug
kubectl get pods -A | grep Evicted kubectl delete pods -A --field-selector=status.phase=Failed
aws · debug
aws logs tail /aws/lambda/my-function --follow --since 1h aws logs describe-log-groups --log-group-name-prefix /ecs/
debug
echo "show stat" | socat stdio /var/run/haproxy/admin.sock curl -I http://backend:port
debug
redis-cli -h <host> -p 6379 ping redis-cli -h <host> INFO server
debug
psql -h <host> -U <user> -d <db> -c "SELECT 1" pg_isready -h <host> -p 5432
debug · nginx
curl -I https://app.example.com curl -v http://localhost:8080/health 2>&1 | head -30
k8s · debug
kubectl describe node <node> kubectl get events -A --field-selector involvedObject.kind=Node # check DiskPressure, MemoryPressure, PIDPressure
ci · debug
# .gitlab-ci.yml tags must match runner # Settings → CI/CD → Runners → note tags # tags: [docker, aws]
k8s · finops · debug
kubectl top pods -A --sort-by=cpu # vs kubectl get pods -A -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,CPU:.spec.containers[*].resources.requests.cpu
debug · nginx
dig app.example.com +short dig app.example.com ANY nslookup app.example.com # compare to ingress LB: kubectl get ingress -n apps
debug
nc -zv backend.example.com 443 curl -v telnet://backend:8080 ss -tlnp | grep 8080
debug
sudo systemctl status <service> sudo systemctl restart <service> sudo journalctl -u <service> -n 50 --no-pager sudo systemctl is-active <service>
debug · nginx · backup
echo | openssl s_client -connect app.example.com:443 -servername app.example.com 2>/dev/null | openssl x509 -noout -dates -subject # days left: echo | openssl s_client -connect app.example.com:443 2>/dev/null | openssl x509 -noout -checkend 604800
debug
vault kv put secret/apps/myapp db_password=new-secret api_key=new-key vault kv get secret/apps/myapp # External Secrets Operator syncs within refreshInterval
k8s · debug
for ctx in staging prod; do echo "=== $ctx ===" kubectl --context=$ctx get deploy -n apps -o custom-columns=NAME:.metadata.name,IMAGE:.spec.template.spec.containers[0].image done
k8s · ci · debug
argocd app list argocd app get <app> --refresh argocd app sync <app> argocd app diff <app>
k8s · ci · debug
argocd app get <app> --hard-refresh
kubectl patch application <app> -n argocd --type merge -p '{"operation":{"initiatedBy":{"username":"admin"},"sync":{"revision":"HEAD"}}}'k8s · ci · debug
flux get sources git -A flux get kustomizations -A flux reconcile source git <name> -n flux-system flux reconcile kustomization <name> -n flux-system --with-source
k8s · debug
# describe shows probe failures
kubectl describe pod <pod> -n <ns> | grep -A3 Liveness
# common fix — separate paths, longer initialDelaySeconds
readinessProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 15
periodSeconds: 10
livenessProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 30
failureThreshold: 3k8s · debug
kubectl get hpa -n <ns> kubectl describe hpa <name> -n <ns> kubectl get --raw "/apis/metrics.k8s.io/v1beta1/namespaces/<ns>/pods" | head # metrics-server must be running
k8s · debug
kubectl auth can-i create deployments -n apps kubectl auth can-i get secrets --as=system:serviceaccount:apps:default -n apps kubectl describe rolebinding -n apps
observe · debug
curl -s http://localhost:9200/_cluster/health?pretty curl -s http://localhost:9200/_cat/indices?v curl -s http://localhost:9200/_cat/shards?v | grep UNASSIGNED
debug · linux
ssh -J bastion@jump.example.com user@10.0.2.50 # ~/.ssh/config: # Host private # ProxyJump bastion@jump.example.com # HostName 10.0.2.50 # User deploy
linux · debug
sudo apt install fail2ban sudo tee /etc/fail2ban/jail.local <<'EOF' [sshd] enabled = true port = ssh maxretry = 5 bantime = 3600 EOF sudo systemctl enable --now fail2ban sudo fail2ban-client status sshd
linux · debug
chronyc tracking chronyc sources -v timedatectl status