k8s-debug

27 commands

27 shown

Ephemeral debug container Attach a debug container (netshoot) to a running pod — shares PID/net namespaces kubectl debug -it <pod> --image=nicolaka/netshoot -n <ns> #debug#network#pods Node resource usage CPU and memory per node (requires metrics-server) kubectl top nodes #nodes#monitoring Pod resource usage sorted Memory-sorted pod usage across all namespaces — find the hungry ones kubectl top pods -A --sort-by=memory #pods#monitoring Check what's in a ConfigMap Print just the data field; pipe to python -m json.tool for formatting kubectl get cm <name> -n <ns> -o jsonpath='{.data}' #config#debug Decode a Secret Dump and base64-decode all fields in a k8s Secret kubectl get secret <name> -n <ns> -o jsonpath='{.data}' | python3 -c "import sys,json,base64; [print(k,'=',base64.b64decode(v).decode()) for k,v in json.load(sys.stdin).items()]" #secrets#debug List all resources in namespace Enumerate every resource type in a namespace — useful for auditing kubectl api-resources --verbs=list --namespaced -o name | xargs -I{} kubectl get {} -n <ns> --ignore-not-found #debug JSONPath on any resource Extract specific fields from k8s API objects without jq kubectl get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}\t{.metadata.name}\t{.status.phase}\n{end}' #debug#pods Debug pod with ephemeral container Attach an ephemeral debug container to a running pod kubectl debug -it mypod --image=busybox --target=app #k8s-debug#kubectl Run debug pod on node Get privileged shell on a cluster node via debug pod kubectl debug node/my-node -it --image=ubuntu #k8s-debug#kubectl Exec into sidecar container Open shell in a specific container of a multi-container pod kubectl exec -it mypod -c sidecar-name -- /bin/sh #k8s-debug#kubectl Check resource requests/limits Show resource requests and limits for each container kubectl get pod mypod -o jsonpath='{range .spec.containers[*]}{.name}{"\t"}{.resources}{"\n"}{end}' #k8s-debug#kubectl Describe node conditions Check node conditions: Ready, MemoryPressure, DiskPressure kubectl describe node my-node | grep -A5 'Conditions:' #k8s-debug#kubectl Find pods by resource usage Top 20 pods sorted by memory consumption kubectl top pods -A --sort-by=memory | head -20 #k8s-debug#kubectl#monitoring Watch pod events Stream events for a specific pod kubectl get events -n mynamespace --field-selector involvedObject.name=mypod --watch #k8s-debug#kubectl Copy file from pod Download a file from a running container kubectl cp mynamespace/mypod:/var/log/app.log ./app.log #k8s-debug#kubectl Top pods by restart count Rank pods cluster-wide by restart count to spot crash loops fast kubectl get po -A --sort-by='.status.containerStatuses[0].restartCount' | tail -20 #k8s-debug#troubleshooting#k8s#observability Detect OOMKilled containers List pods whose last container exit was OOMKilled across all namespaces kubectl get po -A -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.status.containerStatuses[*].lastState.terminated.reason}{"\n"}{end}' | grep OOMKilled #k8s-debug#troubleshooting#performance#k8s Pending pods scheduling reason Extract FailedScheduling reasons for every Pending pod (taints, resources, affinity) kubectl get po -A --field-selector=status.phase=Pending -o name | xargs -I{} kubectl describe {} -n <ns> | grep -A3 FailedScheduling #k8s-debug#troubleshooting#k8s#performance Node NotReady triage Read node Conditions then tail kubelet journal to find why node is NotReady kubectl describe node <node> | grep -A12 Conditions; ssh <node> 'journalctl -u kubelet -n 100 --no-pager' #k8s-debug#troubleshooting#k8s#kernel ImagePullBackOff root cause Surface registry pull failures: bad tag, missing imagePullSecret, or rate limit kubectl get events -n <ns> --field-selector reason=Failed -o wide | grep -i 'pull\|manifest\|unauthorized' #k8s-debug#troubleshooting#k8s#security Cleanup evicted pods Bulk-delete all Evicted pods left behind by node-pressure eviction kubectl get po -A | grep Evicted | awk '{print $2" -n "$1}' | xargs -L1 kubectl delete po #k8s-debug#troubleshooting#k8s#disk Force-remove stuck Terminating pod Clear finalizers to unstick a pod hung in Terminating; note: confirm node is gone first kubectl patch po <pod> -n <ns> -p '{"metadata":{"finalizers":null}}' --type=merge; kubectl delete po <pod> -n <ns> --grace-period=0 --force #k8s-debug#troubleshooting#k8s Raw kubelet stats summary Pull live per-pod memory from kubelet Summary API, bypassing metrics-server kubectl get --raw /api/v1/nodes/<node>/proxy/stats/summary | jq '.pods[] | {name:.podRef.name, mem:.memory.workingSetBytes}' #k8s-debug#monitoring#performance#k8s Conntrack stats on node Check per-CPU conntrack drops/insert_failed; nf_conntrack table exhaustion drops packets kubectl debug node/<node> -it --image=nicolaka/netshoot -- conntrack -S #k8s-debug#network#troubleshooting#kernel In-cluster DNS resolution test Resolve a Service FQDN from an ephemeral pod to isolate CoreDNS vs app DNS issues kubectl run dnstest --rm -it --image=nicolaka/netshoot --restart=Never -- dig +short <name>.<ns>.svc.cluster.local #k8s-debug#dns#network#troubleshooting Ordered events for a pod Stream only Warning events scoped to one pod in chronological order; modern: replaces get events filter kubectl events --for pod/<pod> -n <ns> --types=Warning #k8s-debug#troubleshooting#observability#k8s Previous crash logs with timestamps Tail the crashed container's prior instance logs to catch the pre-restart stack trace kubectl logs <pod> -n <ns> -c <name> --previous --timestamps --tail=200 #k8s-debug#troubleshooting#debug#k8s
no commands match