kubectl debug — How to Troubleshoot Pods That Won't Start

kubectl exec -it my-pod -- /bin/sh
# error: unable to upgrade connection: container not found ("app")
# OR: OCI runtime exec failed: exec failed: unable to start container process: 
# exec: "/bin/sh": stat /bin/sh: no such file or directory

# Attach a debug container to a running pod
kubectl debug -it my-pod --image=busybox --target=app
 
# Now you're inside a busybox container sharing the same:
# - PID namespace (can see app processes)
# - Network namespace (can curl localhost)
# - Filesystem (via /proc/PID/root)

# Create a copy of the pod with debug container, keeps original running
kubectl debug my-pod -it --image=ubuntu --share-processes --copy-to=my-pod-debug
 
# Inside the debug container, inspect the crashed app's filesystem
ls /proc/1/root/app/
cat /proc/1/root/etc/config.yaml
 
# Check environment variables of the crashed process
cat /proc/1/environ | tr '\0' '\n'

# SSH-like access to a node without actually SSH-ing
kubectl debug node/my-node-name -it --image=ubuntu
 
# You get a root shell on the node
# Node filesystem is at /host
ls /host/var/log/
cat /host/etc/kubernetes/kubelet.conf

kubectl debug -it crashing-pod --image=busybox --target=app
# Inside debug container:
cat /proc/1/root/etc/app/config.yaml
# Look for missing required fields, wrong DB URL, etc.

kubectl debug -it my-pod --image=nicolaka/netshoot
# netshoot has curl, nslookup, netstat, tcpdump, etc.
curl http://other-service.namespace.svc.cluster.local:8080/health
nslookup postgres.database.svc.cluster.local

kubectl debug -it my-pod --image=busybox --target=app --share-processes
# Inside debug container:
ps aux  # See app processes
strace -p 1  # Trace syscalls of PID 1

# Basic debug session
kubectl debug -it <pod> --image=busybox
 
# Share process namespace (see app processes)
kubectl debug -it <pod> --image=ubuntu --share-processes --copy-to=<pod>-debug
 
# Debug a specific container in multi-container pod
kubectl debug -it <pod> --image=busybox --target=<container-name>
 
# Debug a node
kubectl debug node/<node-name> -it --image=ubuntu
 
# Useful debug images:
# busybox — minimal tools
# ubuntu — full apt install
# nicolaka/netshoot — all network tools
# curlimages/curl — just curl

# Ephemeral containers are removed when pod restarts
# Debug copy pods must be deleted manually
kubectl delete pod my-pod-debug
 
# Check no debug pods left
kubectl get pods | grep debug