Kubernetes Node NotReady: Fix in 5 Minutes

kubectl get nodes
# NAME       STATUS     ROLES    AGE   VERSION
# worker-3   NotReady   <none>   45d   v1.29.2
 
kubectl describe node worker-3 | grep -A 10 "Conditions:"
# Type             Status  Reason
# ----             ------  ------
# MemoryPressure   False   KubeletHasSufficientMemory
# DiskPressure     True    KubeletHasDiskPressure   ← here's your cause
# PIDPressure      False   KubeletHasSufficientPID
# Ready            False   KubeletNotReady

# SSH to the node (or use a debug pod if SSH isn't available)
ssh worker-3
sudo systemctl status kubelet
 
# If it's crashed, check why
sudo journalctl -u kubelet -n 100 --no-pager

# Certificate expiry is the most common silent killer
sudo journalctl -u kubelet | grep -i "certificate\|x509"
# "x509: certificate has expired or is not yet valid"
 
# Check cert expiry directly
sudo kubeadm certs check-expiration

sudo kubeadm certs renew all
sudo systemctl restart kubelet

df -h /var/lib/kubelet /var/lib/containerd
# If /var is above ~85%, kubelet marks DiskPressure=True and stops scheduling

# Check what's actually eating disk
sudo du -sh /var/lib/containerd/io.containerd.content.v1.content/blobs/* 2>/dev/null | sort -rh | head -10
 
# Prune unused images and containers
sudo crictl rmi --prune
# or for docker runtime:
docker system prune -af --volumes

# kubelet config
evictionHard:
  nodefs.available: "10%"
  imagefs.available: "15%"
imageGCHighThresholdPercent: 80
imageGCLowThresholdPercent: 70

sudo systemctl status containerd
sudo journalctl -u containerd -n 50 --no-pager
 
# kubelet logs will show this pattern when the runtime is unreachable:
# "Failed to get status for pod: rpc error: code = Unavailable
#  desc = connection error: desc = "transport: Error while dialing
#  dial unix /run/containerd/containerd.sock: connect: no such file or directory"

sudo systemctl restart containerd
sudo systemctl restart kubelet    # kubelet needs to re-establish the CRI connection

sudo systemctl stop containerd
sudo mv /var/lib/containerd /var/lib/containerd.bak
sudo systemctl start containerd    # Recreates a clean state — existing containers are lost on this node

# kubelet logs show this when CNI isn't ready:
# "container runtime network not ready: NetworkReady=false
#  reason:NetworkPluginNotReady message:Network plugin returns error:
#  cni plugin not initialized"
 
# Check the CNI daemonset (varies by plugin — this example is Cilium)
kubectl get pods -n kube-system -l k8s-app=cilium -o wide | grep worker-3
kubectl logs -n kube-system cilium-xxxxx --previous

kubectl delete pod -n kube-system cilium-xxxxx
# The daemonset controller recreates it automatically

ls /etc/cni/net.d/
# Should contain a config like 05-cilium.conflist — if empty, the CNI
# daemonset never successfully wrote its config on this node
 
# Usually fixed by deleting and letting the daemonset pod reschedule

# Run these in order — stop at whichever one shows the problem
kubectl describe node NODE_NAME | grep -A 10 "Conditions:"
ssh NODE_NAME "sudo systemctl status kubelet containerd"
ssh NODE_NAME "df -h /var/lib/kubelet"
ssh NODE_NAME "sudo journalctl -u kubelet -n 50 --no-pager"
kubectl get pods -n kube-system -o wide | grep NODE_NAME

# Stop new pods scheduling here while you investigate further,
# and safely move existing workloads off
kubectl cordon worker-3
kubectl drain worker-3 --ignore-daemonsets --delete-emptydir-data
 
# Once fixed and Ready again:
kubectl uncordon worker-3