Kubernetes Pod Evicted Due to Disk Pressure — Fix

# Check node disk pressure condition
kubectl get nodes
# Look for: DiskPressure=True
 
kubectl describe node <node-name> | grep -A5 "Conditions:"
# Look for: DiskPressure True
 
# See which pods got evicted
kubectl get pods --all-namespaces | grep Evicted
 
# Check disk usage on the node
kubectl debug node/<node-name> -it --image=ubuntu -- df -h
# Or SSH into the node if you have access

# Check log sizes on node
kubectl debug node/<node-name> -it --image=ubuntu -- \
  du -sh /host/var/log/containers/* | sort -rh | head -20
 
# Check if a specific pod is logging excessively
kubectl logs <pod-name> --tail=10 | wc -c

# For containerd, edit /etc/containerd/config.toml on each node:
[plugins."io.containerd.grpc.v1.cri".containerd]
  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
    # Log rotation is handled by docker/containerd log driver
 
# For dockerd (older setups), /etc/docker/daemon.json:
{
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m",
    "max-file": "3"
  }
}

# Doesn't directly limit log size but limits resources:
resources:
  limits:
    ephemeral-storage: 2Gi   # Limits total ephemeral storage for pod

# Check image disk usage
kubectl debug node/<node-name> -it --image=ubuntu -- \
  crictl images | head -30
 
# Auto-garbage collection is enabled by default in kubelet
# Check if it's working:
kubectl describe node <node-name> | grep -i "image"

# /var/lib/kubelet/config.yaml on each node
imageGCHighThresholdPercent: 75   # Start GC at 75% (default 85%)
imageGCLowThresholdPercent: 60    # GC down to 60% (default 80%)

# SSH to node
crictl rmi --prune   # Remove unused images

# Check emptyDir sizes
kubectl debug node/<node-name> -it --image=ubuntu -- \
  du -sh /host/var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/* 2>/dev/null | sort -rh | head -20

volumes:
- name: temp-data
  emptyDir:
    sizeLimit: 500Mi   # Pod evicted if it exceeds this

# Check root filesystem
df -h /
# If / is > 80% full, it's OS-level disk usage
 
# Common culprits:
du -sh /var/log/* | sort -rh | head -10
du -sh /tmp/*
journalctl --disk-usage

# Clean journal logs
journalctl --vacuum-size=500M
 
# Clean apt/yum cache (on nodes you can access)
apt clean   # Ubuntu/Debian
yum clean all   # RHEL/CentOS

# In every pod spec — prevent runaway disk usage
resources:
  requests:
    ephemeral-storage: "500Mi"
  limits:
    ephemeral-storage: "2Gi"

# Prometheus alert for node disk pressure
- alert: NodeDiskPressure
  expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
  for: 5m
  annotations:
    summary: "Node {{ $labels.node }} has disk pressure"
 
# Alert before it gets critical
- alert: NodeFilesystemSpaceFilling
  expr: predict_linear(node_filesystem_free_bytes[6h], 24*3600) < 0
  annotations:
    summary: "Node filesystem will fill up in 24h"