Run vLLM on Kubernetes for Fast LLM Inference (2026)

# Create EKS cluster with GPU node group
eksctl create cluster --name llm-cluster --region us-east-1
 
# Add GPU node group (g4dn.xlarge = 1x T4 GPU, 16 GiB VRAM)
eksctl create nodegroup \
  --cluster llm-cluster \
  --name gpu-nodes \
  --node-type g4dn.xlarge \
  --nodes 1 \
  --nodes-min 1 \
  --nodes-max 3 \
  --node-ami-family AmazonLinux2
 
# Install NVIDIA device plugin
kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.14.0/nvidia-device-plugin.yml
 
# Verify GPU is visible
kubectl get nodes -l accelerator=vgpu
kubectl describe node <gpu-node> | grep nvidia.com/gpu

# pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: model-cache
  namespace: vllm
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: gp3
  resources:
    requests:
      storage: 50Gi

kubectl create namespace vllm
kubectl apply -f pvc.yaml

# vllm-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-mistral
  namespace: vllm
spec:
  replicas: 1
  selector:
    matchLabels:
      app: vllm-mistral
  template:
    metadata:
      labels:
        app: vllm-mistral
    spec:
      containers:
      - name: vllm
        image: vllm/vllm-openai:latest
        command: ["python", "-m", "vllm.entrypoints.openai.api_server"]
        args:
          - "--model"
          - "mistralai/Mistral-7B-Instruct-v0.2"
          - "--host"
          - "0.0.0.0"
          - "--port"
          - "8000"
          - "--max-model-len"
          - "4096"
          - "--tensor-parallel-size"
          - "1"
        env:
        - name: HUGGING_FACE_HUB_TOKEN
          valueFrom:
            secretKeyRef:
              name: hf-token
              key: token
        - name: TRANSFORMERS_CACHE
          value: /model-cache
        ports:
        - containerPort: 8000
        resources:
          limits:
            nvidia.com/gpu: "1"
            memory: "24Gi"
            cpu: "4"
          requests:
            nvidia.com/gpu: "1"
            memory: "16Gi"
            cpu: "2"
        volumeMounts:
        - name: model-cache
          mountPath: /model-cache
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 60
          periodSeconds: 10
      tolerations:
      - key: nvidia.com/gpu
        operator: Exists
        effect: NoSchedule
      volumes:
      - name: model-cache
        persistentVolumeClaim:
          claimName: model-cache

# Create HuggingFace token secret
kubectl create secret generic hf-token \
  --from-literal=token=hf_your_token_here \
  -n vllm
 
kubectl apply -f vllm-deployment.yaml

# service.yaml
apiVersion: v1
kind: Service
metadata:
  name: vllm-service
  namespace: vllm
spec:
  selector:
    app: vllm-mistral
  ports:
  - port: 8000
    targetPort: 8000
  type: ClusterIP   # Use LoadBalancer for external access

# Port-forward for testing
kubectl port-forward svc/vllm-service 8000:8000 -n vllm
 
# List available models
curl http://localhost:8000/v1/models
 
# Chat completion (same format as OpenAI)
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "mistralai/Mistral-7B-Instruct-v0.2",
    "messages": [
      {"role": "user", "content": "Write a Kubernetes HPA YAML for a web app"}
    ],
    "max_tokens": 512,
    "temperature": 0.7
  }'

args:
  - "--model"
  - "microsoft/phi-2"       # 2.7B, runs on CPU (slowly)
  - "--device"
  - "cpu"
  - "--max-model-len"
  - "2048"
# Remove GPU resource limits

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: vllm-hpa
  namespace: vllm
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: vllm-mistral
  minReplicas: 1
  maxReplicas: 4
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70

args:
  - "--model"
  - "meta-llama/Meta-Llama-3-70B-Instruct"
  - "--tensor-parallel-size"
  - "4"   # Split across 4 GPUs
resources:
  limits:
    nvidia.com/gpu: "4"