Deploy Qwen2.5-Coder on Kubernetes for Private Code AI (2026)

# karpenter-gpu-nodepool.yaml
apiVersion: karpenter.sh/v1
kind: NodePool
metadata:
  name: gpu-qwen
spec:
  template:
    spec:
      requirements:
      - key: karpenter.k8s.aws/instance-family
        operator: In
        values: ["p4d", "p3", "g5"]   # A100 or A10G instances
      - key: karpenter.sh/capacity-type
        operator: In
        values: ["on-demand"]
      nodeClassRef:
        group: karpenter.k8s.aws
        kind: EC2NodeClass
        name: gpu-nodes
      taints:
      - key: nvidia.com/gpu
        effect: NoSchedule
  limits:
    nvidia.com/gpu: 8
---
apiVersion: karpenter.k8s.aws/v1
kind: EC2NodeClass
metadata:
  name: gpu-nodes
spec:
  amiFamily: AL2
  instanceProfile: KarpenterNodeInstanceProfile
  subnetSelectorTerms:
  - tags:
      karpenter.sh/discovery: my-cluster
  securityGroupSelectorTerms:
  - tags:
      karpenter.sh/discovery: my-cluster

kubectl create namespace llm

# model-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: qwen-model-storage
  namespace: llm
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 120Gi   # ~65GB for 32B model weights + buffer
  storageClassName: gp3  # AWS EBS gp3

kubectl apply -f model-pvc.yaml

# model-downloader.yaml
apiVersion: batch/v1
kind: Job
metadata:
  name: download-qwen-model
  namespace: llm
spec:
  template:
    spec:
      restartPolicy: OnFailure
      containers:
      - name: downloader
        image: python:3.11-slim
        command:
        - /bin/sh
        - -c
        - |
          pip install huggingface_hub -q
          python3 -c "
          from huggingface_hub import snapshot_download
          snapshot_download(
              repo_id='Qwen/Qwen2.5-Coder-32B-Instruct',
              local_dir='/models/qwen2.5-coder-32b',
              ignore_patterns=['*.msgpack', '*.h5', 'original/*']
          )
          print('Download complete')
          "
        env:
        - name: HF_TOKEN
          valueFrom:
            secretKeyRef:
              name: hf-credentials
              key: token
        volumeMounts:
        - name: model-storage
          mountPath: /models
        resources:
          requests:
            memory: "8Gi"
            cpu: "2"
          limits:
            memory: "16Gi"
            cpu: "4"
      volumes:
      - name: model-storage
        persistentVolumeClaim:
          claimName: qwen-model-storage

# Create HuggingFace token secret (needed for gated models)
kubectl create secret generic hf-credentials \
  --from-literal=token=hf_your_token_here \
  -n llm
 
kubectl apply -f model-downloader.yaml
 
# Watch download progress (takes 20-40 minutes for 32B)
kubectl logs -f job/download-qwen-model -n llm

# vllm-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: qwen-vllm
  namespace: llm
spec:
  replicas: 1
  selector:
    matchLabels:
      app: qwen-vllm
  template:
    metadata:
      labels:
        app: qwen-vllm
    spec:
      tolerations:
      - key: nvidia.com/gpu
        operator: Exists
        effect: NoSchedule
      containers:
      - name: vllm
        image: vllm/vllm-openai:latest
        args:
        - "--model=/models/qwen2.5-coder-32b"
        - "--served-model-name=qwen2.5-coder"
        - "--tensor-parallel-size=2"      # Use 2 GPUs for 32B model
        - "--max-model-len=65536"          # 64K context window
        - "--max-num-seqs=64"              # Max concurrent requests
        - "--gpu-memory-utilization=0.92"
        - "--dtype=bfloat16"
        - "--port=8000"
        env:
        - name: NCCL_DEBUG
          value: "WARN"
        ports:
        - containerPort: 8000
          name: http
        resources:
          limits:
            nvidia.com/gpu: "2"
            memory: "80Gi"
            cpu: "16"
          requests:
            nvidia.com/gpu: "2"
            memory: "60Gi"
            cpu: "8"
        volumeMounts:
        - name: model-storage
          mountPath: /models
          readOnly: true
        - name: shm
          mountPath: /dev/shm
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 120
          periodSeconds: 15
          failureThreshold: 20
        livenessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 180
          periodSeconds: 30
      volumes:
      - name: model-storage
        persistentVolumeClaim:
          claimName: qwen-model-storage
      - name: shm
        emptyDir:
          medium: Memory
          sizeLimit: 16Gi
---
apiVersion: v1
kind: Service
metadata:
  name: qwen-vllm
  namespace: llm
spec:
  selector:
    app: qwen-vllm
  ports:
  - port: 8000
    targetPort: 8000
    name: http
  type: ClusterIP

kubectl apply -f vllm-deployment.yaml
 
# Watch startup (model loading takes 5-10 minutes)
kubectl logs -f deployment/qwen-vllm -n llm
 
# Wait for ready
kubectl wait --for=condition=ready pod -l app=qwen-vllm -n llm --timeout=600s

# Port forward for testing
kubectl port-forward svc/qwen-vllm 8000:8000 -n llm &
 
# Test completion
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "qwen2.5-coder",
    "messages": [
      {
        "role": "user",
        "content": "Write a Python function that checks if a binary tree is balanced. Include type hints and docstring."
      }
    ],
    "temperature": 0.1,
    "max_tokens": 1024
  }'

# ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: qwen-ingress
  namespace: llm
  annotations:
    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
    nginx.ingress.kubernetes.io/proxy-body-size: "10m"
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - llm.internal.mycompany.com
    secretName: llm-tls
  rules:
  - host: llm.internal.mycompany.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: qwen-vllm
            port:
              number: 8000

// ~/.continue/config.json
{
  "models": [
    {
      "title": "Qwen2.5-Coder (Private)",
      "provider": "openai",
      "model": "qwen2.5-coder",
      "apiBase": "http://llm.internal.mycompany.com/v1",
      "apiKey": "not-needed"
    }
  ],
  "tabAutocompleteModel": {
    "title": "Qwen2.5-Coder Autocomplete",
    "provider": "openai",
    "model": "qwen2.5-coder",
    "apiBase": "http://llm.internal.mycompany.com/v1",
    "apiKey": "not-needed"
  },
  "contextProviders": [
    { "name": "codebase" },
    { "name": "files" }
  ]
}

# vLLM exposes Prometheus metrics
kubectl port-forward svc/qwen-vllm 8000:8000 -n llm
curl http://localhost:8000/metrics
 
# Key metrics to watch:
# vllm:gpu_cache_usage_perc - GPU KV cache utilization
# vllm:num_requests_running - Active requests
# vllm:time_per_output_token_seconds - Token generation speed
# vllm:e2e_request_latency_seconds - End-to-end latency

- job_name: vllm
  static_configs:
  - targets: ['qwen-vllm.llm.svc.cluster.local:8000']