Deploy Gemma 3 on Kubernetes with GPU — Complete Guide (2026)

kubectl create secret generic huggingface-secret \
  --from-literal=token=hf_your_token_here \
  -n gpu-workloads

# gemma3-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: gemma3-vllm
  namespace: gpu-workloads
spec:
  replicas: 1
  selector:
    matchLabels:
      app: gemma3-vllm
  template:
    metadata:
      labels:
        app: gemma3-vllm
    spec:
      containers:
      - name: vllm
        image: vllm/vllm-openai:latest
        args:
        - "--model"
        - "google/gemma-3-12b-it"
        - "--dtype"
        - "bfloat16"
        - "--max-model-len"
        - "8192"
        - "--gpu-memory-utilization"
        - "0.90"
        - "--port"
        - "8000"
        env:
        - name: HUGGING_FACE_HUB_TOKEN
          valueFrom:
            secretKeyRef:
              name: huggingface-secret
              key: token
        ports:
        - containerPort: 8000
        resources:
          limits:
            nvidia.com/gpu: "1"
            memory: "24Gi"
            cpu: "4"
          requests:
            nvidia.com/gpu: "1"
            memory: "16Gi"
            cpu: "2"
        volumeMounts:
        - name: model-cache
          mountPath: /root/.cache/huggingface
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 120
          periodSeconds: 30
          timeoutSeconds: 10
      volumes:
      - name: model-cache
        emptyDir:
          sizeLimit: 30Gi
      tolerations:
      - key: nvidia.com/gpu
        operator: Exists
        effect: NoSchedule
---
apiVersion: v1
kind: Service
metadata:
  name: gemma3-vllm-svc
  namespace: gpu-workloads
spec:
  selector:
    app: gemma3-vllm
  ports:
  - port: 8000
    targetPort: 8000
  type: ClusterIP

kubectl apply -f gemma3-deployment.yaml
 
# Watch deployment
kubectl get pods -n gpu-workloads -w
 
# Check logs (model download takes 5–10 minutes on first start)
kubectl logs -n gpu-workloads -l app=gemma3-vllm -f

# Port-forward to test locally
kubectl port-forward -n gpu-workloads svc/gemma3-vllm-svc 8000:8000 &
 
# Test with curl
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "google/gemma-3-12b-it",
    "messages": [
      {"role": "user", "content": "Write a Kubernetes readiness probe for an HTTP service on port 8080"}
    ],
    "max_tokens": 500
  }'
 
# List available models
curl http://localhost:8000/v1/models

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: model-cache-pvc
  namespace: gpu-workloads
spec:
  accessModes:
  - ReadWriteOnce
  storageClassName: gp3
  resources:
    requests:
      storage: 40Gi

volumes:
- name: model-cache
  persistentVolumeClaim:
    claimName: model-cache-pvc

# Kubernetes YAML generator
from openai import OpenAI
 
client = OpenAI(
    base_url="http://gemma3-vllm-svc.gpu-workloads:8000/v1",
    api_key="not-needed"
)
 
def generate_k8s_manifest(description: str) -> str:
    response = client.chat.completions.create(
        model="google/gemma-3-12b-it",
        messages=[
            {
                "role": "system",
                "content": "You are a Kubernetes expert. Generate valid YAML manifests. Output only YAML, no explanations."
            },
            {
                "role": "user",
                "content": description
            }
        ],
        max_tokens=1000,
        temperature=0.1  # low temp for structured output
    )
    return response.choices[0].message.content
 
# Example
yaml_output = generate_k8s_manifest(
    "Deployment for a Node.js app on port 3000, 3 replicas, "
    "resource limits 500m CPU / 512Mi memory, readiness probe on /health"
)
print(yaml_output)

apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: gemma3-scaledobject
  namespace: gpu-workloads
spec:
  scaleTargetRef:
    name: gemma3-vllm
  minReplicaCount: 0  # scale to zero when idle
  maxReplicaCount: 3
  triggers:
  - type: prometheus
    metadata:
      serverAddress: http://prometheus:9090
      metricName: vllm_pending_requests
      threshold: "5"
      query: vllm:pending_requests:count