Deploy Code Llama on Kubernetes for Self-Hosted Code Generation (2026)

# 1. Kubernetes cluster with GPU nodes
kubectl get nodes -l nvidia.com/gpu.present=true
 
# 2. NVIDIA GPU Operator installed
kubectl get pods -n gpu-operator
 
# 3. Verify GPU is available
kubectl get node <gpu-node> -o json | jq '.status.capacity | to_entries | .[] | select(.key | startswith("nvidia"))'
# Should show: "nvidia.com/gpu": "1"

kubectl create namespace ai-platform
 
kubectl create secret generic huggingface-token \
  --from-literal=token=hf_YOUR_TOKEN_HERE \
  --namespace ai-platform

# model-storage.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: code-llama-model-cache
  namespace: ai-platform
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 60Gi  # 26GB model + buffer
  storageClassName: gp3  # AWS; adjust for your cloud

kubectl apply -f model-storage.yaml

# code-llama-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: code-llama-server
  namespace: ai-platform
  labels:
    app: code-llama
spec:
  replicas: 1
  selector:
    matchLabels:
      app: code-llama
  template:
    metadata:
      labels:
        app: code-llama
    spec:
      tolerations:
        - key: "nvidia.com/gpu"
          operator: "Exists"
          effect: "NoSchedule"
      containers:
        - name: vllm
          image: vllm/vllm-openai:latest
          command:
            - python3
            - -m
            - vllm.entrypoints.openai.api_server
            - --model
            - codellama/CodeLlama-13b-Instruct-hf
            - --port
            - "8000"
            - --gpu-memory-utilization
            - "0.90"
            - --max-model-len
            - "16384"         # 16K context window
            - --dtype
            - "float16"       # FP16 for better memory efficiency
            - --served-model-name
            - "code-llama-13b"
          ports:
            - containerPort: 8000
              name: http
          env:
            - name: HUGGING_FACE_HUB_TOKEN
              valueFrom:
                secretKeyRef:
                  name: huggingface-token
                  key: token
            - name: TRANSFORMERS_CACHE
              value: "/model-cache"
          resources:
            limits:
              nvidia.com/gpu: 1
              memory: 32Gi
            requests:
              memory: 24Gi
              cpu: "4"
          volumeMounts:
            - name: model-cache
              mountPath: /model-cache
            - name: dshm
              mountPath: /dev/shm
          readinessProbe:
            httpGet:
              path: /health
              port: 8000
            initialDelaySeconds: 120  # Model loading takes time
            periodSeconds: 10
            failureThreshold: 30
          livenessProbe:
            httpGet:
              path: /health
              port: 8000
            initialDelaySeconds: 180
            periodSeconds: 30
            failureThreshold: 5
      volumes:
        - name: model-cache
          persistentVolumeClaim:
            claimName: code-llama-model-cache
        - name: dshm
          emptyDir:
            medium: Memory
            sizeLimit: 8Gi
---
apiVersion: v1
kind: Service
metadata:
  name: code-llama-service
  namespace: ai-platform
spec:
  selector:
    app: code-llama
  ports:
    - port: 8000
      targetPort: 8000
      name: http
  type: ClusterIP

kubectl apply -f code-llama-deployment.yaml
 
# Watch the pod start (first run downloads the model — takes 5-10 minutes)
kubectl logs -f deployment/code-llama-server -n ai-platform

# Port-forward to test locally
kubectl port-forward svc/code-llama-service 8000:8000 -n ai-platform &
 
# List available models
curl http://localhost:8000/v1/models
 
# Test code generation
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "code-llama-13b",
    "messages": [
      {
        "role": "system",
        "content": "You are an expert DevOps engineer. Generate clean, production-ready code."
      },
      {
        "role": "user", 
        "content": "Write a Kubernetes Deployment YAML for a Python Flask app with health checks, resource limits, and 3 replicas."
      }
    ],
    "max_tokens": 1024,
    "temperature": 0.1
  }'

# code-llama-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: code-llama-ingress
  namespace: ai-platform
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: "50m"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
spec:
  ingressClassName: nginx
  rules:
    - host: code-llama.internal.yourdomain.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: code-llama-service
                port:
                  number: 8000

# code_review.py
import openai
import subprocess
import sys
 
client = openai.OpenAI(
    api_key="not-needed",
    base_url="http://code-llama.internal.yourdomain.com/v1"
)
 
def review_diff(diff: str) -> str:
    response = client.chat.completions.create(
        model="code-llama-13b",
        messages=[
            {
                "role": "system",
                "content": """You are an expert code reviewer. Focus on:
                - Security vulnerabilities
                - Performance issues
                - Kubernetes/infrastructure best practices
                - Missing error handling
                Be concise. Format as bullet points."""
            },
            {
                "role": "user",
                "content": f"Review this git diff:\n\n```diff\n{diff}\n```"
            }
        ],
        temperature=0.1,
        max_tokens=1024
    )
    return response.choices[0].message.content
 
# Get diff from git
diff = subprocess.check_output(
    ["git", "diff", "origin/main...HEAD"], 
    text=True
)
 
if diff:
    review = review_diff(diff[:8000])  # Limit context
    print("## AI Code Review\n")
    print(review)

# In your GitHub Actions workflow
- name: AI Code Review
  run: |
    pip install openai
    python code_review.py >> $GITHUB_STEP_SUMMARY

# terraform_generator.py
def generate_terraform(resource_description: str) -> str:
    response = client.chat.completions.create(
        model="code-llama-13b",
        messages=[
            {
                "role": "system",
                "content": "Generate production-ready Terraform HCL code. Include variables, outputs, and follow best practices. Use latest AWS provider syntax."
            },
            {
                "role": "user",
                "content": resource_description
            }
        ],
        temperature=0.1,
        max_tokens=2048
    )
    return response.choices[0].message.content
 
# Example
result = generate_terraform(
    "Create an EKS cluster with managed node groups, VPC with public/private subnets, IAM roles, and OIDC provider for service accounts"
)
print(result)

def generate_k8s_manifest(description: str) -> str:
    response = client.chat.completions.create(
        model="code-llama-13b",
        messages=[
            {
                "role": "system",
                "content": "Generate Kubernetes YAML manifests. Follow best practices: resource limits, health checks, security contexts, proper labels."
            },
            {
                "role": "user",
                "content": description
            }
        ],
        temperature=0.05,
        max_tokens=2048
    )
    return response.choices[0].message.content

spec:
  replicas: 2  # Requires 2 GPU nodes

command:
  - --tensor-parallel-size
  - "2"  # Split model across 2 GPUs on same node

# Check GPU utilization during inference
kubectl exec -it <pod-name> -n ai-platform -- nvidia-smi
 
# Or via DCGM metrics in Prometheus
# DCGM_FI_DEV_GPU_UTIL > 80 means good utilization