Continuous Verification: The Post-Deploy Testing Methodology That Replaces 'Hope-Driven Ops'

# synthetic_monitor.py — runs every 60 seconds via Kubernetes CronJob
import requests
import time
from dataclasses import dataclass
from prometheus_client import Gauge, Counter, push_to_gateway
 
CHECKOUT_SUCCESS = Counter('synthetic_checkout_success_total', 'Successful synthetic checkouts')
CHECKOUT_FAILURE = Counter('synthetic_checkout_failure_total', 'Failed synthetic checkouts', ['reason'])
CHECKOUT_LATENCY = Gauge('synthetic_checkout_latency_seconds', 'Checkout flow latency')
 
def run_checkout_flow():
    start = time.time()
    
    try:
        # Step 1: Search for product
        search = requests.get("https://api.myapp.com/search?q=test-product-synthetic", 
                            headers={"X-Synthetic": "true"}, timeout=5)
        assert search.status_code == 200
        assert len(search.json()["products"]) > 0
        product_id = search.json()["products"][0]["id"]
        
        # Step 2: Add to cart
        cart = requests.post("https://api.myapp.com/cart", 
                           json={"product_id": product_id, "quantity": 1},
                           headers={"X-Synthetic": "true"}, timeout=5)
        assert cart.status_code == 201
        cart_id = cart.json()["cart_id"]
        
        # Step 3: Get pricing (verify not stale)
        pricing = requests.get(f"https://api.myapp.com/cart/{cart_id}/pricing",
                             headers={"X-Synthetic": "true"}, timeout=5)
        assert pricing.status_code == 200
        
        # Business logic validation — not just HTTP status
        price_data = pricing.json()
        assert "total" in price_data
        assert price_data["currency"] == "USD"
        assert price_data["timestamp"] > (time.time() - 300)  # Price not older than 5 minutes
        
        # Step 4: Clean up (don't actually complete checkout with real payment)
        requests.delete(f"https://api.myapp.com/cart/{cart_id}",
                      headers={"X-Synthetic": "true"})
        
        CHECKOUT_SUCCESS.inc()
        CHECKOUT_LATENCY.set(time.time() - start)
        
    except AssertionError as e:
        CHECKOUT_FAILURE.labels(reason="assertion_failed").inc()
        raise
    except requests.Timeout:
        CHECKOUT_FAILURE.labels(reason="timeout").inc()
        raise
    except Exception as e:
        CHECKOUT_FAILURE.labels(reason="exception").inc()
        raise
 
if __name__ == "__main__":
    run_checkout_flow()
    push_to_gateway('pushgateway:9091', job='synthetic-monitor', 
                   registry=prometheus_client.REGISTRY)

# flagger-canary.yaml — automated canary with metric analysis
apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
  name: checkout-service
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: checkout-service
  
  progressDeadlineSeconds: 600
  
  service:
    port: 8080
    targetPort: 8080
  
  analysis:
    interval: 1m
    threshold: 5        # Max failed analysis iterations before rollback
    maxWeight: 50       # Max canary traffic percentage
    stepWeight: 10      # Traffic step per iteration
    
    # These metrics determine if the canary is healthy
    metrics:
    
    - name: request-success-rate
      thresholdRange:
        min: 99          # Canary must have 99%+ success rate
      interval: 1m
    
    - name: request-duration
      thresholdRange:
        max: 500         # P99 latency must be under 500ms
      interval: 1m
    
    # Custom business metric
    - name: checkout-success-rate
      templateRef:
        name: checkout-rate-template
        namespace: monitoring
      thresholdRange:
        min: 97.5        # Business metric gate — not just HTTP success
      interval: 2m
    
    # Run synthetic checks against the canary specifically
    webhooks:
    - name: synthetic-acceptance-test
      type: pre-rollout
      url: http://synthetic-runner.monitoring/run
      timeout: 3m
      metadata:
        type: "bash"
        cmd: "python /scripts/synthetic_monitor.py --target canary"

# production_contract_check.py
import requests
from pact import Consumer, Provider, Term
 
def verify_checkout_api_contract():
    """Verify the checkout API response matches the contract consumers expect."""
    
    response = requests.post(
        "https://api.myapp.com/orders",
        json={"cart_id": "test-cart-synthetic"},
        headers={"X-Synthetic": "true"}
    )
    
    body = response.json()
    
    # These are the fields the mobile app contract requires
    required_fields = ["order_id", "status", "total", "currency", "created_at", "items"]
    
    for field in required_fields:
        assert field in body, f"Contract violation: missing required field '{field}'"
    
    # Type checks
    assert isinstance(body["order_id"], str) and len(body["order_id"]) > 0
    assert body["status"] in ["confirmed", "pending", "failed"]
    assert isinstance(body["total"], (int, float)) and body["total"] >= 0
    assert body["currency"] in ["USD", "EUR", "GBP"]
    
    # The mobile app breaks if items is null (vs empty list)
    assert isinstance(body["items"], list), "items must be an array, not null"
    
    print("Contract verified: checkout API response matches mobile app contract")

# PrometheusRule for SLO-based alerting
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: checkout-slo
spec:
  groups:
  - name: checkout.slo
    rules:
    
    # 30-day error budget
    - record: checkout:error_rate:5m
      expr: |
        sum(rate(http_requests_total{service="checkout", status=~"5.."}[5m]))
        /
        sum(rate(http_requests_total{service="checkout"}[5m]))
    
    # Fast burn — losing 5% of monthly budget in 1 hour
    - alert: CheckoutSLOFastBurn
      expr: |
        checkout:error_rate:5m > (14.4 * 0.001)  # 14.4x the error rate budget
      for: 2m
      labels:
        severity: critical
        team: checkout
      annotations:
        summary: "Checkout SLO burning fast — page immediately"
    
    # Slow burn — on track to exhaust budget in 6 days
    - alert: CheckoutSLOSlowBurn  
      expr: |
        checkout:error_rate:5m > (3 * 0.001)
      for: 15m
      labels:
        severity: warning