Build an AI Feature Flag Risk Analyzer with Claude API

pip install anthropic requests

import anthropic
import requests
import subprocess
import re
 
client = anthropic.Anthropic()
 
 
def get_flag_inventory(api_key: str) -> list[dict]:
    """Example for LaunchDarkly — adapt for your flag provider's API."""
    resp = requests.get(
        "https://app.launchdarkly.com/api/v2/flags/my-project",
        headers={"Authorization": api_key}
    )
    flags = resp.json()["items"]
    return [{
        "key": f["key"],
        "name": f["name"],
        "created_date": f["creationDate"],
        "temporary": f.get("temporary", False),
        "environments": f.get("environments", {}),
    } for f in flags]
 
 
def get_flag_evaluation_stats(flag_key: str, api_key: str, days: int = 30) -> dict:
    """Pull rollout percentage and evaluation counts — a flag stuck at
    100% rollout for months with no variation being served is a strong
    signal it should just be removed from the code."""
    resp = requests.get(
        f"https://app.launchdarkly.com/api/v2/flags/my-project/{flag_key}/status",
        headers={"Authorization": api_key}
    )
    return resp.json()

def find_code_references(flag_key: str, repo_path: str) -> list[dict]:
    """Find every place in the codebase this flag is actually referenced."""
    result = subprocess.run(
        ["grep", "-rn", flag_key, repo_path, "--include=*.py", "--include=*.ts", "--include=*.js"],
        capture_output=True, text=True
    )
    references = []
    for line in result.stdout.splitlines():
        parts = line.split(":", 2)
        if len(parts) == 3:
            references.append({"file": parts[0], "line": parts[1], "code": parts[2].strip()})
    return references

import json
 
ANALYZE_PROMPT = """Analyze this feature flag for cleanup risk.
 
Flag: {flag_key}
Created: {created_date}
Marked temporary: {temporary}
Current rollout state per environment: {environments}
Evaluation stats (last {days} days): {eval_stats}
 
Code references found:
{code_references}
 
Determine:
1. Is this flag safe to remove entirely (100% rolled out everywhere for
   months, code always takes the "on" branch, no A/B test still running)?
2. Is this flag stale but risky to touch (references exist but the logic
   is tangled — removing needs careful review, not a mechanical delete)?
3. Is this flag actively in use for legitimate ongoing purposes (kill switch,
   active experiment, gradual rollout still in progress)?
 
Respond with ONLY valid JSON:
{{"verdict": "safe_to_remove" | "stale_needs_review" | "actively_used",
  "reasoning": "...", "cleanup_complexity": "simple" | "moderate" | "complex" | null}}"""
 
 
def analyze_flag_risk(flag: dict, eval_stats: dict, code_references: list, days: int = 30) -> dict:
    response = client.messages.create(
        model="claude-sonnet-5",
        max_tokens=500,
        messages=[{
            "role": "user",
            "content": ANALYZE_PROMPT.format(
                flag_key=flag["key"], created_date=flag["created_date"],
                temporary=flag["temporary"], environments=flag["environments"],
                eval_stats=eval_stats, code_references=code_references, days=days,
            )
        }]
    )
    text = response.content[0].text.strip()
    if text.startswith("```"):
        text = text.split("```")[1].replace("json", "", 1).strip()
    return json.loads(text)

def generate_cleanup_report(repo_path: str, ld_api_key: str) -> str:
    flags = get_flag_inventory(ld_api_key)
    report = ["# Feature Flag Cleanup Report\n"]
    safe_count = 0
 
    for flag in flags:
        eval_stats = get_flag_evaluation_stats(flag["key"], ld_api_key)
        code_refs = find_code_references(flag["key"], repo_path)
        analysis = analyze_flag_risk(flag, eval_stats, code_refs)
 
        if analysis["verdict"] == "safe_to_remove":
            safe_count += 1
            report.append(f"## ✅ {flag['key']} — Safe to remove ({analysis['cleanup_complexity']})")
            report.append(f"{analysis['reasoning']}")
            report.append(f"References: {len(code_refs)} locations\n")
        elif analysis["verdict"] == "stale_needs_review":
            report.append(f"## ⚠️ {flag['key']} — Stale, needs manual review")
            report.append(f"{analysis['reasoning']}\n")
 
    report.insert(1, f"**{safe_count} flags safe to remove** out of {len(flags)} total\n")
    return "\n".join(report)

python flag_analyzer.py --repo ./myapp --output cleanup-report.md
 
# Feature Flag Cleanup Report
# 8 flags safe to remove out of 34 total
#
# ✅ new-checkout-flow — Safe to remove (simple)
# 100% rolled out in all environments since 2026-02-15 (5+ months),
# code takes the "on" branch unconditionally in all 3 references, no
# A/B test infrastructure detected around it.
# References: 3 locations
#
# ⚠️ payments-retry-logic — Stale, needs manual review
# 100% rolled out for 4 months, but code references show complex
# conditional logic intertwined with error handling — a mechanical
# removal risks changing retry behavior in a way that needs careful testing.