Build an AI Synthetic Test Generator with Claude API

pip install anthropic PyGithub

import anthropic
from github import Github
import subprocess
import json
 
client = anthropic.Anthropic()
 
 
def get_pr_diff(repo_name: str, pr_number: int, github_token: str) -> dict:
    gh = Github(github_token)
    repo = gh.get_repo(repo_name)
    pr = repo.get_pull(pr_number)
 
    changed_files = []
    for file in pr.get_files():
        if file.filename.endswith((".py", ".ts", ".js", ".go")) and not file.filename.startswith("test"):
            changed_files.append({
                "filename": file.filename,
                "patch": file.patch,
                "status": file.status,
            })
 
    return {"files": changed_files, "title": pr.title, "body": pr.body}

ANALYZE_PROMPT = """Review this code diff and identify missing test coverage.
 
PR title: {title}
File: {filename}
Diff:
{patch}
 
Look for:
- New functions/methods with no corresponding test
- Edge cases not covered (null/None inputs, empty collections, boundary values)
- Error paths that aren't tested (what happens when the API call fails, timeout, etc.)
- Changed behavior that existing tests don't verify
 
Respond with ONLY valid JSON:
{{"gaps": [{{"function": "name", "missing_case": "description", "severity": "high"|"medium"|"low"}}]}}"""
 
 
def analyze_gaps(filename: str, patch: str, title: str) -> list[dict]:
    response = client.messages.create(
        model="claude-sonnet-5",
        max_tokens=1000,
        messages=[{
            "role": "user",
            "content": ANALYZE_PROMPT.format(title=title, filename=filename, patch=patch)
        }]
    )
    text = response.content[0].text.strip()
    if text.startswith("```"):
        text = text.split("```")[1].replace("json", "", 1).strip()
    return json.loads(text)["gaps"]

GENERATE_TEST_PROMPT = """Generate a test function for this specific gap in existing coverage.
 
Source file: {filename}
Source diff:
{patch}
 
Gap to cover:
Function: {function}
Missing case: {missing_case}
 
Existing test file style (match this framework and pattern):
{existing_test_sample}
 
Generate ONLY the new test function(s) needed — do not regenerate the whole file.
Use realistic test data, not placeholder values like "foo" or "test123" unless
the domain genuinely calls for it."""
 
 
def generate_test(gap: dict, filename: str, patch: str, existing_test_sample: str) -> str:
    response = client.messages.create(
        model="claude-sonnet-5",
        max_tokens=800,
        messages=[{
            "role": "user",
            "content": GENERATE_TEST_PROMPT.format(
                filename=filename,
                patch=patch,
                function=gap["function"],
                missing_case=gap["missing_case"],
                existing_test_sample=existing_test_sample,
            )
        }]
    )
    return response.content[0].text

def review_pr_test_coverage(repo_name: str, pr_number: int, github_token: str):
    pr_data = get_pr_diff(repo_name, pr_number, github_token)
 
    all_gaps = []
    for file in pr_data["files"]:
        gaps = analyze_gaps(file["filename"], file["patch"], pr_data["title"])
        for gap in gaps:
            gap["filename"] = file["filename"]
            gap["patch"] = file["patch"]
            all_gaps.append(gap)
 
    high_priority = [g for g in all_gaps if g["severity"] == "high"]
    if not high_priority:
        return
 
    comment = "## Missing Test Coverage\n\n"
    for gap in high_priority:
        existing_sample = find_existing_test_sample(gap["filename"])
        test_code = generate_test(gap, gap["filename"], gap["patch"], existing_sample)
        comment += f"**{gap['filename']}** — {gap['missing_case']}\n```python\n{test_code}\n```\n\n"
 
    post_pr_comment(repo_name, pr_number, comment, github_token)
 
 
def post_pr_comment(repo_name: str, pr_number: int, body: str, github_token: str):
    gh = Github(github_token)
    repo = gh.get_repo(repo_name)
    pr = repo.get_pull(pr_number)
    pr.create_issue_comment(body)

name: Test Coverage Check
on:
  pull_request:
    types: [opened, synchronize]
 
jobs:
  suggest-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - run: pip install anthropic PyGithub
      - run: python review_test_coverage.py ${{ github.event.pull_request.number }}
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

# Always require a human to accept generated tests before merge —
# never auto-commit AI-generated test code directly to the PR branch