Getting Started

pip install promptassert

# promptassert.config.yaml
version: 1
default_model: gpt-4o

providers:
  openai:
    api_key: ${OPENAI_API_KEY}
  anthropic:
    api_key: ${ANTHROPIC_API_KEY}

output_dir: .promptassert/snapshots

from promptassert import PromptAssert

pa = PromptAssert(model="gpt-4o")

@pa.test
def test_customer_support_tone():
    response = pa.call(
        system="You are a helpful customer support agent.",
        prompt="A customer is asking about their order status."
    )
    # Tone checks
    pa.assert_sentiment(response, "positive")
    pa.assert_contains(response, ["order", "help", "assist"])
    pa.assert_not_contains(response, ["sorry", "unfortunately", "can't"])

    # Length constraints
    pa.assert_max_tokens(response, 300)

@pa.test
def test_refund_policy_accuracy():
    response = pa.call(
        system="You are a support agent. Our refund window is 30 days.",
        prompt="What is your refund policy?"
    )
    # Factual accuracy
    pa.assert_contains(response, ["30"])
    pa.assert_semantic_similarity(response, "30-day refund window", threshold=0.85)

@pa.test
def test_no_hallucination_on_pricing():
    response = pa.call(
        system="You are a product assistant. Do not invent pricing.",
        prompt="How much does the enterprise plan cost?"
    )
    # Should not make up numbers
    pa.assert_not_regex(response, r"\$[0-9]+")
    pa.assert_contains(response, ["contact", "sales", "pricing"])

# Run all tests
promptassert run tests/

# Run a single file
promptassert run tests/test_support.py

# Run against a different model (comparison mode)
promptassert run tests/ --model claude-3-5-sonnet-20241022

# Generate a snapshot baseline
promptassert snapshot --create tests/

# Check drift against baseline
promptassert snapshot --check tests/

# .github/workflows/prompt-tests.yml
name: Prompt Regression Tests

on:
  push:
    branches: [main]
  pull_request:

jobs:
  prompt-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install PromptAssert
        run: pip install promptassert

      - name: Run prompt test suite
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: promptassert run tests/

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: promptassert-results
          path: .promptassert/results/