DeepEval is a pytest-based LLM unit testing framework. pip install deepeval. from deepeval import assert_test, evaluate. from deepeval.test_case import LLMTestCase. from deepeval.metrics import AnswerRelevancyMetric, FaithfulnessMetric, ContextualPrecisionMetric, ContextualRecallMetric, HallucinationMetric. Test case: tc = LLMTestCase(input="What is RAG?", actual_output="RAG combines retrieval...", expected_output="RAG is...", retrieval_context=["RAG paper text..."]). Single metric: metric = AnswerRelevancyMetric(threshold=0.7, model="gpt-4o"), metric.measure(tc), metric.score, metric.reason. Assert: assert_test(tc, [AnswerRelevancyMetric(threshold=0.7)]). Batch: evaluate([tc1, tc2], [AnswerRelevancyMetric(), FaithfulnessMetric()]). pytest: @pytest.mark.parametrize("test_case", dataset) + def test_rag(test_case): assert_test(test_case, metrics). @deepeval.on_test_run_end hook for post-run actions. GEval custom: from deepeval.metrics import GEval, GEval(name="Correctness", evaluation_steps=["Check if response contains key facts", "Verify no contradictions"], evaluation_params=[LLMTestCaseParams.INPUT, LLMTestCaseParams.ACTUAL_OUTPUT], threshold=0.7). Dataset: from deepeval.dataset import EvaluationDataset, dataset = EvaluationDataset(test_cases=[tc1, tc2]), dataset.evaluate([metric]). Load CSV: dataset.pull(alias="my-dataset") from Confident AI. Conversational: from deepeval.test_case import ConversationalTestCase, Turn, ConversationalTestCase(turns=[Turn(input="Hi", actual_output="Hello!")]). deepeval login to connect Confident AI cloud dashboard. deepeval test run test_llm.py CLI command. Claude Code generates DeepEval test suites, metric configs, custom GEval rubrics, dataset loaders, and pytest CI pipelines for LLM applications.
CLAUDE.md for DeepEval
## DeepEval Stack
- Version: deepeval >= 0.21
- Test: LLMTestCase(input, actual_output, expected_output?, retrieval_context?)
- Assert: assert_test(test_case, [metric1, metric2]) — raises if below threshold
- Batch: evaluate([tc1, tc2], metrics) → EvaluationResult with scores
- RAG metrics: AnswerRelevancyMetric | FaithfulnessMetric | ContextualPrecisionMetric | ContextualRecallMetric
- Hallucination: HallucinationMetric(threshold=0.5) — score=0 means no hallucination
- Custom: GEval(name, evaluation_steps=[...], evaluation_params=[...], threshold)
- pytest: @pytest.mark.parametrize("test_case", dataset) + assert_test inside test fn
- CI: deepeval test run test_file.py — exits non-zero on failuresDeepEval LLM Test Suite
# tests/test_llm_app.py — DeepEval unit tests for a RAG application
from __future__ import annotations
import os
import pytest
import deepeval
from deepeval import assert_test, evaluate
from deepeval.dataset import EvaluationDataset
from deepeval.metrics import (
AnswerRelevancyMetric,
FaithfulnessMetric,
ContextualPrecisionMetric,
ContextualRecallMetric,
HallucinationMetric,
SummarizationMetric,
)
from deepeval.metrics import GEval
from deepeval.metrics.base_metric import BaseMetric
from deepeval.test_case import LLMTestCase, LLMTestCaseParams, ConversationalTestCase, Turn
# ── 1. Test case builders ──────────────────────────────────────────────────────
def make_rag_test_case(
question: str,
answer: str,
contexts: list[str],
expected: str | None = None,
) -> LLMTestCase:
"""Build a RAG test case with retrieval context."""
return LLMTestCase(
input=question,
actual_output=answer,
expected_output=expected,
retrieval_context=contexts,
)
# Curated golden dataset
RAG_TEST_CASES = [
make_rag_test_case(
question="What is Retrieval-Augmented Generation?",
answer=(
"Retrieval-Augmented Generation (RAG) enhances LLM responses by first "
"retrieving relevant documents from a knowledge base and incorporating "
"them as context for the generation step."
),
contexts=[
"RAG was introduced in 'Retrieval-Augmented Generation for Knowledge-Intensive "
"NLP Tasks'. It combines a dense retrieval component with a sequence-to-sequence model.",
"The retrieval step uses dense vector search to find semantically similar documents.",
],
expected="RAG combines retrieval with generation to produce grounded, factual responses.",
),
make_rag_test_case(
question="What is the difference between BM25 and dense retrieval?",
answer=(
"BM25 is a sparse retrieval method using keyword frequency and inverse document "
"frequency. Dense retrieval uses neural embeddings to find semantically similar "
"documents even without keyword overlap."
),
contexts=[
"BM25 ranks documents based on term frequency-inverse document frequency (TF-IDF).",
"Dense retrieval models like DPR embed queries and passages into shared vector spaces.",
],
expected="BM25 uses keyword matching; dense retrieval uses embedding similarity.",
),
make_rag_test_case(
question="How does chunking strategy affect RAG quality?",
answer=(
"Chunk size directly impacts retrieval precision and context completeness. "
"Smaller chunks improve precision but may miss surrounding context; "
"larger chunks retain more context but introduce noise. "
"Semantic chunking preserves meaning boundaries."
),
contexts=[
"Chunking strategies include fixed-size, sentence-level, and semantic chunking. "
"Chunk overlap helps preserve context across boundaries.",
],
),
# Hallucination test — answer contains info NOT in context
make_rag_test_case(
question="What year was BERT released?",
answer="BERT was released by Google in 2018 and revolutionized NLP benchmarks.",
contexts=["Transformer models have significantly advanced natural language processing."],
),
]
# ── 2. Standard RAG metrics tests ─────────────────────────────────────────────
@pytest.mark.parametrize("test_case", RAG_TEST_CASES[:3]) # First 3 have valid contexts
def test_answer_relevancy(test_case: LLMTestCase):
"""Answer must be relevant to the question."""
metric = AnswerRelevancyMetric(
threshold=0.7,
model="gpt-4o-mini",
include_reason=True,
)
assert_test(test_case, [metric])
@pytest.mark.parametrize("test_case", RAG_TEST_CASES[:3])
def test_faithfulness(test_case: LLMTestCase):
"""Answer must be grounded in retrieved contexts — no hallucination."""
metric = FaithfulnessMetric(
threshold=0.7,
model="gpt-4o-mini",
include_reason=True,
)
assert_test(test_case, [metric])
@pytest.mark.parametrize("test_case", RAG_TEST_CASES[:2]) # Need expected_output
def test_contextual_precision(test_case: LLMTestCase):
"""Retrieved contexts should be ranked by relevance."""
metric = ContextualPrecisionMetric(
threshold=0.7,
model="gpt-4o-mini",
)
assert_test(test_case, [metric])
@pytest.mark.parametrize("test_case", RAG_TEST_CASES[:2])
def test_contextual_recall(test_case: LLMTestCase):
"""Retrieved contexts should cover the expected answer."""
metric = ContextualRecallMetric(
threshold=0.6,
model="gpt-4o-mini",
)
assert_test(test_case, [metric])
def test_hallucination_detection():
"""The hallucination test case should score poorly on faithfulness."""
hallucination_case = RAG_TEST_CASES[3] # BERT answer not in context
metric = HallucinationMetric(
threshold=0.5, # score ABOVE threshold = too much hallucination
model="gpt-4o-mini",
)
metric.measure(hallucination_case)
print(f"Hallucination score: {metric.score:.2f} — {metric.reason}")
# Note: HallucinationMetric is inverted — higher score = more hallucination
assert metric.score >= 0.0 # Just verify it runs; expected to flag hallucination
# ── 3. GEval custom metric ────────────────────────────────────────────────────
def test_response_conciseness():
"""Custom GEval metric: response should be concise and to-the-point."""
conciseness_metric = GEval(
name="Conciseness",
evaluation_steps=[
"Check if the response directly answers the question without unnecessary padding",
"Verify the response is under 100 words",
"Confirm there are no repeated points or verbose phrases",
],
evaluation_params=[
LLMTestCaseParams.INPUT,
LLMTestCaseParams.ACTUAL_OUTPUT,
],
threshold=0.6,
model="gpt-4o-mini",
)
test_case = LLMTestCase(
input="What is a vector database?",
actual_output=(
"A vector database stores high-dimensional embeddings and supports "
"approximate nearest neighbor search for semantic similarity queries."
),
)
assert_test(test_case, [conciseness_metric])
def test_citation_quality():
"""Custom GEval: responses should reference retrieved context accurately."""
citation_metric = GEval(
name="CitationQuality",
evaluation_steps=[
"Check if claims in the response can be traced to the retrieval context",
"Verify no facts are introduced that aren't present in the context",
"Assess whether the response properly synthesizes context rather than copying verbatim",
],
evaluation_params=[
LLMTestCaseParams.INPUT,
LLMTestCaseParams.ACTUAL_OUTPUT,
LLMTestCaseParams.RETRIEVAL_CONTEXT,
],
threshold=0.65,
model="gpt-4o-mini",
)
assert_test(RAG_TEST_CASES[0], [citation_metric])
# ── 4. Custom BaseMetric ──────────────────────────────────────────────────────
class ResponseLengthMetric(BaseMetric):
"""Custom metric: response must be within a word count range."""
def __init__(self, min_words: int = 10, max_words: int = 150, threshold: float = 1.0):
self.min_words = min_words
self.max_words = max_words
self.threshold = threshold
self.name = f"ResponseLength({min_words}-{max_words} words)"
def measure(self, test_case: LLMTestCase) -> float:
words = len(test_case.actual_output.split())
in_range = self.min_words <= words <= self.max_words
self.score = 1.0 if in_range else 0.0
self.reason = (
f"Response has {words} words. "
f"{'Within' if in_range else 'Outside'} range [{self.min_words}, {self.max_words}]."
)
self.success = self.score >= self.threshold
return self.score
def is_successful(self) -> bool:
return self.success
def test_response_length():
"""Response must be between 10 and 150 words."""
metric = ResponseLengthMetric(min_words=10, max_words=150)
test_case = RAG_TEST_CASES[0]
assert_test(test_case, [metric])
# ── 5. EvaluationDataset and batch evaluation ─────────────────────────────────
def test_full_rag_suite_batch():
"""Run all RAG metrics across the first 3 test cases in one batch call."""
dataset = EvaluationDataset(test_cases=RAG_TEST_CASES[:3])
results = evaluate(
test_cases=dataset,
metrics=[
AnswerRelevancyMetric(threshold=0.7, model="gpt-4o-mini"),
FaithfulnessMetric(threshold=0.7, model="gpt-4o-mini"),
],
run_async=True, # Parallel metric evaluation
print_results=True,
)
# Check aggregate pass rate
passed = sum(1 for r in results.test_results if r.success)
total = len(results.test_results)
print(f"\nBatch results: {passed}/{total} test cases passed")
assert passed / total >= 0.75, f"Pass rate {passed/total:.0%} below 75% threshold"
# ── 6. Conversational testing ─────────────────────────────────────────────────
def test_multi_turn_conversation():
"""Test a multi-turn conversation for coherence and relevancy."""
conv_case = ConversationalTestCase(
turns=[
Turn(
input="What is a transformer model?",
actual_output=(
"A transformer is a neural network architecture using self-attention "
"mechanisms to process sequences in parallel, introduced in 'Attention is All You Need'."
),
retrieval_context=["Transformers use multi-head self-attention and position encodings."],
),
Turn(
input="How does the attention mechanism work?",
actual_output=(
"Attention computes weighted sums over values, where weights come from "
"query-key dot products scaled by the square root of dimension size, "
"then passed through softmax."
),
retrieval_context=["Attention(Q,K,V) = softmax(QK^T / sqrt(d_k)) * V."],
),
]
)
# Conversational test cases use ConversationalRelevancyMetric
from deepeval.metrics import ConversationalRelevancyMetric
metric = ConversationalRelevancyMetric(threshold=0.7, model="gpt-4o-mini")
assert_test(conv_case, [metric])
# ── 7. Summarization metric ───────────────────────────────────────────────────
def test_summarization_quality():
"""Test that a document summary covers key points faithfully."""
source_doc = (
"BERT (Bidirectional Encoder Representations from Transformers) was developed by Google "
"and published in 2018. It introduced the concept of pre-training deep bidirectional "
"transformers on large text corpora using masked language modeling and next sentence "
"prediction. BERT achieved state-of-the-art results on 11 NLP benchmarks at the time "
"of release, including GLUE, SQuAD, and NER tasks."
)
summary = (
"BERT, released by Google in 2018, is a bidirectional transformer pre-trained on "
"masked language modeling. It set new records on multiple NLP benchmarks."
)
test_case = LLMTestCase(
input=source_doc,
actual_output=summary,
)
metric = SummarizationMetric(threshold=0.7, model="gpt-4o-mini")
assert_test(test_case, [metric])
# ── Post-run hook ──────────────────────────────────────────────────────────────
@deepeval.on_test_run_end
def post_run_callback():
"""Called after all tests complete — log or notify CI."""
print("\nDeepEval test run complete. Results logged to Confident AI.")For the RAGAS alternative when needing reference-free RAG metrics and testset generation from documents — RAGAS provides faithfulness and answer relevancy without ground truth labels while DeepEval’s pytest integration, GEval custom rubrics, and assert_test make it the stronger choice for CI/CD pipelines where LLM quality is a deployment gate. For the LangSmith evaluation alternative when already using LangChain and wanting native trace-linked evaluation — LangSmith ties evaluations directly to LangChain traces while DeepEval’s provider-agnostic LLMTestCase API works with any LLM backend and gives explicit pass/fail thresholds that block deployments exactly like unit tests block code merges. The Claude Skills 360 bundle includes DeepEval skill sets covering RAG test cases, pytest integration, GEval custom metrics, custom BaseMetric, EvaluationDataset batch evaluation, conversational testing, and CI/CD pipeline configuration. Start with the free tier to try LLM unit test generation.