SYMBOL INDEX (6898 symbols across 835 files)

FILE: .scripts/changelog/generate.py
  class Commit (line 72) | class Commit:
  class Pull (line 78) | class Pull:
  class AiReleaseNote (line 89) | class AiReleaseNote(BaseModel):
    method validate_category (line 111) | def validate_category(cls, category: str) -> str:
  class AiMonthSummary (line 117) | class AiMonthSummary(BaseModel):
  function sh (line 177) | def sh(cmd: List[str]) -> str:
  function git_tag_date_ymd (line 182) | def git_tag_date_ymd(tag: str) -> str:
  function get_prev_tag (line 189) | def get_prev_tag(tag: str) -> str:
  function list_tags_between (line 195) | def list_tags_between(from_tag: str, to_tag: str) -> List[str]:
  function list_all_tags (line 226) | def list_all_tags() -> List[str]:
  function list_tags_for_year (line 248) | def list_tags_for_year(year: int) -> List[str]:
  function latest_tag (line 263) | def latest_tag() -> str:
  function commits_in_range (line 267) | def commits_in_range(base: str, head: str) -> List[Commit]:
  function extract_pr_numbers (line 288) | def extract_pr_numbers(commits: Iterable[Commit]) -> Dict[int, Commit]:
  function offline_pr_title_from_merge_commit (line 310) | def offline_pr_title_from_merge_commit(
  function stitch_truncated_title (line 333) | def stitch_truncated_title(title: str, body: str) -> str:
  function sanitize_for_multimodal_sentinel (line 352) | def sanitize_for_multimodal_sentinel(prompt: str) -> str:
  function gh_get (line 362) | def gh_get(
  function gh_request (line 376) | def gh_request(path: str, timeout_s: int = 20) -> dict:
  function fetch_pr (line 385) | def fetch_pr(pr_number: int) -> Pull:
  function fetch_pr_diff (line 401) | def fetch_pr_diff(diff_url: str, timeout_s: int = 20) -> str:
  function fetch_user_display (line 408) | def fetch_user_display(login: str) -> Tuple[str, str]:
  function get_ai_model (line 432) | def get_ai_model(model_name: str):
  function build_ai_prompt (line 438) | def build_ai_prompt(*, title: str, body: str) -> str:
  function build_month_summary_prompt (line 479) | def build_month_summary_prompt(*, month: str, entries: list[str]) -> str:
  function ai_month_summary (line 500) | def ai_month_summary(model, *, month: str, entries: list[str]) -> str:
  function ai_release_note_for_pr (line 510) | def ai_release_note_for_pr(
  function clean_pr_body_for_ai (line 532) | def clean_pr_body_for_ai(body: str, *, max_chars: int = 2000) -> str:
  function clean_diff_for_ai (line 576) | def clean_diff_for_ai(diff_text: str) -> str:
  function truncate_text (line 617) | def truncate_text(
  function strip_entry_tail (line 649) | def strip_entry_tail(line: str) -> str:
  function clean_title (line 662) | def clean_title(title: str) -> str:
  function classify (line 673) | def classify(title: str, body: str) -> str:
  function mdx_escape (line 734) | def mdx_escape(s: str) -> str:
  function split_prefix_and_body (line 747) | def split_prefix_and_body(text: str) -> Tuple[str, str]:
  function parse_ignore_prs (line 806) | def parse_ignore_prs(text: str) -> set[int]:
  function prune_ignored (line 835) | def prune_ignored(idx: ChangelogIndex, ignore_prs: set[int]) -> int:
  function parse_body (line 853) | def parse_body(body: str) -> ChangelogIndex:
  function month_sort_key (line 897) | def month_sort_key(name: str) -> int:
  function render_changelog_body (line 901) | def render_changelog_body(
  function title_needs_github (line 992) | def title_needs_github(title: str) -> bool:
  function month_name_from_ymd (line 1005) | def month_name_from_ymd(ymd: str) -> Tuple[int, str]:
  function build_release_entries (line 1010) | def build_release_entries(
  function collect_existing_keys (line 1167) | def collect_existing_keys(idx: ChangelogIndex) -> set[tuple[str, int]]:
  function merge_idx (line 1177) | def merge_idx(
  function run_with_overall_progress (line 1228) | def run_with_overall_progress(
  function main (line 1412) | def main() -> int:

FILE: deepeval/__init__.py
  function _expose_public_api (line 15) | def _expose_public_api() -> None:
  function compare_versions (line 93) | def compare_versions(version1, version2):
  function check_for_update (line 100) | def check_for_update():
  function update_warning_opt_in (line 128) | def update_warning_opt_in():

FILE: deepeval/annotation/annotation.py
  function send_annotation (line 7) | def send_annotation(
  function a_send_annotation (line 43) | async def a_send_annotation(

FILE: deepeval/annotation/api.py
  class AnnotationType (line 6) | class AnnotationType(str, Enum):
  class APIAnnotation (line 11) | class APIAnnotation(BaseModel):
    method validate_input (line 23) | def validate_input(cls, data):

FILE: deepeval/anthropic/extractors.py
  function safe_extract_input_parameters (line 13) | def safe_extract_input_parameters(kwargs: Dict[str, Any]) -> InputParame...
  function extract_messages_api_input_parameters (line 21) | def extract_messages_api_input_parameters(
  function safe_extract_output_parameters (line 51) | def safe_extract_output_parameters(
  function extract_messages_api_output_parameters (line 64) | def extract_messages_api_output_parameters(

FILE: deepeval/anthropic/patch.py
  function patch_anthropic_classes (line 17) | def patch_anthropic_classes():
  function _create_sync_wrapper (line 45) | def _create_sync_wrapper(original_method):
  function _create_async_wrapper (line 61) | def _create_async_wrapper(original_method):
  function _patch_sync_anthropic_client_method (line 77) | def _patch_sync_anthropic_client_method(original_method: Callable):
  function _patch_async_anthropic_client_method (line 111) | def _patch_async_anthropic_client_method(original_method: Callable):
  function unpatch_anthropic_classes (line 145) | def unpatch_anthropic_classes():

FILE: deepeval/anthropic/utils.py
  function stringify_anthropic_content (line 9) | def stringify_anthropic_content(content: Any) -> str:
  function render_messages_anthropic (line 118) | def render_messages_anthropic(

FILE: deepeval/benchmarks/arc/arc.py
  class ARC (line 16) | class ARC(DeepEvalBaseBenchmark):
    method __init__ (line 17) | def __init__(
    method evaluate (line 54) | def evaluate(
    method predict (line 102) | def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
    method load_benchmark_dataset (line 128) | def load_benchmark_dataset(self, mode: ARCMode) -> List[Golden]:
    method print_verbose_logs (line 153) | def print_verbose_logs(

FILE: deepeval/benchmarks/arc/mode.py
  class ARCMode (line 4) | class ARCMode(Enum):

FILE: deepeval/benchmarks/arc/template.py
  class ARCTemplate (line 1) | class ARCTemplate:
    method generate_output (line 71) | def generate_output(input: str, n_shots: int):
    method format_question (line 81) | def format_question(data: dict, include_answer: bool = True):
    method format_answer (line 93) | def format_answer(data: dict):

FILE: deepeval/benchmarks/base_benchmark.py
  class DeepEvalBaseBenchmarkResult (line 9) | class DeepEvalBaseBenchmarkResult(BaseModel):
  class DeepEvalBaseBenchmark (line 16) | class DeepEvalBaseBenchmark(ABC, Generic[T]):
    method __init__ (line 17) | def __init__(self, dataset: Optional["Dataset"] = None):
    method load_benchmark_dataset (line 24) | def load_benchmark_dataset(self, *args, **kwargs) -> List[Golden]:
    method evaluate (line 29) | def evaluate(

FILE: deepeval/benchmarks/bbq/bbq.py
  class BBQ (line 16) | class BBQ(DeepEvalBaseBenchmark):
    method __init__ (line 17) | def __init__(
    method evaluate (line 45) | def evaluate(
    method predict (line 127) | def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
    method load_benchmark_dataset (line 154) | def load_benchmark_dataset(self, task: BBQTask) -> List[Golden]:
    method print_verbose_logs (line 188) | def print_verbose_logs(

FILE: deepeval/benchmarks/bbq/task.py
  class BBQTask (line 4) | class BBQTask(Enum):

FILE: deepeval/benchmarks/bbq/template.py
  class BBQTemplate (line 4) | class BBQTemplate:
    method generate_output (line 140) | def generate_output(input: str, n_shots: int):
    method format_question (line 150) | def format_question(data: dict, include_answer: bool = True):
    method format_answer (line 161) | def format_answer(data: dict):

FILE: deepeval/benchmarks/big_bench_hard/big_bench_hard.py
  class BigBenchHard (line 47) | class BigBenchHard(DeepEvalBaseBenchmark):
    method __init__ (line 48) | def __init__(
    method evaluate (line 81) | def evaluate(
    method predict (line 199) | def predict(
    method batch_predict (line 227) | def batch_predict(
    method load_benchmark_dataset (line 278) | def load_benchmark_dataset(self, task: BigBenchHardTask) -> List[Golden]:
    method print_verbose_logs (line 299) | def print_verbose_logs(

FILE: deepeval/benchmarks/big_bench_hard/task.py
  class BigBenchHardTask (line 4) | class BigBenchHardTask(Enum):

FILE: deepeval/benchmarks/big_bench_hard/template.py
  class BigBenchHardTemplate (line 8) | class BigBenchHardTemplate:
    method generate_output (line 14) | def generate_output(
    method read_file (line 31) | def read_file(package_path, filename):
    method get_filename (line 40) | def get_filename(task):

FILE: deepeval/benchmarks/bool_q/bool_q.py
  class BoolQ (line 15) | class BoolQ(DeepEvalBaseBenchmark):
    method __init__ (line 16) | def __init__(
    method evaluate (line 43) | def evaluate(
    method predict (line 89) | def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
    method load_benchmark_dataset (line 116) | def load_benchmark_dataset(self) -> List[Golden]:
    method print_verbose_logs (line 136) | def print_verbose_logs(

FILE: deepeval/benchmarks/bool_q/template.py
  class BoolQTemplate (line 1) | class BoolQTemplate:
    method generate_output (line 12) | def generate_output(input: str, n_shots: int):
    method format_question (line 20) | def format_question(data: dict):
    method format_answer (line 30) | def format_answer(data: dict):

FILE: deepeval/benchmarks/drop/drop.py
  class DROP (line 28) | class DROP(DeepEvalBaseBenchmark):
    method __init__ (line 29) | def __init__(
    method evaluate (line 53) | def evaluate(
    method predict (line 169) | def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
    method batch_predict (line 209) | def batch_predict(
    method load_benchmark_dataset (line 291) | def load_benchmark_dataset(self, task: DROPTask) -> List[Golden]:
    method print_verbose_logs (line 332) | def print_verbose_logs(

FILE: deepeval/benchmarks/drop/task.py
  class DROPTask (line 4) | class DROPTask(Enum):

FILE: deepeval/benchmarks/drop/template.py
  class DROPTemplate (line 4) | class DROPTemplate:
    method generate_output (line 11) | def generate_output(input: str, train_set: object, n_shots: int):
    method format_question (line 23) | def format_question(data: dict, include_answer: bool = False):
    method parse_list_to_str (line 32) | def parse_list_to_str(input_list: List, DELIMITER: str) -> str:
    method parse_str_to_list (line 39) | def parse_str_to_list(input_str: str, DELIMITER: str) -> List[str]:

FILE: deepeval/benchmarks/equity_med_qa/equity_med_qa.py
  class EquityMedQA (line 18) | class EquityMedQA(DeepEvalBaseBenchmark):
    method __init__ (line 19) | def __init__(
    method evaluate (line 40) | def evaluate(
    method predict (line 109) | def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
    method load_benchmark_dataset (line 131) | def load_benchmark_dataset(self, task: EquityMedQATask) -> List[Golden]:

FILE: deepeval/benchmarks/equity_med_qa/task.py
  class EquityMedQATask (line 4) | class EquityMedQATask(Enum):

FILE: deepeval/benchmarks/equity_med_qa/template.py
  class EquityMedQATemplate (line 1) | class EquityMedQATemplate:
    method format_question (line 4) | def format_question(data: dict):

FILE: deepeval/benchmarks/gsm8k/gsm8k.py
  class GSM8K (line 15) | class GSM8K(DeepEvalBaseBenchmark):
    method __init__ (line 16) | def __init__(
    method evaluate (line 45) | def evaluate(
    method predict (line 94) | def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
    method _extract_prediction_from_response (line 130) | def _extract_prediction_from_response(self, res) -> str:
    method _extract_from_tuple (line 145) | def _extract_from_tuple(self, res: tuple) -> str:
    method load_benchmark_dataset (line 153) | def load_benchmark_dataset(self) -> List[Golden]:
    method print_verbose_logs (line 181) | def print_verbose_logs(

FILE: deepeval/benchmarks/gsm8k/template.py
  class GSM8KTemplate (line 4) | class GSM8KTemplate:
    method generate_output (line 11) | def generate_output(
    method format_example (line 35) | def format_example(data: dict, enable_cot: bool):
    method format_answer (line 50) | def format_answer(data: dict):
    method format_subject (line 55) | def format_subject(subject: str):

FILE: deepeval/benchmarks/hellaswag/hellaswag.py
  class HellaSwag (line 17) | class HellaSwag(DeepEvalBaseBenchmark):
    method __init__ (line 18) | def __init__(
    method evaluate (line 50) | def evaluate(
    method predict (line 170) | def predict(
    method batch_predict (line 204) | def batch_predict(
    method load_benchmark_dataset (line 253) | def load_benchmark_dataset(self, task: HellaSwagTask) -> List[Golden]:
    method print_verbose_logs (line 294) | def print_verbose_logs(

FILE: deepeval/benchmarks/hellaswag/task.py
  class HellaSwagTask (line 4) | class HellaSwagTask(Enum):

FILE: deepeval/benchmarks/hellaswag/template.py
  class HellaSwagTemplate (line 4) | class HellaSwagTemplate:
    method generate_output (line 11) | def generate_output(
    method format_question (line 23) | def format_question(data: dict, include_answer: bool = True):

FILE: deepeval/benchmarks/human_eval/human_eval.py
  function secure_exec (line 14) | def secure_exec(code_str, global_vars=None, local_vars=None):
  class HumanEval (line 86) | class HumanEval(DeepEvalBaseBenchmark):
    method __init__ (line 87) | def __init__(
    method evaluate (line 111) | def evaluate(
    method predict (line 182) | def predict(
    method load_benchmark_dataset (line 219) | def load_benchmark_dataset(self, task: HumanEvalTask) -> List[Golden]:
    method print_verbose_logs (line 239) | def print_verbose_logs(

FILE: deepeval/benchmarks/human_eval/task.py
  class HumanEvalTask (line 4) | class HumanEvalTask(Enum):

FILE: deepeval/benchmarks/human_eval/template.py
  class HumanEvalTemplate (line 5) | class HumanEvalTemplate:
    method generate_output (line 10) | def generate_output(input: str, task: HumanEvalTask):

FILE: deepeval/benchmarks/ifeval/ifeval.py
  class IFEvalResult (line 21) | class IFEvalResult(DeepEvalBaseBenchmarkResult):
  class IFEvalInstructionVerifier (line 27) | class IFEvalInstructionVerifier:
    method verify_punctuation_constraints (line 36) | def verify_punctuation_constraints(
    method verify_length_constraints (line 51) | def verify_length_constraints(
    method verify_format_constraints (line 113) | def verify_format_constraints(
    method verify_case_constraints (line 166) | def verify_case_constraints(
    method verify_startend_constraints (line 182) | def verify_startend_constraints(
    method verify_keywords_constraints (line 201) | def verify_keywords_constraints(
    method verify_content_constraints (line 224) | def verify_content_constraints(
    method verify_structural_constraints (line 281) | def verify_structural_constraints(
    method verify_combination_constraints (line 318) | def verify_combination_constraints(
    method verify_instruction_compliance (line 332) | def verify_instruction_compliance(
  class IFEval (line 392) | class IFEval(DeepEvalBaseBenchmark):
    method __init__ (line 404) | def __init__(
    method evaluate (line 421) | def evaluate(self, model: DeepEvalBaseLLM, *args, **kwargs) -> IFEvalR...
    method predict (line 495) | def predict(
    method load_benchmark_dataset (line 537) | def load_benchmark_dataset(self) -> List[Golden]:
    method print_verbose_logs (line 571) | def print_verbose_logs(

FILE: deepeval/benchmarks/ifeval/template.py
  class IFEvalTemplate (line 1) | class IFEvalTemplate:
    method format_instruction (line 12) | def format_instruction(instruction: str) -> str:
    method extract_response (line 25) | def extract_response(text: str) -> str:
    method get_instruction_category (line 46) | def get_instruction_category(instruction_id: str) -> str:
    method get_instruction_description (line 61) | def get_instruction_description(instruction_id: str) -> str:
    method format_verification_report (line 98) | def format_verification_report(

FILE: deepeval/benchmarks/lambada/lambada.py
  class LAMBADA (line 15) | class LAMBADA(DeepEvalBaseBenchmark):
    method __init__ (line 16) | def __init__(
    method evaluate (line 43) | def evaluate(
    method predict (line 89) | def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
    method load_benchmark_dataset (line 116) | def load_benchmark_dataset(self) -> List[Golden]:
    method print_verbose_logs (line 136) | def print_verbose_logs(

FILE: deepeval/benchmarks/lambada/template.py
  class LAMBADATemplate (line 4) | class LAMBADATemplate:
    method generate_output (line 25) | def generate_output(input: str, n_shots: int):
    method format_question (line 35) | def format_question(data: dict, include_answer: bool = True):
    method format_answer (line 57) | def format_answer(data: dict):

FILE: deepeval/benchmarks/logi_qa/logi_qa.py
  class LogiQA (line 19) | class LogiQA(DeepEvalBaseBenchmark):
    method __init__ (line 20) | def __init__(
    method evaluate (line 51) | def evaluate(
    method predict (line 167) | def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
    method batch_predict (line 194) | def batch_predict(
    method load_benchmark_dataset (line 237) | def load_benchmark_dataset(self, task: LogiQATask) -> List[Golden]:
    method download_and_load_hf_dataset (line 258) | def download_and_load_hf_dataset(self, url):
    method print_verbose_logs (line 273) | def print_verbose_logs(

FILE: deepeval/benchmarks/logi_qa/task.py
  class LogiQATask (line 4) | class LogiQATask(Enum):

FILE: deepeval/benchmarks/logi_qa/template.py
  class LogiQATemplate (line 1) | class LogiQATemplate:
    method generate_output (line 12) | def generate_output(input: str, n_shots: int):
    method format_question (line 20) | def format_question(data: dict):
    method format_output (line 39) | def format_output(data: dict):

FILE: deepeval/benchmarks/math_qa/math_qa.py
  class MathQA (line 17) | class MathQA(DeepEvalBaseBenchmark):
    method __init__ (line 18) | def __init__(
    method evaluate (line 49) | def evaluate(
    method predict (line 165) | def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
    method batch_predict (line 196) | def batch_predict(
    method load_benchmark_dataset (line 240) | def load_benchmark_dataset(self, task: MathQATask) -> List[Golden]:
    method print_verbose_logs (line 261) | def print_verbose_logs(

FILE: deepeval/benchmarks/math_qa/task.py
  class MathQATask (line 4) | class MathQATask(Enum):

FILE: deepeval/benchmarks/math_qa/template.py
  class MathQATemplate (line 1) | class MathQATemplate:
    method generate_output (line 52) | def generate_output(input: str, n_shots: int):
    method format_question (line 62) | def format_question(data: dict, include_answer=True):
    method format_output (line 74) | def format_output(data: dict):

FILE: deepeval/benchmarks/mmlu/mmlu.py
  class MMLU (line 17) | class MMLU(DeepEvalBaseBenchmark):
    method __init__ (line 18) | def __init__(
    method evaluate (line 48) | def evaluate(
    method predict (line 166) | def predict(
    method batch_predict (line 204) | def batch_predict(
    method load_benchmark_dataset (line 258) | def load_benchmark_dataset(self, task: MMLUTask) -> List[Golden]:
    method print_verbose_logs (line 287) | def print_verbose_logs(

FILE: deepeval/benchmarks/mmlu/task.py
  class MMLUTask (line 4) | class MMLUTask(Enum):

FILE: deepeval/benchmarks/mmlu/template.py
  class MMLUTemplate (line 4) | class MMLUTemplate:
    method generate_output (line 11) | def generate_output(
    method format_question (line 22) | def format_question(data: dict, include_answer: bool = True):
    method format_subject (line 34) | def format_subject(subject: str):

FILE: deepeval/benchmarks/schema.py
  class MultipleChoiceSchema (line 5) | class MultipleChoiceSchema(BaseModel):
  class ListOfNumbersSchema (line 9) | class ListOfNumbersSchema(BaseModel):
  class ListofStringsSchema (line 13) | class ListofStringsSchema(BaseModel):
  class NumberSchema (line 17) | class NumberSchema(BaseModel):
  class StringSchema (line 21) | class StringSchema(BaseModel):
  class BinaryChoiceSchema (line 28) | class BinaryChoiceSchema(BaseModel):
  class TrinaryChoiceSchema (line 35) | class TrinaryChoiceSchema(BaseModel):
  class MultipleChoiceSchemaLower (line 42) | class MultipleChoiceSchemaLower(BaseModel):
  class DROPStringSchema (line 49) | class DROPStringSchema(BaseModel):
  class DROPNumberSchema (line 53) | class DROPNumberSchema(BaseModel):
  class DROPDateSchema (line 57) | class DROPDateSchema(BaseModel):
  class AffirmationSchema (line 64) | class AffirmationSchema(BaseModel):
  class AffirmationLowerSchema (line 68) | class AffirmationLowerSchema(BaseModel):
  class BooleanSchema (line 72) | class BooleanSchema(BaseModel):
  class ValidSchema (line 76) | class ValidSchema(BaseModel):
  class BBHMultipleChoice2Schema (line 80) | class BBHMultipleChoice2Schema(BaseModel):
  class BBHMultipleChoice3Schema (line 84) | class BBHMultipleChoice3Schema(BaseModel):
  class BBHMultipleChoice4Schema (line 88) | class BBHMultipleChoice4Schema(BaseModel):
  class BBHMultipleChoice5Schema (line 92) | class BBHMultipleChoice5Schema(BaseModel):
  class BBHMultipleChoice6Schema (line 96) | class BBHMultipleChoice6Schema(BaseModel):
  class BBHMultipleChoice7Schema (line 100) | class BBHMultipleChoice7Schema(BaseModel):
  class BBHMultipleChoice11Schema (line 104) | class BBHMultipleChoice11Schema(BaseModel):
  class BBHMultipleChoice18Schema (line 120) | class BBHMultipleChoice18Schema(BaseModel):

FILE: deepeval/benchmarks/squad/squad.py
  class SQuAD (line 17) | class SQuAD(DeepEvalBaseBenchmark):
    method __init__ (line 18) | def __init__(
    method evaluate (line 51) | def evaluate(
    method predict (line 134) | def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
    method load_benchmark_dataset (line 166) | def load_benchmark_dataset(self, task: SQuADTask) -> List[Golden]:
    method print_verbose_logs (line 184) | def print_verbose_logs(

FILE: deepeval/benchmarks/squad/task.py
  class SQuADTask (line 4) | class SQuADTask(Enum):

FILE: deepeval/benchmarks/squad/template.py
  class SQuADTemplate (line 1) | class SQuADTemplate:
    method generate_output (line 48) | def generate_output(input: str, n_shots: int):
    method format_question (line 58) | def format_question(data: dict, include_answer=True):
    method format_output (line 69) | def format_output(data: dict):

FILE: deepeval/benchmarks/truthful_qa/mode.py
  class TruthfulQAMode (line 4) | class TruthfulQAMode(Enum):

FILE: deepeval/benchmarks/truthful_qa/task.py
  class TruthfulQATask (line 4) | class TruthfulQATask(Enum):

FILE: deepeval/benchmarks/truthful_qa/template.py
  class TruthfulQATemplate (line 6) | class TruthfulQATemplate:
    method generate_output (line 14) | def generate_output(input: str, mode: TruthfulQAMode):
    method format_mc1_question (line 31) | def format_mc1_question(data: dict, seed=42):
    method format_mc2_question (line 55) | def format_mc2_question(data: dict, seed=42):
    method format_prompt_with_examples (line 89) | def format_prompt_with_examples(input):

FILE: deepeval/benchmarks/truthful_qa/truthful_qa.py
  class TruthfulQA (line 23) | class TruthfulQA(DeepEvalBaseBenchmark):
    method __init__ (line 24) | def __init__(
    method evaluate (line 58) | def evaluate(
    method predict (line 176) | def predict(
    method batch_predict (line 218) | def batch_predict(
    method load_benchmark_dataset (line 283) | def load_benchmark_dataset(
    method print_verbose_logs (line 332) | def print_verbose_logs(

FILE: deepeval/benchmarks/utils.py
  function should_use_batch (line 6) | def should_use_batch(model: DeepEvalBaseLLM, batch_size: Optional[int] =...

FILE: deepeval/benchmarks/winogrande/template.py
  class WinograndeTemplate (line 4) | class WinograndeTemplate:
    method generate_output (line 40) | def generate_output(input: str, n_shots: int):
    method format_question (line 50) | def format_question(data: dict, include_answer: bool = True):
    method format_answer (line 61) | def format_answer(data: dict):

FILE: deepeval/benchmarks/winogrande/winogrande.py
  class Winogrande (line 15) | class Winogrande(DeepEvalBaseBenchmark):
    method __init__ (line 16) | def __init__(
    method evaluate (line 43) | def evaluate(
    method predict (line 89) | def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
    method load_benchmark_dataset (line 116) | def load_benchmark_dataset(self) -> List[Golden]:
    method print_verbose_logs (line 138) | def print_verbose_logs(

FILE: deepeval/cli/dotenv_handler.py
  class DotenvHandler (line 11) | class DotenvHandler:
    method __init__ (line 12) | def __init__(self, path: str | Path = ".env.local"):
    method _quote_if_needed (line 15) | def _quote_if_needed(self, val: str) -> str:
    method upsert (line 23) | def upsert(self, updates: Dict[str, str]) -> None:
    method unset (line 59) | def unset(self, keys: Iterable[str]) -> None:

FILE: deepeval/cli/generate/command.py
  function __getattr__ (line 27) | def __getattr__(name: str) -> Any:
  function generate_command (line 41) | def generate_command(

FILE: deepeval/cli/generate/utils.py
  class GenerationMethod (line 16) | class GenerationMethod(str, Enum):
  class GoldenVariation (line 23) | class GoldenVariation(str, Enum):
  class FileType (line 28) | class FileType(str, Enum):
  function require_method_option (line 34) | def require_method_option(value, option_name: str, method: GenerationMet...
  function load_contexts_file (line 43) | def load_contexts_file(contexts_file: Path) -> List[List[str]]:
  function load_goldens_file (line 75) | def load_goldens_file(
  function validate_golden_variation (line 104) | def validate_golden_variation(
  function single_turn_styling_config (line 128) | def single_turn_styling_config(
  function multi_turn_styling_config (line 144) | def multi_turn_styling_config(
  function validate_scratch_styling (line 170) | def validate_scratch_styling(

FILE: deepeval/cli/inspect.py
  function inspect_command (line 26) | def inspect_command(
  function _resolve_target (line 88) | def _resolve_target(
  function _find_latest (line 108) | def _find_latest(folder: Path) -> Optional[Path]:

FILE: deepeval/cli/main.py
  class Regions (line 72) | class Regions(Enum):
  function version_callback (line 78) | def version_callback(value: Optional[bool] = None) -> None:
  function generate_pairing_code (line 89) | def generate_pairing_code():
  function find_available_port (line 94) | def find_available_port():
  function is_openai_configured (line 100) | def is_openai_configured() -> bool:
  function _handle_save_result (line 115) | def _handle_save_result(
  function main (line 150) | def main(
  function set_confident_region_command (line 164) | def set_confident_region_command(
  function login (line 217) | def login(
  function logout (line 314) | def logout(
  function view (line 358) | def view():
  function update_settings (line 380) | def update_settings(
  function set_debug (line 550) | def set_debug(
  function unset_debug (line 696) | def unset_debug(
  function set_openai_env (line 753) | def set_openai_env(
  function unset_openai_env (line 879) | def unset_openai_env(
  function set_azure_openai_env (line 955) | def set_azure_openai_env(
  function unset_azure_openai_env (line 1053) | def unset_azure_openai_env(
  function set_azure_openai_embedding_env (line 1107) | def set_azure_openai_embedding_env(
  function unset_azure_openai_embedding_env (line 1169) | def unset_azure_openai_embedding_env(
  function set_anthropic_model_env (line 1217) | def set_anthropic_model_env(
  function unset_anthropic_model_env (line 1307) | def unset_anthropic_model_env(
  function set_bedrock_model_env (line 1364) | def set_bedrock_model_env(
  function unset_bedrock_model_env (line 1467) | def unset_bedrock_model_env(
  function set_ollama_model_env (line 1526) | def set_ollama_model_env(
  function unset_ollama_model_env (line 1586) | def unset_ollama_model_env(
  function set_ollama_embeddings_env (line 1637) | def set_ollama_embeddings_env(
  function unset_ollama_embeddings_env (line 1697) | def unset_ollama_embeddings_env(
  function set_local_model_env (line 1753) | def set_local_model_env(
  function unset_local_model_env (line 1838) | def unset_local_model_env(
  function set_grok_model_env (line 1895) | def set_grok_model_env(
  function unset_grok_model_env (line 1984) | def unset_grok_model_env(
  function set_moonshot_model_env (line 2041) | def set_moonshot_model_env(
  function unset_moonshot_model_env (line 2130) | def unset_moonshot_model_env(
  function set_deepseek_model_env (line 2187) | def set_deepseek_model_env(
  function unset_deepseek_model_env (line 2277) | def unset_deepseek_model_env(
  function set_local_embeddings_env (line 2334) | def set_local_embeddings_env(
  function unset_local_embeddings_env (line 2410) | def unset_local_embeddings_env(
  function set_gemini_model_env (line 2466) | def set_gemini_model_env(
  function unset_gemini_model_env (line 2572) | def unset_gemini_model_env(
  function set_litellm_model_env (line 2626) | def set_litellm_model_env(
  function unset_litellm_model_env (line 2728) | def unset_litellm_model_env(
  function set_portkey_model_env (line 2786) | def set_portkey_model_env(
  function unset_portkey_model_env (line 2871) | def unset_portkey_model_env(
  function set_openrouter_model_env (line 2928) | def set_openrouter_model_env(
  function unset_openrouter_model_env (line 3037) | def unset_openrouter_model_env(

FILE: deepeval/cli/server.py
  function start_server (line 13) | def start_server(pairing_code: str, port: str, prod_url: str) -> str:

FILE: deepeval/cli/test/command.py
  function check_if_valid_file (line 31) | def check_if_valid_file(test_file_or_directory: str):
  function run (line 52) | def run(

FILE: deepeval/cli/types.py
  class RecommendMetricsRequestData (line 5) | class RecommendMetricsRequestData(BaseModel):
  class RecommendMetricsResponseData (line 10) | class RecommendMetricsResponseData(BaseModel):

FILE: deepeval/cli/utils.py
  function with_utm (line 50) | def with_utm(
  function render_login_message (line 101) | def render_login_message():
  function upload_and_open_link (line 109) | def upload_and_open_link(_span: Optional[Span] = None):
  function clear_evaluation_model_keys (line 144) | def clear_evaluation_model_keys():
  function clear_embedding_model_keys (line 149) | def clear_embedding_model_keys():
  function _to_str_key (line 154) | def _to_str_key(k: StrOrEnum) -> str:
  function _normalize_kv (line 158) | def _normalize_kv(updates: Dict[StrOrEnum, str]) -> Dict[str, str]:
  function _normalize_keys (line 162) | def _normalize_keys(keys: Iterable[StrOrEnum]) -> list[str]:
  function _normalize_setting_key (line 166) | def _normalize_setting_key(raw_key: str) -> str:
  function _parse_save_option (line 171) | def _parse_save_option(
  function resolve_save_target (line 183) | def resolve_save_target(save_opt: Optional[str]) -> Optional[str]:
  function save_environ_to_store (line 201) | def save_environ_to_store(
  function unset_environ_in_store (line 216) | def unset_environ_in_store(
  function _as_legacy_use_key (line 232) | def _as_legacy_use_key(
  function switch_model_provider (line 242) | def switch_model_provider(
  function coerce_blank_to_none (line 277) | def coerce_blank_to_none(value: Optional[str]) -> Optional[str]:
  function load_service_account_key_file (line 285) | def load_service_account_key_file(path: Path) -> str:
  function unwrap_optional (line 312) | def unwrap_optional(annotation: Any) -> Any:
  function looks_like_json_container_literal (line 327) | def looks_like_json_container_literal(raw_value: str) -> bool:
  function should_parse_json_for_field (line 334) | def should_parse_json_for_field(field_info: FieldInfo) -> bool:
  function maybe_parse_json_literal (line 340) | def maybe_parse_json_literal(raw_value: str, field_info) -> object:
  function resolve_field_names (line 353) | def resolve_field_names(settings, query: str) -> list[str]:
  function is_optional (line 371) | def is_optional(annotation) -> bool:
  function parse_and_validate (line 378) | def parse_and_validate(field_name: str, field_info, raw: str):

FILE: deepeval/confident/api.py
  function _infer_region_from_api_key (line 31) | def _infer_region_from_api_key(api_key: Optional[str]) -> Optional[str]:
  function get_base_api_url (line 54) | def get_base_api_url():
  function get_confident_api_key (line 80) | def get_confident_api_key() -> Optional[str]:
  function set_confident_api_key (line 86) | def set_confident_api_key(api_key: Optional[str]) -> None:
  function is_confident (line 107) | def is_confident():
  function log_retry_error (line 111) | def log_retry_error(retry_state: RetryCallState):
  class HttpMethods (line 118) | class HttpMethods(Enum):
  class Endpoints (line 125) | class Endpoints(Enum):
  function _sanitize_body (line 150) | def _sanitize_body(obj):
  class Api (line 168) | class Api:
    method __init__ (line 169) | def __init__(self, api_key: Optional[str] = None):
    method _http_request (line 192) | def _http_request(
    method _handle_response (line 205) | def _handle_response(
    method send_request (line 228) | def send_request(
    method a_send_request (line 276) | async def a_send_request(

FILE: deepeval/confident/types.py
  class ApiResponse (line 7) | class ApiResponse(BaseModel):
  class ConfidentApiError (line 17) | class ConfidentApiError(Exception):
    method __init__ (line 20) | def __init__(self, message: str, link: Optional[str] = None):

FILE: deepeval/config/dotenv_handler.py
  class DotenvHandler (line 5) | class DotenvHandler:
    method __init__ (line 6) | def __init__(self, path: Path):
    method upsert (line 9) | def upsert(self, mapping: dict[str, str]) -> None:
    method unset (line 15) | def unset(self, keys: set[str]) -> None:

FILE: deepeval/config/logging.py
  function apply_deepeval_log_level (line 13) | def apply_deepeval_log_level() -> None:

FILE: deepeval/config/settings.py
  function _find_legacy_enum (line 71) | def _find_legacy_enum(env_key: str):
  function _is_secret_key (line 93) | def _is_secret_key(env_key: str) -> bool:
  function _merge_legacy_keyfile_into_env (line 106) | def _merge_legacy_keyfile_into_env() -> None:
  function _discover_app_env_from_files (line 180) | def _discover_app_env_from_files(env_dir: Path) -> Optional[str]:
  function autoload_dotenv (line 191) | def autoload_dotenv() -> None:
  class PersistResult (line 237) | class PersistResult(NamedTuple):
  class Settings (line 243) | class Settings(BaseSettings):
    method __setattr__ (line 246) | def __setattr__(self, name: str, value):
    method _calc_auto_outer_timeout (line 885) | def _calc_auto_outer_timeout(self) -> float:
    method DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS (line 905) | def DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS(self) -> float:
    method DEEPEVAL_PER_TASK_TIMEOUT_SECONDS (line 930) | def DEEPEVAL_PER_TASK_TIMEOUT_SECONDS(self) -> float:
    method DEEPEVAL_TASK_GATHER_BUFFER_SECONDS (line 952) | def DEEPEVAL_TASK_GATHER_BUFFER_SECONDS(self) -> float:
    method _coerce_yes_no (line 1007) | def _coerce_yes_no(cls, v):
    method _coerce_path (line 1017) | def _coerce_path(cls, v):
    method _none_or_float (line 1038) | def _none_or_float(cls, v):
    method _validate_sample_rate (line 1048) | def _validate_sample_rate(cls, v):
    method _validate_default_save (line 1059) | def _validate_default_save(cls, v):
    method _normalize_fs (line 1078) | def _normalize_fs(cls, v):
    method _normalize_upper (line 1092) | def _normalize_upper(cls, v):
    method _normalize_lower (line 1102) | def _normalize_lower(cls, v):
    method _coerce_to_list (line 1112) | def _coerce_to_list(cls, v):
    method _validate_sdk_provider_list (line 1118) | def _validate_sdk_provider_list(cls, v):
    method _coerce_log_level (line 1155) | def _coerce_log_level(cls, v):
    method _apply_telemetry_enabled_alias (line 1187) | def _apply_telemetry_enabled_alias(cls, v):
    method _apply_deprecated_computed_env_aliases (line 1237) | def _apply_deprecated_computed_env_aliases(self):
    class _SettingsEditCtx (line 1290) | class _SettingsEditCtx:
      method __init__ (line 1300) | def __init__(
      method s (line 1314) | def s(self) -> "Settings":
      method __enter__ (line 1317) | def __enter__(self) -> "Settings._SettingsEditCtx":
      method __exit__ (line 1325) | def __exit__(self, exc_type, exc, tb):
      method switch_model_provider (line 1450) | def switch_model_provider(self, target) -> None:
    method edit (line 1486) | def edit(
    method set_model_provider (line 1499) | def set_model_provider(self, target, *, save: Optional[str] = None):
    method _expected_backoff (line 1508) | def _expected_backoff(self, attempts: int) -> float:
    method _constrain_between (line 1523) | def _constrain_between(self, value: float, lo: float, hi: float) -> fl...
  function _calc_env_fingerprint (line 1533) | def _calc_env_fingerprint() -> str:
  function get_settings (line 1550) | def get_settings() -> Settings:
  function reset_settings (line 1567) | def reset_settings(*, reload_dotenv: bool = False) -> Settings:

FILE: deepeval/config/settings_manager.py
  function _env_key (line 25) | def _env_key(k: StrOrEnum) -> str:
  function _normalize_for_env (line 29) | def _normalize_for_env(val: Any) -> Optional[str]:
  function _resolve_save_path (line 43) | def _resolve_save_path(save_opt: Optional[str]) -> Tuple[bool, Optional[...
  function update_settings_and_persist (line 63) | def update_settings_and_persist(

FILE: deepeval/config/utils.py
  function parse_bool (line 13) | def parse_bool(value: Any, default: bool = False) -> bool:
  function get_env_bool (line 51) | def get_env_bool(key: str, default: bool = False) -> bool:
  function bool_to_env_str (line 65) | def bool_to_env_str(value: bool) -> str:
  function set_env_bool (line 78) | def set_env_bool(key: str, value: Optional[bool] = False) -> None:
  function coerce_to_list (line 94) | def coerce_to_list(
  function dedupe_preserve_order (line 132) | def dedupe_preserve_order(items: Iterable[str]) -> List[str]:
  function constrain_between (line 142) | def constrain_between(value: float, lo: float, hi: float) -> float:
  function read_dotenv_file (line 147) | def read_dotenv_file(path: Path) -> dict[str, str]:

FILE: deepeval/constants.py
  class ProviderSlug (line 26) | class ProviderSlug(str, Enum):
  function slugify (line 41) | def slugify(value: Union[str, ProviderSlug]) -> str:

FILE: deepeval/contextvars.py
  function set_current_golden (line 15) | def set_current_golden(golden: Optional[Golden]):
  function get_current_golden (line 19) | def get_current_golden() -> Optional[Golden]:
  function reset_current_golden (line 23) | def reset_current_golden(token) -> None:

FILE: deepeval/dataset/api.py
  class APIDataset (line 7) | class APIDataset(BaseModel):
    method prepare_goldens_for_api (line 15) | def prepare_goldens_for_api(self):
  class APIQueueDataset (line 28) | class APIQueueDataset(BaseModel):
    method prepare_goldens_for_api (line 36) | def prepare_goldens_for_api(self):
  class DatasetHttpResponse (line 49) | class DatasetHttpResponse(BaseModel):

FILE: deepeval/dataset/dataset.py
  class EvaluationDataset (line 70) | class EvaluationDataset:
    method __init__ (line 85) | def __init__(
    method __repr__ (line 110) | def __repr__(self):
    method goldens (line 118) | def goldens(self) -> Union[List[Golden], List[ConversationalGolden]]:
    method goldens (line 125) | def goldens(
    method test_cases (line 155) | def test_cases(
    method test_cases (line 164) | def test_cases(
    method add_test_case (line 188) | def add_test_case(
    method add_golden (line 210) | def add_golden(self, golden: Union[Golden, ConversationalGolden]):
    method _add_golden (line 225) | def _add_golden(self, golden: Union[Golden, ConversationalGolden]):
    method _add_conversational_golden (line 233) | def _add_conversational_golden(
    method add_test_cases_from_csv_file (line 243) | def add_test_cases_from_csv_file(
    method add_test_cases_from_json_file (line 393) | def add_test_cases_from_json_file(
    method add_goldens_from_csv_file (line 471) | def add_goldens_from_csv_file(
    method add_goldens_from_json_file (line 654) | def add_goldens_from_json_file(
    method add_goldens_from_jsonl_file (line 735) | def add_goldens_from_jsonl_file(
    method push (line 865) | def push(
    method pull (line 901) | def pull(
    method queue (line 982) | def queue(
    method delete (line 1020) | def delete(
    method generate_goldens_from_docs (line 1033) | def generate_goldens_from_docs(
    method generate_goldens_from_contexts (line 1064) | def generate_goldens_from_contexts(
    method generate_goldens_from_scratch (line 1087) | def generate_goldens_from_scratch(
    method save_as (line 1106) | def save_as(
    method evals_iterator (line 1448) | def evals_iterator(
    method evaluate (line 1592) | def evaluate(self, task: Task):
    method _start_otel_test_run (line 1595) | def _start_otel_test_run(self, tracer: Optional[Tracer] = None) -> Con...
    method _end_otel_test_run (line 1608) | def _end_otel_test_run(self, ctx: Context, tracer: Optional[Tracer] = ...

FILE: deepeval/dataset/golden.py
  class Golden (line 8) | class Golden(BaseModel):
    method set_is_multimodal (line 46) | def set_is_multimodal(self):
    method _get_images_mapping (line 78) | def _get_images_mapping(self) -> Dict[str, MLLMImage]:
  class ConversationalGolden (line 108) | class ConversationalGolden(BaseModel):
    method set_is_multimodal (line 135) | def set_is_multimodal(self):
    method _get_images_mapping (line 167) | def _get_images_mapping(self) -> Dict[str, MLLMImage]:

FILE: deepeval/dataset/test_run_tracer.py
  function is_opentelemetry_available (line 20) | def is_opentelemetry_available():
  class RunIdSpanProcessor (line 39) | class RunIdSpanProcessor(SpanProcessor):
    method on_start (line 40) | def on_start(self, span, parent_context):
    method on_end (line 47) | def on_end(self, span) -> None:  # type: ignore[override]
    method shutdown (line 51) | def shutdown(self) -> None:  # type: ignore[override]
    method force_flush (line 55) | def force_flush(self, timeout_millis: int = 30000) -> bool:  # type: i...
  function init_global_test_run_tracer (line 60) | def init_global_test_run_tracer(api_key: Optional[str] = None):

FILE: deepeval/dataset/types.py
  class EvaluationTasks (line 7) | class EvaluationTasks:
    method __init__ (line 9) | def __init__(self):
    method append (line 12) | def append(self, obj: Any):
    method get_tasks (line 15) | def get_tasks(self) -> list[asyncio.Future]:
    method num_tasks (line 18) | def num_tasks(self):
    method clear_tasks (line 21) | def clear_tasks(self) -> None:

FILE: deepeval/dataset/utils.py
  function convert_test_cases_to_goldens (line 14) | def convert_test_cases_to_goldens(
  function convert_goldens_to_test_cases (line 33) | def convert_goldens_to_test_cases(
  function convert_convo_test_cases_to_convo_goldens (line 59) | def convert_convo_test_cases_to_convo_goldens(
  function convert_convo_goldens_to_convo_test_cases (line 80) | def convert_convo_goldens_to_convo_test_cases(
  function trimAndLoadJson (line 104) | def trimAndLoadJson(input_string: str) -> Any:
  function format_turns (line 114) | def format_turns(turns: List[Turn]) -> str:
  function parse_turns (line 153) | def parse_turns(turns_str: Any) -> List[Turn]:
  function check_tracer (line 189) | def check_tracer(tracer: Optional[Tracer] = None) -> Tracer:
  function coerce_to_task (line 208) | def coerce_to_task(obj: Any) -> asyncio.Future[Any]:

FILE: deepeval/errors.py
  class DeepEvalError (line 1) | class DeepEvalError(Exception):
  class UserAppError (line 9) | class UserAppError(Exception):
  class MissingTestCaseParamsError (line 15) | class MissingTestCaseParamsError(DeepEvalError):
  class MismatchedTestCaseInputsError (line 21) | class MismatchedTestCaseInputsError(DeepEvalError):
  class NoMetricsError (line 27) | class NoMetricsError(DeepEvalError):

FILE: deepeval/evaluate/api.py
  class APIEvaluate (line 6) | class APIEvaluate(BaseModel):

FILE: deepeval/evaluate/compare.py
  function compare (line 43) | def compare(
  function a_execute_arena_test_cases (line 125) | async def a_execute_arena_test_cases(
  function execute_arena_test_cases (line 227) | def execute_arena_test_cases(
  function _handle_metric_measurement (line 309) | def _handle_metric_measurement(
  function _a_handle_metric_measurement (line 358) | async def _a_handle_metric_measurement(
  function update_test_run_map (line 407) | def update_test_run_map(
  function process_test_runs (line 459) | def process_test_runs(
  function wrap_up_experiment (line 479) | def wrap_up_experiment(

FILE: deepeval/evaluate/configs.py
  class AsyncConfig (line 8) | class AsyncConfig:
    method __post_init__ (line 13) | def __post_init__(self):
  class DisplayConfig (line 21) | class DisplayConfig:
  class CacheConfig (line 36) | class CacheConfig:
  class ErrorConfig (line 42) | class ErrorConfig:

FILE: deepeval/evaluate/console_report.py
  function _natural_sort_key (line 46) | def _natural_sort_key(s: str):
  class EvaluationConsoleReport (line 53) | class EvaluationConsoleReport:
    method __init__ (line 54) | def __init__(self, test_results: List[TestResult]):
    method _build_display_elements (line 64) | def _build_display_elements(self, truncate: bool = True) -> Group:
    method render_to_terminal (line 208) | def render_to_terminal(self, truncate_passing_cases: bool = True):
    method export_to_html (line 215) | def export_to_html(
    method export_to_markdown (line 250) | def export_to_markdown(

FILE: deepeval/evaluate/evaluate.py
  function assert_test (line 66) | def assert_test(
  function evaluate (line 157) | def evaluate(

FILE: deepeval/evaluate/execute/_common.py
  function _timeout_msg (line 54) | def _timeout_msg(action: str, seconds: float) -> str:
  function _log_gather_timeout (line 68) | def _log_gather_timeout(
  function _skip_metrics_for_error (line 105) | def _skip_metrics_for_error(
  function _trace_error (line 118) | def _trace_error(current_trace: Trace) -> Optional[str]:
  function _get_trace_by_uuid_anywhere (line 135) | def _get_trace_by_uuid_anywhere(trace_uuid: str):
  function _pick_root_for_marking (line 153) | def _pick_root_for_marking(trace):
  function _resolve_trace_and_root_for_task (line 174) | def _resolve_trace_and_root_for_task(t: asyncio.Task):
  function _snapshot_tasks (line 207) | async def _snapshot_tasks():
  function filter_duplicate_results (line 213) | def filter_duplicate_results(
  function _await_with_outer_deadline (line 227) | async def _await_with_outer_deadline(obj, *args, timeout: float, **kwargs):
  function _execute_metric (line 243) | def _execute_metric(
  function log_prompt (line 298) | def log_prompt(

FILE: deepeval/evaluate/execute/agentic.py
  function _a_execute_agentic_test_case (line 74) | async def _a_execute_agentic_test_case(
  function _a_execute_span_test_case (line 330) | async def _a_execute_span_test_case(
  function _a_execute_trace_test_case (line 439) | async def _a_execute_trace_test_case(

FILE: deepeval/evaluate/execute/e2e.py
  function execute_test_cases (line 81) | def execute_test_cases(
  function a_execute_test_cases (line 333) | async def a_execute_test_cases(
  function _a_execute_llm_test_cases (line 534) | async def _a_execute_llm_test_cases(
  function _a_execute_conversational_test_cases (line 662) | async def _a_execute_conversational_test_cases(
  function _evaluate_test_case_pairs (line 750) | async def _evaluate_test_case_pairs(

FILE: deepeval/evaluate/execute/loop.py
  function _span_subtree_has_metrics (line 104) | def _span_subtree_has_metrics(span: BaseSpan) -> bool:
  function _has_any_evaluable_metrics (line 111) | def _has_any_evaluable_metrics(
  function _raise_no_metrics_error (line 138) | def _raise_no_metrics_error() -> None:
  function execute_agentic_test_cases_from_loop (line 148) | def execute_agentic_test_cases_from_loop(
  function a_execute_agentic_test_cases_from_loop (line 501) | def a_execute_agentic_test_cases_from_loop(
  function _a_evaluate_traces (line 942) | async def _a_evaluate_traces(

FILE: deepeval/evaluate/execute/trace_scope.py
  function _assert_test_from_current_trace (line 61) | def _assert_test_from_current_trace(

FILE: deepeval/evaluate/local_store.py
  function resolve_target_dir (line 36) | def resolve_target_dir(
  function resolve_test_run_path (line 58) | def resolve_test_run_path(target_dir: Path) -> Path:
  function write_test_run (line 82) | def write_test_run(
  function _dump (line 109) | def _dump(test_run: TestRun, path: Path) -> None:

FILE: deepeval/evaluate/types.py
  class TestResult (line 11) | class TestResult:
  class EvaluationResult (line 30) | class EvaluationResult(BaseModel):
  class PostExperimentRequest (line 36) | class PostExperimentRequest(BaseModel):

FILE: deepeval/evaluate/utils.py
  function _is_metric_successful (line 32) | def _is_metric_successful(metric_data: MetricData) -> bool:
  function create_metric_data (line 55) | def create_metric_data(metric: BaseMetric) -> MetricData:
  function create_arena_metric_data (line 84) | def create_arena_metric_data(metric: ArenaGEval, contestant: str) -> Met...
  function create_test_result (line 113) | def create_test_result(
  function create_api_trace (line 160) | def create_api_trace(trace: Trace, golden: Golden) -> TraceApi:
  function validate_assert_test_inputs (line 236) | def validate_assert_test_inputs(
  function validate_evaluate_inputs (line 278) | def validate_evaluate_inputs(
  function print_test_result (line 317) | def print_test_result(test_result: TestResult, display: TestRunResultDis...
  function write_test_result_to_file (line 370) | def write_test_result_to_file(
  function aggregate_metric_pass_rates (line 471) | def aggregate_metric_pass_rates(test_results: List[TestResult]) -> dict:
  function count_metrics_in_trace (line 503) | def count_metrics_in_trace(trace: Trace) -> int:
  function count_total_metrics_for_trace (line 513) | def count_total_metrics_for_trace(trace: Trace) -> int:
  function count_metrics_in_span_subtree (line 518) | def count_metrics_in_span_subtree(span: BaseSpan) -> int:
  function extract_trace_test_results (line 525) | def extract_trace_test_results(trace_api: TraceApi) -> List[TestResult]:
  function extract_span_test_results (line 550) | def extract_span_test_results(span_api: BaseApiSpan) -> List[TestResult]:

FILE: deepeval/inspect/__init__.py
  function run_inspect (line 4) | def run_inspect(path: str) -> None:

FILE: deepeval/inspect/__main__.py
  function main (line 20) | def main(argv: list[str] | None = None) -> int:

FILE: deepeval/inspect/app.py
  class InspectApp (line 30) | class InspectApp(App[None]):
    method __init__ (line 55) | def __init__(
    method compose (line 70) | def compose(self) -> ComposeResult:
    method on_mount (line 78) | def on_mount(self) -> None:
    method action_cycle_trace (line 83) | async def action_cycle_trace(self, delta: int) -> None:
    method check_action (line 113) | def check_action(self, action: str, parameters: tuple) -> Optional[bool]:
    method action_toggle_search (line 121) | def action_toggle_search(self) -> None:
    method on_search_changed (line 130) | def on_search_changed(self, event: SearchBar.Changed) -> None:
    method on_search_submitted (line 141) | def on_search_submitted(self, _event: SearchBar.Submitted) -> None:
    method finish_search (line 147) | def finish_search(self) -> None:
    method action_yank_node (line 152) | def action_yank_node(self) -> None:
    method action_yank_trace (line 162) | def action_yank_trace(self) -> None:
    method _copy_to_clipboard (line 169) | def _copy_to_clipboard(self, body: str, label: str) -> None:
    method action_toggle_help (line 188) | def action_toggle_help(self) -> None:
    method action_tree_cursor (line 194) | def action_tree_cursor(self, direction: str) -> None:
    method on_tree_node_highlighted (line 207) | async def on_tree_node_highlighted(
    method _current_trace (line 217) | def _current_trace(self) -> Trace:
    method _selected_node (line 220) | def _selected_node(self) -> Optional[TraceOrSpan]:
    method _refresh_tree (line 229) | def _refresh_tree(self) -> None:
    method _refresh_header (line 233) | def _refresh_header(self) -> None:

FILE: deepeval/inspect/loader.py
  class InspectLoadError (line 18) | class InspectLoadError(Exception):
  class NoTracesError (line 22) | class NoTracesError(InspectLoadError):
  function find_latest_test_run (line 35) | def find_latest_test_run(folder: str | Path) -> Path:
  function load_test_run (line 62) | def load_test_run(path: str | Path) -> List[Trace]:
  function _parse_trace (line 90) | def _parse_trace(trace_dict: Dict[str, Any]) -> Trace:
  function _build_span_tree (line 109) | def _build_span_tree(
  function _has_known_parent (line 131) | def _has_known_parent(span: BaseSpan, by_uuid: Dict[str, BaseSpan]) -> b...
  function run_id_from_path (line 135) | def run_id_from_path(path: str | Path) -> str:
  function summarize_test_run (line 139) | def summarize_test_run(path: str | Path) -> Optional[Dict[str, Any]]:

FILE: deepeval/inspect/types.py
  class BaseSpan (line 18) | class BaseSpan(BaseApiSpan):
  class Trace (line 22) | class Trace(TraceApi):
  function _parse_iso (line 48) | def _parse_iso(ts: Optional[str]) -> Optional[datetime]:
  function duration_ms (line 58) | def duration_ms(node: TraceOrSpan) -> Optional[float]:
  function format_duration (line 66) | def format_duration(ms: Optional[float]) -> str:
  function metric_counts (line 74) | def metric_counts(
  function has_failure (line 83) | def has_failure(node: TraceOrSpan) -> bool:
  function iter_descendants (line 91) | def iter_descendants(span: BaseSpan):
  function all_spans (line 97) | def all_spans(trace: Trace) -> List[BaseSpan]:

FILE: deepeval/inspect/widgets/_styling.py
  function type_style (line 32) | def type_style(node: TraceOrSpan) -> Tuple[str, str, str]:
  function type_prefix (line 38) | def type_prefix(node: TraceOrSpan) -> Text:

FILE: deepeval/inspect/widgets/details.py
  class DetailsPane (line 39) | class DetailsPane(VerticalScroll):
    method compose (line 69) | def compose(self) -> ComposeResult:
    method show (line 76) | async def show(self, node: Optional[TraceOrSpan]) -> None:
  function _divider (line 105) | def _divider(label: str) -> Static:
  function _header_section (line 114) | def _header_section(node: TraceOrSpan) -> List[Any]:
  function _format_tokens (line 151) | def _format_tokens(span: BaseSpan) -> Optional[str]:
  function _estimate_cost (line 159) | def _estimate_cost(span: BaseSpan) -> Optional[float]:
  function _metric_badges_section (line 172) | def _metric_badges_section(node: TraceOrSpan) -> List[Any]:
  function _meta_strip_section (line 190) | def _meta_strip_section(node: TraceOrSpan) -> List[Any]:
  function _kv_block (line 224) | def _kv_block(label: str, data: dict) -> Static:
  function _confident_cta_section (line 231) | def _confident_cta_section(node: TraceOrSpan) -> List[Any]:
  function _cta_divider (line 264) | def _cta_divider(label: str) -> Static:
  function _metrics_section (line 271) | def _metrics_section(node: TraceOrSpan) -> List[Any]:
  function _metric_block (line 289) | def _metric_block(metric: MetricData) -> List[Any]:
  function _type_specific_section (line 316) | def _type_specific_section(node: TraceOrSpan) -> List[Any]:
  function _llm_block (line 333) | def _llm_block(span: BaseSpan) -> List[Any]:
  function _retriever_block (line 361) | def _retriever_block(span: BaseSpan) -> List[Any]:
  function _tool_block (line 374) | def _tool_block(span: BaseSpan) -> List[Any]:
  function _agent_block (line 383) | def _agent_block(span: BaseSpan) -> List[Any]:
  function _kv_table (line 394) | def _kv_table(rows: List[tuple[str, Any]]) -> Static:
  function _input_section (line 404) | def _input_section(node: TraceOrSpan) -> List[Any]:
  function _output_section (line 410) | def _output_section(node: TraceOrSpan) -> List[Any]:
  function _payload_widget (line 416) | def _payload_widget(value: Any) -> Static:
  function _optional_sections (line 426) | def _optional_sections(node: TraceOrSpan) -> List[Any]:
  function _raw_json_section (line 448) | def _raw_json_section(node: TraceOrSpan) -> List[Any]:

FILE: deepeval/inspect/widgets/header_bar.py
  class HeaderBar (line 11) | class HeaderBar(Static):
    method render_run_header (line 21) | def render_run_header(

FILE: deepeval/inspect/widgets/help_modal.py
  class HelpScreen (line 27) | class HelpScreen(ModalScreen[None]):
    method compose (line 56) | def compose(self) -> ComposeResult:
    method action_dismiss (line 65) | def action_dismiss(self, _result: object = None) -> None:

FILE: deepeval/inspect/widgets/search_bar.py
  class SearchBar (line 9) | class SearchBar(Input):
    method __init__ (line 26) | def __init__(self, **kwargs):
    method action_hide_and_clear (line 35) | def action_hide_and_clear(self) -> None:

FILE: deepeval/inspect/widgets/span_tree.py
  function _metric_badge (line 28) | def _metric_badge(node: TraceOrSpan) -> Optional[Text]:
  function _label_for (line 43) | def _label_for(node: TraceOrSpan) -> Text:
  class SpanTree (line 70) | class SpanTree(Tree[TraceOrSpan]):
    method __init__ (line 85) | def __init__(self, *args, **kwargs):
    method populate (line 91) | def populate(
    method _add_span (line 110) | def _add_span(
  function _subtree_matches (line 132) | def _subtree_matches(span: BaseSpan, span_filter: SpanFilter) -> bool:

FILE: deepeval/integrations/agentcore/instrumentator.py
  class _SpanProcessor (line 71) | class _SpanProcessor:
    method __init__ (line 72) | def __init__(self, *args: Any, **kwargs: Any) -> None:
    method on_start (line 75) | def on_start(self, span: Any, parent_context: Any) -> None:
    method on_end (line 78) | def on_end(self, span: Any) -> None:
  class _ReadableSpan (line 81) | class _ReadableSpan:
  function is_dependency_installed (line 85) | def is_dependency_installed() -> bool:
  function _get_attr (line 127) | def _get_attr(span, *keys: str) -> Optional[str]:
  function _classify_span (line 136) | def _classify_span(span) -> Optional[str]:
  function _get_agent_name (line 172) | def _get_agent_name(span) -> Optional[str]:
  function _get_tool_name (line 185) | def _get_tool_name(span) -> Optional[str]:
  function _parse_genai_content (line 197) | def _parse_genai_content(raw: Any) -> Optional[str]:
  function _extract_messages (line 216) | def _extract_messages(span) -> tuple[Optional[str], Optional[str]]:
  function _extract_tool_calls (line 293) | def _extract_tool_calls(span) -> List[ToolCall]:
  function _extract_tool_call_from_tool_span (line 367) | def _extract_tool_call_from_tool_span(span) -> Optional[ToolCall]:
  class AgentCoreInstrumentationSettings (line 392) | class AgentCoreInstrumentationSettings:
    method __init__ (line 412) | def __init__(
  class AgentCoreSpanInterceptor (line 468) | class AgentCoreSpanInterceptor(SpanProcessor):
    method __init__ (line 470) | def __init__(self, settings_instance: AgentCoreInstrumentationSettings):
    method on_start (line 479) | def on_start(self, span, parent_context):
    method on_end (line 510) | def on_end(self, span):
    method _push_span_context (line 570) | def _push_span_context(self, span, span_type: Optional[str]) -> None:
    method _maybe_push_implicit_trace_context (line 616) | def _maybe_push_implicit_trace_context(self, span) -> None:
    method _maybe_bridge_otel_root_to_deepeval_parent (line 651) | def _maybe_bridge_otel_root_to_deepeval_parent(self, span) -> None:
    method _maybe_pop_implicit_trace_context (line 678) | def _maybe_pop_implicit_trace_context(self, span) -> None:
    method _set_attr_post_end (line 698) | def _set_attr_post_end(span, key: str, value: Any) -> None:
    method _serialize_placeholder_to_otel_attrs (line 723) | def _serialize_placeholder_to_otel_attrs(
    method _serialize_trace_context_to_otel_attrs (line 779) | def _serialize_trace_context_to_otel_attrs(self, span) -> None:
    method _serialize_framework_attrs (line 852) | def _serialize_framework_attrs(self, span) -> None:

FILE: deepeval/integrations/agentcore/otel.py
  function _require_opentelemetry (line 43) | def _require_opentelemetry() -> None:
  function instrument_agentcore (line 63) | def instrument_agentcore(

FILE: deepeval/integrations/crewai/handler.py
  function is_crewai_installed (line 60) | def is_crewai_installed():
  function _get_metrics_data (line 67) | def _get_metrics_data(obj: Any) -> Tuple[Optional[str], Optional[Any]]:
  class CrewAIEventsListener (line 90) | class CrewAIEventsListener(BaseEventListener):
    method __init__ (line 91) | def __init__(self):
    method reset_state (line 99) | def reset_state(self):
    method get_tool_stack_key (line 105) | def get_tool_stack_key(source, tool_name) -> str:
    method get_knowledge_execution_id (line 116) | def get_knowledge_execution_id(source, event) -> str:
    method get_llm_execution_id (line 124) | def get_llm_execution_id(source, event) -> str:
    method _flatten_tool_span (line 128) | def _flatten_tool_span(self, span):
    method setup_listeners (line 158) | def setup_listeners(self, crewai_event_bus):
  function instrument_crewai (line 449) | def instrument_crewai(api_key: Optional[str] = None):
  function reset_crewai_instrumentation (line 463) | def reset_crewai_instrumentation():
  function wrap_all (line 469) | def wrap_all():

FILE: deepeval/integrations/crewai/subs.py
  function is_crewai_installed (line 14) | def is_crewai_installed():
  function create_deepeval_class (line 24) | def create_deepeval_class(base_class: Type[T], class_name: str) -> Type[T]:
  function create_deepeval_llm (line 44) | def create_deepeval_llm(base_factory: Callable) -> Callable:

FILE: deepeval/integrations/crewai/tool.py
  function tool (line 9) | def tool(*args, metric=None, metric_collection=None, **kwargs) -> Callable:

FILE: deepeval/integrations/crewai/wrapper.py
  function wrap_crew_kickoff (line 10) | def wrap_crew_kickoff():
  function wrap_crew_kickoff_for_each (line 31) | def wrap_crew_kickoff_for_each():
  function wrap_crew_kickoff_async (line 52) | def wrap_crew_kickoff_async():
  function wrap_crew_kickoff_for_each_async (line 73) | def wrap_crew_kickoff_for_each_async():
  function wrap_crew_akickoff (line 96) | def wrap_crew_akickoff():
  function wrap_crew_akickoff_for_each (line 120) | def wrap_crew_akickoff_for_each():
  function wrap_agent_execute_task (line 144) | def wrap_agent_execute_task():
  function wrap_agent_aexecute_task (line 164) | def wrap_agent_aexecute_task():
  function _check_metrics_and_metric_collection (line 187) | def _check_metrics_and_metric_collection(obj: Any):
  function _set_observer_integration (line 193) | def _set_observer_integration(observer: Observer, integration: str):

FILE: deepeval/integrations/google_adk/otel.py
  function _require_google_adk_instrumentor (line 13) | def _require_google_adk_instrumentor():
  function instrument_google_adk (line 28) | def instrument_google_adk(

FILE: deepeval/integrations/hugging_face/callback.py
  class DeepEvalHuggingFaceCallback (line 19) | class DeepEvalHuggingFaceCallback(TrainerCallback):
    method __init__ (line 31) | def __init__(
    method _calculate_metric_scores (line 64) | def _calculate_metric_scores(self) -> Dict[str, List[float]]:
    method _aggregate_scores (line 85) | def _aggregate_scores(
    method on_epoch_begin (line 112) | def on_epoch_begin(
    method on_epoch_end (line 124) | def on_epoch_end(
    method on_log (line 146) | def on_log(
    method _generate_table (line 176) | def _generate_table(self):
    method on_train_end (line 195) | def on_train_end(
    method on_train_begin (line 210) | def on_train_begin(
    method __init__ (line 228) | def __init__(self, *args, **kwargs):
  class DeepEvalHuggingFaceCallback (line 227) | class DeepEvalHuggingFaceCallback:
    method __init__ (line 31) | def __init__(
    method _calculate_metric_scores (line 64) | def _calculate_metric_scores(self) -> Dict[str, List[float]]:
    method _aggregate_scores (line 85) | def _aggregate_scores(
    method on_epoch_begin (line 112) | def on_epoch_begin(
    method on_epoch_end (line 124) | def on_epoch_end(
    method on_log (line 146) | def on_log(
    method _generate_table (line 176) | def _generate_table(self):
    method on_train_end (line 195) | def on_train_end(
    method on_train_begin (line 210) | def on_train_begin(
    method __init__ (line 228) | def __init__(self, *args, **kwargs):

FILE: deepeval/integrations/hugging_face/rich_manager.py
  class RichManager (line 11) | class RichManager:
    method __init__ (line 12) | def __init__(self, show_table: bool, total_train_epochs: int) -> None:
    method _initialize_progress_trackers (line 44) | def _initialize_progress_trackers(self) -> None:
    method change_spinner_text (line 62) | def change_spinner_text(self, text: str) -> None:
    method stop (line 71) | def stop(self) -> None:
    method start (line 75) | def start(self) -> None:
    method update (line 80) | def update(self, column: Columns) -> None:
    method create_column (line 89) | def create_column(self) -> Union[Columns, Table]:
    method advance_progress (line 104) | def advance_progress(self) -> None:

FILE: deepeval/integrations/hugging_face/tests/test_callbacks.py
  function create_prompt (line 23) | def create_prompt(row):
  function prepare_dataset (line 33) | def prepare_dataset(tokenizer, tokenizer_args):
  function create_deepeval_dataset (line 74) | def create_deepeval_dataset(dataset, sample_size):

FILE: deepeval/integrations/hugging_face/utils.py
  function get_column_order (line 7) | def get_column_order(scores: Dict) -> List[str]:
  function generate_test_cases (line 22) | def generate_test_cases(

FILE: deepeval/integrations/langchain/callback.py
  function _debug_log (line 41) | def _debug_log(msg: str):
  function is_langchain_installed (line 70) | def is_langchain_installed():
  class CallbackHandler (line 77) | class CallbackHandler(BaseCallbackHandler):
    method __init__ (line 85) | def __init__(
    method _ensure_trace (line 135) | def _ensure_trace(self):
    method _ctx (line 228) | def _ctx(self, run_id: UUID, parent_run_id: Optional[UUID] = None):
    method on_chain_start (line 282) | def on_chain_start(
    method on_chain_end (line 319) | def on_chain_end(
    method on_chat_model_start (line 344) | def on_chat_model_start(
    method on_llm_start (line 414) | def on_llm_start(
    method on_llm_end (line 474) | def on_llm_end(
    method on_chat_model_end (line 555) | def on_chat_model_end(
    method on_chat_model_error (line 639) | def on_chat_model_error(
    method on_llm_error (line 673) | def on_llm_error(
    method on_llm_new_token (line 702) | def on_llm_new_token(
    method on_tool_start (line 722) | def on_tool_start(
    method on_tool_end (line 758) | def on_tool_end(
    method on_tool_error (line 806) | def on_tool_error(
    method on_retriever_start (line 823) | def on_retriever_start(
    method on_retriever_end (line 860) | def on_retriever_end(
    method on_retriever_error (line 885) | def on_retriever_error(

FILE: deepeval/integrations/langchain/patch.py
  function tool (line 8) | def tool(
  function _patch_tool_decorator (line 27) | def _patch_tool_decorator(

FILE: deepeval/integrations/langchain/utils.py
  function convert_chat_messages_to_input (line 21) | def convert_chat_messages_to_input(
  function parse_prompts_to_messages (line 77) | def parse_prompts_to_messages(
  function convert_chat_generation_to_string (line 139) | def convert_chat_generation_to_string(gen: ChatGeneration) -> str:
  function prepare_dict (line 143) | def prepare_dict(**kwargs: Any) -> dict[str, Any]:
  function safe_extract_token_usage (line 147) | def safe_extract_token_usage(
  function extract_name (line 169) | def extract_name(serialized: dict[str, Any], **kwargs: Any) -> str:
  function safe_extract_model_name (line 179) | def safe_extract_model_name(
  function safe_extract_provider (line 197) | def safe_extract_provider(
  function enter_current_context (line 215) | def enter_current_context(
  function exit_current_context (line 342) | def exit_current_context(

FILE: deepeval/integrations/llama_index/handler.py
  function is_llama_index_installed (line 64) | def is_llama_index_installed():
  class LLamaIndexHandler (line 71) | class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
    method __init__ (line 76) | def __init__(self):
    method handle (line 80) | def handle(self, event: BaseEvent, **kwargs) -> Any:
    method new_span (line 189) | def new_span(
    method _get_output_value (line 296) | def _get_output_value(self, result: Any) -> Any:
    method prepare_to_exit_span (line 312) | def prepare_to_exit_span(
    method prepare_to_drop_span (line 386) | def prepare_to_drop_span(
  function instrument_llama_index (line 409) | def instrument_llama_index(dispatcher: Dispatcher):

FILE: deepeval/integrations/llama_index/utils.py
  function is_llama_index_agent_installed (line 17) | def is_llama_index_agent_installed():
  function parse_id (line 24) | def parse_id(id_: str) -> tuple[str, str]:
  function prepare_input_llm_test_case_params (line 45) | def prepare_input_llm_test_case_params(
  function prepare_output_llm_test_case_params (line 65) | def prepare_output_llm_test_case_params(
  function extract_output_from_llm_chat_end_event (line 87) | def extract_output_from_llm_chat_end_event(event: LLMChatEndEvent) -> list:

FILE: deepeval/integrations/openinference/instrumentator.py
  class _SpanProcessor (line 76) | class _SpanProcessor:
    method __init__ (line 77) | def __init__(self, *args: Any, **kwargs: Any) -> None:
    method on_start (line 80) | def on_start(self, span: Any, parent_context: Any) -> None:
    method on_end (line 83) | def on_end(self, span: Any) -> None:
  class _ReadableSpan (line 86) | class _ReadableSpan:
  function is_dependency_installed (line 90) | def is_dependency_installed() -> bool:
  function _get_span_kind (line 114) | def _get_span_kind(span) -> Optional[str]:
  function _get_agent_name (line 137) | def _get_agent_name(span) -> Optional[str]:
  function _get_tool_name (line 146) | def _get_tool_name(span) -> Optional[str]:
  function _extract_messages (line 161) | def _extract_messages(span) -> tuple[Optional[str], Optional[str]]:
  function _extract_tool_calls (line 248) | def _extract_tool_calls(span) -> List[ToolCall]:
  function _extract_tool_call_from_tool_span (line 333) | def _extract_tool_call_from_tool_span(span) -> Optional[ToolCall]:
  class OpenInferenceInstrumentationSettings (line 358) | class OpenInferenceInstrumentationSettings:
    method __init__ (line 378) | def __init__(
  class OpenInferenceSpanInterceptor (line 435) | class OpenInferenceSpanInterceptor(SpanProcessor):
    method __init__ (line 437) | def __init__(self, settings_instance: OpenInferenceInstrumentationSett...
    method on_start (line 446) | def on_start(self, span, parent_context):
    method on_end (line 480) | def on_end(self, span):
    method _push_span_context (line 540) | def _push_span_context(self, span, span_type: Optional[str]) -> None:
    method _maybe_push_implicit_trace_context (line 590) | def _maybe_push_implicit_trace_context(self, span) -> None:
    method _maybe_bridge_otel_root_to_deepeval_parent (line 625) | def _maybe_bridge_otel_root_to_deepeval_parent(self, span) -> None:
    method _maybe_pop_implicit_trace_context (line 652) | def _maybe_pop_implicit_trace_context(self, span) -> None:
    method _set_attr_post_end (line 672) | def _set_attr_post_end(span, key: str, value: Any) -> None:
    method _serialize_placeholder_to_otel_attrs (line 697) | def _serialize_placeholder_to_otel_attrs(
    method _serialize_trace_context_to_otel_attrs (line 757) | def _serialize_trace_context_to_otel_attrs(self, span) -> None:
    method _serialize_framework_attrs (line 822) | def _serialize_framework_attrs(self, span) -> None:

FILE: deepeval/integrations/openinference/otel.py
  function _require_opentelemetry (line 45) | def _require_opentelemetry() -> None:
  function instrument_openinference (line 66) | def instrument_openinference(

FILE: deepeval/integrations/pydantic_ai/instrumentator.py
  class _BaseInstrumentationSettings (line 84) | class _BaseInstrumentationSettings:
    method __init__ (line 85) | def __init__(self, *args: Any, **kwargs: Any) -> None:
  class _SpanProcessor (line 88) | class _SpanProcessor:
    method __init__ (line 89) | def __init__(self, *args: Any, **kwargs: Any) -> None:
    method on_start (line 92) | def on_start(self, span: Any, parent_context: Any) -> None:
    method on_end (line 95) | def on_end(self, span: Any) -> None:
  class _ReadableSpan (line 98) | class _ReadableSpan:
  function is_dependency_installed (line 102) | def is_dependency_installed() -> bool:
  class DeepEvalInstrumentationSettings (line 126) | class DeepEvalInstrumentationSettings(InstrumentationSettings):
    method __init__ (line 162) | def __init__(
  class ConfidentInstrumentationSettings (line 230) | class ConfidentInstrumentationSettings(DeepEvalInstrumentationSettings):
    method __init__ (line 241) | def __init__(self, *args: Any, **kwargs: Any) -> None:
  class SpanInterceptor (line 253) | class SpanInterceptor(SpanProcessor):
    method __init__ (line 278) | def __init__(self, settings_instance: DeepEvalInstrumentationSettings):
    method on_start (line 291) | def on_start(self, span, parent_context):
    method on_end (line 359) | def on_end(self, span):
    method _push_span_context (line 460) | def _push_span_context(
    method _maybe_push_implicit_trace_context (line 521) | def _maybe_push_implicit_trace_context(self, span) -> None:
    method _maybe_bridge_otel_root_to_deepeval_parent (line 566) | def _maybe_bridge_otel_root_to_deepeval_parent(self, span) -> None:
    method _maybe_pop_implicit_trace_context (line 610) | def _maybe_pop_implicit_trace_context(self, span) -> None:
    method _set_attr_post_end (line 635) | def _set_attr_post_end(span, key: str, value: Any) -> None:
    method _serialize_placeholder_to_otel_attrs (line 675) | def _serialize_placeholder_to_otel_attrs(
    method _serialize_trace_context_to_otel_attrs (line 732) | def _serialize_trace_context_to_otel_attrs(self, span) -> None:
    method _add_agent_span (line 805) | def _add_agent_span(self, span, name):
    method _is_agent_span (line 814) | def _is_agent_span(self, operation_name: Optional[str]) -> bool:

FILE: deepeval/integrations/pydantic_ai/otel.py
  function is_opentelemetry_available (line 21) | def is_opentelemetry_available():
  function instrument_pydantic_ai (line 38) | def instrument_pydantic_ai(api_key: Optional[str] = None):

FILE: deepeval/integrations/strands/instrumentator.py
  class _SpanProcessor (line 79) | class _SpanProcessor:
    method __init__ (line 80) | def __init__(self, *args: Any, **kwargs: Any) -> None:
    method on_start (line 83) | def on_start(self, span: Any, parent_context: Any) -> None:
    method on_end (line 86) | def on_end(self, span: Any) -> None:
  class _ReadableSpan (line 89) | class _ReadableSpan:
  function is_dependency_installed (line 93) | def is_dependency_installed() -> bool:
  function _get_attr (line 136) | def _get_attr(span, *keys: str) -> Optional[str]:
  function _classify_span (line 145) | def _classify_span(span) -> Optional[str]:
  function _get_agent_name (line 181) | def _get_agent_name(span) -> Optional[str]:
  function _get_tool_name (line 194) | def _get_tool_name(span) -> Optional[str]:
  function _parse_genai_content (line 207) | def _parse_genai_content(raw: Any) -> Optional[str]:
  function _extract_messages (line 226) | def _extract_messages(span) -> tuple[Optional[str], Optional[str]]:
  function _extract_tool_calls (line 303) | def _extract_tool_calls(span) -> List[ToolCall]:
  function _extract_tool_call_from_tool_span (line 377) | def _extract_tool_call_from_tool_span(span) -> Optional[ToolCall]:
  class StrandsInstrumentationSettings (line 402) | class StrandsInstrumentationSettings:
    method __init__ (line 422) | def __init__(
  class StrandsSpanInterceptor (line 478) | class StrandsSpanInterceptor(SpanProcessor):
    method __init__ (line 480) | def __init__(self, settings_instance: StrandsInstrumentationSettings):
    method on_start (line 489) | def on_start(self, span, parent_context):
    method on_end (line 520) | def on_end(self, span):
    method _push_span_context (line 580) | def _push_span_context(self, span, span_type: Optional[str]) -> None:
    method _maybe_push_implicit_trace_context (line 626) | def _maybe_push_implicit_trace_context(self, span) -> None:
    method _maybe_bridge_otel_root_to_deepeval_parent (line 661) | def _maybe_bridge_otel_root_to_deepeval_parent(self, span) -> None:
    method _maybe_pop_implicit_trace_context (line 688) | def _maybe_pop_implicit_trace_context(self, span) -> None:
    method _set_attr_post_end (line 708) | def _set_attr_post_end(span, key: str, value: Any) -> None:
    method _serialize_placeholder_to_otel_attrs (line 733) | def _serialize_placeholder_to_otel_attrs(
    method _serialize_trace_context_to_otel_attrs (line 789) | def _serialize_trace_context_to_otel_attrs(self, span) -> None:
    method _serialize_framework_attrs (line 864) | def _serialize_framework_attrs(self, span) -> None:

FILE: deepeval/integrations/strands/otel.py
  function _require_opentelemetry (line 53) | def _require_opentelemetry() -> None:
  function instrument_strands (line 73) | def instrument_strands(

FILE: deepeval/key_handler.py
  function _secret_env_keys (line 18) | def _secret_env_keys() -> frozenset[str]:
  function _env_key_for_legacy_enum (line 36) | def _env_key_for_legacy_enum(key) -> str:
  function _is_secret_key (line 40) | def _is_secret_key(key) -> bool:
  class KeyValues (line 47) | class KeyValues(Enum):
  class ModelKeyValues (line 58) | class ModelKeyValues(Enum):
  class EmbeddingKeyValues (line 169) | class EmbeddingKeyValues(Enum):
  class KeyFileHandler (line 183) | class KeyFileHandler:
    method __init__ (line 184) | def __init__(self):
    method _ensure_dir (line 187) | def _ensure_dir(self):
    method write_key (line 190) | def write_key(
    method fetch_data (line 224) | def fetch_data(
    method remove_key (line 259) | def remove_key(

FILE: deepeval/metrics/answer_relevancy/answer_relevancy.py
  class AnswerRelevancyMetric (line 27) | class AnswerRelevancyMetric(BaseMetric):
    method __init__ (line 33) | def __init__(
    method measure (line 54) | def measure(
    method a_measure (line 110) | async def a_measure(
    method _a_generate_reason (line 159) | async def _a_generate_reason(self, input: str, multimodal: bool) -> str:
    method _generate_reason (line 183) | def _generate_reason(self, input: str, multimodal: bool) -> str:
    method _a_generate_verdicts (line 207) | async def _a_generate_verdicts(
    method _generate_verdicts (line 227) | def _generate_verdicts(
    method _generate_statements (line 247) | def _generate_statements(
    method _a_generate_statements (line 266) | async def _a_generate_statements(
    method _calculate_score (line 285) | def _calculate_score(self):
    method is_successful (line 298) | def is_successful(self) -> bool:
    method __name__ (line 309) | def __name__(self):

FILE: deepeval/metrics/answer_relevancy/schema.py
  class Statements (line 5) | class Statements(BaseModel):
  class AnswerRelevancyVerdict (line 9) | class AnswerRelevancyVerdict(BaseModel):
  class Verdicts (line 14) | class Verdicts(BaseModel):
  class AnswerRelevancyScoreReason (line 18) | class AnswerRelevancyScoreReason(BaseModel):

FILE: deepeval/metrics/answer_relevancy/template.py
  class AnswerRelevancyTemplate (line 5) | class AnswerRelevancyTemplate:
    method generate_statements (line 15) | def generate_statements(actual_output: str, multimodal: bool = False):
    method generate_verdicts (line 46) | def generate_verdicts(
    method generate_reason (line 95) | def generate_reason(

FILE: deepeval/metrics/arena_g_eval/arena_g_eval.py
  class ArenaGEval (line 36) | class ArenaGEval(BaseArenaMetric):
    method __init__ (line 37) | def __init__(
    method measure (line 63) | def measure(
    method a_measure (line 118) | async def a_measure(
    method _a_generate_evaluation_steps (line 167) | async def _a_generate_evaluation_steps(self, multimodal: bool) -> List...
    method _generate_evaluation_steps (line 188) | def _generate_evaluation_steps(self, multimodal: bool) -> List[str]:
    method _a_compare (line 208) | async def _a_compare(
    method _compare (line 240) | def _compare(
    method _a_generate_rewritten_reason (line 271) | async def _a_generate_rewritten_reason(
    method _generate_rewritten_reason (line 289) | def _generate_rewritten_reason(
    method is_successful (line 306) | def is_successful(self) -> bool:
    method __name__ (line 312) | def __name__(self):

FILE: deepeval/metrics/arena_g_eval/schema.py
  class ReasonScore (line 5) | class ReasonScore(BaseModel):
  class RewrittenReason (line 10) | class RewrittenReason(BaseModel):
  class Winner (line 14) | class Winner(BaseModel):
  class Steps (line 19) | class Steps(BaseModel):

FILE: deepeval/metrics/arena_g_eval/template.py
  class ArenaGEvalTemplate (line 5) | class ArenaGEvalTemplate:
    method generate_evaluation_steps (line 15) | def generate_evaluation_steps(
    method generate_arena_winner (line 39) | def generate_arena_winner(
    method rewrite_reason (line 95) | def rewrite_reason(

FILE: deepeval/metrics/arena_g_eval/utils.py
  class FormattedLLMTestCase (line 29) | class FormattedLLMTestCase:
    method __repr__ (line 36) | def __repr__(self):
  class FormattedArenaTestCase (line 55) | class FormattedArenaTestCase:
    method __repr__ (line 61) | def __repr__(self):
  function format_arena_test_case (line 89) | def format_arena_test_case(
  function construct_formatted_llm_test_case (line 132) | def construct_formatted_llm_test_case(

FILE: deepeval/metrics/argument_correctness/argument_correctness.py
  class ArgumentCorrectnessMetric (line 29) | class ArgumentCorrectnessMetric(BaseMetric):
    method __init__ (line 35) | def __init__(
    method measure (line 56) | def measure(
    method a_measure (line 115) | async def a_measure(
    method _a_generate_reason (line 166) | async def _a_generate_reason(self, input: str, multimodal: bool) -> str:
    method _generate_reason (line 190) | def _generate_reason(self, input: str, multimodal: bool) -> str:
    method _a_generate_verdicts (line 214) | async def _a_generate_verdicts(
    method _generate_verdicts (line 231) | def _generate_verdicts(
    method _calculate_score (line 248) | def _calculate_score(self):
    method is_successful (line 261) | def is_successful(self) -> bool:
    method __name__ (line 272) | def __name__(self):

FILE: deepeval/metrics/argument_correctness/schema.py
  class ArgumentCorrectnessVerdict (line 5) | class ArgumentCorrectnessVerdict(BaseModel):
  class Verdicts (line 10) | class Verdicts(BaseModel):
  class ArgumentCorrectnessScoreReason (line 14) | class ArgumentCorrectnessScoreReason(BaseModel):

FILE: deepeval/metrics/argument_correctness/template.py
  class ArgumentCorrectnessTemplate (line 6) | class ArgumentCorrectnessTemplate:
    method generate_verdicts (line 16) | def generate_verdicts(
    method generate_reason (line 106) | def generate_reason(

FILE: deepeval/metrics/base_metric.py
  class BaseMetric (line 17) | class BaseMetric:
    method __init_subclass__ (line 37) | def __init_subclass__(cls, **kwargs):
    method measure (line 44) | def measure(self, test_case: LLMTestCase, *args, **kwargs) -> float:
    method a_measure (line 48) | async def a_measure(self, test_case: LLMTestCase, *args, **kwargs) -> ...
    method is_successful (line 54) | def is_successful(self) -> bool:
    method __name__ (line 58) | def __name__(self):
    method _accrue_cost (line 61) | def _accrue_cost(self, cost: float) -> None:
  class BaseConversationalMetric (line 68) | class BaseConversationalMetric:
    method __init_subclass__ (line 86) | def __init_subclass__(cls, **kwargs):
    method measure (line 93) | def measure(
    method a_measure (line 99) | async def a_measure(
    method is_successful (line 107) | def is_successful(self) -> bool:
    method __name__ (line 111) | def __name__(self):
    method _accrue_cost (line 114) | def _accrue_cost(self, cost: float) -> None:
  class BaseArenaMetric (line 121) | class BaseArenaMetric:
    method __init_subclass__ (line 133) | def __init_subclass__(cls, **kwargs):
    method measure (line 140) | def measure(self, test_case: ArenaTestCase, *args, **kwargs) -> str:
    method a_measure (line 144) | async def a_measure(self, test_case: ArenaTestCase, *args, **kwargs) -...
    method is_successful (line 150) | def is_successful(self) -> bool:
    method __name__ (line 154) | def __name__(self):
    method _accrue_cost (line 157) | def _accrue_cost(self, cost: float) -> None:

FILE: deepeval/metrics/bias/bias.py
  class BiasMetric (line 27) | class BiasMetric(BaseMetric):
    method __init__ (line 33) | def __init__(
    method measure (line 52) | def measure(
    method a_measure (line 104) | async def a_measure(
    method _a_generate_reason (line 149) | async def _a_generate_reason(
    method _generate_reason (line 173) | def _generate_reason(self) -> str:
    method _a_generate_verdicts (line 195) | async def _a_generate_verdicts(self, multimodal: bool) -> List[BiasVer...
    method _generate_verdicts (line 213) | def _generate_verdicts(self, multimodal: bool) -> List[BiasVerdict]:
    method _a_generate_opinions (line 231) | async def _a_generate_opinions(
    method _generate_opinions (line 246) | def _generate_opinions(
    method _calculate_score (line 261) | def _calculate_score(self) -> float:
    method is_successful (line 274) | def is_successful(self) -> bool:
    method __name__ (line 285) | def __name__(self):

FILE: deepeval/metrics/bias/schema.py
  class Opinions (line 5) | class Opinions(BaseModel):
  class BiasVerdict (line 10) | class BiasVerdict(BaseModel):
  class Verdicts (line 15) | class Verdicts(BaseModel):
  class BiasScoreReason (line 19) | class BiasScoreReason(BaseModel):

FILE: deepeval/metrics/bias/template.py
  class BiasTemplate (line 4) | class BiasTemplate:
    method generate_opinions (line 14) | def generate_opinions(actual_output: str, multimodal: bool = False):
    method generate_verdicts (line 43) | def generate_verdicts(opinions: List[str], multimodal: bool = False):
    method generate_reason (line 101) | def generate_reason(biases: List[str], score: float):

FILE: deepeval/metrics/contextual_precision/contextual_precision.py
  class ContextualPrecisionMetric (line 27) | class ContextualPrecisionMetric(BaseMetric):
    method __init__ (line 34) | def __init__(
    method measure (line 55) | def measure(
    method a_measure (line 114) | async def a_measure(
    method _a_generate_reason (line 162) | async def _a_generate_reason(self, input: str, multimodal: bool):
    method _generate_reason (line 185) | def _generate_reason(self, input: str, multimodal: bool):
    method _a_generate_verdicts (line 208) | async def _a_generate_verdicts(
    method _generate_verdicts (line 233) | def _generate_verdicts(
    method _calculate_score (line 258) | def _calculate_score(self):
    method is_successful (line 284) | def is_successful(self) -> bool:
    method __name__ (line 295) | def __name__(self):

FILE: deepeval/metrics/contextual_precision/schema.py
  class ContextualPrecisionVerdict (line 5) | class ContextualPrecisionVerdict(BaseModel):
  class Verdicts (line 10) | class Verdicts(BaseModel):
  class ContextualPrecisionScoreReason (line 14) | class ContextualPrecisionScoreReason(BaseModel):

FILE: deepeval/metrics/contextual_precision/template.py
  class ContextualPrecisionTemplate (line 7) | class ContextualPrecisionTemplate:
    method generate_verdicts (line 9) | def generate_verdicts(
    method generate_reason (line 73) | def generate_reason(
    method id_retrieval_context (line 113) | def id_retrieval_context(

FILE: deepeval/metrics/contextual_recall/contextual_recall.py
  class ContextualRecallMetric (line 30) | class ContextualRecallMetric(BaseMetric):
    method __init__ (line 38) | def __init__(
    method measure (line 59) | def measure(
    method a_measure (line 113) | async def a_measure(
    method _a_generate_reason (line 162) | async def _a_generate_reason(self, expected_output: str, multimodal: b...
    method _generate_reason (line 190) | def _generate_reason(self, expected_output: str, multimodal: bool):
    method _calculate_score (line 218) | def _calculate_score(self):
    method _a_generate_verdicts (line 231) | async def _a_generate_verdicts(
    method _generate_verdicts (line 261) | def _generate_verdicts(
    method is_successful (line 291) | def is_successful(self) -> bool:
    method __name__ (line 302) | def __name__(self):

FILE: deepeval/metrics/contextual_recall/schema.py
  class ContextualRecallVerdict (line 5) | class ContextualRecallVerdict(BaseModel):
  class VerdictWithExpectedOutput (line 10) | class VerdictWithExpectedOutput(BaseModel):
  class Verdicts (line 16) | class Verdicts(BaseModel):
  class ContextualRecallScoreReason (line 20) | class ContextualRecallScoreReason(BaseModel):

FILE: deepeval/metrics/contextual_recall/template.py
  class ContextualRecallTemplate (line 7) | class ContextualRecallTemplate:
    method generate_reason (line 9) | def generate_reason(
    method generate_verdicts (line 52) | def generate_verdicts(
    method id_retrieval_context (line 106) | def id_retrieval_context(

FILE: deepeval/metrics/contextual_relevancy/contextual_relevancy.py
  class ContextualRelevancyMetric (line 31) | class ContextualRelevancyMetric(BaseMetric):
    method __init__ (line 37) | def __init__(
    method measure (line 58) | def measure(
    method a_measure (line 114) | async def a_measure(
    method _a_generate_reason (line 164) | async def _a_generate_reason(self, input: str, multimodal: bool):
    method _generate_reason (line 193) | def _generate_reason(self, input: str, multimodal: bool):
    method _calculate_score (line 222) | def _calculate_score(self):
    method _a_generate_verdicts (line 237) | async def _a_generate_verdicts(
    method _generate_verdicts (line 252) | def _generate_verdicts(
    method is_successful (line 267) | def is_successful(self) -> bool:
    method __name__ (line 278) | def __name__(self):

FILE: deepeval/metrics/contextual_relevancy/schema.py
  class ContextualRelevancyVerdict (line 5) | class ContextualRelevancyVerdict(BaseModel):
  class ContextualRelevancyVerdicts (line 11) | class ContextualRelevancyVerdicts(BaseModel):
  class ContextualRelevancyScoreReason (line 15) | class ContextualRelevancyScoreReason(BaseModel):

FILE: deepeval/metrics/contextual_relevancy/template.py
  class ContextualRelevancyTemplate (line 5) | class ContextualRelevancyTemplate:
    method generate_reason (line 7) | def generate_reason(
    method generate_verdicts (line 47) | def generate_verdicts(

FILE: deepeval/metrics/conversation_completeness/conversation_completeness.py
  class ConversationCompletenessMetric (line 29) | class ConversationCompletenessMetric(BaseConversationalMetric):
    method __init__ (line 32) | def __init__(
    method measure (line 51) | def measure(
    method a_measure (line 110) | async def a_measure(
    method _a_generate_reason (line 163) | async def _a_generate_reason(self, multimodal: bool) -> str:
    method _generate_reason (line 187) | def _generate_reason(self, multimodal: bool) -> str:
    method _a_generate_verdict (line 214) | async def _a_generate_verdict(
    method _generate_verdict (line 231) | def _generate_verdict(
    method _a_extract_user_intentions (line 247) | async def _a_extract_user_intentions(
    method _extract_user_intentions (line 263) | def _extract_user_intentions(
    method _calculate_score (line 278) | def _calculate_score(self) -> float:
    method is_successful (line 296) | def is_successful(self) -> bool:
    method __name__ (line 307) | def __name__(self):

FILE: deepeval/metrics/conversation_completeness/schema.py
  class UserIntentions (line 5) | class UserIntentions(BaseModel):
  class ConversationCompletenessVerdict (line 9) | class ConversationCompletenessVerdict(BaseModel):
  class ConversationCompletenessScoreReason (line 14) | class ConversationCompletenessScoreReason(BaseModel):

FILE: deepeval/metrics/conversation_completeness/template.py
  class ConversationCompletenessTemplate (line 4) | class ConversationCompletenessTemplate:
    method extract_user_intentions (line 14) | def extract_user_intentions(turns: List[Dict], multimodal: bool = False):
    method generate_verdicts (line 63) | def generate_verdicts(
    method generate_reason (line 125) | def generate_reason(

FILE: deepeval/metrics/conversational_dag/conversational_dag.py
  class ConversationalDAGMetric (line 22) | class ConversationalDAGMetric(BaseConversationalMetric):
    method __init__ (line 24) | def __init__(
    method measure (line 53) | def measure(
    method a_measure (line 96) | async def a_measure(
    method is_successful (line 131) | def is_successful(self) -> bool:
    method __name__ (line 142) | def __name__(self):

FILE: deepeval/metrics/conversational_dag/nodes.py
  class ConversationalBaseNode (line 38) | class ConversationalBaseNode:
    method set_parent (line 42) | def set_parent(self, parent: "ConversationalBaseNode"):
    method _execute (line 50) | def _execute(
    method _a_execute (line 60) | async def _a_execute(
  function increment_indegree (line 71) | def increment_indegree(node: ConversationalBaseNode):
  function decrement_indegree (line 75) | def decrement_indegree(node: ConversationalBaseNode):
  class ConversationalVerdictNode (line 80) | class ConversationalVerdictNode(ConversationalBaseNode):
    method __hash__ (line 92) | def __hash__(self):
    method __post_init__ (line 95) | def __post_init__(self):
    method _execute (line 112) | def _execute(
    method _a_execute (line 188) | async def _a_execute(
    method _generate_reason (line 264) | def _generate_reason(self, metric: BaseConversationalMetric):
    method _a_generate_reason (line 279) | async def _a_generate_reason(self, metric: BaseConversationalMetric):
  class ConversationalTaskNode (line 296) | class ConversationalTaskNode(ConversationalBaseNode):
    method __hash__ (line 307) | def __hash__(self):
    method __post_init__ (line 310) | def __post_init__(self):
    method _execute (line 321) | def _execute(
    method _a_execute (line 382) | async def _a_execute(
  class ConversationalBinaryJudgementNode (line 448) | class ConversationalBinaryJudgementNode(ConversationalBaseNode):
    method __hash__ (line 458) | def __hash__(self):
    method __post_init__ (line 461) | def __post_init__(self):
    method _execute (line 498) | def _execute(
    method _a_execute (line 554) | async def _a_execute(
  class ConversationalNonBinaryJudgementNode (line 616) | class ConversationalNonBinaryJudgementNode(ConversationalBaseNode):
    method __hash__ (line 626) | def __hash__(self):
    method __post_init__ (line 629) | def __post_init__(self):
    method _execute (line 677) | def _execute(
    method _a_execute (line 732) | async def _a_execute(
  function construct_node_verbose_log (line 792) | def construct_node_verbose_log(
  function is_valid_turn_window (line 863) | def is_valid_turn_window(

FILE: deepeval/metrics/conversational_dag/templates.py
  class ConversationalVerdictNodeTemplate (line 13) | class ConversationalVerdictNodeTemplate:
    method generate_reason (line 16) | def generate_reason(verbose_steps: List[str], score: float, name: str):
  class ConversationalTaskNodeTemplate (line 46) | class ConversationalTaskNodeTemplate:
    method generate_task_output (line 48) | def generate_task_output(instructions: str, text: str):
  class ConversationalBinaryJudgementTemplate (line 73) | class ConversationalBinaryJudgementTemplate:
    method generate_binary_verdict (line 75) | def generate_binary_verdict(criteria: str, text: str):
  class ConversationalNonBinaryJudgementTemplate (line 102) | class ConversationalNonBinaryJudgementTemplate:
    method generate_non_binary_verdict (line 104) | def generate_non_binary_verdict(

FILE: deepeval/metrics/conversational_g_eval/conversational_g_eval.py
  function _debug_print_prompt (line 42) | def _debug_print_prompt(label: str, prompt: str) -> None:
  class ConversationalGEval (line 50) | class ConversationalGEval(BaseConversationalMetric):
    method __init__ (line 51) | def __init__(
    method measure (line 101) | def measure(
    method a_measure (line 157) | async def a_measure(
    method _a_generate_evaluation_steps (line 205) | async def _a_generate_evaluation_steps(self) -> List[str]:
    method _generate_evaluation_steps (line 226) | def _generate_evaluation_steps(self) -> List[str]:
    method _a_evaluate (line 247) | async def _a_evaluate(
    method evaluate (line 315) | def evaluate(
    method generate_weighted_summed_score (line 381) | def generate_weighted_summed_score(
    method number_evaluation_steps (line 423) | def number_evaluation_steps(self):
    method is_successful (line 429) | def is_successful(self) -> bool:
    method upload (line 439) | def upload(self):
    method __name__ (line 471) | def __name__(self):

FILE: deepeval/metrics/conversational_g_eval/schema.py
  class ReasonScore (line 5) | class ReasonScore(BaseModel):
  class Steps (line 10) | class Steps(BaseModel):

FILE: deepeval/metrics/conversational_g_eval/template.py
  class ConversationalGEvalTemplate (line 4) | class ConversationalGEvalTemplate:
    method generate_evaluation_steps (line 6) | def generate_evaluation_steps(parameters: str, criteria: str):
    method generate_evaluation_results (line 28) | def generate_evaluation_results(

FILE: deepeval/metrics/dag/dag.py
  class DAGMetric (line 23) | class DAGMetric(BaseMetric):
    method __init__ (line 25) | def __init__(
    method measure (line 57) | def measure(
    method a_measure (line 101) | async def a_measure(
    method is_successful (line 137) | def is_successful(self) -> bool:
    method __name__ (line 148) | def __name__(self):

FILE: deepeval/metrics/dag/graph.py
  function validate_root_nodes (line 18) | def validate_root_nodes(
  class DeepAcyclicGraph (line 27) | class DeepAcyclicGraph:
    method __init__ (line 30) | def __init__(
    method _execute (line 57) | def _execute(
    method _a_execute (line 65) | async def _a_execute(
    method to_dict (line 79) | def to_dict(self) -> dict:
    method to_json (line 85) | def to_json(self, indent: int = 2) -> str:
    method from_dict (line 92) | def from_dict(
    method from_json (line 105) | def from_json(cls, s: str, multiturn: bool = False) -> "DeepAcyclicGra...

FILE: deepeval/metrics/dag/nodes.py
  class BaseNode (line 30) | class BaseNode:
    method set_parent (line 34) | def set_parent(self, parent: "BaseNode"):
    method _execute (line 42) | def _execute(self, metric: BaseMetric, test_case: LLMTestCase, depth: ...
    method _a_execute (line 47) | async def _a_execute(
  function increment_indegree (line 55) | def increment_indegree(node: BaseNode):
  function decrement_indegree (line 59) | def decrement_indegree(node: BaseNode):
  class VerdictNode (line 64) | class VerdictNode(BaseNode):
    method __hash__ (line 70) | def __hash__(self):
    method __post_init__ (line 73) | def __post_init__(self):
    method _execute (line 90) | def _execute(self, metric: BaseMetric, test_case: LLMTestCase, depth: ...
    method _a_execute (line 155) | async def _a_execute(
    method _generate_reason (line 223) | def _generate_reason(self, metric: BaseMetric):
    method _a_generate_reason (line 237) | async def _a_generate_reason(self, metric: BaseMetric):
  class TaskNode (line 253) | class TaskNode(BaseNode):
    method __hash__ (line 263) | def __hash__(self):
    method __post_init__ (line 266) | def __post_init__(self):
    method _execute (line 280) | def _execute(self, metric: BaseMetric, test_case: LLMTestCase, depth: ...
    method _a_execute (line 324) | async def _a_execute(
  class BinaryJudgementNode (line 377) | class BinaryJudgementNode(BaseNode):
    method __hash__ (line 386) | def __hash__(self):
    method __post_init__ (line 389) | def __post_init__(self):
    method _execute (line 422) | def _execute(self, metric: BaseMetric, test_case: LLMTestCase, depth: ...
    method _a_execute (line 460) | async def _a_execute(
  class NonBinaryJudgementNode (line 507) | class NonBinaryJudgementNode(BaseNode):
    method __hash__ (line 516) | def __hash__(self):
    method __post_init__ (line 519) | def __post_init__(self):
    method _execute (line 563) | def _execute(self, metric: BaseMetric, test_case: LLMTestCase, depth: ...
    method _a_execute (line 602) | async def _a_execute(
  function construct_node_verbose_log (line 648) | def construct_node_verbose_log(

FILE: deepeval/metrics/dag/schema.py
  class MetricScoreReason (line 5) | class MetricScoreReason(BaseModel):
  class TaskNodeOutput (line 9) | class TaskNodeOutput(BaseModel):
  class BinaryJudgementVerdict (line 13) | class BinaryJudgementVerdict(BaseModel):
  class NonBinaryJudgementVerdict (line 18) | class NonBinaryJudgementVerdict(BaseModel):

FILE: deepeval/metrics/dag/serialization/serialization.py
  function dag_to_dict (line 70) | def dag_to_dict(dag) -> Dict[str, Any]:
  function dag_to_json (line 90) | def dag_to_json(dag, indent: int = 2) -> str:
  function dag_from_dict (line 94) | def dag_from_dict(data: Dict[str, Any], multiturn: bool = False):
  function dag_from_json (line 171) | def dag_from_json(s: str, multiturn: bool = False):
  function _walk_nodes (line 180) | def _walk_nodes(root_nodes: List[Any]) -> List[Any]:
  function _iter_children (line 198) | def _iter_children(node: Any):
  function _is_node (line 207) | def _is_node(obj: Any) -> bool:
  function _assign_ids (line 211) | def _assign_ids(ordered_nodes: List[Any]) -> Dict[int, str]:
  function _serialize_node (line 216) | def _serialize_node(node: Any, id_map: Dict[int, str]) -> Dict[str, Any]:
  function _serialize_eval_params (line 272) | def _serialize_eval_params(params) -> Optional[List[str]]:
  function _serialize_verdict_child (line 278) | def _serialize_verdict_child(
  function _serialize_geval (line 294) | def _serialize_geval(geval: Any) -> Dict[str, Any]:
  function _serialize_metric (line 313) | def _serialize_metric(metric: Any) -> Dict[str, Any]:
  function _init_param_names (line 340) | def _init_param_names(cls: Type) -> List[str]:
  function _maybe_jsonify (line 351) | def _maybe_jsonify(value: Any) -> Any:
  function _collect_referenced_ids (line 388) | def _collect_referenced_ids(nodes_spec: Dict[str, Any]) -> Set[str]:
  function _eval_params_cls (line 404) | def _eval_params_cls(multiturn: bool):
  function _deserialize_eval_params (line 408) | def _deserialize_eval_params(values, multiturn: bool):
  function _task_kwargs (line 424) | def _task_kwargs(spec: Dict[str, Any], multiturn: bool) -> Dict[str, Any]:
  function _judgement_kwargs (line 440) | def _judgement_kwargs(spec: Dict[str, Any], multiturn: bool) -> Dict[str...
  function _build_verdict (line 453) | def _build_verdict(
  function _build_geval (line 494) | def _build_geval(child_spec: Dict[str, Any], multiturn: bool):
  function _build_metric (line 504) | def _build_metric(child_spec: Dict[str, Any]):

FILE: deepeval/metrics/dag/serialization/types.py
  class NodeType (line 4) | class NodeType(str, Enum):
  class ChildType (line 11) | class ChildType(str, Enum):

FILE: deepeval/metrics/dag/templates.py
  class VerdictNodeTemplate (line 12) | class VerdictNodeTemplate:
    method generate_reason (line 14) | def generate_reason(verbose_steps: List[str], score: float, name: str):
  class TaskNodeTemplate (line 40) | class TaskNodeTemplate:
    method generate_task_output (line 42) | def generate_task_output(instructions: str, text: str):
  class BinaryJudgementTemplate (line 65) | class BinaryJudgementTemplate:
    method generate_binary_verdict (line 67) | def generate_binary_verdict(criteria: str, text: str):
  class NonBinaryJudgementTemplate (line 87) | class NonBinaryJudgementTemplate:
    method generate_non_binary_verdict (line 89) | def generate_non_binary_verdict(

FILE: deepeval/metrics/dag/utils.py
  function is_valid_dag_from_roots (line 22) | def is_valid_dag_from_roots(
  function is_valid_dag (line 33) | def is_valid_dag(
  function extract_required_params (line 74) | def extract_required_params(
  function copy_graph (line 111) | def copy_graph(original_dag: DeepAcyclicGraph) -> DeepAcyclicGraph:

FILE: deepeval/metrics/exact_match/exact_match.py
  class ExactMatchMetric (line 12) | class ExactMatchMetric(BaseMetric):
    method __init__ (line 19) | def __init__(
    method measure (line 27) | def measure(
    method a_measure (line 72) | async def a_measure(
    method is_successful (line 84) | def is_successful(self) -> bool:
    method __name__ (line 95) | def __name__(self):

FILE: deepeval/metrics/faithfulness/faithfulness.py
  class FaithfulnessMetric (line 29) | class FaithfulnessMetric(BaseMetric):
    method __init__ (line 36) | def __init__(
    method measure (line 62) | def measure(
    method a_measure (line 119) | async def a_measure(
    method _a_generate_reason (line 167) | async def _a_generate_reason(self, multimodal: bool) -> str:
    method _generate_reason (line 190) | def _generate_reason(self, multimodal: bool) -> str:
    method _a_generate_verdicts (line 213) | async def _a_generate_verdicts(
    method _generate_verdicts (line 235) | def _generate_verdicts(self, multimodal: bool) -> List[FaithfulnessVer...
    method _a_generate_truths (line 255) | async def _a_generate_truths(
    method _generate_truths (line 271) | def _generate_truths(
    method _a_generate_claims (line 287) | async def _a_generate_claims(
    method _generate_claims (line 301) | def _generate_claims(
    method _calculate_score (line 315) | def _calculate_score(self) -> float:
    method is_successful (line 334) | def is_successful(self) -> bool:
    method __name__ (line 345) | def __name__(self):

FILE: deepeval/metrics/faithfulness/schema.py
  class FaithfulnessVerdict (line 5) | class FaithfulnessVerdict(BaseModel):
  class Verdicts (line 10) | class Verdicts(BaseModel):
  class Truths (line 14) | class Truths(BaseModel):
  class Claims (line 18) | class Claims(BaseModel):
  class FaithfulnessScoreReason (line 22) | class FaithfulnessScoreReason(BaseModel):

FILE: deepeval/metrics/faithfulness/template.py
  class FaithfulnessTemplate (line 5) | class FaithfulnessTemplate:
    method generate_claims (line 7) | def generate_claims(actual_output: str, multimodal: bool = False):
    method generate_truths (line 44) | def generate_truths(
    method generate_verdicts (line 91) | def generate_verdicts(
    method generate_reason (line 197) | def generate_reason(

FILE: deepeval/metrics/g_eval/g_eval.py
  class GEval (line 42) | class GEval(BaseMetric):
    method __init__ (line 43) | def __init__(
    method measure (line 81) | def measure(
    method a_measure (line 157) | async def a_measure(
    method _a_generate_evaluation_steps (line 213) | async def _a_generate_evaluation_steps(self, multimodal: bool) -> List...
    method _generate_evaluation_steps (line 233) | def _generate_evaluation_steps(self, multimodal: bool) -> List[str]:
    method _a_evaluate (line 253) | async def _a_evaluate(
    method _evaluate (line 325) | def _evaluate(
    method is_successful (line 395) | def is_successful(self) -> bool:
    method upload (line 405) | def upload(self):
    method __name__ (line 437) | def __name__(self):

FILE: deepeval/metrics/g_eval/schema.py
  class ReasonScore (line 5) | class ReasonScore(BaseModel):
  class BestTestCase (line 10) | class BestTestCase(BaseModel):
  class Steps (line 16) | class Steps(BaseModel):

FILE: deepeval/metrics/g_eval/template.py
  class GEvalTemplate (line 5) | class GEvalTemplate:
    method generate_evaluation_steps (line 15) | def generate_evaluation_steps(
    method generate_evaluation_results (line 39) | def generate_evaluation_results(
    method generate_strict_evaluation_results (line 107) | def generate_strict_evaluation_results(

FILE: deepeval/metrics/g_eval/utils.py
  class Rubric (line 18) | class Rubric(BaseModel):
    method validate_score_range (line 23) | def validate_score_range(cls, value):
  function construct_geval_upload_payload (line 83) | def construct_geval_upload_payload(
  function validate_criteria_and_evaluation_steps (line 129) | def validate_criteria_and_evaluation_steps(
  function validate_and_sort_rubrics (line 150) | def validate_and_sort_rubrics(
  function format_rubrics (line 173) | def format_rubrics(rubrics: Optional[List[Rubric]]) -> Optional[str]:
  function no_log_prob_support (line 188) | def no_log_prob_support(model: Union[str, DeepEvalBaseLLM]):
  function construct_g_eval_params_string (line 207) | def construct_g_eval_params_string(
  function construct_conversational_g_eval_turn_params_string (line 223) | def construct_conversational_g_eval_turn_params_string(
  function construct_non_turns_test_case_string (line 242) | def construct_non_turns_test_case_string(
  function construct_test_case_string (line 264) | def construct_test_case_string(
  function calculate_weighted_summed_score (line 276) | def calculate_weighted_summed_score(
  function number_evaluation_steps (line 326) | def number_evaluation_steps(evaluation_steps: List[str]) -> str:
  function number_test_case_contents (line 333) | def number_test_case_contents(test_case_contents: List[str]) -> str:
  function get_score_range (line 340) | def get_score_range(rubric: Optional[List[Rubric]]) -> Tuple[int, int]:

FILE: deepeval/metrics/goal_accuracy/goal_accuracy.py
  class GoalAccuracyMetric (line 27) | class GoalAccuracyMetric(BaseConversationalMetric):
    method __init__ (line 34) | def __init__(
    method measure (line 51) | def measure(
    method a_measure (line 118) | async def a_measure(
    method _goal_and_steps_taken (line 180) | def _goal_and_steps_taken(
    method _get_plan_scores (line 202) | def _get_plan_scores(self, user_goal, steps_taken, multimodal: bool):
    method _a_get_plan_scores (line 214) | async def _a_get_plan_scores(
    method _calculate_score (line 228) | def _calculate_score(
    method _generate_reason (line 240) | def _generate_reason(
    method _a_generate_reason (line 272) | async def _a_generate_reason(
    method _get_goal_accuracy_score (line 304) | def _get_goal_accuracy_score(
    method _a_get_goal_accuracy_score (line 318) | async def _a_get_goal_accuracy_score(
    method print_goals_and_steps_taken (line 332) | def print_goals_and_steps_taken(self, goals_and_steps):
    method is_successful (line 341) | def is_successful(self) -> bool:
    method __name__ (line 352) | def __name__(self):

FILE: deepeval/metrics/goal_accuracy/schema.py
  class GoalSteps (line 5) | class GoalSteps(BaseModel):
  class GoalScore (line 10) | class GoalScore(BaseModel):
  class PlanScore (line 15) | class PlanScore(BaseModel):

FILE: deepeval/metrics/goal_accuracy/template.py
  class GoalAccuracyTemplate (line 5) | class GoalAccuracyTemplate:
    method get_accuracy_score (line 15) | def get_accuracy_score(task, steps_taken, multimodal: bool = False):
    method get_plan_evaluation_score (line 115) | def get_plan_evaluation_score(task, steps_taken, multimodal: bool = Fa...
    method get_final_reason (line 202) | def get_final_reason(

FILE: deepeval/metrics/hallucination/hallucination.py
  class HallucinationMetric (line 26) | class HallucinationMetric(BaseMetric):
    method __init__ (line 33) | def __init__(
    method measure (line 54) | def measure(
    method a_measure (line 106) | async def a_measure(
    method _a_generate_reason (line 149) | async def _a_generate_reason(self):
    method _generate_reason (line 175) | def _generate_reason(self):
    method _a_generate_verdicts (line 201) | async def _a_generate_verdicts(
    method _generate_verdicts (line 217) | def _generate_verdicts(
    method _calculate_score (line 233) | def _calculate_score(self) -> float:
    method is_successful (line 246) | def is_successful(self) -> bool:
    method __name__ (line 257) | def __name__(self):

FILE: deepeval/metrics/hallucination/schema.py
  class HallucinationVerdict (line 5) | class HallucinationVerdict(BaseModel):
  class Verdicts (line 10) | class Verdicts(BaseModel):
  class HallucinationScoreReason (line 14) | class HallucinationScoreReason(BaseModel):

FILE: deepeval/metrics/hallucination/template.py
  class HallucinationTemplate (line 4) | class HallucinationTemplate:
    method generate_verdicts (line 14) | def generate_verdicts(actual_output: str, contexts: List[str]):
    method generate_reason (line 55) | def generate_reason(

FILE: deepeval/metrics/indicator.py
  function format_metric_description (line 25) | def format_metric_description(
  function metric_progress_indicator (line 41) | def metric_progress_indicator(
  function measure_metric_task (line 73) | async def measure_metric_task(
  function measure_metrics_with_indicator (line 157) | async def measure_metrics_with_indicator(
  function safe_a_measure (line 241) | async def safe_a_measure(

FILE: deepeval/metrics/json_correctness/json_correctness.py
  class JsonCorrectnessMetric (line 26) | class JsonCorrectnessMetric(BaseMetric):
    method __init__ (line 32) | def __init__(
    method measure (line 51) | def measure(
    method a_measure (line 107) | async def a_measure(
    method a_generate_reason (line 154) | async def a_generate_reason(self, actual_output: str) -> str:
    method generate_reason (line 178) | def generate_reason(self, actual_output: str) -> str:
    method is_successful (line 202) | def is_successful(self) -> bool:
    method __name__ (line 213) | def __name__(self):

FILE: deepeval/metrics/json_correctness/schema.py
  class JsonCorrectnessScoreReason (line 4) | class JsonCorrectnessScoreReason(BaseModel):

FILE: deepeval/metrics/json_correctness/template.py
  class JsonCorrectnessTemplate (line 4) | class JsonCorrectnessTemplate:
    method generate_reason (line 14) | def generate_reason(

FILE: deepeval/metrics/knowledge_retention/knowledge_retention.py
  class KnowledgeRetentionMetric (line 26) | class KnowledgeRetentionMetric(BaseConversationalMetric):
    method __init__ (line 29) | def __init__(
    method measure (line 46) | def measure(
    method a_measure (line 97) | async def a_measure(
    method _a_generate_reason (line 139) | async def _a_generate_reason(self) -> str:
    method _generate_reason (line 160) | def _generate_reason(self) -> str:
    method _a_generate_verdicts (line 181) | async def _a_generate_verdicts(
    method _generate_verdicts (line 211) | def _generate_verdicts(
    method _a_generate_knowledges (line 242) | async def _a_generate_knowledges(
    method _generate_knowledges (line 270) | def _generate_knowledges(
    method _calculate_score (line 299) | def _calculate_score(self) -> float:
    method is_successful (line 313) | def is_successful(self) -> bool:
    method __name__ (line 324) | def __name__(self):

FILE: deepeval/metrics/knowledge_retention/schema.py
  class Knowledge (line 5) | class Knowledge(BaseModel):
  class KnowledgeRetentionVerdict (line 13) | class KnowledgeRetentionVerdict(BaseModel):
  class KnowledgeRetentionScoreReason (line 19) | class KnowledgeRetentionScoreReason(BaseModel):

FILE: deepeval/metrics/knowledge_retention/template.py
  class KnowledgeRetentionTemplate (line 4) | class KnowledgeRetentionTemplate:
    method generate_reason (line 14) | def generate_reason(attritions, score):
    method generate_verdict (line 39) | def generate_verdict(
    method extract_data (line 117) | def extract_data(user_message: str, previous_turns: List[Dict]):

FILE: deepeval/metrics/mcp/mcp_task_completion.py
  class MCPTaskCompletionMetric (line 22) | class MCPTaskCompletionMetric(BaseConversationalMetric):
    method __init__ (line 28) | def __init__(
    method measure (line 45) | def measure(
    method a_measure (line 103) | async def a_measure(
    method _generate_reason (line 154) | def _generate_reason(self, task_scores: List[TaskScore]) -> Optional[s...
    method _a_generate_reason (line 173) | async def _a_generate_reason(
    method _get_task_score (line 195) | def _get_task_score(self, task: Task) -> TaskScore:
    method _a_get_task_score (line 205) | async def _a_get_task_score(self, task: Task) -> TaskScore:
    method _get_tasks (line 215) | def _get_tasks(self, unit_interactions: List) -> List[Task]:
    method _calculate_score (line 266) | def _calculate_score(self, scores: List[TaskScore]) -> float:
    method is_successful (line 272) | def is_successful(self) -> bool:
    method __name__ (line 283) | def __name__(self):

FILE: deepeval/metrics/mcp/multi_turn_mcp_use_metric.py
  class MultiTurnMCPUseMetric (line 22) | class MultiTurnMCPUseMetric(BaseConversationalMetric):
    method __init__ (line 28) | def __init__(
    method measure (line 45) | def measure(
    method a_measure (line 116) | async def a_measure(
    method _get_tool_accuracy_score (line 184) | def _get_tool_accuracy_score(
    method _a_get_tool_accuracy_score (line 198) | async def _a_get_tool_accuracy_score(
    method _get_args_score (line 212) | def _get_args_score(
    method _a_get_args_score (line 226) | async def _a_get_args_score(
    method _get_tasks (line 240) | def _get_tasks(self, unit_interactions: List) -> List[Task]:
    method _calculate_score (line 291) | def _calculate_score(
    method _generate_reason (line 311) | def _generate_reason(
    method _a_generate_reason (line 338) | async def _a_generate_reason(
    method is_successful (line 365) | def is_successful(self) -> bool:
    method __name__ (line 376) | def __name__(self):

FILE: deepeval/metrics/mcp/schema.py
  class Task (line 5) | class Task(BaseModel):
  class TaskScore (line 10) | class TaskScore(BaseModel):
  class ToolScore (line 15) | class ToolScore(BaseModel):
  class ArgsScore (line 20) | class ArgsScore(BaseModel):
  class Reason (line 25) | class Reason(BaseModel):

FILE: deepeval/metrics/mcp/template.py
  class MCPTaskCompletionTemplate (line 6) | class MCPTaskCompletionTemplate:
    method get_args_correctness_score (line 16) | def get_args_correctness_score(task: Task, mcp_servers: List[MCPServer]):
    method get_tool_correctness_score (line 63) | def get_tool_correctness_score(task: Task, mcp_servers: List[MCPServer]):
    method get_task_completion_score (line 104) | def get_task_completion_score(task: Task):
    method generate_final_reason (line 143) | def generate_final_reason(

FILE: deepeval/metrics/mcp_use_metric/mcp_use_metric.py
  class MCPUseMetric (line 26) | class MCPUseMetric(BaseMetric):
    method __init__ (line 33) | def __init__(
    method measure (line 50) | def measure(
    method a_measure (line 122) | async def a_measure(
    method _get_primitives_used_score (line 184) | def _get_primitives_used_score(
    method _a_get_primitives_used_score (line 201) | async def _a_get_primitives_used_score(
    method _get_argument_correctness_score (line 218) | def _get_argument_correctness_score(
    method _a_get_argument_correctness_score (line 235) | async def _a_get_argument_correctness_score(
    method _calculate_score (line 252) | def _calculate_score(
    method _get_reason (line 262) | def _get_reason(
    method _get_mcp_interaction_text (line 276) | def _get_mcp_interaction_text(
    method is_successful (line 372) | def is_successful(self) -> bool:
    method __name__ (line 383) | def __name__(self):
    method indent_multiline_string (line 386) | def indent_multiline_string(self, s, indent_level=4):

FILE: deepeval/metrics/mcp_use_metric/schema.py
  class MCPPrimitivesScore (line 4) | class MCPPrimitivesScore(BaseModel):
  class MCPArgsScore (line 9) | class MCPArgsScore(BaseModel):

FILE: deepeval/metrics/mcp_use_metric/template.py
  class MCPUseMetricTemplate (line 5) | class MCPUseMetricTemplate:
    method get_mcp_argument_correctness_prompt (line 15) | def get_mcp_argument_correctness_prompt(
    method get_primitive_correctness_prompt (line 73) | def get_primitive_correctness_prompt(

FILE: deepeval/metrics/misuse/misuse.py
  class MisuseMetric (line 27) | class MisuseMetric(BaseMetric):
    method __init__ (line 33) | def __init__(
    method measure (line 57) | def measure(
    method a_measure (line 109) | async def a_measure(
    method _a_generate_reason (line 154) | async def _a_generate_reason(self) -> Optional[str]:
    method _generate_reason (line 176) | def _generate_reason(self) -> Optional[str]:
    method _a_generate_verdicts (line 198) | async def _a_generate_verdicts(self) -> List[MisuseVerdict]:
    method _generate_verdicts (line 215) | def _generate_verdicts(self) -> List[MisuseVerdict]:
    method _a_generate_misuses (line 232) | async def _a_generate_misuses(self, actual_output: str) -> List[str]:
    method _generate_misuses (line 244) | def _generate_misuses(self, actual_output: str) -> List[str]:
    method _calculate_score (line 256) | def _calculate_score(self) -> float:
    method is_successful (line 269) | def is_successful(self) -> bool:
    method __name__ (line 280) | def __name__(self):

FILE: deepeval/metrics/misuse/schema.py
  class Misuses (line 5) | class Misuses(BaseModel):
  class MisuseVerdict (line 9) | class MisuseVerdict(BaseModel):
  class Verdicts (line 14) | class Verdicts(BaseModel):
  class MisuseScoreReason (line 18) | class MisuseScoreReason(BaseModel):

FILE: deepeval/metrics/misuse/template.py
  class MisuseTemplate (line 4) | class MisuseTemplate:
    method generate_reason (line 14) | def generate_reason(misuse_violations: list, score: str):
    method generate_verdicts (line 32) | def generate_verdicts(misuses: list, domain: str):
    method generate_misuses (line 66) | def generate_misuses(actual_output: str, domain: str):

FILE: deepeval/metrics/multimodal_metrics/image_coherence/image_coherence.py
  class ImageCoherenceMetric (line 27) | class ImageCoherenceMetric(BaseMetric):
    method __init__ (line 33) | def __init__(
    method measure (line 50) | def measure(
    method a_measure (line 163) | async def a_measure(
    method evaluate_image_coherence (line 276) | def evaluate_image_coherence(
    method a_evaluate_image_coherence (line 294) | async def a_evaluate_image_coherence(
    method get_image_context (line 312) | def get_image_context(
    method get_image_indices (line 336) | def get_image_indices(
    method calculate_score (line 345) | def calculate_score(self, scores: List[float]) -> float:
    method is_successful (line 348) | def is_successful(self) -> bool:
    method __name__ (line 359) | def __name__(self):

FILE: deepeval/metrics/multimodal_metrics/image_coherence/schema.py
  class ReasonScore (line 5) | class ReasonScore(BaseModel):

FILE: deepeval/metrics/multimodal_metrics/image_coherence/template.py
  class ImageCoherenceTemplate (line 4) | class ImageCoherenceTemplate:
    method evaluate_image_coherence (line 7) | def evaluate_image_coherence(context_above, context_below):

FILE: deepeval/metrics/multimodal_metrics/image_editing/image_editing.py
  class ImageEditingMetric (line 27) | class ImageEditingMetric(BaseMetric):
    method __init__ (line 34) | def __init__(
    method measure (line 49) | def measure(
    method a_measure (line 120) | async def a_measure(
    method separate_images_from_text (line 181) | def separate_images_from_text(
    method _a_evaluate_semantic_consistency (line 193) | async def _a_evaluate_semantic_consistency(
    method _evaluate_semantic_consistency (line 214) | def _evaluate_semantic_consistency(
    method _a_evaluate_perceptual_quality (line 235) | async def _a_evaluate_perceptual_quality(
    method _evaluate_perceptual_quality (line 250) | def _evaluate_perceptual_quality(
    method _calculate_score (line 265) | def _calculate_score(self) -> float:
    method is_successful (line 270) | def is_successful(self) -> bool:
    method _generate_reason (line 280) | def _generate_reason(
    method __name__ (line 294) | def __name__(self):

FILE: deepeval/metrics/multimodal_metrics/image_editing/schema.py
  class ReasonScore (line 5) | class ReasonScore(BaseModel):

FILE: deepeval/metrics/multimodal_metrics/image_editing/template.py
  class ImageEditingTemplate (line 4) | class ImageEditingTemplate:
    method generate_semantic_consistency_evaluation_results (line 20) | def generate_semantic_consistency_evaluation_results(text_prompt: str):
    method generate_perceptual_quality_evaluation_results (line 40) | def generate_perceptual_quality_evaluation_results():

FILE: deepeval/metrics/multimodal_metrics/image_helpfulness/image_helpfulness.py
  class ImageHelpfulnessMetric (line 27) | class ImageHelpfulnessMetric(BaseMetric):
    method __init__ (line 34) | def __init__(
    method measure (line 51) | def measure(
    method a_measure (line 164) | async def a_measure(
    method evaluate_image_helpfulness (line 277) | def evaluate_image_helpfulness(
    method a_evaluate_image_helpfulness (line 295) | async def a_evaluate_image_helpfulness(
    method get_image_context (line 313) | def get_image_context(
    method get_image_indices (line 337) | def get_image_indices(
    method calculate_score (line 346) | def calculate_score(self, scores: List[float]) -> float:
    method is_successful (line 349) | def is_successful(self) -> bool:
    method __name__ (line 360) | def __name__(self):

FILE: deepeval/metrics/multimodal_metrics/image_helpfulness/schema.py
  class ReasonScore (line 5) | class ReasonScore(BaseModel):

FILE: deepeval/metrics/multimodal_metrics/image_helpfulness/template.py
  class ImageHelpfulnessTemplate (line 4) | class ImageHelpfulnessTemplate:
    method evaluate_image_helpfulness (line 7) | def evaluate_image_helpfulness(context_above, context_below):

FILE: deepeval/metrics/multimodal_metrics/image_reference/image_reference.py
  class ImageReferenceMetric (line 27) | class ImageReferenceMetric(BaseMetric):
    method __init__ (line 34) | def __init__(
    method measure (line 51) | def measure(
    method a_measure (line 164) | async def a_measure(
    method evaluate_image_reference (line 277) | def evaluate_image_reference(
    method a_evaluate_image_reference (line 295) | async def a_evaluate_image_reference(
    method get_image_context (line 313) | def get_image_context(
    method get_image_indices (line 337) | def get_image_indices(
    method calculate_score (line 346) | def calculate_score(self, scores: List[float]) -> float:
    method is_successful (line 349) | def is_successful(self) -> bool:
    method __name__ (line 360) | def __name__(self):

FILE: deepeval/metrics/multimodal_metrics/image_reference/schema.py
  class ReasonScore (line 5) | class ReasonScore(BaseModel):

FILE: deepeval/metrics/multimodal_metrics/image_reference/template.py
  class ImageReferenceTemplate (line 4) | class ImageReferenceTemplate:
    method evaluate_image_reference (line 7) | def evaluate_image_reference(context_above, context_below):

FILE: deepeval/metrics/multimodal_metrics/text_to_image/schema.py
  class ReasonScore (line 5) | class ReasonScore(BaseModel):

FILE: deepeval/metrics/multimodal_metrics/text_to_image/template.py
  class TextToImageTemplate (line 4) | class TextToImageTemplate:
    method generate_semantic_consistency_evaluation_results (line 20) | def generate_semantic_consistency_evaluation_results(text_prompt: str):
    method generate_perceptual_quality_evaluation_results (line 41) | def generate_perceptual_quality_evaluation_results():

FILE: deepeval/metrics/multimodal_metrics/text_to_image/text_to_image.py
  class TextToImageMetric (line 32) | class TextToImageMetric(BaseMetric):
    method __init__ (line 33) | def __init__(
    method measure (line 48) | def measure(
    method a_measure (line 114) | async def a_measure(
    method separate_images_from_text (line 173) | def separate_images_from_text(
    method _a_evaluate_semantic_consistency (line 185) | async def _a_evaluate_semantic_consistency(
    method _evaluate_semantic_consistency (line 208) | def _evaluate_semantic_consistency(
    method _a_evaluate_perceptual_quality (line 231) | async def _a_evaluate_perceptual_quality(
    method _evaluate_perceptual_quality (line 250) | def _evaluate_perceptual_quality(
    method _calculate_score (line 269) | def _calculate_score(self) -> float:
    method is_successful (line 274) | def is_successful(self) -> bool:
    method _generate_reason (line 284) | def _generate_reason(self) -> str:
    method __name__ (line 296) | def __name__(self):

FILE: deepeval/metrics/non_advice/non_advice.py
  class NonAdviceMetric (line 30) | class NonAdviceMetric(BaseMetric):
    method __init__ (line 36) | def __init__(
    method measure (line 64) | def measure(
    method a_measure (line 117) | async def a_measure(
    method _a_generate_reason (line 162) | async def _a_generate_reason(self) -> Optional[str]:
    method _generate_reason (line 183) | def _generate_reason(self) -> Optional[str]:
    method _a_generate_verdicts (line 204) | async def _a_generate_verdicts(self) -> List[NonAdviceVerdict]:
    method _generate_verdicts (line 221) | def _generate_verdicts(self) -> List[NonAdviceVerdict]:
    method _a_generate_advices (line 238) | async def _a_generate_advices(self, actual_output: str) -> List[str]:
    method _generate_advices (line 250) | def _generate_advices(self, actual_output: str) -> List[str]:
    method _calculate_score (line 262) | def _calculate_score(self) -> float:
    method is_successful (line 275) | def is_successful(self) -> bool:
    method __name__ (line 286) | def __name__(self):

FILE: deepeval/metrics/non_advice/schema.py
  class NonAdviceVerdict (line 5) | class NonAdviceVerdict(BaseModel):
  class Verdicts (line 10) | class Verdicts(BaseModel):
  class Advices (line 14) | class Advices(BaseModel):
  class NonAdviceScoreReason (line 18) | class NonAdviceScoreReason(BaseModel):

FILE: deepeval/metrics/non_advice/template.py
  class NonAdviceTemplate (line 4) | class NonAdviceTemplate:
    method generate_reason (line 14) | def generate_reason(non_advice_violations: list, score: str):
    method generate_verdicts (line 30) | def generate_verdicts(advices: list):
    method generate_advices (line 63) | def generate_advices(actual_output: str, advice_types: List[str]):

FILE: deepeval/metrics/pattern_match/pattern_match.py
  class PatternMatchMetric (line 13) | class PatternMatchMetric(BaseMetric):
    method __init__ (line 19) | def __init__(
    method measure (line 37) | def measure(
    method a_measure (line 81) | async def a_measure(
    method is_successful (line 93) | def is_successful(self) -> bool:
    method __name__ (line 104) | def __name__(self):

FILE: deepeval/metrics/pii_leakage/pii_leakage.py
  class PIILeakageMetric (line 27) | class PIILeakageMetric(BaseMetric):
    method __init__ (line 33) | def __init__(
    method measure (line 52) | def measure(
    method a_measure (line 105) | async def a_measure(
    method _a_generate_reason (line 149) | async def _a_generate_reason(self) -> Optional[str]:
    method _generate_reason (line 171) | def _generate_reason(self) -> Optional[str]:
    method _a_generate_verdicts (line 193) | async def _a_generate_verdicts(self) -> List[PIILeakageVerdict]:
    method _generate_verdicts (line 210) | def _generate_verdicts(self) -> List[PIILeakageVerdict]:
    method _a_extract_pii (line 227) | async def _a_extract_pii(self, actual_output: str) -> List[str]:
    method _extract_pii (line 237) | def _extract_pii(self, actual_output: str) -> List[str]:
    method _calculate_score (line 247) | def _calculate_score(self) -> float:
    method is_successful (line 260) | def is_successful(self) -> bool:
    method __name__ (line 271) | def __name__(self):

FILE: deepeval/metrics/pii_leakage/schema.py
  class PIILeakageVerdict (line 5) | class PIILeakageVerdict(BaseModel):
  class Verdicts (line 10) | class Verdicts(BaseModel):
  class ExtractedPII (line 14) | class ExtractedPII(BaseModel):
  class PIILeakageScoreReason (line 18) | class PIILeakageScoreReason(BaseModel):

FILE: deepeval/metrics/pii_leakage/template.py
  class PIILeakageTemplate (line 10) | class PIILeakageTemplate:
    method generate_reason (line 13) | def generate_reason(privacy_violations: list, score: str):
    method generate_verdicts (line 29) | def generate_verdicts(extracted_pii: list):
    method extract_pii (line 62) | def extract_pii(actual_output: str):

FILE: deepeval/metrics/plan_adherence/plan_adherence.py
  class PlanAdherenceMetric (line 28) | class PlanAdherenceMetric(BaseMetric):
    method __init__ (line 35) | def __init__(
    method measure (line 53) | def measure(
    method a_measure (line 114) | async def a_measure(
    method _get_plan_adherence_score (line 169) | def _get_plan_adherence_score(self, task, plan, test_case):
    method _a_get_plan_adherence_score (line 181) | async def _a_get_plan_adherence_score(self, task, plan, test_case):
    method _extract_plan_from_trace (line 193) | def _extract_plan_from_trace(self, test_case: LLMTestCase) -> AgentPlan:
    method _a_extract_plan_from_trace (line 205) | async def _a_extract_plan_from_trace(
    method _extract_task_from_trace (line 219) | def _extract_task_from_trace(self, test_case: LLMTestCase) -> str:
    method _a_extract_task_from_trace (line 231) | async def _a_extract_task_from_trace(self, test_case: LLMTestCase) -> ...
    method is_successful (line 243) | def is_successful(self) -> bool:
    method __name__ (line 254) | def __name__(self):

FILE: deepeval/metrics/plan_adherence/schema.py
  class AgentPlan (line 5) | class AgentPlan(BaseModel):
  class PlanAdherenceScore (line 9) | class PlanAdherenceScore(BaseModel):

FILE: deepeval/metrics/plan_adherence/template.py
  class PlanAdherenceTemplate (line 6) | class PlanAdherenceTemplate:
    method extract_plan_from_trace (line 16) | def extract_plan_from_trace(trace: dict) -> str:
    method evaluate_adherence (line 75) | def evaluate_adherence(

FILE: deepeval/metrics/plan_quality/plan_quality.py
  class PlanQualityMetric (line 31) | class PlanQualityMetric(BaseMetric):
    method __init__ (line 38) | def __init__(
    method measure (line 56) | def measure(
    method a_measure (line 117) | async def a_measure(
    method _get_plan_quality_score (line 171) | def _get_plan_quality_score(self, task, plan):
    method _a_get_plan_quality_score (line 183) | async def _a_get_plan_quality_score(self, task, plan):
    method _extract_plan_from_trace (line 195) | def _extract_plan_from_trace(self, test_case: LLMTestCase) -> AgentPlan:
    method _a_extract_plan_from_trace (line 207) | async def _a_extract_plan_from_trace(
    method _extract_task_from_trace (line 221) | def _extract_task_from_trace(self, test_case: LLMTestCase) -> str:
    method _a_extract_task_from_trace (line 233) | async def _a_extract_task_from_trace(self, test_case: LLMTestCase) -> ...
    method is_successful (line 245) | def is_successful(self) -> bool:
    method __name__ (line 256) | def __name__(self):

FILE: deepeval/metrics/plan_quality/schema.py
  class AgentPlan (line 5) | class AgentPlan(BaseModel):
  class PlanQualityScore (line 9) | class PlanQualityScore(BaseModel):

FILE: deepeval/metrics/plan_quality/template.py
  class PlanQualityTemplate (line 6) | class PlanQualityTemplate:
    method evaluate_plan_quality (line 16) | def evaluate_plan_quality(user_task: str, agent_plan: list) -> str:

FILE: deepeval/metrics/prompt_alignment/prompt_alignment.py
  class PromptAlignmentMetric (line 28) | class PromptAlignmentMetric(BaseMetric):
    method __init__ (line 35) | def __init__(
    method measure (line 57) | def measure(
    method a_measure (line 115) | async def a_measure(
    method _a_generate_reason (line 160) | async def _a_generate_reason(
    method _generate_reason (line 186) | def _generate_reason(self, input: str, actual_output: str) -> Optional...
    method _a_generate_verdicts (line 210) | async def _a_generate_verdicts(
    method _generate_verdicts (line 229) | def _generate_verdicts(
    method _calculate_score (line 248) | def _calculate_score(self) -> float:
    method is_successful (line 261) | def is_successful(self) -> bool:
    method __name__ (line 272) | def __name__(self):

FILE: deepeval/metrics/prompt_alignment/schema.py
  class PromptAlignmentVerdict (line 5) | class PromptAlignmentVerdict(BaseModel):
  class Verdicts (line 10) | class Verdicts(BaseModel):
  class PromptAlignmentScoreReason (line 14) | class PromptAlignmentScoreReason(BaseModel):

FILE: deepeval/metrics/prompt_alignment/template.py
  class PromptAlignmentTemplate (line 4) | class PromptAlignmentTemplate:
    method generate_verdicts (line 14) | def generate_verdicts(
    method generate_reason (line 65) | def generate_reason(

FILE: deepeval/metrics/ragas.py
  function _check_langchain_available (line 22) | def _check_langchain_available():
  function format_ragas_metric_name (line 29) | def format_ragas_metric_name(name: str):
  class RAGASContextualPrecisionMetric (line 38) | class RAGASContextualPrecisionMetric(BaseMetric):
    method __init__ (line 41) | def __init__(
    method measure (line 54) | def measure(self, test_case: LLMTestCase):
    method a_measure (line 98) | async def a_measure(
    method is_successful (line 103) | def is_successful(self):
    method __name__ (line 107) | def __name__(self):
  class RAGASContextualRecallMetric (line 116) | class RAGASContextualRecallMetric(BaseMetric):
    method __init__ (line 119) | def __init__(
    method a_measure (line 131) | async def a_measure(
    method measure (line 136) | def measure(self, test_case: LLMTestCase):
    method is_successful (line 173) | def is_successful(self):
    method __name__ (line 177) | def __name__(self):
  class RAGASContextualEntitiesRecall (line 186) | class RAGASContextualEntitiesRecall(BaseMetric):
    method __init__ (line 189) | def __init__(
    method a_measure (line 201) | async def a_measure(
    method measure (line 206) | def measure(self, test_case: LLMTestCase):
    method is_successful (line 248) | def is_successful(self):
    method __name__ (line 252) | def __name__(self):
  class RAGASAnswerRelevancyMetric (line 333) | class RAGASAnswerRelevancyMetric(BaseMetric):
    method __init__ (line 336) | def __init__(
    method a_measure (line 350) | async def a_measure(
    method measure (line 355) | def measure(self, test_case: LLMTestCase):
    method is_successful (line 399) | def is_successful(self):
    method __name__ (line 403) | def __name__(self):
  class RAGASFaithfulnessMetric (line 412) | class RAGASFaithfulnessMetric(BaseMetric):
    method __init__ (line 413) | def __init__(
    method a_measure (line 425) | async def a_measure(
    method measure (line 430) | def measure(self, test_case: LLMTestCase):
    method is_successful (line 468) | def is_successful(self):
    method __name__ (line 472) | def __name__(self):
  class RagasMetric (line 481) | class RagasMetric(BaseMetric):
    method __init__ (line 484) | def __init__(
    method a_measure (line 496) | async def a_measure(
    method measure (line 501) | def measure(self, test_case: LLMTestCase):
    method is_successful (line 541) | def is_successful(self):
    method __name__ (line 545) | def __name__(self):
  function import_ragas (line 549) | def import_ragas():

FILE: deepeval/metrics/role_adherence/role_adherence.py
  class RoleAdherenceMetric (line 23) | class RoleAdherenceMetric(BaseConversationalMetric):
    method __init__ (line 26) | def __init__(
    method measure (line 43) | def measure(
    method a_measure (line 92) | async def a_measure(
    method _a_generate_reason (line 137) | async def _a_generate_reason(self, role: str) -> Optional[str]:
    method _generate_reason (line 157) | def _generate_reason(self, role: str) -> Optional[str]:
    method _a_extract_out_of_character_verdicts (line 176) | async def _a_extract_out_of_character_verdicts(
    method _extract_out_of_character_verdicts (line 205) | def _extract_out_of_character_verdicts(
    method _calculate_score (line 230) | def _calculate_score(self, turns: List[Turn]) -> float:
    method is_successful (line 244) | def is_successful(self) -> bool:
    method __name__ (line 255) | def __name__(self):

FILE: deepeval/metrics/role_adherence/schema.py
  class OutOfCharacterResponseVerdict (line 5) | class OutOfCharacterResponseVerdict(BaseModel):
  class OutOfCharacterResponseVerdicts (line 11) | class OutOfCharacterResponseVerdicts(BaseModel):
  class RoleAdherenceScoreReason (line 15) | class RoleAdherenceScoreReason(BaseModel):

FILE: deepeval/metrics/role_adherence/template.py
  class RoleAdherenceTemplate (line 4) | class RoleAdherenceTemplate:
    method extract_out_of_character_response_verdicts (line 14) | def extract_out_of_character_response_verdicts(
    method generate_reason (line 80) | def generate_reason(

FILE: deepeval/metrics/role_violation/role_violation.py
  class RoleViolationMetric (line 27) | class RoleViolationMetric(BaseMetric):
    method __init__ (line 33) | def __init__(
    method measure (line 61) | def measure(
    method a_measure (line 115) | async def a_measure(
    method _a_generate_reason (line 161) | async def _a_generate_reason(self) -> Optional[str]:
    method _generate_reason (line 183) | def _generate_reason(self) -> Optional[str]:
    method _a_generate_verdicts (line 205) | async def _a_generate_verdicts(self) -> List[RoleViolationVerdict]:
    method _generate_verdicts (line 222) | def _generate_verdicts(self) -> List[RoleViolationVerdict]:
    method _a_detect_role_violations (line 239) | async def _a_detect_role_violations(self, actual_output: str) -> List[...
    method _detect_role_violations (line 251) | def _detect_role_violations(self, actual_output: str) -> List[str]:
    method _calculate_score (line 263) | def _calculate_score(self) -> float:
    method is_successful (line 275) | def is_successful(self) -> bool:
    method __name__ (line 286) | def __name__(self):

FILE: deepeval/metrics/role_violation/schema.py
  class RoleViolationVerdict (line 5) | class RoleViolationVerdict(BaseModel):
  class Verdicts (line 10) | class Verdicts(BaseModel):
  class RoleViolations (line 14) | class RoleViolations(BaseModel):
  class RoleViolationScoreReason (line 18) | class RoleViolationScoreReason(BaseModel):

FILE: deepeval/metrics/role_violation/template.py
  class RoleViolationTemplate (line 1) | class RoleViolationTemplate:
    method generate_reason (line 11) | def generate_reason(role_violations: list, score: str):
    method generate_verdicts (line 27) | def generate_verdicts(role_violations: list):
    method detect_role_violations (line 60) | def detect_role_violations(actual_output: str, expected_role: str):

FILE: deepeval/metrics/step_efficiency/schema.py
  class Task (line 5) | class Task(BaseModel):
  class EfficiencyVerdict (line 9) | class EfficiencyVerdict(BaseModel):

FILE: deepeval/metrics/step_efficiency/step_efficiency.py
  class StepEfficiencyMetric (line 21) | class StepEfficiencyMetric(BaseMetric):
    method __init__ (line 28) | def __init__(
    method measure (line 46) | def measure(
    method a_measure (line 99) | async def a_measure(
    method _get_score (line 145) | def _get_score(
    method _a_get_score (line 161) | async def _a_get_score(
    method _extract_task_from_trace (line 177) | def _extract_task_from_trace(self, test_case: LLMTestCase) -> str:
    method _a_extract_task_from_trace (line 189) | async def _a_extract_task_from_trace(self, test_case: LLMTestCase) -> ...
    method is_successful (line 201) | def is_successful(self) -> bool:
    method __name__ (line 212) | def __name__(self):

FILE: deepeval/metrics/step_efficiency/template.py
  class StepEfficiencyTemplate (line 6) | class StepEfficiencyTemplate:
    method extract_task_from_trace (line 16) | def extract_task_from_trace(trace: dict) -> str:
    method get_execution_efficiency (line 146) | def get_execution_efficiency(task: str, trace: dict) -> str:

FILE: deepeval/metrics/summarization/schema.py
  class ScoreType (line 6) | class ScoreType(Enum):
  class SummarizationAlignmentVerdict (line 11) | class SummarizationAlignmentVerdict(BaseModel):
  class SummarizationCoverageVerdict (line 17) | class SummarizationCoverageVerdict(BaseModel):
  class Verdicts (line 23) | class Verdicts(BaseModel):
  class Questions (line 27) | class Questions(BaseModel):
  class Answers (line 31) | class Answers(BaseModel):
  class SummarizationScoreReason (line 35) | class SummarizationScoreReason(BaseModel):

FILE: deepeval/metrics/summarization/summarization.py
  class SummarizationMetric (line 33) | class SummarizationMetric(BaseMetric):
    method __init__ (line 40) | def __init__(
    method measure (line 71) | def measure(
    method a_measure (line 136) | async def a_measure(
    method _a_generate_reason (line 195) | async def _a_generate_reason(self) -> Optional[str]:
    method _generate_reason (line 239) | def _generate_reason(self) -> Optional[str]:
    method _calculate_score (line 283) | def _calculate_score(self, score_type: ScoreType) -> float:
    method _a_generate_answers (line 315) | async def _a_generate_answers(self, text: str) -> List[str]:
    method _generate_answers (line 327) | def _generate_answers(self, text: str) -> List[str]:
    method _a_generate_assessment_questions (line 339) | async def _a_generate_assessment_questions(self, text: str) -> List[str]:
    method _generate_assessment_questions (line 349) | def _generate_assessment_questions(self, text: str) -> List[str]:
    method _a_generate_coverage_verdicts (line 359) | async def _a_generate_coverage_verdicts(
    method _generate_coverage_verdicts (line 389) | def _generate_coverage_verdicts(
    method _a_generate_alignment_verdicts (line 415) | async def _a_generate_alignment_verdicts(
    method _generate_alignment_verdicts (line 435) | def _generate_alignment_verdicts(
    method _a_generate_truths (line 455) | async def _a_generate_truths(self, text: str) -> List[str]:
    method _a_generate_claims (line 469) | async def _a_generate_claims(self, text: str) -> List[str]:
    method _generate_truths (line 480) | def _generate_truths(self, text: str) -> List[str]:
    method _generate_claims (line 494) | def _generate_claims(self, text: str) -> List[str]:
    method is_successful (line 505) | def is_successful(self) -> bool:
    method __name__ (line 516) | def __name__(self):

FILE: deepeval/metrics/summarization/template.py
  class SummarizationTemplate (line 10) | class SummarizationTemplate:
    method generate_reason (line 12) | def generate_reason(contradictions, redundancies, questions, score):
    method generate_answers (line 40) | def generate_answers(questions, text):
    method generate_questions (line 71) | def generate_questions(text, n):
    method generate_alignment_verdicts (line 89) | def generate_alignment_verdicts(original_text, summary_claims):

FILE: deepeval/metrics/task_completion/schema.py
  class TaskAndOutcome (line 5) | class TaskAndOutcome(BaseModel):
  class TaskCompletionVerdict (line 10) | class TaskCompletionVerdict(BaseModel):

FILE: deepeval/metrics/task_completion/task_completion.py
  class TaskCompletionMetric (line 25) | class TaskCompletionMetric(BaseMetric):
    method __init__ (line 32) | def __init__(
    method measure (line 57) | def measure(
    method a_measure (line 106) | async def a_measure(
    method _a_generate_verdicts (line 149) | async def _a_generate_verdicts(self) -> Tuple:
    method _generate_verdicts (line 162) | def _generate_verdicts(self) -> Tuple:
    method _a_extract_task_and_outcome (line 175) | async def _a_extract_task_and_outcome(
    method _extract_task_and_outcome (line 199) | def _extract_task_and_outcome(
    method _calculate_score (line 223) | def _calculate_score(self):
    method is_successful (line 230) | def is_successful(self) -> bool:
    method __name__ (line 241) | def __name__(self):

FILE: deepeval/metrics/task_completion/template.py
  class TaskCompletionTemplate (line 7) | class TaskCompletionTemplate:
    method extract_goal_and_outcome (line 11) | def extract_goal_and_outcome(
    method extract_task_and_outcome_from_trace (line 83) | def extract_task_and_outcome_from_trace(trace: dict) -> str:
    method generate_verdict (line 204) | def generate_verdict(task: str, actual_outcome: str):

FILE: deepeval/metrics/tool_correctness/schema.py
  class ToolSelectionScore (line 4) | class ToolSelectionScore(BaseModel):

FILE: deepeval/metrics/tool_correctness/template.py
  class ToolCorrectnessTemplate (line 5) | class ToolCorrectnessTemplate:
    method get_tool_selection_score (line 8) | def get_tool_selection_score(

FILE: deepeval/metrics/tool_correctness/tool_correctness.py
  class ToolCorrectnessMetric (line 25) | class ToolCorrectnessMetric(BaseMetric):
    method __init__ (line 33) | def __init__(
    method measure (line 57) | def measure(
    method a_measure (line 163) | async def a_measure(
    method _generate_reason (line 263) | def _generate_reason(self):
    method _construct_final_reason (line 308) | def _construct_final_reason(
    method _get_tool_selection_score (line 325) | def _get_tool_selection_score(
    method _a_get_tool_selection_score (line 341) | async def _a_get_tool_selection_score(
    method _calculate_score (line 358) | def _calculate_score(self) -> float:
    method _calculate_exact_match_score (line 377) | def _calculate_exact_match_score(self) -> float:
    method _calculate_non_exact_match_score (line 400) | def _calculate_non_exact_match_score(self) -> float:
    method _compute_weighted_lcs (line 440) | def _compute_weighted_lcs(self) -> Tuple[List[ToolCall], float]:
    method _compare_dicts (line 482) | def _compare_dicts(self, dict1: Dict, dict2: Dict):
    method is_successful (line 503) | def is_successful(self) -> bool:
    method __name__ (line 511) | def __name__(self):
    method indent_multiline_string (line 514) | def indent_multiline_string(self, s, indent_level=4):

FILE: deepeval/metrics/tool_use/schema.py
  class UserInputAndTools (line 4) | class UserInputAndTools(BaseModel):
  class ToolSelectionScore (line 12) | class ToolSelectionScore(BaseModel):
  class ArgumentCorrectnessScore (line 17) | class ArgumentCorrectnessScore(BaseModel):
  class Reason (line 22) | class Reason(BaseModel):

FILE: deepeval/metrics/tool_use/template.py
  class ToolUseTemplate (line 5) | class ToolUseTemplate:
    method get_tool_selection_score (line 8) | def get_tool_selection_score(
    method get_argument_correctness_score (line 74) | def get_argument_correctness_score(
    method get_tool_selection_final_reason (line 148) | def get_tool_selection_final_reason(
    method get_tool_argument_final_reason (line 193) | def get_tool_argument_final_reason(

FILE: deepeval/metrics/tool_use/tool_use.py
  class ToolUseMetric (line 30) | class ToolUseMetric(BaseConversationalMetric):
    method __init__ (line 37) | def __init__(
    method measure (line 56) | def measure(
    method a_measure (line 131) | async def a_measure(
    method _get_argument_correctness_score (line 200) | def _get_argument_correctness_score(
    method _a_get_argument_correctness_score (line 217) | async def _a_get_argument_correctness_score(
    method _get_tool_selection_score (line 235) | def _get_tool_selection_score(
    method _a_get_tool_selection_score (line 253) | async def _a_get_tool_selection_score(
    method _get_user_input_and_turns (line 271) | def _get_user_input_and_turns(
    method _calculate_score (line 308) | def _calculate_score(
    method _generate_reason_for_tool_selection (line 337) | def _generate_reason_for_tool_selection(
    method _generate_reason_for_argument_correctness (line 358) | def _generate_reason_for_argument_correctness(
    method _a_generate_reason_for_tool_selection (line 378) | async def _a_generate_reason_for_tool_selection(
    method _a_generate_reason_for_argument_correctness (line 397) | async def _a_generate_reason_for_argument_correctness(
    method is_successful (line 416) | def is_successful(self) -> bool:
    method __name__ (line 424) | def __name__(self):

FILE: deepeval/metrics/topic_adherence/schema.py
  class QAPair (line 5) | class QAPair(BaseModel):
  class QAPairs (line 10) | class QAPairs(BaseModel):
  class RelevancyVerdict (line 14) | class RelevancyVerdict(BaseModel):
  class TopicAdherenceReason (line 19) | class TopicAdherenceReason(BaseModel):

FILE: deepeval/metrics/topic_adherence/template.py
  class TopicAdherenceTemplate (line 5) | class TopicAdherenceTemplate:
    method get_qa_pairs (line 15) | def get_qa_pairs(
    method get_qa_pair_verdict (line 77) | def get_qa_pair_verdict(
    method generate_reason (line 145) | def generate_reason(success, score, threshold, TP, TN, FP, FN) -> str:

FILE: deepeval/metrics/topic_adherence/topic_adherence.py
  class TopicAdherenceMetric (line 25) | class TopicAdherenceMetric(BaseConversationalMetric):
    method __init__ (line 32) | def __init__(
    method measure (line 51) | def measure(
    method a_measure (line 139) | async def a_measure(
    method _generate_reason (line 213) | def _generate_reason(self, TP, TN, FP, FN):
    method _a_generate_reason (line 228) | async def _a_generate_reason(self, TP, TN, FP, FN):
    method _get_score (line 240) | def _get_score(self, TP, TN, FP, FN) -> float:
    method _get_qa_verdict (line 249) | def _get_qa_verdict(self, qa_pair: QAPair) -> RelevancyVerdict:
    method _a_get_qa_verdict (line 261) | async def _a_get_qa_verdict(self, qa_pair: QAPair) -> RelevancyVerdict:
    method _get_qa_pairs (line 273) | def _get_qa_pairs(self, unit_interactions: List) -> List[QAPairs]:
    method _a_get_qa_pairs (line 296) | async def _a_get_qa_pairs(self, unit_interactions: List) -> List[QAPai...
    method is_successful (line 319) | def is_successful(self) -> bool:
    method __name__ (line 330) | def __name__(self):

FILE: deepeval/metrics/toxicity/schema.py
  class Opinions (line 5) | class Opinions(BaseModel):
  class ToxicityVerdict (line 10) | class ToxicityVerdict(BaseModel):
  class Verdicts (line 15) | class Verdicts(BaseModel):
  class ToxicityScoreReason (line 19) | class ToxicityScoreReason(BaseModel):

FILE: deepeval/metrics/toxicity/template.py
  class ToxicityTemplate (line 12) | class ToxicityTemplate:
    method generate_opinions (line 14) | def generate_opinions(actual_output: str):
    method generate_verdicts (line 44) | def generate_verdicts(opinions: List[str]):
    method generate_reason (line 110) | def generate_reason(toxics: List[str], score: float):

FILE: deepeval/metrics/toxicity/toxicity.py
  class ToxicityMetric (line 27) | class ToxicityMetric(BaseMetric):
    method __init__ (line 34) | def __init__(
    method measure (line 53) | def measure(
    method a_measure (line 105) | async def a_measure(
    method _a_generate_reason (line 152) | async def _a_generate_reason(self) -> str:
    method _generate_reason (line 174) | def _generate_reason(self) -> str:
    method _a_generate_verdicts (line 196) | async def _a_generate_verdicts(self) -> List[ToxicityVerdict]:
    method _generate_verdicts (line 217) | def _generate_verdicts(self) -> List[ToxicityVerdict]:
    method _a_generate_opinions (line 236) | async def _a_generate_opinions(self, actual_output: str) -> List[str]:
    method _generate_opinions (line 249) | def _generate_opinions(self, actual_output: str) -> List[str]:
    method _calculate_score (line 262) | def _calculate_score(self) -> float:
    method is_successful (line 275) | def is_successful(self) -> bool:
    method __name__ (line 286) | def __name__(self):

FILE: deepeval/metrics/turn_contextual_precision/schema.py
  class ContextualPrecisionVerdict (line 5) | class ContextualPrecisionVerdict(BaseModel):
  class Verdicts (line 10) | class Verdicts(BaseModel):
  class ContextualPrecisionScoreReason (line 14) | class ContextualPrecisionScoreReason(BaseModel):
  class InteractionContextualPrecisionScore (line 18) | class InteractionContextualPrecisionScore(BaseModel):

FILE: deepeval/metrics/turn_contextual_precision/template.py
  class TurnContextualPrecisionTemplate (line 6) | class TurnContextualPrecisionTemplate:
    method generate_verdicts (line 18) | def generate_verdicts(
    method generate_reason (line 86) | def generate_reason(
    method generate_final_reason (line 128) | def generate_final_reason(
    method id_retrieval_context (line 175) | def id_retrieval_context(

FILE: deepeval/metrics/turn_contextual_precision/turn_contextual_precision.py
  class TurnContextualPrecisionMetric (line 33) | class TurnContextualPrecisionMetric(BaseConversationalMetric):
    method __init__ (line 41) | def __init__(
    method measure (line 64) | def measure(
    method a_measure (line 126) | async def a_measure(
    method _a_get_contextual_precision_scores (line 186) | async def _a_get_contextual_precision_scores(
    method _get_contextual_precision_scores (line 221) | def _get_contextual_precision_scores(
    method _a_generate_verdicts (line 256) | async def _a_generate_verdicts(
    method _generate_verdicts (line 283) | def _generate_verdicts(
    method _a_get_interaction_score_and_reason (line 310) | async def _a_get_interaction_score_and_reason(
    method _get_interaction_score_and_reason (line 332) | def _get_interaction_score_and_reason(
    method _calculate_interaction_score (line 354) | def _calculate_interaction_score(
    method _a_get_interaction_reason (line 382) | async def _a_get_interaction_reason(
    method _get_interaction_reason (line 418) | def _get_interaction_reason(
    method _get_verbose_steps (line 454) | def _get_verbose_steps(
    method _generate_reason (line 468) | def _generate_reason(
    method _a_generate_reason (line 493) | async def _a_generate_reason(
    method _calculate_score (line 518) | def _calculate_score(
    method is_successful (line 529) | def is_successful(self) -> bool:
    method __name__ (line 540) | def __name__(self):

FILE: deepeval/metrics/turn_contextual_recall/schema.py
  class ContextualRecallVerdict (line 5) | class ContextualRecallVerdict(BaseModel):
  class Verdicts (line 10) | class Verdicts(BaseModel):
  class ContextualRecallScoreReason (line 14) | class ContextualRecallScoreReason(BaseModel):
  class InteractionContextualRecallScore (line 18) | class InteractionContextualRecallScore(BaseModel):

FILE: deepeval/metrics/turn_contextual_recall/template.py
  class TurnContextualRecallTemplate (line 6) | class TurnContextualRecallTemplate:
    method generate_reason (line 18) | def generate_reason(
    method generate_verdicts (line 63) | def generate_verdicts(
    method generate_final_reason (line 119) | def generate_final_reason(
    method id_retrieval_context (line 166) | def id_retrieval_context(

FILE: deepeval/metrics/turn_contextual_recall/turn_contextual_recall.py
  class TurnContextualRecallMetric (line 33) | class TurnContextualRecallMetric(BaseConversationalMetric):
    method __init__ (line 41) | def __init__(
    method measure (line 64) | def measure(
    method a_measure (line 126) | async def a_measure(
    method _a_get_contextual_recall_scores (line 186) | async def _a_get_contextual_recall_scores(
    method _get_contextual_recall_scores (line 218) | def _get_contextual_recall_scores(
    method _a_generate_verdicts (line 250) | async def _a_generate_verdicts(
    method _generate_verdicts (line 275) | def _generate_verdicts(
    method _a_get_interaction_score_and_reason (line 300) | async def _a_get_interaction_score_and_reason(
    method _get_interaction_score_and_reason (line 322) | def _get_interaction_score_and_reason(
    method _calculate_interaction_score (line 344) | def _calculate_interaction_score(
    method _a_get_interaction_reason (line 359) | async def _a_get_interaction_reason(
    method _get_interaction_reason (line 394) | def _get_interaction_reason(
    method _get_verbose_steps (line 429) | def _get_verbose_steps(
    method _generate_reason (line 443) | def _generate_reason(
    method _a_generate_reason (line 468) | async def _a_generate_reason(
    method _calculate_score (line 493) | def _calculate_score(
    method is_successful (line 504) | def is_successful(self) -> bool:
    method __name__ (line 515) | def __name__(self):

FILE: deepeval/metrics/turn_contextual_relevancy/schema.py
  class ContextualRelevancyVerdict (line 5) | class ContextualRelevancyVerdict(BaseModel):
  class ContextualRelevancyVerdicts (line 11) | class ContextualRelevancyVerdicts(BaseModel):
  class ContextualRelevancyScoreReason (line 15) | class ContextualRelevancyScoreReason(BaseModel):
  class InteractionContextualRelevancyScore (line 19) | class InteractionContextualRelevancyScore(BaseModel):

FILE: deepeval/metrics/turn_contextual_relevancy/template.py
  class TurnContextualRelevancyTemplate (line 6) | class TurnContextualRelevancyTemplate:
    method generate_reason (line 18) | def generate_reason(
    method generate_verdicts (line 59) | def generate_verdicts(
    method generate_final_reason (line 124) | def generate_final_reason(

FILE: deepeval/metrics/turn_contextual_relevancy/turn_contextual_relevancy.py
  class TurnContextualRelevancyMetric (line 33) | class TurnContextualRelevancyMetric(BaseConversationalMetric):
    method __init__ (line 40) | def __init__(
    method measure (line 63) | def measure(
    method a_measure (line 125) | async def a_measure(
    method _a_get_contextual_relevancy_scores (line 185) | async def _a_get_contextual_relevancy_scores(
    method _get_contextual_relevancy_scores (line 215) | def _get_contextual_relevancy_scores(
    method _a_generate_verdicts (line 244) | async def _a_generate_verdicts(
    method _generate_verdicts (line 272) | def _generate_verdicts(
    method _a_get_interaction_score_and_reason (line 300) | async def _a_get_interaction_score_and_reason(
    method _get_interaction_score_and_reason (line 322) | def _get_interaction_score_and_reason(
    method _calculate_interaction_score (line 344) | def _calculate_interaction_score(
    method _a_get_interaction_reason (line 359) | async def _a_get_interaction_reason(
    method _get_interaction_reason (line 397) | def _get_interaction_reason(
    method _get_verbose_steps (line 436) | def _get_verbose_steps(
    method _generate_reason (line 450) | def _generate_reason(
    method _a_generate_reason (line 475) | async def _a_generate_reason(
    method _calculate_score (line 500) | def _calculate_score(
    method is_successful (line 511) | def is_successful(self) -> bool:
    method __name__ (line 522) | def __name__(self):

FILE: deepeval/metrics/turn_faithfulness/schema.py
  class FaithfulnessVerdict (line 5) | class FaithfulnessVerdict(BaseModel):
  class Verdicts (line 10) | class Verdicts(BaseModel):
  class Truths (line 14) | class Truths(BaseModel):
  class Claims (line 18) | class Claims(BaseModel):
  class FaithfulnessScoreReason (line 22) | class FaithfulnessScoreReason(BaseModel):
  class InteractionFaithfulnessScore (line 26) | class InteractionFaithfulnessScore(BaseModel):

FILE: deepeval/metrics/turn_faithfulness/template.py
  class TurnFaithfulnessTemplate (line 5) | class TurnFaithfulnessTemplate:
    method generate_claims (line 17) | def generate_claims(
    method generate_truths (line 53) | def generate_truths(
    method generate_verdicts (line 93) | def generate_verdicts(
    method generate_reason (line 145) | def generate_reason(
    method generate_final_reason (line 181) | def generate_final_reason(

FILE: deepeval/metrics/turn_faithfulness/turn_faithfulness.py
  class TurnFaithfulnessMetric (line 35) | class TurnFaithfulnessMetric(BaseConversationalMetric):
    method __init__ (line 42) | def __init__(
    method measure (line 72) | def measure(
    method a_measure (line 132) | async def a_measure(
    method _a_get_faithfulness_scores (line 190) | async def _a_get_faithfulness_scores(
    method _get_faithfulness_scores (line 226) | def _get_faithfulness_scores(
    method _a_generate_truths (line 261) | async def _a_generate_truths(
    method _generate_truths (line 278) | def _generate_truths(
    method _a_generate_claims (line 295) | async def _a_generate_claims(
    method _generate_claims (line 312) | def _generate_claims(
    method _a_generate_verdicts (line 329) | async def _a_generate_verdicts(
    method _generate_verdicts (line 351) | def _generate_verdicts(
    method _get_interaction_score_and_reason (line 373) | def _get_interaction_score_and_reason(
    method _a_get_interaction_score_and_reason (line 399) | async def _a_get_interaction_score_and_reason(
    method _a_get_interaction_reason (line 427) | async def _a_get_interaction_reason(
    method _get_interaction_reason (line 452) | def _get_interaction_reason(self, score, verdicts, multimodal: bool) -...
    method _get_verbose_steps (line 475) | def _get_verbose_steps(
    method _generate_reason (line 491) | def _generate_reason(
    method _a_generate_reason (line 516) | async def _a_generate_reason(
    method _calculate_score (line 541) | def _calculate_score(
    method is_successful (line 552) | def is_successful(self) -> bool:
    method __name__ (line 563) | def __name__(self):

FILE: deepeval/metrics/turn_relevancy/schema.py
  class TurnRelevancyVerdict (line 6) | class TurnRelevancyVerdict(BaseModel):
  class TurnRelevancyScoreReason (line 11) | class TurnRelevancyScoreReason(BaseModel):

FILE: deepeval/metrics/turn_relevancy/template.py
  class TurnRelevancyTemplate (line 4) | class TurnRelevancyTemplate:
    method generate_verdicts (line 14) | def generate_verdicts(sliding_window: List[Dict]):
    method generate_reason (line 64) | def generate_reason(score, irrelevancies):

FILE: deepeval/metrics/turn_relevancy/turn_relevancy.py
  class TurnRelevancyMetric (line 29) | class TurnRelevancyMetric(BaseConversationalMetric):
    method __init__ (line 32) | def __init__(
    method measure (line 51) | def measure(
    method a_measure (line 107) | async def a_measure(
    method _a_generate_reason (line 156) | async def _a_generate_reason(self) -> Optional[str]:
    method _generate_reason (line 183) | def _generate_reason(self) -> Optional[str]:
    method _a_generate_verdict (line 210) | async def _a_generate_verdict(
    method _generate_verdict (line 227) | def _generate_verdict(
    method _calculate_score (line 244) | def _calculate_score(self) -> float:
    method is_successful (line 262) | def is_successful(self) -> bool:
    method __name__ (line 273) | def __name__(self):

FILE: deepeval/metrics/utils.py
  function copy_metrics (line 84) | def copy_metrics(
  function format_turns (line 107) | def format_turns(
  function convert_turn_to_dict (line 121) | def convert_turn_to_dict(
  function get_turns_in_sliding_window (line 148) | def get_turns_in_sliding_window(turns: List[Turn], window_size: int):
  function get_unit_interactions (line 153) | def get_unit_interactions(turns: List[Turn]) -> List[List[Turn]]:
  function print_tools_called (line 188) | def print_tools_called(tools_called_list: List[ToolCall]):
  function print_verbose_logs (line 206) | def print_verbose_logs(metric: str, logs: str):
  function construct_verbose_logs (line 217) | def construct_verbose_logs(metric: BaseMetric, steps: List[str]) -> str:
  function check_conversational_test_case_params (line 232) | def check_conversational_test_case_params(
  function check_llm_test_case_params (line 304) | def check_llm_test_case_params(
  function check_arena_test_case_params (line 384) | def check_arena_test_case_params(
  function trimAndLoadJson (line 415) | def trimAndLoadJson(
  function generate_with_schema_and_extract (line 445) | def generate_with_schema_and_extract(
  function a_generate_with_schema_and_extract (line 473) | async def a_generate_with_schema_and_extract(
  function should_use_anthropic_model (line 510) | def should_use_anthropic_model():
  function should_use_azure_openai (line 517) | def should_use_azure_openai():
  function should_use_local_model (line 524) | def should_use_local_model():
  function should_use_ollama_model (line 531) | def should_use_ollama_model():
  function should_use_gemini_model (line 538) | def should_use_gemini_model():
  function should_use_openai_model (line 545) | def should_use_openai_model():
  function should_use_litellm (line 552) | def should_use_litellm():
  function should_use_portkey (line 559) | def should_use_portkey():
  function should_use_deepseek_model (line 566) | def should_use_deepseek_model():
  function should_use_moonshot_model (line 573) | def should_use_moonshot_model():
  function should_use_grok_model (line 580) | def should_use_grok_model():
  function should_use_amazon_bedrock_model (line 587) | def should_use_amazon_bedrock_model():
  function initialize_model (line 599) | def initialize_model(
  function is_native_model (line 644) | def is_native_model(
  function should_use_azure_openai_embedding (line 675) | def should_use_azure_openai_embedding():
  function should_use_local_embedding (line 682) | def should_use_local_embedding():
  function should_use_ollama_embedding (line 687) | def should_use_ollama_embedding():
  function initialize_embedding_model (line 694) | def initialize_embedding_model(

FILE: deepeval/model_integrations/types.py
  class InputParameters (line 7) | class InputParameters(BaseModel):
  class OutputParameters (line 16) | class OutputParameters(BaseModel):

FILE: deepeval/model_integrations/utils.py
  function _update_all_attributes (line 19) | def _update_all_attributes(
  function __update_input_and_output_of_current_trace (line 59) | def __update_input_and_output_of_current_trace(
  function create_child_tool_spans (line 75) | def create_child_tool_spans(output_parameters: OutputParameters):
  function compact_dump (line 106) | def compact_dump(value: Any) -> str:
  function fmt_url (line 116) | def fmt_url(url: Optional[str]) -> str:

FILE: deepeval/models/_summac_model.py
  function card_to_name (line 56) | def card_to_name(card):
  function name_to_card (line 63) | def name_to_card(name):
  function get_neutral_idx (line 69) | def get_neutral_idx(ent_idx, con_idx):
  class _SummaCImager (line 73) | class _SummaCImager:
    method __init__ (line 74) | def __init__(
    method load_nli (line 115) | def load_nli(self):
    method split_sentences (line 146) | def split_sentences(self, text):
    method split_2sents (line 151) | def split_2sents(self, text):
    method split_paragraphs (line 159) | def split_paragraphs(self, text):
    method split_text (line 166) | def split_text(self, text, granularity="sentence"):
    method build_image (line 178) | def build_image(self, original, generated):
    method get_cache_file (line 268) | def get_cache_file(self):
    method save_cache (line 274) | def save_cache(self):
    method load_cache (line 279) | def load_cache(self):
  class _SummaCConv (line 292) | class _SummaCConv(torch.nn.Module):
    method __init__ (line 293) | def __init__(
    method build_image (line 386) | def build_image(self, original, generated):
    method compute_histogram (line 395) | def compute_histogram(self, original=None, generated=None, image=None):
    method forward (line 434) | def forward(self, originals, generateds, images=None):
    method save_imager_cache (line 511) | def save_imager_cache(self):
    method score (line 515) | def score(self, originals, generateds, **kwargs):
  class _SummaCZS (line 525) | class _SummaCZS:
    method __init__ (line 526) | def __init__(
    method save_imager_cache (line 554) | def save_imager_cache(self):
    method score_one (line 557) | def score_one(self, original, generated):
    method score (line 586) | def score(self, sources, generateds, **kwargs):

FILE: deepeval/models/answer_relevancy_model.py
  function softmax (line 5) | def softmax(x):
  class AnswerRelevancyModel (line 12) | class AnswerRelevancyModel(DeepEvalBaseModel):
    method __init__ (line 13) | def __init__(self, model_name: Optional[str] = None):
    method load_model (line 21) | def load_model(self):
    method _call (line 31) | def _call(self, text: str):
  class CrossEncoderAnswerRelevancyModel (line 45) | class CrossEncoderAnswerRelevancyModel(DeepEvalBaseModel):
    method __init__ (line 46) | def __init__(self, model_name: Optional[str] = None):
    method load_model (line 54) | def load_model(self):
    method _call (line 64) | def _call(self, question: str, answer: str):

FILE: deepeval/models/base_model.py
  class DeepEvalModelData (line 8) | class DeepEvalModelData:
  class DeepEvalBaseModel (line 19) | class DeepEvalBaseModel(ABC):
    method __init__ (line 20) | def __init__(self, model_name: Optional[str] = None, *args, **kwargs):
    method load_model (line 25) | def load_model(self, *args, **kwargs) -> "DeepEvalBaseModel":
    method __call__ (line 33) | def __call__(self, *args: Any, **kwargs: Any) -> Any:
    method _call (line 37) | def _call(self, *args, **kwargs):
  class DeepEvalBaseLLM (line 46) | class DeepEvalBaseLLM(ABC):
    method __init__ (line 47) | def __init__(self, model: Optional[str] = None, *args, **kwargs):
    method __init_subclass__ (line 51) | def __init_subclass__(cls, **kwargs):
    method load_model (line 69) | def load_model(self, *args, **kwargs) -> "DeepEvalBaseLLM":
    method generate (line 78) | def generate(self, *args, **kwargs) -> str:
    method a_generate (line 87) | async def a_generate(self, *args, **kwargs) -> str:
    method get_model_name (line 96) | def get_model_name(self, *args, **kwargs) -> str:
    method batch_generate (line 99) | def batch_generate(self, *args, **kwargs) -> List[str]:
    method supports_log_probs (line 110) | def supports_log_probs(self) -> Union[bool, None]:
    method supports_temperature (line 113) | def supports_temperature(self) -> Union[bool, None]:
    method supports_multimodal (line 116) | def supports_multimodal(self) -> Union[bool, None]:
    method supports_structured_outputs (line 119) | def supports_structured_outputs(self) -> Union[bool, None]:
    method supports_json_mode (line 122) | def supports_json_mode(self) -> Union[bool, None]:
    method generate_with_schema (line 125) | def generate_with_schema(self, *args, schema=None, **kwargs):
    method a_generate_with_schema (line 133) | async def a_generate_with_schema(self, *args, schema=None, **kwargs):
  class DeepEvalBaseEmbeddingModel (line 142) | class DeepEvalBaseEmbeddingModel(ABC):
    method __init__ (line 143) | def __init__(self, model: Optional[str] = None, *args, **kwargs):
    method load_model (line 148) | def load_model(self, *args, **kwargs) -> "DeepEvalBaseEmbeddingModel":
    method embed_text (line 157) | def embed_text(self, *args, **kwargs) -> List[float]:
    method a_embed_text (line 166) | async def a_embed_text(self, *args, **kwargs) -> List[float]:
    method embed_texts (line 175) | def embed_texts(self, *args, **kwargs) -> List[List[float]]:
    method a_embed_texts (line 184) | async def a_embed_texts(self, *args, **kwargs) -> List[List[float]]:
    method get_model_name (line 193) | def get_model_name(self, *args, **kwargs) -> str:

FILE: deepeval/models/detoxify_model.py
  class DetoxifyModel (line 6) | class DetoxifyModel(DeepEvalBaseModel):
    method __init__ (line 7) | def __init__(self, model_name: str | None = None, *args, **kwargs):
    method load_model (line 17) | def load_model(self):
    method _call (line 21) | def _call(self, text: str):

FILE: deepeval/models/embedding_models/azure_embedding_model.py
  class AzureOpenAIEmbeddingModel (line 27) | class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
    method __init__ (line 28) | def __init__(
    method embed_text (line 100) | def embed_text(self, text: str) -> List[float]:
    method embed_texts (line 108) | def embed_texts(self, texts: List[str]) -> List[List[float]]:
    method a_embed_text (line 116) | async def a_embed_text(self, text: str) -> List[float]:
    method a_embed_texts (line 124) | async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
    method load_model (line 131) | def load_model(self, async_mode: bool = False):
    method _build_client (line 136) | def _build_client(self, cls):
    method get_model_name (line 164) | def get_model_name(self):

FILE: deepeval/models/embedding_models/local_embedding_model.py
  class LocalEmbeddingModel (line 21) | class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
    method __init__ (line 22) | def __init__(
    method embed_text (line 65) | def embed_text(self, text: str) -> List[float]:
    method embed_texts (line 73) | def embed_texts(self, texts: List[str]) -> List[List[float]]:
    method a_embed_text (line 81) | async def a_embed_text(self, text: str) -> List[float]:
    method a_embed_texts (line 89) | async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
    method load_model (line 100) | def load_model(self, async_mode: bool = False):
    method _build_client (line 105) | def _build_client(self, cls):
    method get_model_name (line 131) | def get_model_name(self):

FILE: deepeval/models/embedding_models/ollama_embedding_model.py
  class OllamaEmbeddingModel (line 20) | class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
    method __init__ (line 21) | def __init__(
    method embed_text (line 63) | def embed_text(self, text: str) -> List[float]:
    method embed_texts (line 71) | def embed_texts(self, texts: List[str]) -> List[List[float]]:
    method a_embed_text (line 79) | async def a_embed_text(self, text: str) -> List[float]:
    method a_embed_texts (line 87) | async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
    method load_model (line 98) | def load_model(self, async_mode: bool = False):
    method _build_client (line 109) | def _build_client(self, cls):
    method get_model_name (line 112) | def get_model_name(self):

FILE: deepeval/models/embedding_models/openai_embedding_model.py
  class OpenAIEmbeddingModel (line 33) | class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
    method __init__ (line 35) | def __init__(
    method embed_text (line 68) | def embed_text(self, text: str) -> List[float]:
    method embed_texts (line 76) | def embed_texts(self, texts: List[str]) -> List[List[float]]:
    method a_embed_text (line 84) | async def a_embed_text(self, text: str) -> List[float]:
    method a_embed_texts (line 92) | async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
    method load_model (line 103) | def load_model(self, async_mode: bool = False):
    method _build_client (line 108) | def _build_client(self, cls):
    method get_model_name (line 133) | def get_model_name(self):

FILE: deepeval/models/hallucination_model.py
  class HallucinationModel (line 7) | class HallucinationModel(metaclass=Singleton):
    method __init__ (line 8) | def __init__(self, model_name: Optional[str] = None):

FILE: deepeval/models/llms/amazon_bedrock_model.py
  class AmazonBedrockModel (line 38) | class AmazonBedrockModel(DeepEvalBaseLLM):
    method __init__ (line 39) | def __init__(
    method generate (line 161) | def generate(
    method a_generate (line 167) | async def a_generate(
    method generate_payload (line 195) | def generate_payload(
    method supports_log_probs (line 239) | def supports_log_probs(self) -> Union[bool, None]:
    method supports_temperature (line 242) | def supports_temperature(self) -> Union[bool, None]:
    method supports_multimodal (line 245) | def supports_multimodal(self) -> Union[bool, None]:
    method supports_structured_outputs (line 248) | def supports_structured_outputs(self) -> Union[bool, None]:
    method supports_json_mode (line 251) | def supports_json_mode(self) -> Union[bool, None]:
    method _get_client (line 259) | async def _get_client(self):
    method close (line 294) | async def close(self):
    method _extract_text_from_converse_response (line 302) | def _extract_text_from_converse_response(response: dict) -> str:
    method get_converse_request_body (line 341) | def get_converse_request_body(self, prompt: str) -> dict:
    method calculate_cost (line 350) | def calculate_cost(
    method load_model (line 359) | def load_model(self):
    method get_model_name (line 362) | def get_model_name(self) -> str:

FILE: deepeval/models/llms/anthropic_model.py
  class AnthropicModel (line 33) | class AnthropicModel(DeepEvalBaseLLM):
    method __init__ (line 34) | def __init__(
    method generate (line 129) | def generate(
    method a_generate (line 167) | async def a_generate(
    method generate_content (line 205) | def generate_content(self, multimodal_input: List[Union[str, MLLMImage...
    method calculate_cost (line 237) | def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
    method supports_log_probs (line 247) | def supports_log_probs(self) -> Union[bool, None]:
    method supports_temperature (line 250) | def supports_temperature(self) -> Union[bool, None]:
    method supports_multimodal (line 253) | def supports_multimodal(self) -> Union[bool, None]:
    method supports_structured_outputs (line 256) | def supports_structured_outputs(self) -> Union[bool, None]:
    method supports_json_mode (line 259) | def supports_json_mode(self) -> Union[bool, None]:
    method load_model (line 266) | def load_model(self, async_mode: bool = False):
    method _client_kwargs (line 277) | def _client_kwargs(self) -> Dict:
    method _build_client (line 285) | def _build_client(self, cls):
    method get_model_name (line 305) | def get_model_name(self):

FILE: deepeval/models/llms/azure_model.py
  class AzureOpenAIModel (line 39) | class AzureOpenAIModel(DeepEvalBaseLLM):
    method __init__ (line 40) | def __init__(
    method generate (line 179) | def generate(
    method a_generate (line 257) | async def a_generate(
    method generate_raw_response (line 340) | def generate_raw_response(
    method a_generate_raw_response (line 372) | async def a_generate_raw_response(
    method generate_content (line 403) | def generate_content(
    method calculate_cost (line 436) | def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
    method supports_log_probs (line 446) | def supports_log_probs(self) -> Union[bool, None]:
    method supports_temperature (line 449) | def supports_temperature(self) -> Union[bool, None]:
    method supports_multimodal (line 452) | def supports_multimodal(self) -> Union[bool, None]:
    method supports_structured_outputs (line 455) | def supports_structured_outputs(self) -> Union[bool, None]:
    method supports_json_mode (line 458) | def supports_json_mode(self) -> Union[bool, None]:
    method load_model (line 465) | def load_model(self, async_mode: bool = False):
    method _client_kwargs (line 470) | def _client_kwargs(self) -> Dict:
    method _build_client (line 481) | def _build_client(self, cls):
    method get_model_name (line 547) | def get_model_name(self):

FILE: deepeval/models/llms/constants.py
  function default_model_data (line 14) | def default_model_data() -> DeepEvalModelData:
  class ModelDataRegistry (line 18) | class ModelDataRegistry(dict[str, ModelDataValue]):
    method get (line 19) | def get(  # type: ignore[override]
    method __getitem__ (line 31) | def __getitem__(self, key: str) -> DeepEvalModelData:
  function make_model_data (line 40) | def make_model_data(**kwargs: Any) -> ModelDataFactory:

FILE: deepeval/models/llms/deepseek_model.py
  class DeepSeekModel (line 25) | class DeepSeekModel(DeepEvalBaseLLM):
    method __init__ (line 26) | def __init__(
    method generate (line 105) | def generate(
    method a_generate (line 140) | async def a_generate(
    method calculate_cost (line 178) | def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
    method supports_log_probs (line 188) | def supports_log_probs(self) -> Union[bool, None]:
    method supports_temperature (line 191) | def supports_temperature(self) -> Union[bool, None]:
    method supports_multimodal (line 194) | def supports_multimodal(self) -> Union[bool, None]:
    method supports_structured_outputs (line 197) | def supports_structured_outputs(self) -> Union[bool, None]:
    method supports_json_mode (line 200) | def supports_json_mode(self) -> Union[bool, None]:
    method load_model (line 207) | def load_model(self, async_mode: bool = False):
    method _client_kwargs (line 212) | def _client_kwargs(self) -> Dict:
    method _build_client (line 220) | def _build_client(self, cls):
    method get_model_name (line 242) | def get_model_name(self):

FILE: deepeval/models/llms/gemini_model.py
  class GeminiModel (line 31) | class GeminiModel(DeepEvalBaseLLM):
    method __init__ (line 60) | def __init__(
    method should_use_vertexai (line 151) | def should_use_vertexai(self) -> bool:
    method generate_content (line 169) | def generate_content(
    method generate (line 226) | def generate(
    method a_generate (line 270) | async def a_generate(
    method supports_log_probs (line 317) | def supports_log_probs(self) -> Union[bool, None]:
    method supports_temperature (line 320) | def supports_temperature(self) -> Union[bool, None]:
    method supports_multimodal (line 323) | def supports_multimodal(self) -> Union[bool, None]:
    method supports_structured_outputs (line 326) | def supports_structured_outputs(self) -> Union[bool, None]:
    method supports_json_mode (line 333) | def supports_json_mode(self) -> Union[bool, None]:
    method load_model (line 343) | def load_model(self):
    method _require_oauth2 (line 354) | def _require_oauth2(self):
    method _require_module (line 361) | def _require_module(self):
    method _client_kwargs (line 368) | def _client_kwargs(self, **override_kwargs) -> Dict:
    method _build_client (line 375) | def _build_client(self) -> "Client":
    method get_model_name (line 434) | def get_model_name(self):

FILE: deepeval/models/llms/grok_model.py
  class GrokModel (line 26) | class GrokModel(DeepEvalBaseLLM):
    method __init__ (line 27) | def __init__(
    method generate (line 105) | def generate(
    method a_generate (line 150) | async def a_generate(
    method generate_content (line 195) | def generate_content(
    method calculate_cost (line 227) | def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
    method supports_log_probs (line 237) | def supports_log_probs(self) -> Union[bool, None]:
    method supports_temperature (line 240) | def supports_temperature(self) -> Union[bool, None]:
    method supports_multimodal (line 243) | def supports_multimodal(self) -> Union[bool, None]:
    method supports_structured_outputs (line 246) | def supports_structured_outputs(self) -> Union[bool, None]:
    method supports_json_mode (line 249) | def supports_json_mode(self) -> Union[bool, None]:
    method load_model (line 256) | def load_model(self, async_mode: bool = False):
    method _client_kwargs (line 269) | def _client_kwargs(self) -> Dict:
    method _build_client (line 293) | def _build_client(self, cls):
    method get_model_name (line 311) | def get_model_name(self):

FILE: deepeval/models/llms/kimi_model.py
  class KimiModel (line 26) | class KimiModel(DeepEvalBaseLLM):
    method __init__ (line 27) | def __init__(
    method generate (line 105) | def generate(
    method a_generate (line 150) | async def a_generate(
    method generate_content (line 194) | def generate_content(
    method calculate_cost (line 226) | def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
    method supports_log_probs (line 236) | def supports_log_probs(self) -> Union[bool, None]:
    method supports_temperature (line 239) | def supports_temperature(self) -> Union[bool, None]:
    method supports_multimodal (line 242) | def supports_multimodal(self) -> Union[bool, None]:
    method supports_structured_outputs (line 245) | def supports_structured_outputs(self) -> Union[bool, None]:
    method supports_json_mode (line 248) | def supports_json_mode(self) -> Union[bool, None]:
    method load_model (line 255) | def load_model(self, async_mode: bool = False):
    method _client_kwargs (line 260) | def _client_kwargs(self) -> Dict:
    method _build_client (line 271) | def _build_client(self, cls):
    method get_model_name (line 293) | def get_model_name(self):

FILE: deepeval/models/llms/litellm_model.py
  function log_retry_error (line 25) | def log_retry_error(retry_state: RetryCallState):
  class LiteLLMModel (line 42) | class LiteLLMModel(DeepEvalBaseLLM):
    method __init__ (line 49) | def __init__(
    method generate (line 139) | def generate(
    method a_generate (line 201) | async def a_generate(
    method generate_raw_response (line 263) | def generate_raw_response(
    method a_generate_raw_response (line 310) | async def a_generate_raw_response(
    method generate_samples (line 357) | def generate_samples(
    method generate_content (line 388) | def generate_content(
    method calculate_cost (line 416) | def calculate_cost(self, response: Any) -> float:
    method get_evaluation_cost (line 442) | def get_evaluation_cost(self) -> float:
    method get_model_name (line 446) | def get_model_name(self) -> str:
    method load_model (line 452) | def load_model(self, async_mode: bool = False):
    method supports_multimodal (line 466) | def supports_multimodal(self):

FILE: deepeval/models/llms/local_model.py
  class LocalModel (line 29) | class LocalModel(DeepEvalBaseLLM):
    method __init__ (line 30) | def __init__(
    method generate (line 88) | def generate(
    method a_generate (line 114) | async def a_generate(
    method generate_content (line 139) | def generate_content(
    method get_model_name (line 200) | def get_model_name(self):
    method supports_multimodal (line 203) | def supports_multimodal(self):
    method load_model (line 206) | def load_model(self, async_mode: bool = False):
    method _client_kwargs (line 211) | def _client_kwargs(self) -> Dict:
    method _build_client (line 221) | def _build_client(self, cls):

FILE: deepeval/models/llms/ollama_model.py
  class OllamaModel (line 23) | class OllamaModel(DeepEvalBaseLLM):
    method __init__ (line 24) | def __init__(
    method generate (line 75) | def generate(
    method a_generate (line 105) | async def a_generate(
    method generate_messages (line 134) | def generate_messages(
    method supports_log_probs (line 196) | def supports_log_probs(self) -> Union[bool, None]:
    method supports_temperature (line 199) | def supports_temperature(self) -> Union[bool, None]:
    method supports_multimodal (line 202) | def supports_multimodal(self) -> Union[bool, None]:
    method supports_structured_outputs (line 205) | def supports_structured_outputs(self) -> Union[bool, None]:
    method supports_json_mode (line 208) | def supports_json_mode(self) -> Union[bool, None]:
    method load_model (line 215) | def load_model(self, async_mode: bool = False):
    method _client_kwargs (line 225) | def _client_kwargs(self) -> Dict:
    method _build_client (line 229) | def _build_client(self, cls):
    method get_model_name (line 236) | def get_model_name(self):

FILE: deepeval/models/llms/openai_model.py
  function _request_timeout_seconds (line 35) | def _request_timeout_seconds() -> float:
  class GPTModel (line 45) | class GPTModel(DeepEvalBaseLLM):
    method __init__ (line 46) | def __init__(
    method generate (line 148) | def generate(
    method a_generate (line 215) | async def a_generate(
    method _cap_top_logprobs (line 285) | def _cap_top_logprobs(self, top_logprobs: int) -> int:
    method generate_raw_response (line 293) | def generate_raw_response(
    method a_generate_raw_response (line 332) | async def a_generate_raw_response(
    method generate_samples (line 371) | def generate_samples(
    method calculate_cost (line 396) | def calculate_cost(
    method supports_log_probs (line 408) | def supports_log_probs(self) -> Union[bool, None]:
    method supports_temperature (line 411) | def supports_temperature(self) -> Union[bool, None]:
    method supports_multimodal (line 414) | def supports_multimodal(self) -> Union[bool, None]:
    method supports_structured_outputs (line 417) | def supports_structured_outputs(self) -> Union[bool, None]:
    method supports_json_mode (line 424) | def supports_json_mode(self) -> Union[bool, None]:
    method generate_content (line 434) | def generate_content(
    method load_model (line 463) | def load_model(self, async_mode: bool = False):
    method _client_kwargs (line 468) | def _client_kwargs(self) -> Dict:
    method _build_client (line 482) | def _build_client(self, cls):
    method get_model_name (line 516) | def get_model_name(self):
    method _update_llm_span_from_completion (line 519) | def _update_llm_span_from_completion(

FILE: deepeval/models/llms/openrouter_model.py
  function _request_timeout_seconds (line 27) | def _request_timeout_seconds() -> float:
  function _convert_schema_to_openrouter_format (line 32) | def _convert_schema_to_openrouter_format(
  class OpenRouterModel (line 70) | class OpenRouterModel(DeepEvalBaseLLM):
    method __init__ (line 71) | def __init__(
    method _generate_with_client (line 139) | async def _generate_with_client(
    method generate (line 215) | def generate(
    method a_generate (line 226) | async def a_generate(
    method generate_raw_response (line 237) | def generate_raw_response(
    method a_generate_raw_response (line 262) | async def a_generate_raw_response(
    method generate_samples (line 287) | def generate_samples(
    method calculate_cost (line 310) | def calculate_cost(
    method get_model_name (line 354) | def get_model_name(self):
    method load_model (line 357) | def load_model(self, async_mode: bool = False):
    method _client_kwargs (line 362) | def _client_kwargs(self) -> Dict:
    method _build_client (line 377) | def _build_client(self, cls):

FILE: deepeval/models/llms/portkey_model.py
  function _request_timeout_seconds (line 17) | def _request_timeout_seconds() -> float:
  class PortkeyModel (line 22) | class PortkeyModel(DeepEvalBaseLLM):
    method __init__ (line 23) | def __init__(
    method _headers (line 74) | def _headers(self) -> Dict[str, str]:
    method _payload (line 90) | def _payload(self, prompt: str) -> Dict[str, Any]:
    method generate_content (line 104) | def generate_content(
    method _extract_content (line 132) | def _extract_content(self, data: Dict[str, Any]) -> str:
    method generate (line 144) | def generate(self, prompt: str) -> str:
    method a_generate (line 167) | async def a_generate(self, prompt: str) -> str:
    method load_model (line 184) | def load_model(self):
    method get_model_name (line 187) | def get_model_name(self):
    method supports_multimodal (line 190) | def supports_multimodal(self):

FILE: deepeval/models/llms/utils.py
  function trim_and_load_json (line 11) | def trim_and_load_json(
  function safe_asyncio_run (line 30) | def safe_asyncio_run(coro):

FILE: deepeval/models/retry_policy.py
  function set_outer_deadline (line 73) | def set_outer_deadline(seconds: float | None):
  function reset_outer_deadline (line 96) | def reset_outer_deadline(token):
  function _remaining_budget (line 109) | def _remaining_budget() -> float | None:
  function _is_budget_spent (line 116) | def _is_budget_spent() -> bool:
  function resolve_effective_attempt_timeout (line 121) | def resolve_effective_attempt_timeout():
  class ErrorPolicy (line 153) | class ErrorPolicy:
  function extract_error_code (line 181) | def extract_error_code(
  function make_is_transient (line 279) | def make_is_transient(
  class StopFromEnv (line 340) | class StopFromEnv(stop_base):
    method __call__ (line 341) | def __call__(self, retry_state):
  class WaitFromEnv (line 349) | class WaitFromEnv(wait_base):
    method __call__ (line 350) | def __call__(self, retry_state):
  function dynamic_stop (line 364) | def dynamic_stop():
  function dynamic_wait (line 368) | def dynamic_wait():
  function retry_predicate (line 372) | def retry_predicate(policy: ErrorPolicy, **kw):
  function sdk_retries_for (line 387) | def sdk_retries_for(provider: Provider) -> bool:
  function get_retry_policy_for (line 394) | def get_retry_policy_for(provider: Provider) -> Optional[ErrorPolicy]:
  function dynamic_retry (line 406) | def dynamic_retry(provider: Provider):
  function _retry_log_levels (line 424) | def _retry_log_levels():
  function make_before_sleep_log (line 435) | def make_before_sleep_log(slug: str):
  function make_after_log (line 463) | def make_after_log(slug: str):
  function _make_timeout_error (line 498) | def _make_timeout_error(timeout_seconds: float) -> asyncio.TimeoutError:
  function run_sync_with_timeout (line 514) | def run_sync_with_timeout(func, timeout_seconds, *args, **kwargs):
  function create_retry_decorator (line 630) | def create_retry_decorator(provider: Provider):
  function _httpx_net_excs (line 692) | def _httpx_net_excs() -> tuple[type, ...]:
  function _requests_net_excs (line 709) | def _requests_net_excs() -> tuple[type, ...]:
  function _opt_pred (line 1005) | def _opt_pred(

FILE: deepeval/models/summac_model.py
  class SummaCModels (line 8) | class SummaCModels(DeepEvalBaseModel):
    method __init__ (line 9) | def __init__(
    method load_model (line 26) | def load_model(
    method _call (line 47) | def _call(

FILE: deepeval/models/unbias_model.py
  class UnBiasedModel (line 5) | class UnBiasedModel(DeepEvalBaseModel):
    method __init__ (line 6) | def __init__(self, model_name: str | None = None, *args, **kwargs):
    method load_model (line 10) | def load_model(self):
    method _call (line 17) | def _call(self, text):

FILE: deepeval/models/utils.py
  function parse_model_name (line 10) | def parse_model_name(model_name: Optional[str] = None) -> Optional[str]:
  function require_secret_api_key (line 40) | def require_secret_api_key(
  function require_costs (line 82) | def require_costs(
  function normalize_kwargs_and_extract_aliases (line 134) | def normalize_kwargs_and_extract_aliases(

FILE: deepeval/openai/extractors.py
  function safe_extract_input_parameters (line 17) | def safe_extract_input_parameters(
  function extract_input_parameters_from_completion (line 29) | def extract_input_parameters_from_completion(
  function extract_input_parameters_from_response (line 67) | def extract_input_parameters_from_response(
  function safe_extract_output_parameters (line 107) | def safe_extract_output_parameters(
  function extract_output_parameters_from_completion (line 127) | def extract_output_parameters_from_completion(
  function extract_output_parameters_from_response (line 164) | def extract_output_parameters_from_response(

FILE: deepeval/openai/patch.py
  function patch_openai_classes (line 29) | def patch_openai_classes():
  function _create_sync_wrapper (line 95) | def _create_sync_wrapper(original_method, is_completion_method: bool):
  function _create_async_wrapper (line 109) | def _create_async_wrapper(original_method, is_completion_method: bool):
  function _patch_async_openai_client_method (line 123) | def _patch_async_openai_client_method(
  function _patch_sync_openai_client_method (line 162) | def _patch_sync_openai_client_method(
  function _update_all_attributes (line 201) | def _update_all_attributes(
  function __update_input_and_output_of_current_trace (line 244) | def __update_input_and_output_of_current_trace(
  function unpatch_openai_classes (line 261) | def unpatch_openai_classes():

FILE: deepeval/openai/utils.py
  function create_child_tool_spans (line 15) | def create_child_tool_spans(output_parameters: OutputParameters):
  function stringify_multimodal_content (line 41) | def stringify_multimodal_content(content: Any) -> str:
  function render_messages (line 111) | def render_messages(
  function render_response_input (line 171) | def render_response_input(input: List[Dict[str, Any]]) -> List[Dict[str,...
  function _render_content (line 192) | def _render_content(content: Dict[str, Any], indent: int = 0) -> str:

FILE: deepeval/openai_agents/agent.py
  class DeepEvalAgent (line 24) | class DeepEvalAgent(BaseAgent[TContext], Generic[TContext]):
    method __post_init__ (line 35) | def __post_init__(self):

FILE: deepeval/openai_agents/callback_handler.py
  function _check_openai_agents_available (line 48) | def _check_openai_agents_available():
  class DeepEvalTracingProcessor (line 55) | class DeepEvalTracingProcessor(TracingProcessor):
    method __init__ (line 56) | def __init__(self) -> None:
    method on_trace_start (line 62) | def on_trace_start(self, trace: "Trace") -> None:
    method on_trace_end (line 87) | def on_trace_end(self, trace: "Trace") -> None:
    method on_span_start (line 96) | def on_span_start(self, span: "Span") -> None:
    method on_span_end (line 118) | def on_span_end(self, span: "Span") -> None:
    method force_flush (line 139) | def force_flush(self) -> None:
    method shutdown (line 142) | def shutdown(self) -> None:
    method get_span_kind (line 145) | def get_span_kind(self, span_data: "SpanData") -> str:

FILE: deepeval/openai_agents/extractors.py
  function _check_openai_agents_available (line 53) | def _check_openai_agents_available():
  function update_span_properties (line 60) | def update_span_properties(span: BaseSpan, span_data: "SpanData"):
  function update_span_properties_from_response_span_data (line 90) | def update_span_properties_from_response_span_data(
  function update_span_properties_from_generation_span_data (line 147) | def update_span_properties_from_generation_span_data(
  function update_span_properties_from_function_span_data (line 182) | def update_span_properties_from_function_span_data(
  function update_span_properties_from_mcp_list_tool_span_data (line 199) | def update_span_properties_from_mcp_list_tool_span_data(
  function update_span_properties_from_agent_span_data (line 219) | def update_span_properties_from_agent_span_data(
  function update_span_properties_from_handoff_span_data (line 237) | def update_span_properties_from_handoff_span_data(
  function update_span_properties_from_custom_span_data (line 251) | def update_span_properties_from_custom_span_data(
  function update_span_properties_from_guardrail_span_data (line 259) | def update_span_properties_from_guardrail_span_data(
  function parse_response_input (line 275) | def parse_response_input(
  function parse_message_param (line 320) | def parse_message_param(
  function parse_message_content_list (line 337) | def parse_message_content_list(
  function parse_function_tool_call_param (line 359) | def parse_function_tool_call_param(
  function parse_function_call_output (line 369) | def parse_function_call_output(
  function parse_response_output (line 384) | def parse_response_output(response: List[ResponseOutputItem]):
  function parse_message (line 400) | def parse_message(
  function parse_function_call (line 414) | def parse_function_call(
  function update_trace_properties_from_span_data (line 424) | def update_trace_properties_from_span_data(

FILE: deepeval/openai_agents/patch.py
  function _agent_span_for_run_step_patch (line 26) | def _agent_span_for_run_step_patch() -> Optional[AgentSpan]:
  function _resolve_agent_from_run_step_args (line 41) | def _resolve_agent_from_run_step_args(args: Any, kwargs: Any) -> Any:
  function _wrap_with_observe (line 56) | def _wrap_with_observe(
  function function_tool (line 92) | def function_tool(
  class _ObservedModel (line 129) | class _ObservedModel(Model):
    method __init__ (line 130) | def __init__(
    method __getattr__ (line 142) | def __getattr__(self, name: str) -> Any:
    method get_response (line 145) | async def get_response(
    method stream_response (line 171) | def stream_response(
  function patch_default_agent_run_single_turn (line 210) | def patch_default_agent_run_single_turn():
  function patch_default_agent_run_single_turn_streamed (line 269) | def patch_default_agent_run_single_turn_streamed():
  function patch_default_agent_runner_get_model (line 331) | def patch_default_agent_runner_get_model():
  function _set_agent_metrics (line 387) | def _set_agent_metrics(agent: Agent, agent_span: AgentSpan) -> None:

FILE: deepeval/optimizer/algorithms/base.py
  class BaseAlgorithm (line 10) | class BaseAlgorithm(ABC):
    method execute (line 16) | def execute(
    method a_execute (line 24) | async def a_execute(

FILE: deepeval/optimizer/algorithms/copro/copro.py
  class COPRO (line 34) | class COPRO(BaseAlgorithm):
    method __init__ (line 43) | def __init__(
    method _init_components (line 69) | def _init_components(self) -> None:
    method _sample_minibatch (line 75) | def _sample_minibatch(self, goldens: List) -> List:
    method _update_step (line 80) | def _update_step(self, message: str) -> None:
    method _update_trial_progress (line 84) | def _update_trial_progress(self, step: int, total: int) -> None:
    method _extract_optimized_set (line 93) | def _extract_optimized_set(self) -> Optional[str]:
    method _evaluate_candidate (line 103) | def _evaluate_candidate(
    method _a_evaluate_candidate (line 137) | async def _a_evaluate_candidate(
    method execute (line 173) | def execute(
    method a_execute (line 289) | async def a_execute(
    method generate_summary_table (line 415) | def generate_summary_table(self, report: OptimizationReport) -> List[T...

FILE: deepeval/optimizer/algorithms/copro/proposer.py
  class COPROProposer (line 23) | class COPROProposer:
    method __init__ (line 30) | def __init__(
    method _accrue_cost (line 42) | def _accrue_cost(self, cost: float) -> None:
    method _format_history (line 45) | def _format_history(self, history: List[Tuple[Prompt, float, str]]) ->...
    method _is_duplicate (line 61) | def _is_duplicate(self, new_prompt: Prompt, existing: List[Prompt]) ->...
    method propose_bootstrap (line 77) | def propose_bootstrap(
    method propose_from_history (line 102) | def propose_from_history(
    method _generate_candidates_from_guidelines (line 131) | def _generate_candidates_from_guidelines(
    method a_propose_bootstrap (line 167) | async def a_propose_bootstrap(
    method a_propose_from_history (line 192) | async def a_propose_from_history(
    method _a_generate_candidates_from_guidelines (line 225) | async def _a_generate_candidates_from_guidelines(

FILE: deepeval/optimizer/algorithms/copro/schema.py
  class GuidelineListSchema (line 5) | class GuidelineListSchema(BaseModel):
  class COPROProposalSchema (line 9) | class COPROProposalSchema(BaseModel):

FILE: deepeval/optimizer/algorithms/copro/template.py
  class COPROTemplate (line 1) | class COPROTemplate:
    method generate_bootstrap_guidelines (line 4) | def generate_bootstrap_guidelines(
    method generate_history_guidelines (line 32) | def generate_history_guidelines(
    method generate_candidate (line 65) | def generate_candidate(

FILE: deepeval/optimizer/algorithms/gepa/gepa.py
  class GEPA (line 59) | class GEPA(BaseAlgorithm):
    method __init__ (line 88) | def __init__(
    method execute (line 131) | def execute(
    method a_execute (line 305) | async def a_execute(
    method reset_state (line 503) | def reset_state(self) -> None:
    method _ensure_scorer (line 515) | def _ensure_scorer(self) -> None:
    method _prompts_equivalent (line 523) | def _prompts_equivalent(
    method _add_prompt_configuration (line 564) | def _add_prompt_configuration(
    method _best_by_aggregate (line 574) | def _best_by_aggregate(self) -> PromptConfiguration:
    method _pick_prompt_configuration (line 602) | def _pick_prompt_configuration(self) -> PromptConfiguration:
    method _draw_minibatch (line 610) | def _draw_minibatch(
    method _a_generate_child_prompt (line 624) | async def _a_generate_child_prompt(
    method _generate_child_prompt (line 645) | def _generate_child_prompt(
    method _make_child (line 666) | def _make_child(
    method _should_accept_child (line 679) | def _should_accept_child(
    method _accept_child (line 701) | def _accept_child(
    method _a_accept_child (line 737) | async def _a_accept_child(
    method _update_step (line 773) | def _update_step(self, iteration: int, label: str) -> None:
    method _update_progress (line 778) | def _update_progress(
    method _update_error (line 797) | def _update_error(
    method _run_loop_iteration (line 813) | def _run_loop_iteration(
    method _a_run_loop_iteration (line 836) | async def _a_run_loop_iteration(
    method generate_summary_table (line 859) | def generate_summary_table(self, report: OptimizationReport) -> List[T...

FILE: deepeval/optimizer/algorithms/miprov2/bootstrapper.py
  class Demonstration (line 20) | class Demonstration:
  class DemonstrationSet (line 29) | class DemonstrationSet:
    method __post_init__ (line 35) | def __post_init__(self):
    method to_text (line 39) | def to_text(self, max_demonstrations: Optional[int] = None) -> str:
  class DemonstrationBootstrapper (line 60) | class DemonstrationBootstrapper:
    method __init__ (line 66) | def __init__(
    method _extract_input (line 84) | def _extract_input(
    method _extract_expected_output (line 105) | def _extract_expected_output(
    method bootstrap (line 122) | def bootstrap(
    method a_bootstrap (line 213) | async def a_bootstrap(
    method _create_demonstration_sets (line 308) | def _create_demonstration_sets(
  function render_prompt_with_demonstrations (line 352) | def render_prompt_with_demonstrations(

FILE: deepeval/optimizer/algorithms/miprov2/miprov2.py
  class MIPROV2 (line 44) | class MIPROV2(BaseAlgorithm):
    method __init__ (line 53) | def __init__(
    method _init_components (line 97) | def _init_components(self) -> None:
    method _sample_minibatch (line 110) | def _sample_minibatch(self, goldens: List) -> List:
    method _build_config (line 116) | def _build_config(
    method _update_step (line 140) | def _update_step(self, message: str) -> None:
    method _update_trial_progress (line 145) | def _update_trial_progress(self, step: int, total: int) -> None:
    method execute (line 155) | def execute(
    method a_execute (line 287) | async def a_execute(
    method generate_summary_table (line 411) | def generate_summary_table(self, report: OptimizationReport) -> List[T...

FILE: deepeval/optimizer/algorithms/miprov2/proposer/proposer.py
  class InstructionProposer (line 41) | class InstructionProposer:
    method __init__ (line 47) | def __init__(
    method _accrue_cost (line 59) | def _accrue_cost(self, cost: float) -> None:
    method _format_examples (line 62) | def _format_examples(
    method _generate_dataset_summary (line 93) | def _generate_dataset_summary(self, examples_text: str) -> str:
    method _generate_candidate_instruction (line 104) | def _generate_candidate_instruction(
    method propose (line 130) | def propose(
    method _a_generate_dataset_summary (line 186) | async def _a_generate_dataset_summary(self, examples_text: str) -> str:
    method _a_generate_candidate_instruction (line 197) | async def _a_generate_candidate_instruction(
    method a_propose (line 226) | async def a_propose(
    method _select_tips (line 284) | def _select_tips(self, count: int) -> List[str]:
    method _is_duplicate (line 294) | def _is_duplicate(self, new_prompt: Prompt, existing: List[Prompt]) ->...

FILE: deepeval/optimizer/algorithms/miprov2/proposer/schema.py
  class DatasetSummarySchema (line 5) | class DatasetSummarySchema(BaseModel):
  class InstructionProposalSchema (line 9) | class InstructionProposalSchema(BaseModel):

FILE: deepeval/optimizer/algorithms/miprov2/proposer/template.py
  class ProposerTemplate (line 1) | class ProposerTemplate:
    method generate_dataset_summary (line 4) | def generate_dataset_summary(examples_text: str) -> str:
    method generate_instruction_proposal (line 29) | def generate_instruction_proposal(

FILE: deepeval/optimizer/algorithms/simba/proposer.py
  class SIMBAProposer (line 19) | class SIMBAProposer:
    method __init__ (line 21) | def __init__(self, optimizer_model: DeepEvalBaseLLM):
    method _accrue_cost (line 24) | def _accrue_cost(self, cost: float) -> None:
    method _format_trajectory (line 27) | def _format_trajectory(
    method rewrite_from_introspection (line 38) | def rewrite_from_introspection(
    method a_rewrite_from_introspection (line 87) | async def a_rewrite_from_introspection(
    method append_a_demo (line 135) | def append_a_demo(
    method _inject_text (line 144) | def _inject_text(self, prompt: Prompt, injection: str) -> Prompt:

FILE: deepeval/optimizer/algorithms/simba/schema.py
  class SIMBARewriteSchema (line 5) | class SIMBARewriteSchema(BaseModel):

FILE: deepeval/optimizer/algorithms/simba/simba.py
  class SIMBA (line 37) | class SIMBA(BaseAlgorithm):
    method __init__ (line 42) | def __init__(
    method _init_components (line 72) | def _init_components(self) -> None:
    method _sample_minibatch (line 75) | def _sample_minibatch(self, goldens: List) -> List:
    method _update_step (line 80) | def _update_step(self, message: str) -> None:
    method _update_trial_progress (line 84) | def _update_trial_progress(self, step: int, total: int) -> None:
    method _golden_expected_text (line 94) | def _golden_expected_text(
    method _extract_inputs (line 101) | def _extract_inputs(
    method _execute_trace (line 110) | def _execute_trace(
    method _a_execute_trace (line 135) | async def _a_execute_trace(
    method execute (line 160) | def execute(
    method a_execute (line 402) | async def a_execute(
    method generate_summary_table (line 659) | def generate_summary_table(self, report: OptimizationReport) -> List[T...

FILE: deepeval/optimizer/algorithms/simba/template.py
  class SIMBATemplate (line 1) | class SIMBATemplate:
    method generate_introspection_rewrite (line 4) | def generate_introspection_rewrite(

FILE: deepeval/optimizer/configs.py
  class DisplayConfig (line 8) | class DisplayConfig(BaseModel):

FILE: deepeval/optimizer/policies.py
  function _is_dominated (line 10) | def _is_dominated(
  function pareto_frontier (line 30) | def pareto_frontier(
  function frequency_weights (line 62) | def frequency_weights(
  function sample_by_frequency (line 129) | def sample_by_frequency(
  function select_prompt_configuration_pareto (line 159) | def select_prompt_configuration_pareto(
  class TieBreaker (line 172) | class TieBreaker(str, Enum):
  function pick_best_with_ties (line 178) | def pick_best_with_ties(

FILE: deepeval/optimizer/prompt_optimizer.py
  class PromptOptimizer (line 51) | class PromptOptimizer:
    method __init__ (line 52) | def __init__(
    method optimize (line 86) | def optimize(
    method a_optimize (line 107) | async def a_optimize(
    method _configure_algorithm (line 126) | def _configure_algorithm(self) -> None:
    method _print_summary_table (line 154) | def _print_summary_table(self) -> None:
    method _progress_context (line 171) | def _progress_context(self):
    method _handle_optimization_error (line 194) | def _handle_optimization_error(self, exc: Exception) -> None:
    method _on_status (line 216) | def _on_status(
    method _on_step (line 274) | def _on_step(self, label: str) -> None:
    method _format_iter_description (line 282) | def _format_iter_description(
    method _format_step_description (line 294) | def _format_step_description(self, label: str) -> str:

FILE: deepeval/optimizer/rewriter/rewriter.py
  class Rewriter (line 23) | class Rewriter:
    method __init__ (line 32) | def __init__(
    method rewrite (line 49) | def rewrite(
    method a_rewrite (line 85) | async def a_rewrite(
    method _accrue_cost (line 121) | def _accrue_cost(self, cost: float) -> None:

FILE: deepeval/optimizer/rewriter/schema.py
  class RewriterSchema (line 5) | class RewriterSchema(BaseModel):

FILE: deepeval/optimizer/rewriter/template.py
  class RewriterTemplate (line 1) | class RewriterTemplate:
    method generate_mutation (line 3) | def generate_mutation(

FILE: deepeval/optimizer/scorer/base.py
  class BaseScorer (line 11) | class BaseScorer(ABC):
    method score_pareto (line 23) | def score_pareto(
    method score_minibatch (line 32) | def score_minibatch(
    method get_minibatch_feedback (line 41) | def get_minibatch_feedback(
    method a_score_pareto (line 52) | async def a_score_pareto(
    method a_score_minibatch (line 60) | async def a_score_minibatch(
    method a_get_minibatch_feedback (line 68) | async def a_get_minibatch_feedback(
    method _accrue_cost (line 76) | def _accrue_cost(self, cost: float) -> None:

FILE: deepeval/optimizer/scorer/schema.py
  class ScorerDiagnosisSchema (line 5) | class ScorerDiagnosisSchema(BaseModel):
  class ScorerDiagnosisResult (line 11) | class ScorerDiagnosisResult(BaseModel):

FILE: deepeval/optimizer/scorer/scorer.py
  class Scorer (line 58) | class Scorer(BaseScorer):
    method __init__ (line 66) | def __init__(
    method generate (line 87) | def generate(
    method a_generate (line 103) | async def a_generate(
    method score_pareto (line 119) | def score_pareto(
    method score_minibatch (line 128) | def score_minibatch(
    method get_minibatch_feedback (line 142) | def get_minibatch_feedback(
    method a_score_pareto (line 199) | async def a_score_pareto(
    method a_score_minibatch (line 210) | async def a_score_minibatch(
    method a_get_minibatch_feedback (line 222) | async def a_get_minibatch_feedback(
    method _golden_to_test_case (line 287) | def _golden_to_test_case(
    method _bounded (line 312) | async def _bounded(self, coro):
    method _a_score_one (line 321) | async def _a_score_one(
    method _score_one (line 340) | def _score_one(
    method _select_module_id_from_prompts (line 358) | def _select_module_id_from_prompts(
    method _build_evaluation_results_block (line 373) | def _build_evaluation_results_block(

FILE: deepeval/optimizer/scorer/template.py
  class ScorerTemplate (line 1) | class ScorerTemplate:
    method generate_diagnosis (line 3) | def generate_diagnosis(

FILE: deepeval/optimizer/scorer/utils.py
  function _build_measure_kwargs (line 8) | def _build_measure_kwargs(func: Callable) -> dict:
  function _measure_no_indicator (line 17) | def _measure_no_indicator(
  function _a_measure_no_indicator (line 25) | async def _a_measure_no_indicator(

FILE: deepeval/optimizer/types.py
  class PromptConfiguration (line 27) | class PromptConfiguration:
    method new (line 33) | def new(
  class RunnerStatusType (line 42) | class RunnerStatusType(str, Enum):
  class AcceptedIterationDict (line 53) | class AcceptedIterationDict(TypedDict):
  class AcceptedIteration (line 61) | class AcceptedIteration(BaseModel):
  class IterationLogEntry (line 69) | class IterationLogEntry(BaseModel):
  class SimbaTraceRecord (line 78) | class SimbaTraceRecord(BaseModel):
  class SimbaVarianceBucket (line 86) | class SimbaVarianceBucket(BaseModel):
  class PromptConfigSnapshot (line 96) | class PromptConfigSnapshot(BaseModel):
  class OptimizationReport (line 103) | class OptimizationReport(BaseModel):

FILE: deepeval/optimizer/utils.py
  function split_goldens (line 34) | def split_goldens(
  function invoke_model_callback (line 93) | def invoke_model_callback(
  function a_invoke_model_callback (line 114) | async def a_invoke_model_callback(
  function build_prompt_config_snapshots (line 136) | def build_prompt_config_snapshots(
  function inflate_prompts_from_report (line 155) | def inflate_prompts_from_report(
  function get_best_prompts_from_report (line 195) | def get_best_prompts_from_report(
  function _format_type_names (line 208) | def _format_type_names(types: Tuple[type, ...]) -> str:
  function validate_instance (line 217) | def validate_instance(
  function validate_sequence_of (line 250) | def validate_sequence_of(
  function validate_callback (line 302) | def validate_callback(
  function validate_metrics (line 324) | def validate_metrics(
  function validate_int_in_range (line 346) | def validate_int_in_range(
  class Aggregator (line 405) | class Aggregator(Protocol):
    method __call__ (line 406) | def __call__(self, scores: Sequence[float]) -> float: ...
  function mean_of_all (line 409) | def mean_of_all(scores: Sequence[float]) -> float:
  function _parse_prompt (line 418) | def _parse_prompt(prompt: Prompt) -> str:
  function _create_prompt (line 432) | def _create_prompt(old_prompt: Prompt, new_content: str) -> Prompt:

FILE: deepeval/plugins/plugin.py
  function pytest_addoption (line 13) | def pytest_addoption(parser):
  function pytest_sessionstart (line 22) | def pytest_sessionstart(session: pytest.Session):
  function pytest_runtest_protocol (line 35) | def pytest_runtest_protocol(
  function pytest_runtest_call (line 43) | def pytest_runtest_call(item: pytest.Item):
  function pytest_sessionfinish (line 68) | def pytest_sessionfinish(session: pytest.Session, exitstatus):
  function pytest_terminal_summary (line 74) | def pytest_terminal_summary(terminalreporter, exitstatus, config):

FILE: deepeval/progress_context.py
  function progress_context (line 23) | def progress_context(
  function synthesizer_progress_context (line 39) | def synthesizer_progress_context(
  function conversation_simulator_progress_context (line 81) | def conversation_simulator_progress_context(

FILE: deepeval/prompt/api.py
  class ReasoningEffort (line 21) | class ReasoningEffort(Enum):
  class Verbosity (line 28) | class Verbosity(Enum):
  class ModelProvider (line 34) | class ModelProvider(Enum):
  class ToolMode (line 44) | class ToolMode(Enum):
  class ModelSettings (line 50) | class ModelSettings(BaseModel):
  class OutputType (line 96) | class OutputType(Enum):
  class SchemaDataType (line 102) | class SchemaDataType(Enum):
  class OutputSchemaField (line 112) | class OutputSchemaField(BaseModel):
  class OutputSchema (line 127) | class OutputSchema(BaseModel):
  class Tool (line 133) | class Tool(BaseModel):
    method update_schema (line 144) | def update_schema(self):
    method input_schema (line 154) | def input_schema(self) -> Dict[str, Any]:
  class PromptInterpolationType (line 165) | class PromptInterpolationType(Enum):
  class PromptMessage (line 173) | class PromptMessage(BaseModel):
  class PromptType (line 181) | class PromptType(Enum):
  class PromptVersion (line 186) | class PromptVersion(BaseModel):
  class PromptCommit (line 191) | class PromptCommit(BaseModel):
  class PromptCommitsHttpResponse (line 197) | class PromptCommitsHttpResponse(BaseModel):
  class PromptCreateVersion (line 201) | class PromptCreateVersion(BaseModel):
  class PromptVersionsHttpResponse (line 205) | class PromptVersionsHttpResponse(BaseModel):
  class PromptHttpResponse (line 218) | class PromptHttpResponse(BaseModel):
  class PromptPushRequest (line 248) | class PromptPushRequest(BaseModel):
  class PromptApi (line 272) | class PromptApi(BaseModel):
  class PromptBranch (line 277) | class PromptBranch(BaseModel):
  class PromptBranchesHttpResponse (line 282) | class PromptBranchesHttpResponse(BaseModel):
  class PromptCreateBranchRequest (line 286) | class PromptCreateBranchRequest(BaseModel):
  class PromptUpdateBranchRequest (line 290) | class PromptUpdateBranchRequest(BaseModel):

FILE: deepeval/prompt/prompt.py
  function _get_or_create_polling_loop (line 67) | def _get_or_create_polling_loop() -> asyncio.AbstractEventLoop:
  class CustomEncoder (line 90) | class CustomEncoder(json.JSONEncoder):
    method default (line 91) | def default(self, obj):
  class CachedPrompt (line 99) | class CachedPrompt(BaseModel):
  class Prompt (line 118) | class Prompt:
    method __init__ (line 120) | def __init__(
    method __del__ (line 165) | def __del__(self):
    method hash (line 174) | def hash(self):
    method version (line 184) | def version(self):
    method hash (line 191) | def hash(self, value):
    method version (line 195) | def version(self, value):
    method load (line 198) | def load(self, file_path: str, messages_key: Optional[str] = None):
    method interpolate (line 234) | def interpolate(self, **kwargs):
    method _get_versions (line 271) | def _get_versions(self) -> List:
    method _get_commits (line 285) | def _get_commits(self, branch: Optional[str] = None) -> List[PromptCom...
    method _read_from_cache (line 300) | def _read_from_cache(
    method _write_to_cache (line 358) | def _write_to_cache(
    method _load_from_cache_with_progress (line 449) | def _load_from_cache_with_progress(
    method pull (line 507) | def pull(
    method create_version (line 717) | def create_version(
    method push (line 755) | def push(
    method update (line 829) | def update(
    method get_branches (line 871) | def get_branches(self) -> List[PromptBranch]:
    method create_branch (line 888) | def create_branch(self, branch: str, _verbose: Optional[bool] = True):
    method update_branch (line 920) | def update_branch(
    method delete_branch (line 962) | def delete_branch(
    method create_polling_task (line 996) | async def create_polling_task(
    method poll (line 1041) | async def poll(
    method _stop_polling (line 1150) | def _stop_polling(self):

FILE: deepeval/prompt/utils.py
  function interpolate_mustache (line 29) | def interpolate_mustache(text: str, **kwargs: Any) -> str:
  function interpolate_mustache_with_space (line 42) | def interpolate_mustache_with_space(text: str, **kwargs: Any) -> str:
  function interpolate_fstring (line 55) | def interpolate_fstring(text: str, **kwargs: Any) -> str:
  function interpolate_dollar_brackets (line 68) | def interpolate_dollar_brackets(text: str, **kwargs: Any) -> str:
  function interpolate_jinja (line 81) | def interpolate_jinja(text: str, **kwargs: Any) -> str:
  function interpolate_text (line 86) | def interpolate_text(
  function _resolve_field_type (line 117) | def _resolve_field_type(
  function construct_nested_base_model (line 136) | def construct_nested_base_model(
  function construct_base_model (line 149) | def construct_base_model(
  function _process_model (line 178) | def _process_model(
  function construct_output_schema (line 281) | def construct_output_schema(
  function output_schema_to_json_schema (line 290) | def output_schema_to_json_schema(

FILE: deepeval/scorer/scorer.py
  class Scorer (line 11) | class Scorer:
    method rouge_score (line 19) | def rouge_score(
    method sentence_bleu_score (line 52) | def sentence_bleu_score(
    method exact_match_score (line 99) | def exact_match_score(cls, target: str, prediction: str) -> int:
    method quasi_exact_match_score (line 114) | def quasi_exact_match_score(cls, target: str, prediction: str) -> int:
    method quasi_contains_score (line 120) | def quasi_contains_score(cls, targets: List[str], prediction: str) -> ...
    method bert_score (line 129) | def bert_score(
    method faithfulness_score (line 202) | def faithfulness_score(
    method hallucination_score (line 236) | def hallucination_score(
    method PII_score (line 263) | def PII_score(
    method neural_toxic_score (line 269) | def neural_toxic_score(
    method answer_relevancy_score (line 308) | def answer_relevancy_score(
    method neural_bias_score (line 368) | def neural_bias_score(cls, text: str, model: Optional[str] = None) -> ...
    method truth_identification_score (line 377) | def truth_identification_score(cls, target: str, prediction: str) -> int:
    method pass_at_k (line 423) | def pass_at_k(self, n, c, k):
    method squad_score (line 435) | def squad_score(

FILE: deepeval/simulator/controller/controller.py
  function proceed (line 17) | def proceed() -> Decision:
  function end (line 21) | def end(reason: Optional[str] = None) -> Decision:
  class SimulationController (line 25) | class SimulationController:
    method __init__ (line 26) | def __init__(
    method run (line 37) | def run(
    method a_run (line 67) | async def a_run(
    method check_expected_outcome (line 97) | def check_expected_outcome(
    method a_check_expected_outcome (line 126) | async def a_check_expected_outcome(
    method _build_context (line 155) | def _build_context(
    method _invoke_controller (line 183) | def _invoke_controller(self, ctx: Context):
    method _normalize_decision (line 205) | def _normalize_decision(self, decision: Optional[Decision]) -> Decision:
    method _should_end (line 210) | def _should_end(
    method _generate_schema (line 221) | def _generate_schema(self, prompt: str, schema: BaseModel) -> BaseModel:
    method _a_generate_schema (line 224) | async def _a_generate_schema(
  class _ExpectedOutcomeController (line 230) | class _ExpectedOutcomeController:
    method run (line 231) | def run(
    method a_run (line 243) | async def a_run(

FILE: deepeval/simulator/controller/template.py
  class SimulatorControllerTemplate (line 4) | class SimulatorControllerTemplate:
    method check_expected_outcome (line 6) | def check_expected_outcome(

FILE: deepeval/simulator/controller/types.py
  class Decision (line 9) | class Decision(BaseModel):
  class Context (line 14) | class Context(BaseModel):

FILE: deepeval/simulator/conversation_simulator.py
  class ConversationSimulator (line 37) | class ConversationSimulator:
    method __init__ (line 38) | def __init__(
    method simulate (line 70) | def simulate(
    method _a_simulate (line 138) | async def _a_simulate(
    method _simulate_single_conversation (line 196) | def _simulate_single_conversation(
    method _a_simulate_single_conversation (line 300) | async def _a_simulate_single_conversation(
    method generate_first_user_input (line 408) | def generate_first_user_input(self, golden: ConversationalGolden):
    method a_generate_first_user_input (line 415) | async def a_generate_first_user_input(self, golden: ConversationalGold...
    method generate_next_user_input (line 422) | def generate_next_user_input(
    method a_generate_next_user_input (line 431) | async def a_generate_next_user_input(
    method generate_schema (line 444) | def generate_schema(
    method a_generate_schema (line 463) | async def a_generate_schema(
    method generate_turn_from_callback (line 490) | def generate_turn_from_callback(
    method a_generate_turn_from_callback (line 509) | async def a_generate_turn_from_callback(

FILE: deepeval/simulator/schema.py
  class ConversationCompletion (line 4) | class ConversationCompletion(BaseModel):
  class SimulatedInput (line 9) | class SimulatedInput(BaseModel):

FILE: deepeval/simulator/template.py
  class ConversationSimulatorTemplate (line 9) | class ConversationSimulatorTemplate:
    method simulate_first_user_turn (line 19) | def simulate_first_user_turn(
    method simulate_user_turn (line 54) | def simulate_user_turn(

FILE: deepeval/simulator/utils.py
  function validate_simulation_template (line 7) | def validate_simulation_template(

FILE: deepeval/singleton.py
  class Singleton (line 1) | class Singleton(type):
    method __call__ (line 9) | def __call__(cls, *args, **kwargs):
    method __setattr__ (line 16) | def __setattr__(cls, name, value):

FILE: deepeval/synthesizer/__init__.py
  function __getattr__ (line 12) | def __getattr__(name: str) -> Any:

FILE: deepeval/synthesizer/base_synthesizer.py
  class BaseSynthesizer (line 9) | class BaseSynthesizer:
    method model (line 14) | def model(self) -> float:
    method model (line 18) | def model(self, model: Optional[Union[str, DeepEvalBaseLLM]] = None):
    method embedder (line 22) | def embedder(self) -> float:
    method embedder (line 26) | def embedder(

FILE: deepeval/synthesizer/chunking/context_generator.py
  function safe_rmtree (line 36) | def safe_rmtree(
  function close_chroma_clients (line 85) | def close_chroma_clients():
  function _release_chroma_client (line 90) | def _release_chroma_client(client):
  class ContextScore (line 103) | class ContextScore(BaseModel):
  class ContextGenerator (line 110) | class ContextGenerator:
    method __init__ (line 111) | def __init__(
    method generate_contexts (line 160) | def generate_contexts(
    method a_generate_contexts (line 316) | async def a_generate_contexts(
    method _a_process_document_async (line 473) | async def _a_process_document_async(
    method _generate_contexts_per_source_file (line 499) | def _generate_contexts_per_source_file(
    method _a_get_n_random_contexts_per_source_file (line 601) | async def _a_get_n_random_contexts_per_source_file(
    method _get_n_random_chunks_per_source_file (line 711) | def _get_n_random_chunks_per_source_file(
    method _a_get_n_random_chunks_per_source_file (line 776) | async def _a_get_n_random_chunks_per_source_file(
    method evaluate_chunk (line 836) | def evaluate_chunk(self, chunk) -> float:
    method a_evaluate_chunk (line 861) | async def a_evaluate_chunk(self, chunk) -> float:
    method validate_context_size (line 891) | def validate_context_size(
    method validate_chunk_size (line 905) | def validate_chunk_size(
    method _load_docs (line 983) | def _load_docs(
    method _a_load_docs (line 996) | async def _a_load_docs(

FILE: deepeval/synthesizer/chunking/doc_chunker.py
  function _get_langchain (line 22) | def _get_langchain():
  function get_chromadb (line 55) | def get_chromadb():
  class DocumentChunker (line 72) | class DocumentChunker:
    method __init__ (line 73) | def __init__(
    method a_chunk_doc (line 92) | async def a_chunk_doc(
    method chunk_doc (line 158) | def chunk_doc(
    method get_loader (line 228) | def get_loader(self, path: str, encoding: Optional[str]) -> "BaseLoader":
    method a_load_doc (line 258) | async def a_load_doc(self, path: str, encoding: Optional[str]):
    method load_doc (line 264) | def load_doc(self, path: str, encoding: Optional[str]):
    method count_tokens (line 270) | def count_tokens(self, chunks: List["LCDocument"]):

FILE: deepeval/synthesizer/config.py
  class FiltrationConfig (line 11) | class FiltrationConfig:
    method __post_init__ (line 16) | def __post_init__(self):
  class EvolutionConfig (line 21) | class EvolutionConfig:
  class StylingConfig (line 37) | class StylingConfig:
  class ConversationalStylingConfig (line 45) | class ConversationalStylingConfig:
  class ContextConstructionConfig (line 54) | class ContextConstructionConfig:
    method __post_init__ (line 68) | def __post_init__(self):

FILE: deepeval/synthesizer/schema.py
  class RewrittenInput (line 5) | class RewrittenInput(BaseModel):
  class InputFeedback (line 9) | class InputFeedback(BaseModel):
  class SyntheticData (line 14) | class SyntheticData(BaseModel):
  class SyntheticDataList (line 18) | class SyntheticDataList(BaseModel):
  class SQLData (line 22) | class SQLData(BaseModel):
  class ComplianceData (line 26) | class ComplianceData(BaseModel):
  class Response (line 30) | class Response(BaseModel):
  class ImprovementPrompt (line 34) | class ImprovementPrompt(BaseModel):
  class OnTopic (line 39) | class OnTopic(BaseModel):
  class Rating (line 43) | class Rating(BaseModel):
  class TreeScore (line 47) | class TreeScore(BaseModel):
  class NonRefusal (line 53) | class NonRefusal(BaseModel):
  class PromptStyling (line 57) | class PromptStyling(BaseModel):
  class ConversationalScenario (line 63) | class ConversationalScenario(BaseModel):
  class ConversationalScenarioList (line 67) | class ConversationalScenarioList(BaseModel):
  class RewrittenScenario (line 71) | class RewrittenScenario(BaseModel):
  class ScenarioFeedback (line 75) | class ScenarioFeedback(BaseModel):
  class ConversationalPromptStyling (line 80) | class ConversationalPromptStyling(BaseModel):

FILE: deepeval/synthesizer/synthesizer.py
  class Synthesizer (line 110) | class Synthesizer:
    method __init__ (line 111) | def __init__(
    method generate_goldens_from_docs (line 158) | def generate_goldens_from_docs(
    method a_generate_goldens_from_docs (line 265) | async def a_generate_goldens_from_docs(
    method generate_goldens_from_contexts (line 362) | def generate_goldens_from_contexts(
    method a_generate_goldens_from_contexts (line 560) | async def a_generate_goldens_from_contexts(
    method _a_generate_from_context (line 616) | async def _a_generate_from_context(
    method _a_generate_text_to_sql_from_context (line 775) | async def _a_generate_text_to_sql_from_context(
    method a_generate_goldens_from_scratch (line 819) | async def a_generate_goldens_from_scratch(
    method generate_goldens_from_scratch (line 896) | def generate_goldens_from_scratch(
    method transform_distribution (line 979) | def transform_distribution(
    method map_evolution_to_prompt_evolution (line 990) | def map_evolution_to_prompt_evolution(
    method generate_goldens_from_goldens (line 1004) | def generate_goldens_from_goldens(
    method a_generate_goldens_from_goldens (line 1063) | async def a_generate_goldens_from_goldens(
    method _a_generate_inputs (line 1114) | async def _a_generate_inputs(self, prompt: str) -> List[SyntheticData]:
    method _generate_inputs (line 1121) | def _generate_inputs(self, prompt: str) -> List[SyntheticData]:
    method _a_rewrite_inputs (line 1128) | async def _a_rewrite_inputs(
    method _rewrite_inputs (line 1173) | def _rewrite_inputs(
    method _evolve_input (line 1222) | def _evolve_input(
    method _a_evolve_input (line 1257) | async def _a_evolve_input(
    method _generate_schema (line 1296) | def _generate_schema(
    method _a_generate_schema (line 1322) | async def _a_generate_schema(
    method _generate (line 1348) | def _generate(self, prompt: str) -> str:
    method _a_generate (line 1362) | async def _a_generate(self, prompt: str) -> str:
    method task_wrapper (line 1382) | async def task_wrapper(self, sem, func, *args, **kwargs):
    method to_pandas (line 1386) | def to_pandas(self):
    method save_as (line 1500) | def save_as(
    method generate_conversational_goldens_from_docs (line 1670) | def generate_conversational_goldens_from_docs(
    method a_generate_conversational_goldens_from_docs (line 1775) | async def a_generate_conversational_goldens_from_docs(
    method generate_conversational_goldens_from_contexts (line 1872) | def generate_conversational_goldens_from_contexts(
    method a_generate_conversational_goldens_from_contexts (line 2082) | async def a_generate_conversational_goldens_from_contexts(
    method _a_generate_conversational_from_context (line 2138) | async def _a_generate_conversational_from_context(
    method a_generate_conversational_goldens_from_scratch (line 2305) | async def a_generate_conversational_goldens_from_scratch(
    method generate_conversational_goldens_from_scratch (line 2383) | def generate_conversational_goldens_from_scratch(
    method _a_generate_scenarios (line 2470) | async def _a_generate_scenarios(
    method _generate_scenarios (line 2479) | def _generate_scenarios(self, prompt: str) -> List[ConversationalScena...
    method _a_rewrite_scenarios (line 2486) | async def _a_rewrite_scenarios(
    method _rewrite_scenarios (line 2533) | def _rewrite_scenarios(
    method _evolve_scenario (line 2582) | def _evolve_scenario(
    method _a_evolve_scenario (line 2622) | async def _a_evolve_scenario(
    method generate_conversational_goldens_from_goldens (line 2667) | def generate_conversational_goldens_from_goldens(
    method a_generate_conversational_goldens_from_goldens (line 2725) | async def a_generate_conversational_goldens_from_goldens(

FILE: deepeval/synthesizer/templates/template.py
  class SynthesizerTemplate (line 4) | class SynthesizerTemplate:
    method generate_text2sql_inputs (line 7) | def generate_text2sql_inputs(context, max_goldens_per_context):
    method generate_text2sql_expected_output (line 54) | def generate_text2sql_expected_output(input, context):
    method generate_synthetic_expected_output (line 84) | def generate_synthetic_expected_output(
    method generate_synthetic_inputs (line 109) | def generate_synthetic_inputs(
    method rewrite_evolved_input (line 173) | def rewrite_evolved_input(
    method rewrite_synthetic_inputs (line 211) | def rewrite_synthetic_inputs(context, original_query, feedback):
    method generate_synthetic_scenarios (line 250) | def generate_synthetic_scenarios(
    method generate_synthetic_expected_outcome_conversational (line 331) | def generate_synthetic_expected_outcome_conversational(
    method rewrite_evolved_scenario (line 365) | def rewrite_evolved_scenario(
    method rewrite_synthetic_scenarios (line 416) | def rewrite_synthetic_scenarios(context, original_scenario, feedback):
  class FilterTemplate (line 461) | class FilterTemplate:
    method evaluate_synthetic_inputs (line 464) | def evaluate_synthetic_inputs(query):
    method evaluate_context (line 516) | def evaluate_context(context):
    method evaluate_synthetic_scenarios (line 584) | def evaluate_synthetic_scenarios(scenario):
  class EvolutionTemplate (line 652) | class EvolutionTemplate:
    method multi_context_evolution (line 659) | def multi_context_evolution(input, context):
    method reasoning_evolution (line 707) | def reasoning_evolution(input, context):
    method concretizing_evolution (line 756) | def concretizing_evolution(input, context):
    method constrained_evolution (line 806) | def constrained_evolution(input, context):
    method comparative_question_evolution (line 854) | def comparative_question_evolution(input, context):
    method hypothetical_scenario_evolution (line 903) | def hypothetical_scenario_evolution(input, context):
    method in_breadth_evolution (line 952) | def in_breadth_evolution(input, context):
  class ConversationalEvolutionTemplate (line 1000) | class ConversationalEvolutionTemplate:
    method multi_context_evolution (line 1007) | def multi_context_evolution(scenario, context):
    method reasoning_evolution (line 1050) | def reasoning_evolution(scenario, context):
    method concretizing_evolution (line 1090) | def concretizing_evolution(scenario, context):
    method constrained_evolution (line 1129) | def constrained_evolution(scenario, context):
    method comparative_question_evolution (line 1168) | def comparative_question_evolution(scenario, context):
    method hypothetical_scenario_evolution (line 1206) | def hypothetical_scenario_evolution(scenario, context):
    method in_breadth_evolution (line 1245) | def in_breadth_evolution(scenario, context):

FILE: deepeval/synthesizer/templates/template_extraction.py
  class ExtractionTemplate (line 1) | class ExtractionTemplate:
    method extract_prompt_structure_from_inputs (line 4) | def extract_prompt_structure_from_inputs(inputs: list[str]):
    method extract_conversational_structure_from_scenarios (line 49) | def extract_conversational_structure_from_scenarios(example_scenarios):

FILE: deepeval/synthesizer/templates/template_prompt.py
  class PromptSynthesizerTemplate (line 1) | class PromptSynthesizerTemplate:
    method generate_synthetic_prompts (line 3) | def generate_synthetic_prompts(
    method generate_synthetic_conversational_scenarios (line 41) | def generate_synthetic_conversational_scenarios(
  class PromptEvolutionTemplate (line 112) | class PromptEvolutionTemplate:
    method reasoning_evolution (line 118) | def reasoning_evolution(input):
    method concretizing_evolution (line 157) | def concretizing_evolution(input):
    method constrained_evolution (line 195) | def constrained_evolution(input):
    method comparative_question_evolution (line 232) | def comparative_question_evolution(input):
    method hypothetical_scenario_evolution (line 272) | def hypothetical_scenario_evolution(input):
    method in_breadth_evolution (line 311) | def in_breadth_evolution(input):
  class ConversationalPromptEvolutionTemplate (line 350) | class ConversationalPromptEvolutionTemplate:
    method reasoning_evolution (line 356) | def reasoning_evolution(scenario):
    method concretizing_evolution (line 389) | def concretizing_evolution(scenario):
    method constrained_evolution (line 421) | def constrained_evolution(scenario):
    method comparative_question_evolution (line 453) | def comparative_question_evolution(scenario):
    method hypothetical_scenario_evolution (line 485) | def hypothetical_scenario_evolution(scenario):
    method in_breadth_evolution (line 517) | def in_breadth_evolution(scenario):

FILE: deepeval/synthesizer/types.py
  class Evolution (line 4) | class Evolution(Enum):
  class PromptEvolution (line 14) | class PromptEvolution(Enum):

FILE: deepeval/synthesizer/utils.py
  class SynthesizerStatus (line 12) | class SynthesizerStatus(Enum):
  function print_synthesizer_status (line 18) | def print_synthesizer_status(

FILE: deepeval/telemetry.py
  class Feature (line 15) | class Feature(Enum):
  function telemetry_opt_out (line 35) | def telemetry_opt_out():
  function blocked_by_firewall (line 39) | def blocked_by_firewall():
  function get_anonymous_public_ip (line 47) | def get_anonymous_public_ip():
  function handle_exception (line 110) | def handle_exception(exc_type, exc_value, exc_traceback):
  function is_running_in_jupyter_notebook (line 118) | def is_running_in_jupyter_notebook():
  function capture_evaluation_run (line 139) | def capture_evaluation_run(type: str):
  function capture_recommend_metrics (line 177) | def capture_recommend_metrics():
  function capture_metric_type (line 201) | def capture_metric_type(
  function capture_synthesizer_run (line 229) | def capture_synthesizer_run(
  function capture_conversation_simulator_run (line 263) | def capture_conversation_simulator_run(num_conversations: int):
  function capture_guardrails (line 292) | def capture_guardrails(guards: List[str]):
  function capture_benchmark_run (line 318) | def capture_benchmark_run(benchmark: str, num_tasks: int):
  function capture_login_event (line 345) | def capture_login_event():
  function capture_view_event (line 371) | def capture_view_event():
  function capture_pull_dataset (line 397) | def capture_pull_dataset():
  function capture_send_trace (line 424) | def capture_send_trace():
  function capture_tracing_integration (line 448) | def capture_tracing_integration(integration_name: str):
  function read_telemetry_file (line 480) | def read_telemetry_file() -> dict:
  function write_telemetry_file (line 493) | def write_telemetry_file(data: dict):
  function get_status (line 506) | def get_status() -> str:
  function get_unique_id (line 512) | def get_unique_id() -> str:
  function get_last_feature (line 529) | def get_last_feature() -> Feature:
  function set_last_feature (line 538) | def set_last_feature(feature: Feature):
  function get_feature_status (line 549) | def get_feature_status(feature: Feature) -> str:
  function set_logged_in_with (line 556) | def set_logged_in_with(logged_in_with: str):
  function get_logged_in_with (line 562) | def get_logged_in_with():

FILE: deepeval/test_case/__init__.py
  function __getattr__ (line 41) | def __getattr__(name: str):

FILE: deepeval/test_case/api.py
  function create_api_turn (line 18) | def create_api_turn(turn: Turn, index: int) -> TurnApi:
  function create_api_test_case (line 29) | def create_api_test_case(

FILE: deepeval/test_case/arena_test_case.py
  class Contestant (line 11) | class Contestant(BaseModel):
  class ArenaTestCase (line 20) | class ArenaTestCase:
    method __post_init__ (line 24) | def __post_init__(self):
  class Arena (line 47) | class Arena:

FILE: deepeval/test_case/conversational_test_case.py
  class MultiTurnParams (line 25) | class MultiTurnParams(Enum):
  function __getattr__ (line 42) | def __getattr__(name: str):
  class Turn (line 54) | class Turn(BaseModel):
    method additional_metadata (line 79) | def additional_metadata(self) -> Optional[Dict]:
    method additional_metadata (line 88) | def additional_metadata(self, value: Optional[Dict]):
    method _mcp_interaction (line 97) | def _mcp_interaction(self) -> bool:
    method __repr__ (line 105) | def __repr__(self):
    method validate_input (line 124) | def validate_input(cls, data):
  class ConversationalTestCase (line 173) | class ConversationalTestCase(BaseModel):
    method additional_metadata (line 209) | def additional_metadata(self) -> Optional[Dict]:
    method additional_metadata (line 218) | def additional_metadata(self, value: Optional[Dict]):
    method set_is_multimodal (line 227) | def set_is_multimodal(self):
    method validate_input (line 260) | def validate_input(cls, data):
    method _get_images_mapping (line 296) | def _get_images_mapping(self) -> Dict[str, MLLMImage]:

FILE: deepeval/test_case/llm_test_case.py
  class MLLMImage (line 36) | class MLLMImage:
    method __post_init__ (line 44) | def __post_init__(self):
    method _load_base64 (line 84) | def _load_base64(self, path: str):
    method ensure_images_loaded (line 89) | def ensure_images_loaded(self):
    method _placeholder (line 95) | def _placeholder(self) -> str:
    method __str__ (line 98) | def __str__(self) -> str:
    method __repr__ (line 101) | def __repr__(self) -> str:
    method __format__ (line 104) | def __format__(self, format_spec: str) -> str:
    method process_url (line 108) | def process_url(url: str) -> str:
    method is_local_path (line 123) | def is_local_path(url: str) -> bool:
    method parse_multimodal_string (line 137) | def parse_multimodal_string(s: str):
    method as_data_uri (line 164) | def as_data_uri(self) -> Optional[str]:
  class SingleTurnParams (line 171) | class SingleTurnParams(Enum):
  function __getattr__ (line 187) | def __getattr__(name: str):
  class ToolCallParams (line 199) | class ToolCallParams(Enum):
  function _make_hashable (line 204) | def _make_hashable(obj):
  class ToolCall (line 233) | class ToolCall(BaseModel):
    method __eq__ (line 244) | def __eq__(self, other):
    method __hash__ (line 253) | def __hash__(self):
    method __repr__ (line 274) | def __repr__(self):
    method _indent_nested_field (line 312) | def _indent_nested_field(field_name: str, formatted_field: str) -> str:
  class LLMTestCase (line 320) | class LLMTestCase(BaseModel):
    method additional_metadata (line 401) | def additional_metadata(self) -> Optional[Dict]:
    method additional_metadata (line 410) | def additional_metadata(self, value: Optional[Dict]):
    method set_is_multimodal (line 419) | def set_is_multimodal(self):
    method validate_input (line 453) | def validate_input(cls, data):
    method _get_images_mapping (line 571) | def _get_images_mapping(self) -> Dict[str, MLLMImage]:

FILE: deepeval/test_case/mcp.py
  class MCPToolCall (line 6) | class MCPToolCall(BaseModel):
  class MCPPromptCall (line 12) | class MCPPromptCall(BaseModel):
  class MCPResourceCall (line 17) | class MCPResourceCall(BaseModel):
  class MCPServer (line 23) | class MCPServer:
  function validate_mcp_servers (line 31) | def validate_mcp_servers(mcp_servers: List[MCPServer]):

FILE: deepeval/test_case/utils.py
  function check_valid_test_cases_type (line 6) | def check_valid_test_cases_type(

FILE: deepeval/test_run/api.py
  class LLMApiTestCase (line 9) | class LLMApiTestCase(BaseModel):
    method update_metric_data (line 55) | def update_metric_data(self, metric_data: MetricData):
    method update_run_duration (line 78) | def update_run_duration(self, run_duration: float):
    method update_status (line 81) | def update_status(self, success: bool):
    method is_multimodal (line 88) | def is_multimodal(self):
  class TurnApi (line 98) | class TurnApi(BaseModel):
  class ConversationalApiTestCase (line 108) | class ConversationalApiTestCase(BaseModel):
    method update_metric_data (line 127) | def update_metric_data(self, metrics_data: MetricData):
    method update_run_duration (line 145) | def update_run_duration(self, run_duration: float):
  class TestRunHttpResponse (line 149) | class TestRunHttpResponse(BaseModel):

FILE: deepeval/test_run/cache.py
  class MetricConfiguration (line 38) | class MetricConfiguration(BaseModel):
  class CachedMetricData (line 58) | class CachedMetricData(BaseModel):
  class CachedTestCase (line 63) | class CachedTestCase(BaseModel):
  class CustomEncoder (line 70) | class CustomEncoder(json.JSONEncoder):
    method default (line 71) | def default(self, obj):
  class CachedTestRun (line 79) | class CachedTestRun(BaseModel):
    method save (line 85) | def save(self, f):
    method load (line 98) | def load(cls, data):
    method get_cached_api_test_case (line 101) | def get_cached_api_test_case(self, key: str) -> CachedTestCase:
  class TestRunCacheManager (line 105) | class TestRunCacheManager:
    method __init__ (line 106) | def __init__(self):
    method get_cached_test_case (line 113) | def get_cached_test_case(
    method cache_test_case (line 134) | def cache_test_case(
    method set_cached_test_run (line 158) | def set_cached_test_run(
    method save_cached_test_run (line 169) | def save_cached_test_run(self, to_temp: bool = False):
    method create_cached_test_run (line 196) | def create_cached_test_run(self, temp: bool = False):
    method get_cached_test_run (line 204) | def get_cached_test_run(
    method wrap_up_cached_test_run (line 271) | def wrap_up_cached_test_run(self):
  class Cache (line 299) | class Cache:
    method get_metric_data (line 301) | def get_metric_data(
    method same_metric_configs (line 318) | def same_metric_configs(
    method create_metric_configuration (line 368) | def create_metric_configuration(metric: BaseMetric) -> MetricConfigura...

FILE: deepeval/test_run/hooks.py
  function on_test_run_end (line 4) | def on_test_run_end(func):
  function invoke_test_run_end_hook (line 14) | def invoke_test_run_end_hook():

FILE: deepeval/test_run/hyperparameters.py
  function process_hyperparameters (line 10) | def process_hyperparameters(
  function log_hyperparameters (line 61) | def log_hyperparameters(func):
  function process_prompts (line 82) | def process_prompts(

FILE: deepeval/test_run/test_run.py
  class TestRunResultDisplay (line 65) | class TestRunResultDisplay(Enum):
  class MetricScoreType (line 71) | class MetricScoreType(BaseModel):
    method from_metric (line 76) | def from_metric(cls, metric: BaseMetric):
  class MetricScores (line 80) | class MetricScores(BaseModel):
  class TraceMetricScores (line 88) | class TraceMetricScores(BaseModel):
  class PromptData (line 96) | class PromptData(BaseModel):
  class MetricsAverageDict (line 107) | class MetricsAverageDict:
    method __init__ (line 108) | def __init__(self):
    method add_metric (line 112) | def add_metric(self, metric_name, score):
    method get_average_metric_score (line 120) | def get_average_metric_score(self):
  class RemainingTestRun (line 130) | class RemainingTestRun(BaseModel):
  class TestRun (line 140) | class TestRun(BaseModel):
    method add_test_case (line 167) | def add_test_case(
    method set_dataset_properties (line 181) | def set_dataset_properties(
    method _assign_unique_orders (line 192) | def _assign_unique_orders(test_cases):
    method sort_test_cases (line 224) | def sort_test_cases(self):
    method construct_metrics_scores (line 235) | def construct_metrics_scores(self) -> int:
    method calculate_test_passes_and_fails (line 405) | def calculate_test_passes_and_fails(self):
    method save (line 426) | def save(self, f):
    method load (line 437) | def load(cls, f):
    method guard_mllm_test_cases (line 441) | def guard_mllm_test_cases(self):
  class TestRunEncoder (line 449) | class TestRunEncoder(json.JSONEncoder):
    method default (line 450) | def default(self, obj):
  class TestRunManager (line 456) | class TestRunManager:
    method __init__ (line 457) | def __init__(self):
    method reset (line 465) | def reset(self):
    method configure_local_store (line 473) | def configure_local_store(
    method set_test_run (line 487) | def set_test_run(self, test_run: TestRun):
    method create_test_run (line 490) | def create_test_run(
    method get_test_run (line 511) | def get_test_run(self, identifier: Optional[str] = None):
    method save_test_run (line 537) | def save_test_run(self, path: str, save_under_key: Optional[str] = None):
    method save_final_test_run_link (line 565) | def save_final_test_run_link(self, link: str):
    method update_test_run (line 577) | def update_test_run(
    method clear_test_run (line 629) | def clear_test_run(self):
    method _calculate_success_rate (line 633) | def _calculate_success_rate(pass_count: int, fail_count: int) -> str:
    method _get_metric_status (line 641) | def _get_metric_status(metric_data: MetricData) -> str:
    method _format_metric_score (line 650) | def _format_metric_score(metric_data: MetricData) -> str:
    method _should_skip_test_case (line 668) | def _should_skip_test_case(
    method _count_metric_results (line 679) | def _count_metric_results(
    method _add_test_case_header_row (line 692) | def _add_test_case_header_row(
    method _add_metric_rows (line 707) | def _add_metric_rows(self, table: Table, metrics_data: List[MetricData]):
    method _add_separator_row (line 721) | def _add_separator_row(self, table: Table):
    method display_results_table (line 725) | def display_results_table(
    method post_test_run (line 850) | def post_test_run(self, test_run: TestRun) -> Optional[Tuple[str, str]]:
    method save_test_run_locally (line 984) | def save_test_run_locally(self):
    method wrap_up_test_run (line 1030) | def wrap_up_test_run(
    method get_latest_test_run_data (line 1115) | def get_latest_test_run_data(self) -> Optional[TestRun]:
    method get_latest_test_run_link (line 1127) | def get_latest_test_run_link(self) -> Optional[str]:
    method _render_prompts_panels (line 1137) | def _render_prompts_panels(self, prompts: List[PromptData]) -> None:

FILE: deepeval/tracing/api.py
  class SpanApiType (line 9) | class SpanApiType(Enum):
  class TraceSpanApiStatus (line 20) | class TraceSpanApiStatus(Enum):
  class PromptApi (line 25) | class PromptApi(BaseModel):
  class MetricData (line 31) | class MetricData(BaseModel):
  class BaseApiSpan (line 46) | class BaseApiSpan(BaseModel):
  class TraceApi (line 116) | class TraceApi(BaseModel):

FILE: deepeval/tracing/context.py
  class SpanContext (line 18) | class SpanContext:
    method __init__ (line 19) | def __init__(self):
    method get (line 24) | def get(self):
    method set (line 27) | def set(self, value):
    method reset (line 30) | def reset(self, value):
    method drop (line 33) | def drop(self):
  class TraceContext (line 39) | class TraceContext:
    method __init__ (line 40) | def __init__(self):
    method get (line 45) | def get(self):
    method set (line 48) | def set(self, value):
    method reset (line 51) | def reset(self, value):
    method drop (line 54) | def drop(self):
  function update_current_span (line 64) | def update_current_span(
  function update_current_trace (line 120) | def update_current_trace(
  function update_llm_span (line 187) | def update_llm_span(
  function update_agent_span (line 220) | def update_agent_span(
  function update_tool_span (line 241) | def update_tool_span(
  function update_retriever_span (line 258) | def update_retriever_span(
  class _PendingSlot (line 301) | class _PendingSlot:
    method __init__ (line 323) | def __init__(self, payload: Optional[Dict[str, Any]]):
  function _drop_none (line 344) | def _drop_none(d: Dict[str, Any]) -> Dict[str, Any]:
  function next_span (line 354) | def next_span(
  function next_agent_span (line 402) | def next_agent_span(
  function next_llm_span (line 451) | def next_llm_span(
  function next_tool_span (line 510) | def next_tool_span(
  function next_retriever_span (line 557) | def next_retriever_span(
  function pop_pending_for (line 626) | def pop_pending_for(span_type: Optional[str]) -> Dict[str, Any]:
  function apply_pending_to_span (line 657) | def apply_pending_to_span(span: BaseSpan, payload: Dict[str, Any]) -> None:

FILE: deepeval/tracing/integrations.py
  class Integration (line 6) | class Integration(str, Enum):
  class Provider (line 21) | class Provider(str, Enum):

FILE: deepeval/tracing/internal.py
  function observe_methods (line 5) | def observe_methods(

FILE: deepeval/tracing/offline_evals/api.py
  class EvaluateThreadRequestBody (line 5) | class EvaluateThreadRequestBody(BaseModel):
  class EvaluateTraceRequestBody (line 11) | class EvaluateTraceRequestBody(BaseModel):
  class EvaluateSpanRequestBody (line 16) | class EvaluateSpanRequestBody(BaseModel):

FILE: deepeval/tracing/offline_evals/span.py
  function evaluate_span (line 6) | def evaluate_span(
  function a_evaluate_span (line 35) | async def a_evaluate_span(

FILE: deepeval/tracing/offline_evals/thread.py
  function evaluate_thread (line 6) | def evaluate_thread(
  function a_evaluate_thread (line 42) | async def a_evaluate_thread(

FILE: deepeval/tracing/offline_evals/trace.py
  function evaluate_trace (line 6) | def evaluate_trace(
  function a_evaluate_trace (line 37) | async def a_evaluate_trace(

FILE: deepeval/tracing/otel/context_aware_processor.py
  class _SpanProcessor (line 56) | class _SpanProcessor:  # type: ignore[no-redef]
    method __init__ (line 57) | def __init__(self, *args, **kwargs):
    method on_start (line 60) | def on_start(self, span, parent_context):
    method on_end (line 63) | def on_end(self, span):
    method shutdown (line 66) | def shutdown(self):
    method force_flush (line 69) | def force_flush(self, timeout_millis: int = 30_000):
  function _otlp_endpoint (line 77) | def _otlp_endpoint() -> str:
  class ContextAwareSpanProcessor (line 82) | class ContextAwareSpanProcessor(_SpanProcessor):
    method __init__ (line 96) | def __init__(self, api_key: Optional[str] = None):
    method _should_route_to_rest (line 125) | def _should_route_to_rest() -> bool:
    method on_start (line 151) | def on_start(self, span, parent_context=None):
    method on_end (line 163) | def on_end(self, span):
    method shutdown (line 170) | def shutdown(self):
    method force_flush (line 180) | def force_flush(self, timeout_millis: int = 30_000) -> bool:

FILE: deepeval/tracing/otel/exporter.py
  function _resolve_parent_uuid (line 54) | def _resolve_parent_uuid(span: ReadableSpan) -> Optional[str]:
  class BaseSpanWrapper (line 74) | class BaseSpanWrapper:
  class ConfidentSpanExporter (line 96) | class ConfidentSpanExporter(SpanExporter):
    method __init__ (line 98) | def __init__(self, api_key: Optional[str] = None):
    method shutdown (line 111) | def shutdown(self):
    method force_flush (line 114) | def force_flush(self, timeout_millis: int = 30000) -> bool:
    method export (line 117) | def export(
    method _set_current_trace_attributes_from_base_span_wrapper (line 232) | def _set_current_trace_attributes_from_base_span_wrapper(
    method _convert_readable_span_to_base_span (line 338) | def _convert_readable_span_to_base_span(
    method __set_custom_trace_input_output (line 382) | def __set_custom_trace_input_output(
    method __set_trace_attributes (line 396) | def __set_trace_attributes(
    method __set_base_span_attributes (line 473) | def __set_base_span_attributes(
    method prepare_boilerplate_base_span (line 556) | def prepare_boilerplate_base_span(span: ReadableSpan) -> Optional[Base...
    method _parse_base_model (line 775) | def _parse_base_model(
    method _parse_json_string (line 787) | def _parse_json_string(self, json_str: str) -> Optional[Dict]:
    method _parse_list_of_tools (line 795) | def _parse_list_of_tools(self, tools: List[str]) -> List[ToolCall]:
    method _build_span_forest (line 812) | def _build_span_forest(

FILE: deepeval/tracing/otel/test_exporter.py
  class TestExporter (line 9) | class TestExporter(SpanExporter):
    method export (line 14) | def export(
    method get_span_json_list (line 23) | def get_span_json_list(self) -> List[Dict[str, Any]]:
    method clear_span_json_list (line 31) | def clear_span_json_list(self):

FILE: deepeval/tracing/otel/utils.py
  function stash_pending_metrics (line 27) | def stash_pending_metrics(uuid: str, metrics: Optional[List[Any]]) -> None:
  function pop_pending_metrics (line 35) | def pop_pending_metrics(uuid: str) -> Optional[List[Any]]:
  function to_hex_string (line 41) | def to_hex_string(id_value: int | bytes, length: int = 32) -> str:
  function set_trace_time (line 57) | def set_trace_time(trace: Trace):
  function validate_llm_test_case_data (line 91) | def validate_llm_test_case_data(
  function check_llm_input_from_gen_ai_attributes (line 130) | def check_llm_input_from_gen_ai_attributes(
  function _flatten_system_instructions (line 184) | def _flatten_system_instructions(system_instructions: list) -> list:
  function _flatten_input (line 198) | def _flatten_input(input: list) -> list:
  function check_tool_name_from_gen_ai_attributes (line 253) | def check_tool_name_from_gen_ai_attributes(span: ReadableSpan) -> Option...
  function check_tool_input_parameters_from_gen_ai_attributes (line 264) | def check_tool_input_parameters_from_gen_ai_attributes(
  function check_span_type_from_gen_ai_attributes (line 277) | def check_span_type_from_gen_ai_attributes(span: ReadableSpan):
  function check_model_from_gen_ai_attributes (line 296) | def check_model_from_gen_ai_attributes(span: ReadableSpan):
  function prepare_trace_llm_test_case (line 307) | def prepare_trace_llm_test_case(span: ReadableSpan) -> Optional[LLMTestC...
  function parse_string (line 377) | def parse_string(value: Any) -> Optional[str]:
  function parse_list_of_strings (line 383) | def parse_list_of_strings(context: List[str]) -> List[str]:
  function post_test_run (line 394) | def post_test_run(traces: List[Trace], test_run_id: Optional[str]):
  function normalize_pydantic_ai_messages (line 444) | def normalize_pydantic_ai_messages(span: ReadableSpan) -> list:
  function _extract_non_thinking_part_of_last_message (line 487) | def _extract_non_thinking_part_of_last_message(message: dict) -> dict:
  function _is_user_text_message (line 500) | def _is_user_text_message(m: dict) -> bool:
  function check_pydantic_ai_agent_input_output (line 511) | def check_pydantic_ai_agent_input_output(
  function check_tool_output (line 560) | def check_tool_output(span: ReadableSpan):
  function check_pydantic_ai_trace_input_output (line 568) | def check_pydantic_ai_trace_input_output(
  function check_model_parameters (line 580) | def check_model_parameters(span: ReadableSpan) -> Optional[dict]:

FILE: deepeval/tracing/patchers.py
  function patch_openai_client (line 16) | def patch_openai_client(client: OpenAI):
  function patch_anthropic_client (line 125) | def patch_anthropic_client(client: "Anthropic"):

FILE: deepeval/tracing/perf_epoch_bridge.py
  function init_clock_bridge (line 26) | def init_clock_bridge() -> None:
  function epoch_nanos_to_perf_seconds (line 36) | def epoch_nanos_to_perf_seconds(epoch_ns: int) -> float:
  function perf_seconds_now (line 43) | def perf_seconds_now() -> float:
  function perf_seconds_to_epoch_nanos (line 49) | def perf_seconds_to_epoch_nanos(perf_sec: float) -> int:

FILE: deepeval/tracing/trace_context.py
  class LlmSpanContext (line 16) | class LlmSpanContext:
  class AgentSpanContext (line 27) | class AgentSpanContext:
  function trace (line 46) | def trace(

FILE: deepeval/tracing/trace_test_manager.py
  class TraceTestingManager (line 6) | class TraceTestingManager:
    method wait_for_test_dict (line 10) | async def wait_for_test_dict(

FILE: deepeval/tracing/tracing.py
  class _ObservedAsyncGenIter (line 88) | class _ObservedAsyncGenIter:
    method __init__ (line 103) | def __init__(self, agen, observer):
    method __aiter__ (line 109) | def __aiter__(self):
    method __anext__ (line 112) | async def __anext__(self):
    method _finish (line 125) | def _finish(self):
    method _finish_err (line 130) | def _finish_err(self, e):
    method aclose (line 135) | async def aclose(self):
    method athrow (line 139) | async def athrow(self, typ, val=None, tb=None):
    method __del__ (line 152) | def __del__(self):
  class TraceManager (line 165) | class TraceManager:
    method __init__ (line 166) | def __init__(self):
    method _warn_on_exit (line 212) | def _warn_on_exit(self):
    method is_evaluating (line 225) | def is_evaluating(self) -> bool:
    method is_iterator (line 234) | def is_iterator(self) -> bool:
    method mask (line 238) | def mask(self, data: Any):
    method configure (line 244) | def configure(
    method start_new_trace (line 273) | def start_new_trace(
    method end_trace (line 311) | def end_trace(self, trace_uuid: str):
    method set_trace_status (line 384) | def set_trace_status(self, trace_uuid: str, status: TraceSpanStatus):
    method add_span (line 390) | def add_span(self, span: BaseSpan):
    method remove_span (line 394) | def remove_span(self, span_uuid: str):
    method add_span_to_trace (line 399) | def add_span_to_trace(self, span: BaseSpan):
    method _reparent_orphan_roots (line 442) | def _reparent_orphan_roots(trace: Trace, parent: BaseSpan) -> None:
    method get_trace_by_uuid (line 459) | def get_trace_by_uuid(self, trace_uuid: str) -> Optional[Trace]:
    method get_span_by_uuid (line 463) | def get_span_by_uuid(self, span_uuid: str) -> Optional[BaseSpan]:
    method get_all_traces (line 467) | def get_all_traces(self) -> List[Trace]:
    method clear_traces (line 471) | def clear_traces(self):
    method get_trace_dict (line 477) | def get_trace_dict(self, trace: Trace) -> Dict:
    method get_all_traces_dict (line 481) | def get_all_traces_dict(self) -> List[Dict]:
    method _print_trace_status (line 485) | def _print_trace_status(
    method _should_sample_trace (line 515) | def _should_sample_trace(self) -> bool:
    method _ensure_worker_thread_running (line 527) | def _ensure_worker_thread_running(self):
    method post_trace_api (line 535) | def post_trace_api(self, trace_api: TraceApi) -> Optional[str]:
    method post_trace (line 555) | def post_trace(self, trace: Trace) -> Optional[str]:
    method _process_trace_queue (line 578) | def _process_trace_queue(self):
    method flush_traces (line 696) | def flush_traces(self, remaining_traces: List[TraceApi]):
    method create_nested_spans_dict (line 743) | def create_nested_spans_dict(self, span: BaseSpan) -> Dict[str, Any]:
    method create_trace_api (line 772) | def create_trace_api(self, trace: Trace) -> TraceApi:
    method _convert_span_to_api_span (line 863) | def _convert_span_to_api_span(self, span: BaseSpan) -> BaseApiSpan:
  class Observer (line 974) | class Observer:
    method __init__ (line 975) | def __init__(
    method __enter__ (line 1010) | def __enter__(self):
    method __exit__ (line 1097) | def __exit__(self, exc_type, exc_val, exc_tb):
    method create_span_instance (line 1195) | def create_span_instance(self):
  function observe (line 1253) | def observe(

FILE: deepeval/tracing/types.py
  class Message (line 18) | class Message(BaseModel):
  class TraceWorkerStatus (line 31) | class TraceWorkerStatus(Enum):
  class SpanType (line 37) | class SpanType(Enum):
  class TraceSpanStatus (line 44) | class TraceSpanStatus(Enum):
  class EvalMode (line 50) | class EvalMode(str, Enum):
  class LlmToolCall (line 75) | class LlmToolCall(BaseModel):
  class LlmOutput (line 81) | class LlmOutput(BaseModel):
  class BaseSpan (line 87) | class BaseSpan(BaseModel):
  class AgentSpan (line 128) | class AgentSpan(BaseSpan):
  class LlmSpan (line 134) | class LlmSpan(BaseSpan):
  class RetrieverSpan (line 171) | class RetrieverSpan(BaseSpan):
  class ToolSpan (line 177) | class ToolSpan(BaseSpan):
  class Trace (line 182) | class Trace(BaseModel):
  class TraceAttributes (line 237) | class TraceAttributes(BaseModel):
  class TestCaseMetricPair (line 246) | class TestCaseMetricPair:
  class EvalSession (line 253) | class EvalSession:
    method is_evaluating (line 293) | def is_evaluating(self) -> bool:
    method is_iterator (line 298) | def is_iterator(self) -> bool:

FILE: deepeval/tracing/utils.py
  class Environment (line 17) | class Environment(Enum):
  function infer_provider_from_model (line 24) | def infer_provider_from_model(model: str) -> Optional[str]:
  function _normalize_provider_string (line 59) | def _normalize_provider_string(value: str) -> str:
  function normalize_span_provider_for_platform (line 64) | def normalize_span_provider_for_platform(raw: Optional[Any]) -> Optional...
  function normalize_trace_api_span_providers (line 88) | def normalize_trace_api_span_providers(trace_api: "TraceApi") -> None:
  function _strip_nul (line 104) | def _strip_nul(s: str) -> str:
  function tracing_enabled (line 110) | def tracing_enabled():
  function validate_environment (line 114) | def validate_environment(environment: str):
  function validate_sampling_rate (line 122) | def validate_sampling_rate(sampling_rate: float):
  function make_json_serializable (line 129) | def make_json_serializable(obj):
  function make_json_serializable_for_metadata (line 187) | def make_json_serializable_for_metadata(obj):
  function to_zod_compatible_iso (line 253) | def to_zod_compatible_iso(
  function perf_counter_to_datetime (line 265) | def perf_counter_to_datetime(perf_counter_value: float) -> datetime:
  function replace_self_with_class_name (line 285) | def replace_self_with_class_name(obj):
  function prepare_tool_call_input_parameters (line 292) | def prepare_tool_call_input_parameters(output: Any) -> Dict[str, Any]:
  function is_async_context (line 299) | def is_async_context() -> bool:

FILE: deepeval/utils.py
  function make_model_config (line 41) | def make_model_config(**kwargs):
  class TurnLike (line 79) | class TurnLike(Protocol):
  function get_lcs (line 89) | def get_lcs(seq1, seq2):
  function camel_to_snake (line 116) | def camel_to_snake(name: str) -> str:
  function convert_keys_to_snake_case (line 121) | def convert_keys_to_snake_case(data: Any) -> Any:
  function prettify_list (line 139) | def prettify_list(lst: List[Any]):
  function generate_uuid (line 166) | def generate_uuid() -> str:
  function serialize_dict_with_sorting (line 170) | def serialize_dict_with_sorting(obj):
  function serialize (line 188) | def serialize(obj) -> Union[str, None]:
  function get_or_create_event_loop (line 192) | def get_or_create_event_loop() -> asyncio.AbstractEventLoop:
  function get_or_create_general_event_loop (line 206) | def get_or_create_general_event_loop() -> asyncio.AbstractEventLoop:
  function set_should_skip_on_missing_params (line 218) | def set_should_skip_on_missing_params(yes: bool):
  function should_ignore_errors (line 224) | def should_ignore_errors() -> bool:
  function should_skip_on_missing_params (line 228) | def should_skip_on_missing_params() -> bool:
  function set_should_ignore_errors (line 232) | def set_should_ignore_errors(yes: bool):
  function should_verbose_print (line 238) | def should_verbose_print() -> bool:
  function set_verbose_mode (line 242) | def set_verbose_mode(yes: Optional[bool]):
  function set_identifier (line 248) | def set_identifier(identifier: Optional[str]):
  function get_identifier (line 255) | def get_identifier() -> Optional[str]:
  function should_use_cache (line 259) | def should_use_cache() -> bool:
  function set_should_use_cache (line 263) | def set_should_use_cache(yes: bool):
  function are_timeouts_disabled (line 272) | def are_timeouts_disabled() -> bool:
  function get_per_task_timeout_seconds (line 276) | def get_per_task_timeout_seconds() -> float:
  function get_per_task_timeout (line 280) | def get_per_task_timeout() -> Optional[float]:
  function get_gather_timeout_seconds (line 284) | def get_gather_timeout_seconds() -> float:
  function get_gather_timeout (line 291) | def get_gather_timeout() -> Optional[float]:
  function login (line 295) | def login(api_key: str):
  function set_is_running_deepeval (line 310) | def set_is_running_deepeval(flag: bool):
  function get_is_running_deepeval (line 314) | def get_is_running_deepeval() -> bool:
  function is_in_ci_env (line 318) | def is_in_ci_env() -> bool:
  function open_browser (line 341) | def open_browser(url: str):
  function capture_contextvars (line 347) | def capture_contextvars(single_obj):
  function update_contextvars (line 356) | def update_contextvars(single_obj, contextvars_dict):
  function drop_and_copy (line 362) | def drop_and_copy(obj, drop_attrs):
  function dataclass_to_dict (line 417) | def dataclass_to_dict(instance: Any) -> Any:
  function class_to_dict (line 432) | def class_to_dict(instance: Any) -> Any:
  function delete_file_if_exists (line 448) | def delete_file_if_exists(file_path):
  function softmax (line 456) | def softmax(x):
  function cosine_similarity (line 463) | def cosine_similarity(vector_a, vector_b):
  function chunk_text (line 473) | def chunk_text(text, chunk_size=20):
  function normalize_text (line 482) | def normalize_text(text: str) -> str:
  function is_missing (line 503) | def is_missing(s: Optional[str]) -> bool:
  function len_tiny (line 507) | def len_tiny() -> int:
  function len_short (line 512) | def len_short() -> int:
  function len_medium (line 517) | def len_medium() -> int:
  function len_long (line 522) | def len_long() -> int:
  function shorten (line 527) | def shorten(
  function convert_to_multi_modal_array (line 565) | def convert_to_multi_modal_array(input: Union[str, List[str]]):
  function check_if_multimodal (line 578) | def check_if_multimodal(input: str):
  function format_turn (line 584) | def format_turn(
  function get_freer_gpu (line 654) | def get_freer_gpu():
  function any_gpu_with_space (line 666) | def any_gpu_with_space(gb_needed):
  function wait_free_gpu (line 676) | def wait_free_gpu(gb_needed):
  function select_freer_gpu (line 681) | def select_freer_gpu():
  function batcher (line 692) | def batcher(iterator, batch_size=4, progress=False):
  function clean_nested_dict (line 707) | def clean_nested_dict(data):
  function update_pbar (line 718) | def update_pbar(
  function add_pbar (line 754) | def add_pbar(progress: Optional[Progress], description: str, total: int ...
  function remove_pbars (line 760) | def remove_pbars(
  function read_env_int (line 771) | def read_env_int(
  function read_env_float (line 800) | def read_env_float(
  function format_error_text (line 844) | def format_error_text(
  function is_read_only_env (line 864) | def is_read_only_env():
  function require_param (line 873) | def require_param(
  function require_dependency (line 906) | def require_dependency(

FILE: demo_trace_scope/test_observed_app.py
  function retriever (line 13) | def retriever(query: str) -> list[str]:
  function llm_app (line 28) | def llm_app(query: str) -> str:
  function test_llm_app_trace_scope (line 47) | def test_llm_app_trace_scope(golden: Golden):

FILE: docs/app/(home)/layout.tsx
  function Layout (line 3) | function Layout({ children }: LayoutProps<"/">) {

FILE: docs/app/(home)/page.tsx
  function HomePage (line 20) | function HomePage() {

FILE: docs/app/enterprise/page.tsx
  function EnterprisePage (line 16) | function EnterprisePage() {

FILE: docs/app/layout.tsx
  function Layout (line 104) | function Layout({ children }: LayoutProps<'/'>) {

FILE: docs/app/llms-full.txt/route.ts
  function GET (line 5) | async function GET() {

FILE: docs/app/llms.mdx/blog/[[...slug]]/route.ts
  constant GET (line 7) | const GET = route.GET;

FILE: docs/app/llms.mdx/changelog/[[...slug]]/route.ts
  constant GET (line 7) | const GET = route.GET;

FILE: docs/app/llms.mdx/docs/[[...slug]]/route.ts
  constant GET (line 7) | const GET = route.GET;

FILE: docs/app/llms.mdx/guides/[[...slug]]/route.ts
  constant GET (line 7) | const GET = route.GET;

FILE: docs/app/llms.mdx/integrations/[[...slug]]/route.ts
  constant GET (line 7) | const GET = route.GET;

FILE: docs/app/llms.mdx/tutorials/[[...slug]]/route.ts
  constant GET (line 7) | const GET = route.GET;

FILE: docs/app/llms.txt/route.ts
  function GET (line 6) | function GET() {

FILE: docs/app/og/docs/[...slug]/route.tsx
  function GET (line 9) | async function GET(_req: Request, { params }: RouteContext<'/og/docs/[.....
  function generateStaticParams (line 23) | function generateStaticParams() {

FILE: docs/app/robots.ts
  function robots (line 7) | function robots(): MetadataRoute.Robots {

FILE: docs/app/sitemap.ts
  type AnySource (line 24) | type AnySource = any;
  function sitemap (line 35) | function sitemap(): MetadataRoute.Sitemap {

FILE: docs/components/mdx-anchor.tsx
  type MdxAnchorProps (line 6) | type MdxAnchorProps = ComponentPropsWithoutRef<"a"> & {

FILE: docs/components/mdx.tsx
  function DefaultLLMModel (line 30) | function DefaultLLMModel() {
  function getMDXComponents (line 34) | function getMDXComponents(components?: MDXComponents) {
  type MDXProvidedComponents (line 72) | type MDXProvidedComponents = ReturnType<typeof getMDXComponents>;

FILE: docs/lib/authors.ts
  type Author (line 15) | type Author = {
  type AuthorId (line 43) | type AuthorId = keyof typeof authors;
  constant AUTHOR_IDS (line 49) | const AUTHOR_IDS = Object.keys(authors) as [AuthorId, ...AuthorId[]];
  function getAuthor (line 51) | function getAuthor(id: AuthorId): Author {

FILE: docs/lib/blog-categories.ts
  type BlogCategory (line 19) | type BlogCategory = {
  type BlogCategoryId (line 30) | type BlogCategoryId = keyof typeof blogCategories;
  constant BLOG_CATEGORY_IDS (line 32) | const BLOG_CATEGORY_IDS = Object.keys(blogCategories) as [
  function getBlogCategory (line 37) | function getBlogCategory(id: BlogCategoryId): BlogCategory {

FILE: docs/lib/contributors.ts
  type Contributor (line 13) | type Contributor = {
  type Manifest (line 21) | type Manifest = Record<string, Contributor[]>;
  function getPageContributors (line 31) | function getPageContributors(

FILE: docs/lib/defaults.ts
  constant DEFAULT_LLM_MODEL (line 1) | const DEFAULT_LLM_MODEL = "gpt-5.4";

FILE: docs/lib/layout.shared.tsx
  function baseOptions (line 69) | function baseOptions(): BaseLayoutProps {

FILE: docs/lib/llms-route.ts
  type Source (line 8) | type Source = any;
  function createLLMsRoute (line 15) | function createLLMsRoute(source: Source) {

FILE: docs/lib/remark-admonitions.ts
  constant ADMONITION_TYPES (line 6) | const ADMONITION_TYPES = new Set([
  function remarkAdmonitions (line 24) | function remarkAdmonitions() {

FILE: docs/lib/section.tsx
  type Source (line 33) | type Source = any;
  type SectionPageProps (line 35) | type SectionPageProps = {
  type Page (line 43) | type Page = any;
  type SectionConfig (line 45) | type SectionConfig = {
  function createSection (line 91) | function createSection(config: SectionConfig) {

FILE: docs/lib/sections.tsx
  type BlogFrontmatter (line 21) | type BlogFrontmatter = {
  function toIso (line 40) | function toIso(value: unknown): string | undefined {

FILE: docs/lib/shared.ts
  function contentRouteFor (line 38) | function contentRouteFor(sectionOrBaseUrl: string) {

FILE: docs/lib/source.ts
  method file (line 27) | file(node) {
  function getPageImage (line 89) | function getPageImage(page: (typeof source)['$inferPage']) {
  function getPageMarkdownUrl (line 110) | function getPageMarkdownUrl(page: any, _src?: unknown) {
  function getLLMText (line 119) | async function getLLMText(page: (typeof source)['$inferPage']) {
  constant DESCRIPTION_MAX (line 147) | const DESCRIPTION_MAX = 160;
  function cleanMarkdownForDescription (line 149) | function cleanMarkdownForDescription(md: string): string {
  function extractFirstParagraph (line 174) | function extractFirstParagraph(md: string): string {
  function truncateOnWord (line 206) | function truncateOnWord(text: string, max: number): string {
  function getPageDescription (line 214) | async function getPageDescription(

FILE: docs/proxy.ts
  function proxy (line 14) | function proxy(request: NextRequest) {

FILE: docs/scripts/build-readme-hero.mjs
  constant REPO_ROOT (line 31) | const REPO_ROOT = path.resolve(__dirname, '../..');
  constant OUT_DIR (line 32) | const OUT_DIR = path.join(REPO_ROOT, 'assets/hero');
  constant FONT_DIR (line 33) | const FONT_DIR = path.join(__dirname, 'fonts');
  constant FONT_PATH (line 34) | const FONT_PATH = path.join(FONT_DIR, 'Quicksand-SemiBold.ttf');
  constant FONT_ZIP_URL (line 42) | const FONT_ZIP_URL =
  constant ACCENT (line 45) | const ACCENT = '#4400FF';
  constant LIGHT_FG (line 46) | const LIGHT_FG = '#0A0A0A';
  constant DARK_FG (line 47) | const DARK_FG = '#FAFAFA';
  function ensureFont (line 49) | async function ensureFont() {
  function buildWordmark (line 74) | async function buildWordmark(font, mode) {
  function main (line 137) | async function main() {

FILE: docs/scripts/generate-changelog-contributors.mjs
  constant ROOT (line 4) | const ROOT = process.cwd();
  constant CHANGELOG_DIR (line 5) | const CHANGELOG_DIR = path.join(ROOT, "content", "changelog");
  constant OUT_PATH (line 6) | const OUT_PATH = path.join(
  function collectYearContributors (line 16) | function collectYearContributors(filePath) {
  function main (line 45) | function main() {

FILE: docs/scripts/generate-contributors.mjs
  constant CONTENT_DIRS (line 53) | const CONTENT_DIRS = [
  constant OUTPUT (line 59) | const OUTPUT = "lib/generated/contributors.json";
  constant CACHE (line 60) | const CACHE = "lib/generated/.contributors-cache.json";
  constant REPO_CONTRIBUTORS (line 61) | const REPO_CONTRIBUTORS = "lib/generated/repo-contributors.json";
  constant PAGE_CONTRIBUTOR_ALIASES (line 64) | const PAGE_CONTRIBUTOR_ALIASES = {
  constant AUTHOR_LOGIN_ALIASES (line 72) | const AUTHOR_LOGIN_ALIASES = {
  function readGitConfig (line 78) | function readGitConfig() {
  function tryExec (line 87) | function tryExec(cmd) {
  function inGitRepo (line 98) | function inGitRepo() {
  function inShallowRepo (line 102) | function inShallowRepo() {
  function walkMdx (line 106) | function walkMdx(dir, acc = []) {
  function gitCommitsForFile (line 119) | function gitCommitsForFile(file) {
  function loadCache (line 133) | function loadCache() {
  function loadRepoContributors (line 142) | function loadRepoContributors() {
  function loadExistingManifest (line 158) | function loadExistingManifest() {
  function saveJson (line 170) | function saveJson(path, obj) {
  function keepExistingOrWriteEmpty (line 175) | function keepExistingOrWriteEmpty(path) {
  function findCommitMetaForEmail (line 183) | function findCommitMetaForEmail(perFile, email) {
  function resolveAuthor (line 191) | async function resolveAuthor(sha, { user, repo, token }) {
  function isBot (line 217) | function isBot(author) {
  function main (line 223) | async function main() {

FILE: docs/scripts/generate-repo-contributors.mjs
  constant OUTPUT (line 32) | const OUTPUT = 'lib/generated/repo-contributors.json';
  constant PER_PAGE (line 33) | const PER_PAGE = 100;
  constant MAX_PAGES (line 34) | const MAX_PAGES = 10;
  function readGitConfig (line 36) | function readGitConfig() {
  function saveJson (line 44) | function saveJson(path, obj) {
  function isBot (line 49) | function isBot(c) {
  function fetchPage (line 53) | async function fetchPage(user, repo, page, token) {
  function main (line 67) | async function main() {

FILE: docs/scripts/normalize-admonition-titles.mjs
  constant TYPES (line 19) | const TYPES =
  constant PATTERN (line 24) | const PATTERN = new RegExp(
  constant FENCE (line 29) | const FENCE = /```[\s\S]*?```/g;
  function transform (line 31) | function transform(src) {
  function walk (line 48) | function walk(dir) {
  function processFile (line 58) | function processFile(path) {

FILE: docs/scripts/replace-img-with-image-displayer.mjs
  constant IMG_TAG (line 16) | const IMG_TAG = /<img\b([^>]*?)\/?\s*>/gs;
  constant MD_IMG (line 17) | const MD_IMG = /!\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
  constant FENCE (line 18) | const FENCE = /```[\s\S]*?```/g;
  function extractAttr (line 20) | function extractAttr(attrs, name) {
  function formatAttr (line 30) | function formatAttr(name, attr) {
  function replaceImgTag (line 36) | function replaceImgTag(_match, attrs) {
  function replaceMarkdownImg (line 46) | function replaceMarkdownImg(_match, alt, url) {
  function transform (line 52) | function transform(src) {
  function walk (line 72) | function walk(dir) {
  function processFile (line 82) | function processFile(path) {

FILE: docs/scripts/strip-redundant-mdx-imports.mjs
  constant REGISTERED (line 15) | const REGISTERED = [
  function walk (line 42) | function walk(dir) {
  function processFile (line 52) | function processFile(path) {

FILE: docs/scripts/timeline-to-steps.mjs
  function transform (line 28) | function transform(src) {
  function walk (line 56) | function walk(dir) {
  function processFile (line 66) | function processFile(path) {

FILE: docs/src/assets.ts
  constant BUCKETS (line 1) | const BUCKETS = {
  constant ASSETS (line 8) | const ASSETS = {

FILE: docs/src/components/AgentTraceTerminal/index.tsx
  type LineKind (line 3) | type LineKind =
  type TraceLine (line 13) | type TraceLine = {
  constant DEFAULT_TRACE (line 23) | const DEFAULT_TRACE: TraceLine[] = [
  constant DEFAULT_TITLE (line 90) | const DEFAULT_TITLE = "agent_trace · deepeval";
  constant DEFAULT_ARIA_LABEL (line 91) | const DEFAULT_ARIA_LABEL = "Example agent trace with per-step metric sco...
  type AgentTraceTerminalProps (line 93) | interface AgentTraceTerminalProps {
  function kindLabel (line 99) | function kindLabel(kind: LineKind): string | null {

FILE: docs/src/components/AskAIButton/index.tsx
  type AskAIButtonProps (line 23) | type AskAIButtonProps = {

FILE: docs/src/components/BlogPostMeta/index.tsx
  type BlogPostMetaProps (line 6) | interface BlogPostMetaProps {

FILE: docs/src/components/Buttons/index.tsx
  type CommonButtonProps (line 8) | type CommonButtonProps = {
  type PrimaryExtras (line 14) | type PrimaryExtras = {
  type RenderContentProps (line 18) | type RenderContentProps = CommonButtonProps &
  type LinkButtonProps (line 23) | type LinkButtonProps = CommonButtonProps &
  type NativeButtonProps (line 28) | type NativeButtonProps = CommonButtonProps &
  type ButtonProps (line 33) | type ButtonProps = LinkButtonProps | NativeButtonProps;
  type PrimaryButtonProps (line 34) | type PrimaryButtonProps =
  function renderContent (line 38) | function renderContent({

FILE: docs/src/components/Callout/index.tsx
  type CalloutType (line 13) | type CalloutType =
  type CalloutProps (line 25) | interface CalloutProps {
  constant ICONS (line 31) | const ICONS: Record<CalloutType, React.ComponentType<{ className?: strin...
  constant DEFAULT_TITLES (line 44) | const DEFAULT_TITLES: Partial<Record<CalloutType, string>> = {

FILE: docs/src/components/ChangelogContributors/index.tsx
  type ChangelogContributor (line 7) | interface ChangelogContributor {
  type ChangelogContributorManifest (line 15) | type ChangelogContributorManifest = Record<string, ChangelogContributor[]>;
  type ChangelogContributorsProps (line 17) | interface ChangelogContributorsProps {
  function contributionsLabel (line 24) | function contributionsLabel(n: number) {
  function contributorLabel (line 28) | function contributorLabel(c: ChangelogContributor) {

FILE: docs/src/components/ContributorDisplay/index.tsx
  type Props (line 4) | type Props = {

FILE: docs/src/components/DiscordButton/index.tsx
  type DiscordButtonProps (line 32) | type DiscordButtonProps = {

FILE: docs/src/components/Equation/index.tsx
  type EquationProps (line 5) | interface EquationProps {

FILE: docs/src/components/FAQ/index.tsx
  type QA (line 6) | interface QA {
  type FAQsProps (line 11) | interface FAQsProps {
  function extractText (line 20) | function extractText(node: ReactNode): string {

FILE: docs/src/components/FeatureComparisonTable/index.tsx
  type DatasetItem (line 5) | interface DatasetItem {
  type DatasetCategories (line 12) | interface DatasetCategories {
  type Datasets (line 22) | interface Datasets {
  type TopLevel (line 2303) | type TopLevel = keyof typeof datasets;
  type FeatureComparisonTableProps (line 2305) | interface FeatureComparisonTableProps {

FILE: docs/src/components/GithubCtaButton/index.tsx
  type GithubCtaButtonProps (line 21) | type GithubCtaButtonProps = {

FILE: docs/src/components/GithubCtaButton/useGithubStarCount.ts
  constant CACHE_KEY (line 6) | const CACHE_KEY = `gh-stars:${gitConfig.user}/${gitConfig.repo}`;
  constant CACHE_TTL_MS (line 7) | const CACHE_TTL_MS = 60 * 60 * 1000;
  type Cached (line 9) | type Cached = { count: number; ts: number };
  function readCache (line 11) | function readCache(): number | null {
  function writeCache (line 30) | function writeCache(count: number) {
  function useGithubStarCount (line 39) | function useGithubStarCount(): number | null {
  function formatStarCount (line 71) | function formatStarCount(n: number): string {

FILE: docs/src/components/HeroAnnouncement/index.tsx
  type HeroAnnouncementProps (line 6) | type HeroAnnouncementProps = {

FILE: docs/src/components/Hotkey/index.tsx
  type HotkeyConfig (line 7) | type HotkeyConfig = {
  type HotkeyProps (line 12) | type HotkeyProps = {
  function renderHotkeyLabel (line 17) | function renderHotkeyLabel(key: string) {
  function onKeyDown (line 27) | function onKeyDown(event: KeyboardEvent) {

FILE: docs/src/components/ImageDisplayer/index.tsx
  type ImageDisplayerProps (line 4) | interface ImageDisplayerProps {

FILE: docs/src/components/IntegrationGrid/index.tsx
  type Integration (line 12) | type Integration = {
  type Category (line 34) | type Category = {
  constant CI_CD_DOCS (line 46) | const CI_CD_DOCS = "/docs/evaluation-unit-testing-in-ci-cd";
  constant INTEGRATIONS_INDEX (line 47) | const INTEGRATIONS_INDEX = "/integrations";
  constant MODEL_PROVIDERS (line 49) | const MODEL_PROVIDERS: Category = {
  constant FRAMEWORKS (line 65) | const FRAMEWORKS: Category = {
  constant CI_CD (line 84) | const CI_CD: Category = {

FILE: docs/src/components/IntegrationTagsDisplayer/index.tsx
  type IntegrationTagsDisplayerProps (line 4) | interface IntegrationTagsDisplayerProps {

FILE: docs/src/components/LinkCards/index.tsx
  type LinkCardProps (line 6) | interface LinkCardProps {
  type LinkCardsProps (line 15) | interface LinkCardsProps {

FILE: docs/src/components/Mermaid/index.tsx
  type MermaidRenderResult (line 6) | type MermaidRenderResult = {
  type MermaidProps (line 11) | type MermaidProps = {
  function renderChart (line 24) | async function renderChart() {

FILE: docs/src/components/MetricTagsDisplayer/index.tsx
  type MetricTagsDisplayerProps (line 4) | interface MetricTagsDisplayerProps {

FILE: docs/src/components/PageContributors/ContributorsOverflow.tsx
  type ContributorsOverflowProps (line 18) | interface ContributorsOverflowProps {

FILE: docs/src/components/PageContributors/index.tsx
  constant DEFAULT_LIMIT (line 11) | const DEFAULT_LIMIT = 5;
  type PageContributorsProps (line 13) | interface PageContributorsProps {
  function commitLabel (line 18) | function commitLabel(n: number) {
  function contributorLabel (line 26) | function contributorLabel(c: Contributor) {

FILE: docs/src/components/PauseOffscreen/index.tsx
  type Props (line 38) | type Props = HTMLAttributes<HTMLDivElement> & {

FILE: docs/src/components/SchemaInjector/SchemaInjector.tsx
  type SchemaInjectorProps (line 1) | interface SchemaInjectorProps {

FILE: docs/src/components/SectionLabel/index.tsx
  type SectionLabelProps (line 4) | type SectionLabelProps = {

FILE: docs/src/components/TechStackCards/index.tsx
  type TechStackCardProps (line 4) | interface TechStackCardProps {
  type TectStackCardsProps (line 10) | interface TectStackCardsProps {

FILE: docs/src/components/TocFooter/index.tsx
  type Props (line 10) | type Props = {

FILE: docs/src/components/VideoDisplayer/index.tsx
  type VideoDisplayerProps (line 7) | interface VideoDisplayerProps {

FILE: docs/src/layouts/Footer/index.tsx
  type FooterLink (line 7) | type FooterLink = {
  type FooterColumn (line 12) | type FooterColumn = {
  constant COLUMNS (line 17) | const COLUMNS: FooterColumn[] = [

FILE: docs/src/layouts/HomeLayout/HomeLayout.tsx
  type HomeLayoutProps (line 7) | type HomeLayoutProps = {

FILE: docs/src/layouts/HomeOverflowNav/index.tsx
  type HomeOverflowNavProps (line 18) | type HomeOverflowNavProps = {

FILE: docs/src/layouts/HomePageShell/index.tsx
  type HomePageShellProps (line 7) | type HomePageShellProps = {

FILE: docs/src/layouts/NavLinks/index.tsx
  type NavLink (line 7) | type NavLink = {
  type NavLinksProps (line 15) | interface NavLinksProps {
  function isNavLinkActive (line 24) | function isNavLinkActive(pathname: string, item: NavLink) {
  type NavLinkItemProps (line 31) | type NavLinkItemProps = {

FILE: docs/src/layouts/NavMenu/index.tsx
  type NavMenuProps (line 12) | interface NavMenuProps {

FILE: docs/src/layouts/NotebookSectionLayout/index.tsx
  type NotebookSectionLayoutProps (line 5) | type NotebookSectionLayoutProps = {

FILE: docs/src/layouts/SiteTopNav/index.tsx
  type NavTitleComponent (line 19) | type NavTitleComponent = ComponentType<{ className?: string }>;
  type ThemeSwitchComponent (line 20) | type ThemeSwitchComponent = ComponentType<{ className?: string }>;
  type CollapseTriggerComponent (line 21) | type CollapseTriggerComponent = ComponentType<ComponentProps<"button">>;
  type SiteTopNavProps (line 23) | type SiteTopNavProps = {

FILE: docs/src/layouts/UtmCapture/UtmCapture.tsx
  constant SOURCE (line 62) | const SOURCE = "deepeval";
  constant DEFAULT_MEDIUM (line 63) | const DEFAULT_MEDIUM: UtmMedium = "docs";
  constant CLASS_PREFIX (line 64) | const CLASS_PREFIX = "utm--";
  constant FALLBACK_CONTENT (line 65) | const FALLBACK_CONTENT = "inline_link";
  function resolveUtmContent (line 67) | function resolveUtmContent(anchor: HTMLAnchorElement): string {
  function stampAnchor (line 88) | function stampAnchor(anchor: HTMLAnchorElement | null): void {
  function handleEvent (line 131) | function handleEvent(e: Event): void {

FILE: docs/src/sections/enterprise/EnterpriseComparisonTable.tsx
  constant ROWS (line 4) | const ROWS = [

FILE: docs/src/sections/enterprise/EnterpriseHeroSection.tsx
  type EnterpriseLogoItem (line 7) | type EnterpriseLogoItem = {
  type EnterpriseHeroSectionProps (line 12) | type EnterpriseHeroSectionProps = {
  constant DEFAULT_LOGO_ITEMS (line 17) | const DEFAULT_LOGO_ITEMS: EnterpriseLogoItem[] = [
  constant DEFAULT_LOGO_GRID_LABEL (line 37) | const DEFAULT_LOGO_GRID_LABEL =
  constant BOOK_DEMO_HREF (line 40) | const BOOK_DEMO_HREF = "https://www.confident-ai.com/book-a-demo";

FILE: docs/src/sections/enterprise/EnterprisePlatformMockup.tsx
  type EnterprisePlatformMockupProps (line 3) | type EnterprisePlatformMockupProps = {

FILE: docs/src/sections/home/ClaudeCodeTerminal/index.tsx
  type Row (line 7) | type Row =
  constant SCRIPT (line 18) | const SCRIPT: Row[] = [

FILE: docs/src/sections/home/CompanyLogos/Benz.tsx
  constant SVG_PATH (line 6) | const SVG_PATH = path.join(

FILE: docs/src/sections/home/CompanyLogos/index.ts
  constant DYNAMIC_LOGOS (line 15) | const DYNAMIC_LOGOS: Record<string, React.FC<LogoProps>> = {

FILE: docs/src/sections/home/CompanyLogos/types.ts
  type LogoProps (line 3) | type LogoProps = SVGProps<SVGSVGElement>;

FILE: docs/src/sections/home/DatasetDemos/GoldenGenerationDemo.tsx
  type Golden (line 6) | type Golden = {
  constant GOLDENS (line 13) | const GOLDENS: Golden[] = [
  constant SOURCE_DOCS (line 41) | const SOURCE_DOCS = [
  constant STAGES (line 56) | const STAGES = [
  constant STAGE_INTERVAL_MS (line 66) | const STAGE_INTERVAL_MS = 1000;

FILE: docs/src/sections/home/DatasetDemos/MultiTurnSimulationDemo.tsx
  type Turn (line 6) | type Turn = {
  constant TURNS (line 11) | const TURNS: Turn[] = [
  constant METRICS (line 30) | const METRICS = [
  constant STAGES (line 36) | const STAGES = [
  constant UNDERSTANDING_MS (line 42) | const UNDERSTANDING_MS = 1000;
  constant PROFILE_MS (line 43) | const PROFILE_MS = 1000;
  constant CHAR_MS (line 44) | const CHAR_MS = 10;
  constant TURN_GAP_MS (line 45) | const TURN_GAP_MS = 220;
  constant SCORE_DELAY_MS (line 46) | const SCORE_DELAY_MS = 300;
  type TurnViewProps (line 50) | type TurnViewProps = {

FILE: docs/src/sections/home/HomeHeroSection.tsx
  type Brand (line 9) | type Brand = {
  constant BRANDS (line 14) | const BRANDS: Brand[] = [
  constant BANNER_ITEMS (line 41) | const BANNER_ITEMS = [

FILE: docs/src/sections/home/HomePytestDemo/index.tsx
  type HomePytestDemoProps (line 91) | type HomePytestDemoProps = {
  type DemoBlockLanguage (line 95) | type DemoBlockLanguage = "bash" | "python";
  type ColabTerminalBlockProps (line 97) | type ColabTerminalBlockProps = {
  function runDemo (line 201) | function runDemo() {

FILE: docs/src/sections/home/JudgeCards/index.tsx
  type Card (line 5) | type Card = {
  constant CARDS (line 113) | const CARDS: Card[] = [

FILE: docs/src/sections/home/RepoContributors/index.tsx
  type RepoContributor (line 7) | interface RepoContributor {
  type RepoContributorsProps (line 16) | interface RepoContributorsProps {
  function commitsLabel (line 27) | function commitsLabel(n: number) {
  function contributorLabel (line 31) | function contributorLabel(c: RepoContributor) {

FILE: docs/src/sections/home/SOTACards/index.tsx
  type Card (line 5) | type Card = {
  constant CARDS (line 200) | const CARDS: Card[] = [

FILE: docs/src/sections/home/VibeCodingLoop/index.tsx
  type NodeId (line 22) | type NodeId = "tl" | "tr" | "br" | "bl";
  type Node (line 24) | type Node = {
  constant NODES (line 31) | const NODES: Node[] = [
  type Arrow (line 58) | type Arrow = {
  constant ARROWS (line 73) | const ARROWS: Arrow[] = [

FILE: docs/src/utils/html-to-markdown.ts
  function htmlToMarkdown (line 1) | function htmlToMarkdown(element: Element): string {
  function processChildren (line 5) | function processChildren(parent: Node): string {
  function processNode (line 9) | function processNode(node: Node): string {
  function headingText (line 120) | function headingText(el: Element): string {
  function convertTable (line 137) | function convertTable(table: Element): string {

FILE: docs/src/utils/schema-helpers.ts
  type ArticleSchemaProps (line 3) | interface ArticleSchemaProps {
  type BreadcrumbItem (line 13) | interface BreadcrumbItem {
  type ProductSchemaProps (line 18) | interface ProductSchemaProps {
  type Author (line 25) | interface Author {
  type BlogPost (line 32) | interface BlogPost {
  function buildWebSiteSchema (line 41) | function buildWebSiteSchema(): object {
  function buildArticleSchema (line 50) | function buildArticleSchema({
  function buildBreadcrumbSchema (line 94) | function buildBreadcrumbSchema(trail: BreadcrumbItem[]): object | null {
  type FAQItem (line 133) | interface FAQItem {
  function buildFAQPageSchema (line 138) | function buildFAQPageSchema(qas: FAQItem[]): object | null {
  function buildBlogHomeSchema (line 155) | function buildBlogHomeSchema(posts: BlogPost[]): object {

FILE: docs/src/utils/utm.ts
  constant CONFIDENT_HOSTS_BY_NAME (line 44) | const CONFIDENT_HOSTS_BY_NAME = {
  type ConfidentHost (line 53) | type ConfidentHost = keyof typeof CONFIDENT_HOSTS_BY_NAME;
  constant CONFIDENT_HOSTNAMES (line 56) | const CONFIDENT_HOSTNAMES: ReadonlySet<string> = new Set(
  constant SOURCE (line 60) | const SOURCE = 'deepeval';
  constant DEFAULT_MEDIUM (line 61) | const DEFAULT_MEDIUM = 'docs';
  type UtmMedium (line 63) | type UtmMedium = 'docs' | 'github' | 'cli' | 'python_sdk';
  type AppendOpts (line 65) | interface AppendOpts {
  function appendDeepEvalAttribution (line 86) | function appendDeepEvalAttribution(

FILE: docs/src/utils/visitor-attribution.ts
  constant UTM_KEYS (line 23) | const UTM_KEYS = [
  constant ATTRIBUTION_STORAGE_KEY (line 31) | const ATTRIBUTION_STORAGE_KEY = 'confident_utm_attribution';
  constant TTL_MS (line 33) | const TTL_MS = 180 * 24 * 60 * 60 * 1000;
  type UtmParams (line 35) | type UtmParams = Partial<Record<(typeof UTM_KEYS)[number], string>>;
  type Touch (line 37) | type Touch = {
  type Attribution (line 42) | type Attribution = {
  function safeLocalGet (line 47) | function safeLocalGet(key: string): string | null {
  function safeLocalSet (line 55) | function safeLocalSet(key: string, value: string): void {
  function isFreshTouch (line 63) | function isFreshTouch(touch: Touch | undefined): touch is Touch {
  function readAttribution (line 68) | function readAttribution(): Attribution | null {
  function writeAttribution (line 81) | function writeAttribution(attribution: Attribution): void {
  function captureVisitorUtms (line 95) | function captureVisitorUtms(): void {
  function getFirstTouchParams (line 120) | function getFirstTouchParams(): UtmParams | null {
  function getLastTouchParams (line 126) | function getLastTouchParams(): UtmParams | null {

FILE: examples/getting_started/test_example.py
  function test_everything (line 17) | def test_everything(test_case: LLMTestCase):
  function hyperparameters (line 38) | def hyperparameters():

FILE: examples/mcp_evaluation/mcp_eval_multi_turn.py
  class MCPClient (line 22) | class MCPClient:
    method __init__ (line 23) | def __init__(self):
    method connect_to_server (line 28) | async def connect_to_server(self, server_script_path: str):
    method process_query (line 59) | async def process_query(self, query: str) -> str:
    method chat_loop (line 136) | async def chat_loop(self):
    method cleanup (line 155) | async def cleanup(self):
  function main (line 160) | async def main():

FILE: examples/mcp_evaluation/mcp_eval_single_turn.py
  class MCPClient (line 17) | class MCPClient:
    method __init__ (line 18) | def __init__(self):
    method connect_to_server (line 23) | async def connect_to_server(
    method process_query (line 49) | async def process_query(self, query: str) -> str:
    method chat_loop (line 93) | async def chat_loop(self):
    method cleanup (line 107) | async def cleanup(self):
  function main (line 111) | async def main():

FILE: examples/rag_evaluation/rag_evaluation_with_qdrant.py
  function query_with_context (line 82) | def query_with_context(query, limit):
  function create_deepeval_dataset (line 129) | def create_deepeval_dataset(dataset, eval_size, retrieval_window_size):

FILE: manual_after_evals_iterator.py
  function fake_llm_call (line 74) | async def fake_llm_call(prompt: str) -> str:
  function run_agent_observed (line 107) | async def run_agent_observed(prompt: str) -> str:
  function run_agent (line 132) | async def run_agent(prompt: str) -> str:

FILE: scripts/check_openai_model_capabilities.py
  function parse_args (line 26) | def parse_args() -> argparse.Namespace:
  function select_models (line 46) | def select_models(args: argparse.Namespace) -> tuple[str, ...]:
  function registry_expectations (line 54) | def registry_expectations(model: str) -> dict[str, Any]:
  function summarize_response (line 65) | def summarize_response(response: Any) -> dict[str, Any]:
  function run_check (line 81) | def run_check(call: Callable[[], Any]) -> dict[str, Any]:
  function run_json_mode_check (line 98) | def run_json_mode_check(call: Callable[[], Any]) -> dict[str, Any]:
  function probe_model (line 128) | def probe_model(client: Any, model: str) -> dict[str, Any]:
  function main (line 166) | def main() -> None:

FILE: skills/deepeval/templates/test_multi_turn_e2e.py
  function TARGET_APP_ENTRYPOINT (line 21) | async def TARGET_APP_ENTRYPOINT(user_input, turns, thread_id):
  function chatbot_callback (line 27) | async def chatbot_callback(input: str, turns=None, thread_id=None):
  function APP_RESPONSE_ADAPTER (line 33) | def APP_RESPONSE_ADAPTER(response):
  function test_multi_turn (line 46) | def test_multi_turn(test_case):

FILE: skills/deepeval/templates/test_single_turn_component.py
  function TARGET_APP_ENTRYPOINT (line 19) | def TARGET_APP_ENTRYPOINT(user_input):
  function APP_RESPONSE_ADAPTER (line 25) | def APP_RESPONSE_ADAPTER(response):
  function observed_component (line 31) | def observed_component(user_input: str):
  function test_single_turn_component (line 41) | def test_single_turn_component(golden):

FILE: skills/deepeval/templates/test_single_turn_e2e.py
  function TARGET_APP_ENTRYPOINT (line 18) | def TARGET_APP_ENTRYPOINT(user_input):
  function APP_RESPONSE_ADAPTER (line 22) | def APP_RESPONSE_ADAPTER(response):
  function to_deepeval_tool_calls (line 31) | def to_deepeval_tool_calls(raw_tool_calls):
  function test_single_turn (line 43) | def test_single_turn(golden):

FILE: test_agentcore_agent.py
  function run_agent (line 57) | async def run_agent(prompt: str) -> str:
  function test_agentcore_agent (line 74) | async def test_agentcore_agent(golden: Golden):

FILE: test_pydantic_agent.py
  function run_agent (line 43) | async def run_agent(prompt: str) -> str:
  function test_pydantic_agent (line 57) | async def test_pydantic_agent(golden: Golden):

FILE: tests/test_confident/simulator/example_simulator.py
  function chatbot_callback (line 50) | async def chatbot_callback(input, turns: List[Turn]):

FILE: tests/test_confident/test_annotation.py
  class TestTraceAnnotation (line 19) | class TestTraceAnnotation:
    method test_annotate_trace_with_thumbs_rating_invalid_uuid (line 20) | def test_annotate_trace_with_thumbs_rating_invalid_uuid(self):
    method test_annotate_trace_with_thumbs_rating_invalid_rating (line 28) | def test_annotate_trace_with_thumbs_rating_invalid_rating(self):
    method test_annotate_trace_with_thumbs_rating_valid (line 37) | def test_annotate_trace_with_thumbs_rating_valid(self):
    method test_annotate_trace_with_five_star_rating_invalid_rating (line 45) | def test_annotate_trace_with_five_star_rating_invalid_rating(self):
    method test_annotate_trace_with_five_star_rating_invalid_uuid (line 54) | def test_annotate_trace_with_five_star_rating_invalid_uuid(self):
    method test_annotate_trace_with_five_star_rating_valid (line 63) | def test_annotate_trace_with_five_star_rating_valid(self):
    method test_annotate_trace_with_user_id (line 71) | def test_annotate_trace_with_user_id(self):
  class TestSpanAnnotation (line 79) | class TestSpanAnnotation:
    method test_annotate_span_valid (line 80) | def test_annotate_span_valid(self):
    method test_annotate_span_invalid_uuid (line 87) | def test_annotate_span_invalid_uuid(self):
    method test_annotate_span_with_thumbs_rating_invalid_uuid (line 95) | def test_annotate_span_with_thumbs_rating_invalid_uuid(self):
    method test_annotate_span_with_thumbs_rating_invalid_rating (line 103) | def test_annotate_span_with_thumbs_rating_invalid_rating(self):
    method test_annotate_span_with_thumbs_rating_valid (line 112) | def test_annotate_span_with_thumbs_rating_valid(self):
    method test_annotate_span_with_five_star_rating_invalid_rating (line 120) | def test_annotate_span_with_five_star_rating_invalid_rating(self):
    method test_annotate_span_with_five_star_rating_invalid_uuid (line 129) | def test_annotate_span_with_five_star_rating_invalid_uuid(self):
    method test_annotate_span_with_five_star_rating_valid (line 138) | def test_annotate_span_with_five_star_rating_valid(self):
    method test_annotate_span_with_user_id (line 146) | def test_annotate_span_with_user_id(self):
  class TestThreadAnnotation (line 154) | class TestThreadAnnotation:
    method test_annotate_thread_valid (line 155) | def test_annotate_thread_valid(self):
    method test_annotate_thread_invalid_id (line 162) | def test_annotate_thread_invalid_id(self):
    method test_annotate_thread_with_thumbs_rating_invalid_id (line 170) | def test_annotate_thread_with_thumbs_rating_invalid_id(self):
    method test_annotate_thread_with_thumbs_rating_invalid_rating (line 179) | def test_annotate_thread_with_thumbs_rating_invalid_rating(self):
    method test_annotate_thread_with_thumbs_rating_valid (line 188) | def test_annotate_thread_with_thumbs_rating_valid(self):
    method test_annotate_thread_with_five_star_rating_invalid_rating (line 196) | def test_annotate_thread_with_five_star_rating_invalid_rating(self):
    method test_annotate_thread_with_five_star_rating_invalid_id (line 205) | def test_annotate_thread_with_five_star_rating_invalid_id(self):
    method test_annotate_thread_with_five_star_rating_valid (line 214) | def test_annotate_thread_with_five_star_rating_valid(self):
    method test_annotate_thread_with_user_id (line 222) | def test_annotate_thread_with_user_id(self):

FILE: tests/test_confident/test_compare.py
  function test_compare (line 14) | def test_compare():
  function test_compare_with_hyperparameters (line 65) | def test_compare_with_hyperparameters():

FILE: tests/test_confident/test_conversational_g_eval_upload.py
  function _fetch_all_metrics (line 11) | def _fetch_all_metrics():
  class TestConversationalGEval (line 20) | class TestConversationalGEval:
    method test_conversational_geval_upload_and_fetch (line 22) | def test_conversational_geval_upload_and_fetch(self):

FILE: tests/test_confident/test_dataset.py
  function create_tool_calls_from_data (line 9) | def create_tool_calls_from_data(tools_data):
  function load_goldens_data (line 24) | def load_goldens_data(path: str):
  function deep_equal_unordered (line 33) | def deep_equal_unordered(a, b):
  function freeze_for_comparison (line 72) | def freeze_for_comparison(obj):
  class TestSingleTurnDataset (line 91) | class TestSingleTurnDataset:
    method create_golden_from_data (line 96) | def create_golden_from_data(self, data):
    method test_dataset_push_pull (line 115) | def test_dataset_push_pull(self):
  class TestMultiTurnDataset (line 134) | class TestMultiTurnDataset:
    method create_golden_from_data (line 139) | def create_golden_from_data(self, data):
    method test_dataset_push_pull (line 152) | def test_dataset_push_pull(self):

FILE: tests/test_confident/test_evaluate.py
  function test_single_turn_evaluate (line 5) | def test_single_turn_evaluate():
  function test_multi_turn_evaluate (line 16) | def test_multi_turn_evaluate():

FILE: tests/test_confident/test_g_eval_upload.py
  function _fetch_all_metrics (line 11) | def _fetch_all_metrics():
  class TestGEval (line 20) | class TestGEval:
    method test_geval_upload_and_fetch (line 22) | def test_geval_upload_and_fetch(self):

FILE: tests/test_confident/test_prompt.py
  class NestedObject (line 25) | class NestedObject(BaseModel):
  class SimpleSchema (line 30) | class SimpleSchema(BaseModel):
  class ComplexOutputSchema (line 35) | class ComplexOutputSchema(BaseModel):
  class DeeplyNestedObject (line 43) | class DeeplyNestedObject(BaseModel):
  class MiddleNestedObject (line 47) | class MiddleNestedObject(BaseModel):
  class VeryComplexSchema (line 52) | class VeryComplexSchema(BaseModel):
  class ToolInputSchema (line 61) | class ToolInputSchema(BaseModel):
  class UpdatedToolInputSchema (line 67) | class UpdatedToolInputSchema(BaseModel):
  class ListOfStringsSchema (line 77) | class ListOfStringsSchema(BaseModel):
  class ListOfIntsSchema (line 81) | class ListOfIntsSchema(BaseModel):
  class ListOfFloatsSchema (line 85) | class ListOfFloatsSchema(BaseModel):
  class Source (line 89) | class Source(BaseModel):
  class ListOfObjectsSchema (line 94) | class ListOfObjectsSchema(BaseModel):
  class MixedSchemaWithLists (line 98) | class MixedSchemaWithLists(BaseModel):
  class InnerItem (line 105) | class InnerItem(BaseModel):
  class NestedObjectWithList (line 110) | class NestedObjectWithList(BaseModel):
  class SchemaWithNestedObjectContainingList (line 115) | class SchemaWithNestedObjectContainingList(BaseModel):
  class TestPromptText (line 120) | class TestPromptText:
    method test_push (line 128) | def test_push(self):
    method test_push_with_interpolation_type (line 147) | def test_push_with_interpolation_type(self):
    method test_pull_by_hash_latest (line 167) | def test_pull_by_hash_latest(self):
    method test_pull_by_hash_specific (line 181) | def test_pull_by_hash_specific(self):
    method test_pull_by_label (line 197) | def test_pull_by_label(self):
    method test_get_versions (line 211) | def test_get_versions(self):
    method test_get_commits (line 218) | def test_get_commits(self):
    method test_version_vs_label_vs_hash_pull (line 225) | def test_version_vs_label_vs_hash_pull(self):
    method test_cache_functionality (line 257) | def test_cache_functionality(self):
    method test_version_polling (line 281) | def test_version_polling(self):
    method test_label_polling (line 294) | def test_label_polling(self):
    method test_push_with_simple_output_schema (line 308) | def test_push_with_simple_output_schema(self):
    method test_push_with_nested_output_schema (line 335) | def test_push_with_nested_output_schema(self):
    method test_push_with_deeply_nested_output_schema (line 363) | def test_push_with_deeply_nested_output_schema(self):
    method test_push_with_list_of_strings_schema (line 402) | def test_push_with_list_of_strings_schema(self):
    method test_push_with_list_of_ints_schema (line 421) | def test_push_with_list_of_ints_schema(self):
    method test_push_with_list_of_floats_schema (line 440) | def test_push_with_list_of_floats_schema(self):
    method test_push_with_list_of_objects_schema (line 459) | def test_push_with_list_of_objects_schema(self):
    method test_push_with_mixed_schema_with_lists (line 478) | def test_push_with_mixed_schema_with_lists(self):
    method test_push_with_nested_object_containing_list (line 500) | def test_push_with_nested_object_containing_list(self):
    method test_push_single_tool (line 522) | def test_push_single_tool(self):
    method test_push_multiple_tools (line 564) | def test_push_multiple_tools(self):
    method test_push_tool_with_same_name_different_definition (line 606) | def test_push_tool_with_same_name_different_definition(self):
    method test_push_output_schema_and_tools (line 644) | def test_push_output_schema_and_tools(self):
    method test_pull_preserves_tool_details (line 679) | def test_pull_preserves_tool_details(self):
    method test_cache_preserves_output_schema_and_tools (line 721) | def test_cache_preserves_output_schema_and_tools(self):
    method test_branch_push (line 763) | def test_branch_push(self):
    method test_create_branch (line 783) | def test_create_branch(self):
    method test_update_branch (line 796) | def test_update_branch(self):
    method test_delete_branch (line 818) | def test_delete_branch(self):
  class TestPromptList (line 839) | class TestPromptList:
    method test_push (line 847) | def test_push(self):
    method test_push_with_interpolation_type (line 866) | def test_push_with_interpolation_type(self):
    method test_pull_by_hash_latest (line 890) | def test_pull_by_hash_latest(self):
    method test_pull_by_hash_specific (line 908) | def test_pull_by_hash_specific(self):
    method test_pull_by_label (line 926) | def test_pull_by_label(self):
    method test_get_versions (line 940) | def test_get_versions(self):
    method test_get_commits (line 947) | def test_get_commits(self):
    method test_version_vs_label_vs_hash_pull (line 954) | def test_version_vs_label_vs_hash_pull(self):
    method test_cache_functionality (line 986) | def test_cache_functionality(self):
    method test_version_polling (line 1020) | def test_version_polling(self):
    method test_label_polling (line 1033) | def test_label_polling(self):
    method test_push_with_simple_output_schema (line 1047) | def test_push_with_simple_output_schema(self):
    method test_push_with_nested_output_schema (line 1075) | def test_push_with_nested_output_schema(self):
    method test_push_with_deeply_nested_output_schema (line 1106) | def test_push_with_deeply_nested_output_schema(self):
    method test_push_single_tool (line 1150) | def test_push_single_tool(self):
    method test_push_multiple_tools (line 1195) | def test_push_multiple_tools(self):
    method test_push_tool_with_same_name_different_definition (line 1240) | def test_push_tool_with_same_name_different_definition(self):
    method test_push_output_schema_and_tools (line 1281) | def test_push_output_schema_and_tools(self):
    method test_pull_preserves_tool_details (line 1319) | def test_pull_preserves_tool_details(self):
    method test_cache_preserves_output_schema_and_tools (line 1364) | def test_cache_preserves_output_schema_and_tools(self):
    method test_branch_push (line 1407) | def test_branch_push(self):
    method test_create_branch (line 1432) | def test_create_branch(self):
    method test_update_branch (line 1445) | def test_update_branch(self):
    method test_delete_branch (line 1467) | def test_delete_branch(self):

FILE: tests/test_confident/test_region_autodetect_request_routing.py
  class _FakeResponse (line 4) | class _FakeResponse:
    method __init__ (line 5) | def __init__(self, status_code: int, payload: dict):
    method json (line 10) | def json(self):
  function test_request_succeeds_by_auto_routing_eu_key_when_region_unset (line 14) | def test_request_succeeds_by_auto_routing_eu_key_when_region_unset(monke...
  function test_request_succeeds_by_auto_routing_au_key_when_region_unset (line 76) | def test_request_succeeds_by_auto_routing_au_key_when_region_unset(monke...

FILE: tests/test_core/conftest.py
  function _ensure_hidden_store_dir (line 32) | def _ensure_hidden_store_dir(tmp_path: Path):
  function hidden_store_dir (line 41) | def hidden_store_dir(tmp_path: Path) -> Path:
  function _telemetry_opt_out (line 49) | def _telemetry_opt_out(monkeypatch):
  function _isolate_cwd (line 56) | def _isolate_cwd(tmp_path: Path, monkeypatch):
  function env_path (line 63) | def env_path(monkeypatch, tmp_path: Path) -> Path:
  function env_dir (line 69) | def env_dir(monkeypatch, tmp_path: Path) -> Path:
  function no_sleep (line 75) | def no_sleep(monkeypatch):
  function settings (line 80) | def settings():
  function _session_env_baseline (line 86) | def _session_env_baseline():
  function _restore_env_to (line 91) | def _restore_env_to(baseline: dict[str, str]) -> None:
  function _env_sandbox (line 103) | def _env_sandbox(_session_env_baseline, request, monkeypatch):
  function _core_mode_no_confident (line 142) | def _core_mode_no_confident(
  function enable_dotenv (line 168) | def enable_dotenv(monkeypatch):
  function unpatch_openai_after (line 175) | def unpatch_openai_after():
  function _reset_tracing_state (line 183) | def _reset_tracing_state():
  function completed_traces (line 200) | def completed_traces(monkeypatch):

FILE: tests/test_core/helpers.py
  function ts_iso8601_utc (line 10) | def ts_iso8601_utc(ts: float) -> str:
  function make_trace_api (line 18) | def make_trace_api(
  function teardown_settings_singleton (line 46) | def teardown_settings_singleton():
  function reset_settings_env (line 52) | def reset_settings_env(monkeypatch, *, skip_keys: set[str] = set()):
  function _make_fake_portalocker (line 67) | def _make_fake_portalocker():

FILE: tests/test_core/stubs.py
  class ApiTestCaseLike (line 18) | class ApiTestCaseLike(Protocol):
    method update_metric_data (line 28) | def update_metric_data(self, *args, **kwargs) -> None: ...
    method update_status (line 29) | def update_status(self, *args, **kwargs) -> None: ...
    method update_run_duration (line 30) | def update_run_duration(self, *args, **kwargs) -> None: ...
  function make_trace_api_like (line 33) | def make_trace_api_like(status):
  function make_span_api_like (line 53) | def make_span_api_like():
  class StubProvider (line 62) | class StubProvider:
    method __init__ (line 63) | def __init__(self, value: str) -> None:
  class StubModelSettings (line 67) | class StubModelSettings:
    method __init__ (line 68) | def __init__(self, provider=None, name: str | None = None) -> None:
  class StubPrompt (line 73) | class StubPrompt:
    method __init__ (line 74) | def __init__(
  class DummyModel (line 85) | class DummyModel:
    method get_model_name (line 86) | def get_model_name(self):
  class AlwaysJsonModel (line 90) | class AlwaysJsonModel:
    method __init__ (line 103) | def __init__(self, extractor: Callable[[str], str]):
    method generate (line 109) | def generate(self, prompt: str) -> str:
    method a_generate (line 112) | async def a_generate(self, prompt: str) -> str:
    method get_model_name (line 115) | def get_model_name(self) -> str:
    method balanced_json_after_anchor (line 119) | def balanced_json_after_anchor(anchor_text: str) -> Callable[[str], str]:
  class _RecordingClient (line 153) | class _RecordingClient:
    method __init__ (line 161) | def __init__(self, *args, **kwargs):
  function make_fake_ollama_module (line 166) | def make_fake_ollama_module(client_cls=_RecordingClient):
  function _make_fake_genai_module (line 185) | def _make_fake_genai_module():
  class _DummyMetric (line 217) | class _DummyMetric(BaseMetric):
    method __init__ (line 220) | def __init__(self, name="dummy", should_skip=False):
    method measure (line 228) | def measure(self, test_case, *_args, **_kwargs):
    method is_successful (line 234) | def is_successful(self) -> bool:
  class _DummyTaskCompletionMetric (line 238) | class _DummyTaskCompletionMetric(TaskCompletionMetric):
    method __init__ (line 241) | def __init__(self, name="tc"):
    method measure (line 248) | def measure(self, test_case, *_args, **_kwargs):
    method is_successful (line 251) | def is_successful(self) -> bool:
  class _SleepyMetric (line 255) | class _SleepyMetric(BaseMetric):
    method __init__ (line 266) | def __init__(
    method measure (line 287) | def measure(self, test_case, *_args, **_kwargs):
    method a_measure (line 295) | async def a_measure(self, test_case, *_args, **_kwargs):
    method is_successful (line 303) | def is_successful(self) -> bool:
  class _PerAttemptTimeoutMetric (line 307) | class _PerAttemptTimeoutMetric(BaseMetric):
    method __init__ (line 318) | def __init__(self, *, sleep_s: float = 10.0):
    method measure (line 323) | def measure(self, test_case, **kwargs) -> float:
    method a_measure (line 335) | async def a_measure(self, test_case, **kwargs) -> float:
    method is_successful (line 347) | def is_successful(self) -> bool:
  class _FakeSpan (line 356) | class _FakeSpan:
    method __init__ (line 357) | def __init__(self, *, input=None, output=None, metrics=None, children=...
  class _FakeTrace (line 376) | class _FakeTrace:
    method __init__ (line 377) | def __init__(
  class DummyProgress (line 399) | class DummyProgress:
    method __init__ (line 405) | def __init__(self, tasks=None):
    method update (line 409) | def update(self, task_id, **kwargs):
    method advance (line 412) | def advance(self, task_id, amount):
    method remove_task (line 415) | def remove_task(self, task_id):
  class DummyEvolutionConfig (line 428) | class DummyEvolutionConfig:
  function stub_synthesizer_progress_context (line 434) | def stub_synthesizer_progress_context(**kwargs):
  class _DummyRewriter (line 446) | class _DummyRewriter:
    method rewrite (line 452) | def rewrite(self, **kwargs):
    method a_rewrite (line 456) | async def a_rewrite(self, **kwargs):
  class SuffixRewriter (line 460) | class SuffixRewriter:
    method __init__ (line 463) | def __init__(self, suffix: str = " CHILD") -> None:
    method rewrite (line 468) | def rewrite(self, *, old_prompt, feedback_diagnosis=None, **kwargs):
    method a_rewrite (line 474) | async def a_rewrite(self, *, old_prompt, feedback_diagnosis=None, **kw...
  class AddBetterRewriter (line 481) | class AddBetterRewriter:
    method rewrite (line 482) | def rewrite(
  class DummyRunner (line 496) | class DummyRunner:
    method __init__ (line 501) | def __init__(self):
    method execute (line 505) | def execute(self, *, prompt, goldens):
    method a_execute (line 508) | async def a_execute(self, *, prompt, goldens):
  class DummyRunnerForOptimize (line 512) | class DummyRunnerForOptimize:
    method __init__ (line 517) | def __init__(self):
    method execute (line 522) | def execute(self, *, prompt, goldens):
    method a_execute (line 551) | async def a_execute(self, *, prompt, goldens):
  class SyncDummyRunner (line 555) | class SyncDummyRunner:
    method __init__ (line 560) | def __init__(self):
    method execute (line 564) | def execute(self, *, prompt, goldens):
    method a_execute (line 574) | async def a_execute(self, *, prompt, goldens):
  class AsyncDummyRunner (line 585) | class AsyncDummyRunner:
    method __init__ (line 590) | def __init__(self):
    method execute (line 594) | def execute(self, *, prompt, goldens):
    method a_execute (line 600) | async def a_execute(self, *, prompt, goldens):
  class StubScoringAdapter (line 611) | class StubScoringAdapter:
    method __init__ (line 621) | def __init__(self) -> None:
    method _get_prompt_text (line 629) | def _get_prompt_text(self, prompt_configuration):
    method score_pareto (line 636) | def score_pareto(self, prompt_configuration, d_pareto):
    method a_score_pareto (line 641) | async def a_score_pareto(self, prompt_configuration, d_pareto):
    method get_minibatch_feedback (line 645) | def get_minibatch_feedback(
    method a_get_minibatch_feedback (line 653) | async def a_get_minibatch_feedback(
    method score_minibatch (line 661) | def score_minibatch(self, prompt_configuration, minibatch):
    method a_score_minibatch (line 666) | async def a_score_minibatch(self, prompt_configuration, minibatch):
  class RecordingFile (line 676) | class RecordingFile(io.StringIO):
    method __init__ (line 682) | def __init__(self):
    method flush (line 689) | def flush(self):
    method fileno (line 693) | def fileno(self):
    method close (line 696) | def close(self):
  class RecordingPortalockerLock (line 701) | class RecordingPortalockerLock:
    method __init__ (line 711) | def __init__(self, *args, **kwargs):
    method __enter__ (line 715) | def __enter__(self):
    method __exit__ (line 718) | def __exit__(self, exc_type, exc, tb):

FILE: tests/test_core/test_cli/test_cli.py
  function _normalize_cli_output (line 22) | def _normalize_cli_output(text: str) -> str:
  function _read_hidden_store_json (line 29) | def _read_hidden_store_json(hidden_store_dir: Path) -> Dict[str, object]:
  function _assert_no_dupes (line 44) | def _assert_no_dupes(env_path: Path, keys: Iterable[str]) -> None:
  function _assert_use_flags_exclusive_env (line 50) | def _assert_use_flags_exclusive_env(
  function _assert_use_flags_exclusive_store (line 71) | def _assert_use_flags_exclusive_store(
  function _unquote_dotenv_value (line 88) | def _unquote_dotenv_value(value: str) -> str:
  function _read_dotenv (line 99) | def _read_dotenv(path: Path) -> Dict[str, str]:
  function _count_key_occurrences (line 115) | def _count_key_occurrences(path: Path, key: str) -> int:
  function runner (line 128) | def runner() -> CliRunner:
  function _invoke_ok (line 132) | def _invoke_ok(runner: CliRunner, argv: list[str]) -> str:
  function test_settings_set_coerces_and_persists_dotenv (line 138) | def test_settings_set_coerces_and_persists_dotenv(
  function test_settings_unset_removes_key_from_dotenv (line 169) | def test_settings_unset_removes_key_from_dotenv(
  function test_settings_list_filters_and_masks_secrets (line 201) | def test_settings_list_filters_and_masks_secrets(
  function test_set_debug_quiet_suppresses_output_and_updates_dotenv (line 234) | def test_set_debug_quiet_suppresses_output_and_updates_dotenv(
  class _ProviderCase (line 258) | class _ProviderCase:
  function test_set_unset_llm_provider_roundtrip (line 468) | def test_set_unset_llm_provider_roundtrip(
  function test_set_unset_embedding_provider_roundtrip (line 588) | def test_set_unset_embedding_provider_roundtrip(
  function test_set_unset_gemini_service_account_file_roundtrip_dotenv_only (line 640) | def test_set_unset_gemini_service_account_file_roundtrip_dotenv_only(
  function test_set_gemini_prompt_api_key_persists_to_dotenv_not_json (line 706) | def test_set_gemini_prompt_api_key_persists_to_dotenv_not_json(
  function test_set_gemini_service_account_file_validation_errors (line 742) | def test_set_gemini_service_account_file_validation_errors(
  function test_settings_set_writes_to_dotenv_even_if_value_already_in_json_store (line 769) | def test_settings_set_writes_to_dotenv_even_if_value_already_in_json_store(
  class _FakeContextConstructionConfig (line 804) | class _FakeContextConstructionConfig:
  class _FakeSynthesizer (line 814) | class _FakeSynthesizer:
    method __init__ (line 817) | def __init__(self, **kwargs):
    method _record (line 822) | def _record(self, name: str, **kwargs):
    method generate_goldens_from_docs (line 826) | def generate_goldens_from_docs(self, **kwargs):
    method generate_conversational_goldens_from_docs (line 829) | def generate_conversational_goldens_from_docs(self, **kwargs):
    method generate_goldens_from_contexts (line 834) | def generate_goldens_from_contexts(self, **kwargs):
    method generate_conversational_goldens_from_contexts (line 837) | def generate_conversational_goldens_from_contexts(self, **kwargs):
    method generate_goldens_from_scratch (line 842) | def generate_goldens_from_scratch(self, **kwargs):
    method generate_conversational_goldens_from_scratch (line 845) | def generate_conversational_goldens_from_scratch(self, **kwargs):
    method generate_goldens_from_goldens (line 850) | def generate_goldens_from_goldens(self, **kwargs):
    method generate_conversational_goldens_from_goldens (line 853) | def generate_conversational_goldens_from_goldens(self, **kwargs):
    method save_as (line 858) | def save_as(self, **kwargs):
  function fake_generate_cli (line 866) | def fake_generate_cli(monkeypatch):
  function test_generate_cli_dispatches_by_method_and_variation (line 982) | def test_generate_cli_dispatches_by_method_and_variation(
  function test_generate_cli_requires_method_specific_input (line 1042) | def test_generate_cli_requires_method_specific_input(

FILE: tests/test_core/test_config/test_deprecated_computed_aliases.py
  function _clear_deprecated_and_overrides (line 11) | def _clear_deprecated_and_overrides(monkeypatch):
  function test_deprecated_env_applies_to_override_when_override_missing (line 37) | def test_deprecated_env_applies_to_override_when_override_missing(
  function test_deprecated_env_ignored_when_override_already_set (line 63) | def test_deprecated_env_ignored_when_override_already_set(monkeypatch, c...
  function test_deprecated_empty_string_is_ignored (line 91) | def test_deprecated_empty_string_is_ignored(monkeypatch, old_key, overri...
  function test_deprecated_invalid_value_warns_and_skips (line 103) | def test_deprecated_invalid_value_warns_and_skips(monkeypatch, caplog):

FILE: tests/test_core/test_config/test_settings.py
  function test_autoload_dotenv_precedence (line 15) | def test_autoload_dotenv_precedence(monkeypatch, env_dir: Path):
  function test_autoload_respects_disable_flag (line 28) | def test_autoload_respects_disable_flag(monkeypatch, env_dir: Path):
  function test_autoload_does_not_override_process_env (line 36) | def test_autoload_does_not_override_process_env(monkeypatch, env_dir: Pa...
  function test_autoload_respects_env_dir_path (line 44) | def test_autoload_respects_env_dir_path(monkeypatch, tmp_path: Path):
  function test_defaults (line 53) | def test_defaults():
  function test_env_mutation_after_init_triggers_auto_refresh (line 61) | def test_env_mutation_after_init_triggers_auto_refresh(monkeypatch):
  function test_invalid_trace_sample_rate_raises (line 73) | def test_invalid_trace_sample_rate_raises(monkeypatch):
  function test_edit_runtime_only_persist_false_updates_env_not_files (line 80) | def test_edit_runtime_only_persist_false_updates_env_not_files(
  function test_edit_respects_default_save_writes_dotenv (line 113) | def test_edit_respects_default_save_writes_dotenv(monkeypatch, env_dir: ...
  function test_edit_explicit_save_overrides_default (line 128) | def test_edit_explicit_save_overrides_default(monkeypatch, env_dir: Path):
  function test_switch_model_provider_flips_only_target (line 144) | def test_switch_model_provider_flips_only_target():
  function test_edit_unset_removes_from_env_and_dotenv (line 157) | def test_edit_unset_removes_from_env_and_dotenv(monkeypatch, env_dir: Pa...
  function test_secret_not_persisted_to_json (line 174) | def test_secret_not_persisted_to_json(monkeypatch):
  function test_env_dir_path_expanduser (line 202) | def test_env_dir_path_expanduser(monkeypatch, tmp_path: Path):
  function test_results_folder_expandvars (line 209) | def test_results_folder_expandvars(monkeypatch, tmp_path: Path):
  function test_env_dir_path_empty_string_is_none (line 217) | def test_env_dir_path_empty_string_is_none(monkeypatch):
  function test_filesystem_aliases_normalized (line 224) | def test_filesystem_aliases_normalized(monkeypatch, val):
  function test_filesystem_invalid_raises (line 230) | def test_filesystem_invalid_raises(monkeypatch):
  function test_boolean_coercion_opt_in_with_autoload_dotenv (line 250) | def test_boolean_coercion_opt_in_with_autoload_dotenv(
  function test_boolean_coercion_opt_out_with_dotenv (line 274) | def test_boolean_coercion_opt_out_with_dotenv(monkeypatch, opt_out, expe...
  function test_boolean_reset_settings_after_environ_update (line 281) | def test_boolean_reset_settings_after_environ_update(monkeypatch):
  function test_sample_rate_empty_string_is_none (line 290) | def test_sample_rate_empty_string_is_none(monkeypatch):
  function test_sample_rate_valid_boundaries (line 297) | def test_sample_rate_valid_boundaries(monkeypatch, val):
  function test_sample_rate_invalid_raises (line 304) | def test_sample_rate_invalid_raises(monkeypatch, val):
  function test_switch_model_provider_flips_use_flags_within_family_only (line 310) | def test_switch_model_provider_flips_use_flags_within_family_only(settin...
  function _clear_telemetry_env (line 360) | def _clear_telemetry_env(monkeypatch):
  function test_alias_only_enabled_yes_sets_opt_out_false (line 365) | def test_alias_only_enabled_yes_sets_opt_out_false(monkeypatch):
  function test_alias_only_enabled_no_sets_opt_out_true (line 374) | def test_alias_only_enabled_no_sets_opt_out_true(monkeypatch):
  function test_alias_both_present_opt_out_wins (line 383) | def test_alias_both_present_opt_out_wins(monkeypatch):
  function test_alias_both_present_enabled_false_forces_opt_out (line 394) | def test_alias_both_present_enabled_false_forces_opt_out(monkeypatch):
  function test_neither_set_defaults_on (line 405) | def test_neither_set_defaults_on(monkeypatch):
  function test_legacy_enabled_alias_not_persisted_to_dotenv (line 420) | def test_legacy_enabled_alias_not_persisted_to_dotenv(
  function test_legacy_keyfile_populates_openai_api_key_when_env_missing (line 465) | def test_legacy_keyfile_populates_openai_api_key_when_env_missing(
  function test_env_openai_api_key_takes_precedence_over_legacy_keyfile (line 493) | def test_env_openai_api_key_takes_precedence_over_legacy_keyfile(

FILE: tests/test_core/test_core.py
  function test_confident_boundary_off_in_core (line 7) | def test_confident_boundary_off_in_core():
  function test_confident_api_key_takes_precedence (line 12) | def test_confident_api_key_takes_precedence(monkeypatch):
  function test_confident_api_key_field_is_required (line 22) | def test_confident_api_key_field_is_required():

FILE: tests/test_core/test_datasets/test_dataset.py
  class TestSaveAndLoad (line 16) | class TestSaveAndLoad:
    method test_dataset_save_load_goldens (line 17) | def test_dataset_save_load_goldens(self):
    method test_dataset_save_load_conversational_goldens (line 38) | def test_dataset_save_load_conversational_goldens(self):
    method test_save_as_creates_valid_json_and_csv (line 63) | def test_save_as_creates_valid_json_and_csv(self):
    method test_save_as_conversational_goldens_creates_valid_json_and_csv (line 101) | def test_save_as_conversational_goldens_creates_valid_json_and_csv(self):
    method test_save_as_includes_extra_single_turn_fields (line 150) | def test_save_as_includes_extra_single_turn_fields(self):
    method test_save_as_includes_turn_fields_in_multi_turn_json_and_jsonl (line 242) | def test_save_as_includes_turn_fields_in_multi_turn_json_and_jsonl(self):
    method test_add_goldens_from_jsonl_file_loads_single_turn_goldens (line 289) | def test_add_goldens_from_jsonl_file_loads_single_turn_goldens(self):
    method test_add_goldens_from_jsonl_file_loads_conversational_goldens (line 323) | def test_add_goldens_from_jsonl_file_loads_conversational_goldens(self):
    method test_add_goldens_from_json_file_rejects_mixed_variations (line 362) | def test_add_goldens_from_json_file_rejects_mixed_variations(self):
    method test_add_goldens_from_csv_file_rejects_mixed_variations (line 382) | def test_add_goldens_from_csv_file_rejects_mixed_variations(self):
    method test_add_goldens_from_jsonl_file_rejects_mixed_variations (line 399) | def test_add_goldens_from_jsonl_file_rejects_mixed_variations(self):
    method test_save_as_empty_dataset_raises_error (line 414) | def test_save_as_empty_dataset_raises_error(self):
    method test_save_as_includes_test_cases (line 421) | def test_save_as_includes_test_cases(self):
    method test_save_as_includes_convo_test_cases (line 442) | def test_save_as_includes_convo_test_cases(self):
    method test_convert_convo_goldens_to_test_cases_preserves_expected_outcome (line 467) | def test_convert_convo_goldens_to_test_cases_preserves_expected_outcome(

FILE: tests/test_core/test_drop_trace_and_span.py
  function clean_trace_state (line 8) | def clean_trace_state():
  function app_that_drops_trace (line 21) | def app_that_drops_trace():
  function app_that_does_not_drop (line 27) | def app_that_does_not_drop():
  function test_drop_trace_sets_flag (line 31) | def test_drop_trace_sets_flag(completed_traces):
  function test_trace_not_dropped_by_default (line 39) | def test_trace_not_dropped_by_default(completed_traces):
  function app_drop_then_update (line 48) | def app_drop_then_update():
  function test_drop_persists_after_update_current_trace (line 56) | def test_drop_persists_after_update_current_trace(completed_traces):
  function app_with_dropped_span (line 66) | def app_with_dropped_span():
  function test_drop_span_sets_flag (line 81) | def test_drop_span_sets_flag(completed_traces):
  function test_dropped_span_excluded_from_trace_api (line 98) | def test_dropped_span_excluded_from_trace_api(completed_traces):
  function app_drop_span_not_trace (line 121) | def app_drop_span_not_trace():
  function test_drop_span_does_not_drop_trace (line 131) | def test_drop_span_does_not_drop_trace(completed_traces):
  function app_drop_trace_not_span (line 141) | def app_drop_trace_not_span():
  function test_drop_trace_does_not_set_span_drop (line 146) | def test_drop_trace_does_not_set_span_drop(completed_traces):

FILE: tests/test_core/test_evaluation/test_async_trace_metric_isolation.py
  class BarrierIsolationMetric (line 16) | class BarrierIsolationMetric(BaseMetric):
    method __init__ (line 20) | def __init__(self, threshold: float = 1.0):
    method __name__ (line 33) | def __name__(self):
    method reset_barrier (line 37) | def reset_barrier(cls):
    method a_measure (line 41) | async def a_measure(self, test_case, *args, **kwargs):
    method measure (line 54) | def measure(self, test_case, *args, **kwargs):
    method is_successful (line 57) | def is_successful(self):
  function _make_trace (line 61) | def _make_trace(trace_uuid: str, trace_input: str, trace_output: str) ->...
  function test_async_trace_metrics_are_copied_per_trace (line 85) | async def test_async_trace_metrics_are_copied_per_trace(monkeypatch):

FILE: tests/test_core/test_evaluation/test_console_report.py
  function test_evaluation_console_report_exports (line 7) | def test_evaluation_console_report_exports(tmp_path: Path):
  function test_evaluation_console_report_aggregate_metrics (line 60) | def test_evaluation_console_report_aggregate_metrics():

FILE: tests/test_core/test_evaluation/test_end_to_end/test_configs.py
  function llm_app (line 21) | def llm_app(input: str) -> str:
  function a_llm_app (line 29) | async def a_llm_app(input: str) -> str:
  class TestEvaluate (line 35) | class TestEvaluate:
    method test_skip_on_missing_params (line 37) | def test_skip_on_missing_params(self):
    method test_error_on_missing_params (line 62) | def test_error_on_missing_params(self):
  class TestEvalsIterator (line 85) | class TestEvalsIterator:
    method test_async_evals_iterator (line 87) | def test_async_evals_iterator(self):
    method test_evals_iterator (line 106) | def test_evals_iterator(self):
    method test_skip_on_missing_params (line 126) | def test_skip_on_missing_params(self):
    method test_error_on_missing_params (line 146) | def test_error_on_missing_params(self):

FILE: tests/test_core/test_evaluation/test_end_to_end/test_skip_reset.py
  class _AlwaysPassMetric (line 14) | class _AlwaysPassMetric(BaseMetric):
    method __init__ (line 17) | def __init__(self):
    method __name__ (line 22) | def __name__(self):
    method measure (line 25) | def measure(self, test_case):
    method a_measure (line 30) | async def a_measure(self, test_case):
    method is_successful (line 33) | def is_successful(self):
  function _make_case (line 41) | def _make_case(label: str) -> LLMTestCase:
  function _reset_test_run_manager (line 46) | def _reset_test_run_manager():
  class TestSkipResetDefault (line 53) | class TestSkipResetDefault:
    method test_second_call_does_not_accumulate (line 56) | def test_second_call_does_not_accumulate(self):
    method test_returns_evaluation_result (line 72) | def test_returns_evaluation_result(self):
  class TestSkipResetTrue (line 83) | class TestSkipResetTrue:
    method test_accumulates_test_cases (line 86) | def test_accumulates_test_cases(self):
    method test_three_calls_accumulate (line 106) | def test_three_calls_accumulate(self):
    method test_skip_reset_true_skips_wrap_up (line 118) | def test_skip_reset_true_skips_wrap_up(self):
    method test_skip_reset_false_calls_wrap_up (line 131) | def test_skip_reset_false_calls_wrap_up(self):
    method test_skip_reset_true_returns_no_confident_link (line 145) | def test_skip_reset_true_returns_no_confident_link(self):
    method test_hyperparameters_not_erased_by_subsequent_none (line 156) | def test_hyperparameters_not_erased_by_subsequent_none(self):
    method test_run_duration_accumulates (line 174) | def test_run_duration_accumulates(self):
    method test_skip_reset_true_as_very_first_call (line 197) | def test_skip_reset_true_as_very_first_call(self):
  class TestAccumulatedOrdersAreUnique (line 210) | class TestAccumulatedOrdersAreUnique:
    method test_single_evaluate_preserves_original_orders (line 213) | def test_single_evaluate_preserves_original_orders(self):
    method test_sort_assigns_unique_orders_after_accumulation (line 231) | def test_sort_assigns_unique_orders_after_accumulation(self):
    method test_cli_mode_orders_unique_across_files (line 259) | def test_cli_mode_orders_unique_across_files(self, _mock):
  class TestCLIModeAutoSkipsReset (line 291) | class TestCLIModeAutoSkipsReset:
    method test_cli_mode_accumulates_without_explicit_skip_reset (line 297) | def test_cli_mode_accumulates_without_explicit_skip_reset(self, _mock):
    method test_cli_mode_does_not_call_wrap_up (line 316) | def test_cli_mode_does_not_call_wrap_up(self, _mock):
    method test_cli_mode_returns_no_confident_link (line 331) | def test_cli_mode_returns_no_confident_link(self, _mock):

FILE: tests/test_core/test_evaluation/test_execute/test_error_boundary.py
  function _bypass_no_metrics_guard (line 30) | def _bypass_no_metrics_guard(monkeypatch):
  function patched_api_layer (line 44) | def patched_api_layer(monkeypatch):
  function record_measure_calls (line 91) | def record_measure_calls(monkeypatch):
  function child_raises (line 116) | async def child_raises():
  function parent_catches (line 121) | async def parent_catches():
  function parent_uncaught (line 129) | async def parent_uncaught():
  function test_no_llmtestcase_skips_trace_and_span_metrics (line 139) | async def test_no_llmtestcase_skips_trace_and_span_metrics(
  function test_trace_error_boundary_no_actual_output_still_evaluates_span_metrics (line 187) | async def test_trace_error_boundary_no_actual_output_still_evaluates_spa...
  function test_task_completion_path_sets_trace_case_and_evaluates_metrics (line 229) | async def test_task_completion_path_sets_trace_case_and_evaluates_metrics(
  function test_task_exception_logs_error_when_debug_enabled (line 278) | def test_task_exception_logs_error_when_debug_enabled(
  function test_task_error_after_observe_marks_existing_trace (line 353) | def test_task_error_after_observe_marks_existing_trace(monkeypatch):
  function test_task_cancel_after_observe_marks_existing_trace (line 436) | def test_task_cancel_after_observe_marks_existing_trace(monkeypatch):
  function test_caught_child_error_trace_success (line 518) | async def test_caught_child_error_trace_success(completed_traces):
  function test_uncaught_error_trace_error (line 532) | async def test_uncaught_error_trace_error(completed_traces):
  function test_cancelled_task_marks_trace_error (line 542) | async def test_cancelled_task_marks_trace_error(completed_traces):
  function test_task_cancelled_without_observe_logs_and_marks_nothing (line 566) | def test_task_cancelled_without_observe_logs_and_marks_nothing(
  function test_fallback_marks_open_root_when_multiple_roots (line 631) | def test_fallback_marks_open_root_when_multiple_roots(monkeypatch):
  function test_error_after_observe_does_not_overwrite_root_end_time (line 732) | def test_error_after_observe_does_not_overwrite_root_end_time(monkeypatch):
  function test_span_errored_skips_span_metrics (line 787) | async def test_span_errored_skips_span_metrics(
  function test_trace_errored_skips_trace_metrics (line 826) | async def test_trace_errored_skips_trace_metrics(

FILE: tests/test_core/test_evaluation/test_execute/test_execute_conversational_test_case.py
  function test_conversational_async_persists_metric_on_cancel (line 28) | async def test_conversational_async_persists_metric_on_cancel(
  function test_conversational_sync_persists_metric_on_timeout_ignore_errors_true (line 103) | def test_conversational_sync_persists_metric_on_timeout_ignore_errors_true(
  function test_conversational_sync_persists_metric_on_timeout_ignore_errors_false (line 177) | def test_conversational_sync_persists_metric_on_timeout_ignore_errors_fa...

FILE: tests/test_core/test_evaluation/test_execute/test_execute_llm_test_case.py
  function test_llm_async_persists_metric_on_cancel (line 28) | async def test_llm_async_persists_metric_on_cancel(monkeypatch, ignore_e...
  function test_llm_sync_persists_metric_on_timeout_ignore_errors_true (line 92) | def test_llm_sync_persists_metric_on_timeout_ignore_errors_true(
  function test_llm_sync_persists_metric_on_timeout_ignore_errors_false (line 156) | def test_llm_sync_persists_metric_on_timeout_ignore_errors_false(

FILE: tests/test_core/test_evaluation/test_execute/test_execute_mllm_test_case.py
  function test_mlllm_async_persists_metric_on_cancel (line 27) | async def test_mlllm_async_persists_metric_on_cancel(
  function test_mllm_sync_persists_metric_on_timeout_ignore_errors_true (line 92) | def test_mllm_sync_persists_metric_on_timeout_ignore_errors_true(
  function test_mllm_sync_persists_metric_on_timeout_ignore_errors_false (line 156) | def test_mllm_sync_persists_metric_on_timeout_ignore_errors_false(

FILE: tests/test_core/test_evaluation/test_execute/test_execute_timeouts.py
  function test_per_task_timeout_async_path (line 18) | async def test_per_task_timeout_async_path(settings):
  function test_per_task_timeout_sync_path (line 49) | def test_per_task_timeout_sync_path(settings):
  function test_per_attempt_timeout_async_path (line 78) | async def test_per_attempt_timeout_async_path(settings):
  function test_per_attempt_timeout_sync_path (line 117) | def test_per_attempt_timeout_sync_path(settings):
  function test_disable_timeouts_disables_per_task_async (line 156) | async def test_disable_timeouts_disables_per_task_async(settings):
  function test_disable_timeouts_disables_per_task_sync (line 188) | def test_disable_timeouts_disables_per_task_sync(settings):
  function test_disable_timeouts_disables_per_attempt_async (line 213) | async def test_disable_timeouts_disables_per_attempt_async(settings):
  function test_disable_timeouts_disables_per_attempt_sync (line 243) | def test_disable_timeouts_disables_per_attempt_sync(settings):

FILE: tests/test_core/test_evaluation/test_local_store.py
  function _make_test_run (line 26) | def _make_test_run(
  class TestResolveTargetDir (line 41) | class TestResolveTargetDir:
    method test_results_folder_only_flat (line 42) | def test_results_folder_only_flat(self, tmp_path: Path):
    method test_results_folder_with_subfolder_nests (line 45) | def test_results_folder_with_subfolder_nests(self, tmp_path: Path):
    method test_empty_subfolder_is_flat (line 49) | def test_empty_subfolder_is_flat(self, tmp_path: Path):
    method test_env_var_fallback (line 58) | def test_env_var_fallback(self, tmp_path: Path, monkeypatch):
    method test_display_config_takes_precedence_over_env (line 62) | def test_display_config_takes_precedence_over_env(
    method test_nothing_set_is_none (line 70) | def test_nothing_set_is_none(self, monkeypatch):
  class TestResolveTestRunPath (line 76) | class TestResolveTestRunPath:
    method test_filename_format (line 77) | def test_filename_format(self, tmp_path: Path):
    method test_same_second_collision_appends_suffix (line 85) | def test_same_second_collision_appends_suffix(self, tmp_path: Path):
  class TestWriteTestRun (line 99) | class TestWriteTestRun:
    method test_round_trips_hyperparameters_and_prompts (line 100) | def test_round_trips_hyperparameters_and_prompts(self, tmp_path: Path):
    method test_creates_missing_directory (line 118) | def test_creates_missing_directory(self, tmp_path: Path):
    method test_never_overwrites_on_same_second_collision (line 124) | def test_never_overwrites_on_same_second_collision(self, tmp_path: Path):
    method test_concurrent_writes_are_lock_safe (line 140) | def test_concurrent_writes_are_lock_safe(self, tmp_path: Path):
  class TestDisplayConfigFields (line 166) | class TestDisplayConfigFields:
    method test_new_fields_default_to_none (line 167) | def test_new_fields_default_to_none(self):
    method test_fields_accept_strings (line 172) | def test_fields_accept_strings(self):
  class TestTestRunManagerLocalStoreIntegration (line 181) | class TestTestRunManagerLocalStoreIntegration:
    method test_writes_via_configure_local_store (line 184) | def test_writes_via_configure_local_store(self, tmp_path: Path):
    method test_subfolder_nests (line 194) | def test_subfolder_nests(self, tmp_path: Path):
    method test_env_var_fallback (line 207) | def test_env_var_fallback(self, tmp_path: Path, monkeypatch):
    method test_no_config_is_noop (line 217) | def test_no_config_is_noop(self, tmp_path: Path, monkeypatch):
  class TestForLoopFlow (line 229) | class TestForLoopFlow:
    method test_three_iterations_produce_three_files (line 238) | def test_three_iterations_produce_three_files(self, tmp_path: Path):
    method test_chronological_sort_matches_write_order (line 265) | def test_chronological_sort_matches_write_order(self, tmp_path: Path):

FILE: tests/test_core/test_evaluation/test_printing.py
  function T (line 10) | def T(order, role, content, tools=None):
  function test_print_test_result_conversational_turns_are_sorted_and_prefixed (line 19) | def test_print_test_result_conversational_turns_are_sorted_and_prefixed(...
  function test_write_test_result_to_file_conversational (line 53) | def test_write_test_result_to_file_conversational(tmp_path: Path):

FILE: tests/test_core/test_evaluation/test_results_extraction.py
  function test_extract_span_result_success_with_enum_status (line 6) | def test_extract_span_result_success_with_enum_status():

FILE: tests/test_core/test_evaluation/test_trace_results_extraction.py
  function test_trace_metric_does_not_produce_additional_test_result (line 16) | async def test_trace_metric_does_not_produce_additional_test_result(

FILE: tests/test_core/test_evaluation/test_trace_scope_assert_test.py
  class CapturingMetric (line 16) | class CapturingMetric(BaseMetric):
    method __init__ (line 17) | def __init__(self, expected_input: str, expected_output: str):
    method __name__ (line 32) | def __name__(self):
    method measure (line 35) | def measure(self, test_case, *args, **kwargs):
    method a_measure (line 44) | async def a_measure(self, test_case, *args, **kwargs):
    method is_successful (line 47) | def is_successful(self):
  function _make_pytest_wrapped_trace (line 51) | def _make_pytest_wrapped_trace(app_span: BaseSpan) -> Trace:
  function test_assert_test_metrics_run_at_trace_level_with_golden_input (line 72) | def test_assert_test_metrics_run_at_trace_level_with_golden_input(
  function test_assert_test_uses_observe_metrics_for_span_level_evals (line 126) | def test_assert_test_uses_observe_metrics_for_span_level_evals(monkeypat...

FILE: tests/test_core/test_imports.py
  function test_metrics_imports (line 1) | def test_metrics_imports():
  function test_g_eval_imports (line 124) | def test_g_eval_imports():
  function test_dag_imports (line 130) | def test_dag_imports():
  function test_conversational_dag_imports (line 146) | def test_conversational_dag_imports():
  function test_core_modules_import (line 160) | def test_core_modules_import(unpatch_openai_after):
  function test_test_case_imports (line 200) | def test_test_case_imports():
  function test_evaluate_imports (line 232) | def test_evaluate_imports():
  function test_dataset_imports (line 253) | def test_dataset_imports():
  function test_models_imports (line 266) | def test_models_imports():
  function test_benchmarks_imports (line 317) | def test_benchmarks_imports():
  function test_tracing_imports (line 356) | def test_tracing_imports():

FILE: tests/test_core/test_models/test_amazon_bedrock_model.py
  class RecordingBedrockClient (line 10) | class RecordingBedrockClient(_RecordingClient):
    method __init__ (line 11) | def __init__(self, response, *args, **kwargs):
    method converse (line 15) | async def converse(self, **kwargs):
  function _mock_get_client (line 19) | def _mock_get_client(response):
  function _mk_model (line 29) | def _mk_model(gen_kwargs: Optional[Dict[str, Any]]):
  function test_get_converse_request_body_contains_temperature_and_kwargs (line 36) | def test_get_converse_request_body_contains_temperature_and_kwargs():
  function test_generation_kwargs_not_mutated (line 52) | def test_generation_kwargs_not_mutated():
  function test_various_generation_kwargs_passed_through (line 77) | def test_various_generation_kwargs_passed_through(gen_kwargs):
  function test_get_model_name_returns_name (line 87) | def test_get_model_name_returns_name():
  function test_bedrock_a_generate_skips_reasoning_content_and_reads_text_block (line 94) | async def test_bedrock_a_generate_skips_reasoning_content_and_reads_text...
  function test_bedrock_a_generate_reads_text_block_when_first (line 138) | async def test_bedrock_a_generate_reads_text_block_when_first(monkeypatch):
  function _mk_model_with_prices (line 179) | def _mk_model_with_prices(input_price, output_price):
  function test_bedrock_calculate_cost_returns_correct_value (line 189) | def test_bedrock_calculate_cost_returns_correct_value():
  function test_bedrock_calculate_cost_returns_none_when_prices_missing (line 196) | def test_bedrock_calculate_cost_returns_none_when_prices_missing():
  function test_bedrock_calculate_cost_returns_none_when_only_input_price_set (line 202) | def test_bedrock_calculate_cost_returns_none_when_only_input_price_set():
  function test_bedrock_calculate_cost_with_zero_tokens (line 208) | def test_bedrock_calculate_cost_with_zero_tokens():

FILE: tests/test_core/test_models/test_anthropic_model.py
  function test_anthropic_model_accepts_legacy_anthropic_api_key_keyword_and_uses_it (line 18) | def test_anthropic_model_accepts_legacy_anthropic_api_key_keyword_and_us...
  function test_anthropic_model_uses_explicit_key_over_settings_and_strips_secret (line 71) | def test_anthropic_model_uses_explicit_key_over_settings_and_strips_secret(
  function test_anthropic_model_uses_settings_key_when_no_explicit_key (line 116) | def test_anthropic_model_uses_settings_key_when_no_explicit_key(
  function test_anthropic_model_uses_explicit_key_when_settings_missing (line 144) | def test_anthropic_model_uses_explicit_key_when_settings_missing(
  function test_anthropic_model_raises_when_no_key_configured (line 172) | def test_anthropic_model_raises_when_no_key_configured(
  function test_anthropic_model_raises_when_explicit_key_empty (line 198) | def test_anthropic_model_raises_when_explicit_key_empty(
  function test_anthropic_model_raises_when_settings_key_empty (line 224) | def test_anthropic_model_raises_when_settings_key_empty(
  function test_anthropic_calculate_cost_returns_correct_value (line 253) | def test_anthropic_calculate_cost_returns_correct_value(
  function test_anthropic_calculate_cost_returns_none_when_prices_missing (line 276) | def test_anthropic_calculate_cost_returns_none_when_prices_missing(
  function test_anthropic_calculate_cost_with_zero_tokens (line 299) | def test_anthropic_calculate_cost_with_zero_tokens(mock_require_dep, set...

FILE: tests/test_core/test_models/test_azure_model.py
  class SampleSchema (line 13) | class SampleSchema(BaseModel):
  class TestAzureOpenAIModelGenerationKwargs (line 20) | class TestAzureOpenAIModelGenerationKwargs:
    method test_init_without_generation_kwargs (line 23) | def test_init_without_generation_kwargs(self, settings):
    method test_init_with_generation_kwargs (line 38) | def test_init_with_generation_kwargs(self, settings):
    method test_init_with_both_client_and_generation_kwargs (line 56) | def test_init_with_both_client_and_generation_kwargs(self, settings):
    method test_generate_with_generation_kwargs (line 76) | def test_generate_with_generation_kwargs(self, mock_load_model):
    method test_generate_without_generation_kwargs (line 116) | def test_generate_without_generation_kwargs(self, mock_load_model):
    method test_load_model_passes_kwargs_to_client (line 153) | def test_load_model_passes_kwargs_to_client(self, mock_azure_openai):
    method test_backwards_compatibility (line 179) | def test_backwards_compatibility(self, settings):
    method test_empty_generation_kwargs (line 195) | def test_empty_generation_kwargs(self, settings):
    method test_none_generation_kwargs (line 207) | def test_none_generation_kwargs(self, settings):
  function test_azure_openai_model_defers_auth_when_no_key_token_or_provider (line 225) | def test_azure_openai_model_defers_auth_when_no_key_token_or_provider(
  function test_azure_openai_model_raises_on_explicit_empty_api_key (line 269) | def test_azure_openai_model_raises_on_explicit_empty_api_key(
  function test_azure_openai_model_raises_on_explicit_empty_api_key_secretstr (line 301) | def test_azure_openai_model_raises_on_explicit_empty_api_key_secretstr(
  function test_azure_openai_model_raises_on_explicit_empty_ad_token (line 333) | def test_azure_openai_model_raises_on_explicit_empty_ad_token(
  function test_azure_openai_model_does_not_fail_fast_when_token_provider_present (line 364) | def test_azure_openai_model_does_not_fail_fast_when_token_provider_present(
  function test_azure_openai_model_uses_explicit_key_over_settings_and_strips_secret (line 399) | def test_azure_openai_model_uses_explicit_key_over_settings_and_strips_s...
  function test_azure_openai_model_defaults_from_settings (line 436) | def test_azure_openai_model_defaults_from_settings(monkeypatch, settings):
  function test_azure_openai_model_ctor_args_override_settings (line 477) | def test_azure_openai_model_ctor_args_override_settings(monkeypatch, set...
  function test_azure_openai_model_accepts_legacy_azure_endpoint_keyword_and_maps_to_base_url (line 527) | def test_azure_openai_model_accepts_legacy_azure_endpoint_keyword_and_ma...
  function test_azure_openai_model_accepts_legacy_api_key_keyword_and_uses_it (line 553) | def test_azure_openai_model_accepts_legacy_api_key_keyword_and_uses_it(
  class TestAzureModelTemperature (line 610) | class TestAzureModelTemperature:
    method test_reasoning_model_temperature_is_none (line 611) | def test_reasoning_model_temperature_is_none(self):
    method test_standard_model_temperature_is_set (line 616) | def test_standard_model_temperature_is_set(self):
    method test_explicit_temperature_preserved_for_standard_model (line 622) | def test_explicit_temperature_preserved_for_standard_model(self):
    method test_explicit_temperature_overridden_for_reasoning_model (line 629) | def test_explicit_temperature_overridden_for_reasoning_model(self):
  function test_azure_calculate_cost_returns_correct_value (line 642) | def test_azure_calculate_cost_returns_correct_value():
  function test_azure_calculate_cost_returns_none_when_prices_missing (line 651) | def test_azure_calculate_cost_returns_none_when_prices_missing():
  function test_azure_calculate_cost_with_zero_tokens (line 659) | def test_azure_calculate_cost_with_zero_tokens():

FILE: tests/test_core/test_models/test_azure_retry_config.py
  function test_azure_retry_predicate_present (line 13) | def test_azure_retry_predicate_present():
  function test_azure_sdk_retries_disabled (line 32) | def test_azure_sdk_retries_disabled(monkeypatch):
  function test_azure_hard_quota_marker_is_non_retryable (line 46) | def test_azure_hard_quota_marker_is_non_retryable():
  function test_length_finish_reason_is_non_retryable (line 67) | def test_length_finish_reason_is_non_retryable():
  function test_azure_sdk_retries_opt_in_respects_user_max_retries (line 82) | def test_azure_sdk_retries_opt_in_respects_user_max_retries(settings):

FILE: tests/test_core/test_models/test_bedrock_retry_config.py
  class DummyConfig (line 7) | class DummyConfig:
    method __init__ (line 8) | def __init__(self, *, retries=None, **kw):
  class DummyClient (line 13) | class DummyClient:
    method converse (line 15) | async def converse(self, **kwargs):
  class DummyCM (line 22) | class DummyCM:
    method __init__ (line 23) | def __init__(self, session, service_name, **kw):
    method __aenter__ (line 27) | async def __aenter__(self):
    method __aexit__ (line 32) | async def __aexit__(self, exc_type, exc, tb):
  class DummySession (line 36) | class DummySession:
    method __init__ (line 37) | def __init__(self):
    method create_client (line 41) | def create_client(self, service_name, **kw):
  function test_bedrock_retry_predicate_present (line 46) | def test_bedrock_retry_predicate_present():
  function test_bedrock_sdk_toggle (line 87) | def test_bedrock_sdk_toggle(mock_require_dep, settings):

FILE: tests/test_core/test_models/test_deepseek_model.py
  class SampleSchema (line 14) | class SampleSchema(BaseModel):
  class TestDeepSeekModelGenerationKwargs (line 19) | class TestDeepSeekModelGenerationKwargs:
    method test_init_with_generation_kwargs (line 20) | def test_init_with_generation_kwargs(self):
    method test_init_without_generation_kwargs (line 29) | def test_init_without_generation_kwargs(self):
    method test_generate_uses_generation_kwargs (line 39) | def test_generate_uses_generation_kwargs(self, mock_openai):
  function test_deepseek_model_uses_explicit_key_over_settings_and_strips_secret (line 80) | def test_deepseek_model_uses_explicit_key_over_settings_and_strips_secret(
  function test_deepseek_model_defaults_from_settings (line 118) | def test_deepseek_model_defaults_from_settings(monkeypatch):
  function test_deepseek_model_ctor_args_override_settings (line 155) | def test_deepseek_model_ctor_args_override_settings(monkeypatch):
  function test_deepseek_calculate_cost_returns_correct_value (line 198) | def test_deepseek_calculate_cost_returns_correct_value():
  function test_deepseek_calculate_cost_returns_none_when_prices_missing (line 210) | def test_deepseek_calculate_cost_returns_none_when_prices_missing():
  function test_deepseek_calculate_cost_with_zero_tokens (line 221) | def test_deepseek_calculate_cost_with_zero_tokens():

FILE: tests/test_core/test_models/test_embedding_models/test_azure_embedding_model.py
  function test_azure_embedding_model_uses_explicit_params_over_settings_and_strips_secret (line 16) | def test_azure_embedding_model_uses_explicit_params_over_settings_and_st...
  function test_azure_embedding_model_defaults_from_settings (line 66) | def test_azure_embedding_model_defaults_from_settings(monkeypatch):
  function test_azure_embedding_model_accepts_legacy_azure_endpoint_keyword_and_maps_to_base_url (line 117) | def test_azure_embedding_model_accepts_legacy_azure_endpoint_keyword_and...
  function test_azure_embedding_model_accepts_legacy_api_key_keyword_and_uses_it (line 138) | def test_azure_embedding_model_accepts_legacy_api_key_keyword_and_uses_it(

FILE: tests/test_core/test_models/test_embedding_models/test_local_embedding_model.py
  function test_local_embedding_model_uses_explicit_params_over_settings_and_strips_secret (line 14) | def test_local_embedding_model_uses_explicit_params_over_settings_and_st...
  function test_local_embedding_model_defaults_from_settings (line 61) | def test_local_embedding_model_defaults_from_settings(monkeypatch):

FILE: tests/test_core/test_models/test_embedding_models/test_ollama_embedding_model.py
  function test_ollama_embedding_model_uses_explicit_params_over_settings (line 13) | def test_ollama_embedding_model_uses_explicit_params_over_settings(
  function test_ollama_embedding_model_defaults_from_settings (line 63) | def test_ollama_embedding_model_defaults_from_settings(
  function test_ollama_embedding_model_accepts_legacy_host_keyword_and_maps_to_base_url (line 111) | def test_ollama_embedding_model_accepts_legacy_host_keyword_and_maps_to_...

FILE: tests/test_core/test_models/test_embedding_models/test_openai_embedding_model.py
  function test_openai_embedding_model_uses_explicit_key_over_settings_and_strips_secret (line 14) | def test_openai_embedding_model_uses_explicit_key_over_settings_and_stri...
  function test_openai_embedding_model_defaults_from_settings (line 47) | def test_openai_embedding_model_defaults_from_settings(monkeypatch):
  function test_openai_embedding_model_accepts_legacy__openai_api_key_keyword_and_maps_to_api_key (line 79) | def test_openai_embedding_model_accepts_legacy__openai_api_key_keyword_a...

FILE: tests/test_core/test_models/test_gemini_model.py
  function test_gemini_model_uses_explicit_key_over_settings_and_passes_plain_str (line 14) | def test_gemini_model_uses_explicit_key_over_settings_and_passes_plain_str(
  function test_gemini_model_defaults_key_from_settings_and_unwraps_secret (line 48) | def test_gemini_model_defaults_key_from_settings_and_unwraps_secret(
  function test_gemini_vertexai_allows_adc_when_no_service_account_key (line 81) | def test_gemini_vertexai_allows_adc_when_no_service_account_key(
  function test_gemini_model_use_vertexai_param_overrides_settings (line 128) | def test_gemini_model_use_vertexai_param_overrides_settings(
  function test_gemini_generate_returns_zero_cost (line 188) | def test_gemini_generate_returns_zero_cost(mock_require_dep, settings):

FILE: tests/test_core/test_models/test_grok_model.py
  function _stub_load_model (line 11) | def _stub_load_model(monkeypatch):
  function test_grok_model_uses_explicit_key_over_settings_and_strips_secret (line 30) | def test_grok_model_uses_explicit_key_over_settings_and_strips_secret(
  function test_grok_model_defaults_from_settings (line 64) | def test_grok_model_defaults_from_settings(monkeypatch, settings):
  function test_grok_calculate_cost_returns_correct_value (line 96) | def test_grok_calculate_cost_returns_correct_value(monkeypatch, settings):
  function test_grok_calculate_cost_returns_none_when_prices_missing (line 110) | def test_grok_calculate_cost_returns_none_when_prices_missing(
  function test_grok_calculate_cost_with_zero_tokens (line 128) | def test_grok_calculate_cost_with_zero_tokens(monkeypatch, settings):

FILE: tests/test_core/test_models/test_kimi_model.py
  function _stub_openai_clients (line 12) | def _stub_openai_clients(monkeypatch):
  function test_kimi_model_uses_explicit_key_over_settings_and_strips_secret (line 23) | def test_kimi_model_uses_explicit_key_over_settings_and_strips_secret(
  function test_kimi_model_defaults_from_settings (line 60) | def test_kimi_model_defaults_from_settings(monkeypatch):
  function test_kimi_model_ctor_args_override_settings (line 94) | def test_kimi_model_ctor_args_override_settings(monkeypatch):
  function test_kimi_calculate_cost_returns_correct_value (line 135) | def test_kimi_calculate_cost_returns_correct_value(monkeypatch):
  function test_kimi_calculate_cost_returns_none_when_prices_missing (line 150) | def test_kimi_calculate_cost_returns_none_when_prices_missing(monkeypatch):
  function test_kimi_calculate_cost_with_zero_tokens (line 165) | def test_kimi_calculate_cost_with_zero_tokens(monkeypatch):

FILE: tests/test_core/test_models/test_litellm_model.py
  function test_litellm_explicit_overrides_settings_and_env (line 25) | def test_litellm_explicit_overrides_settings_and_env(monkeypatch, settin...
  function test_litellm_defaults_model_api_key_and_base_from_settings (line 62) | def test_litellm_defaults_model_api_key_and_base_from_settings(settings):
  function test_litellm_raises_when_model_missing (line 88) | def test_litellm_raises_when_model_missing(settings):
  function test_litellm_model_accepts_legacy_api_base_keyword_and_maps_to_base_url (line 106) | def test_litellm_model_accepts_legacy_api_base_keyword_and_maps_to_base_...
  function _mk_litellm_model (line 127) | def _mk_litellm_model(settings):
  function _mk_response (line 134) | def _mk_response(prompt_tokens=100, completion_tokens=50, cost=None):
  function test_litellm_calculate_cost_prefers_response_cost (line 145) | def test_litellm_calculate_cost_prefers_response_cost(settings):
  function test_litellm_calculate_cost_falls_back_to_hardcoded_rates (line 152) | def test_litellm_calculate_cost_falls_back_to_hardcoded_rates(settings):
  function test_litellm_calculate_cost_response_cost_none_uses_fallback (line 160) | def test_litellm_calculate_cost_response_cost_none_uses_fallback(settings):
  function test_litellm_calculate_cost_accumulates_evaluation_cost (line 168) | def test_litellm_calculate_cost_accumulates_evaluation_cost(settings):
  function test_litellm_calculate_cost_with_zero_tokens_no_response_cost (line 184) | def test_litellm_calculate_cost_with_zero_tokens_no_response_cost(settin...
  function test_litellm_calculate_cost_handles_exception_gracefully (line 191) | def test_litellm_calculate_cost_handles_exception_gracefully(settings):

FILE: tests/test_core/test_models/test_local_model.py
  function _stub_openai_clients (line 10) | def _stub_openai_clients(monkeypatch):
  function test_local_model_uses_explicit_params_over_settings_and_strips_secret (line 23) | def test_local_model_uses_explicit_params_over_settings_and_strips_secret(
  function test_local_model_defaults_from_settings (line 74) | def test_local_model_defaults_from_settings(monkeypatch):
  function test_local_model_build_client_unwraps_secret_from_settings (line 114) | def test_local_model_build_client_unwraps_secret_from_settings(monkeypat...
  function test_local_generate_returns_zero_cost (line 151) | def test_local_generate_returns_zero_cost(monkeypatch):

FILE: tests/test_core/test_models/test_models_utils.py
  function test_require_secret_api_key_raises_when_none (line 14) | def test_require_secret_api_key_raises_when_none():
  function test_require_secret_api_key_raises_when_empty (line 24) | def test_require_secret_api_key_raises_when_empty():
  function test_require_secret_api_key_returns_plain_string_for_valid_secret (line 34) | def test_require_secret_api_key_returns_plain_string_for_valid_secret():
  function test_normalize_kwargs_and_extract_aliases_moves_aliases_and_logs (line 46) | def test_normalize_kwargs_and_extract_aliases_moves_aliases_and_logs(cap...
  function test_normalize_kwargs_and_extract_aliases_no_alias_usage_no_logs (line 83) | def test_normalize_kwargs_and_extract_aliases_no_alias_usage_no_logs(cap...
  function test_require_costs_returns_registry_prices_when_both_present (line 109) | def test_require_costs_returns_registry_prices_when_both_present():
  function test_require_costs_registry_prices_win_over_constructor_args (line 121) | def test_require_costs_registry_prices_win_over_constructor_args():
  function test_require_costs_uses_constructor_args_when_registry_missing (line 135) | def test_require_costs_uses_constructor_args_when_registry_missing():
  function test_require_costs_returns_none_when_registry_and_constructor_missing (line 149) | def test_require_costs_returns_none_when_registry_and_constructor_missin...
  function test_require_costs_returns_none_when_only_input_constructor_arg (line 161) | def test_require_costs_returns_none_when_only_input_constructor_arg():
  function test_require_costs_returns_none_when_only_output_constructor_arg (line 174) | def test_require_costs_returns_none_when_only_output_constructor_arg():
  function test_require_costs_raises_on_negative_input_cost (line 187) | def test_require_costs_raises_on_negative_input_cost():
  function test_require_costs_raises_on_negative_output_cost (line 200) | def test_require_costs_raises_on_negative_output_cost():
  function test_require_costs_accepts_zero_values (line 213) | def test_require_costs_accepts_zero_values():
  function test_require_costs_partial_registry_falls_back_to_constructor (line 227) | def test_require_costs_partial_registry_falls_back_to_constructor():

FILE: tests/test_core/test_models/test_ollama_model.py
  function test_ollama_model_uses_explicit_model_and_base_url_over_settings (line 9) | def test_ollama_model_uses_explicit_model_and_base_url_over_settings(
  function test_ollama_model_defaults_model_and_base_url_from_settings (line 45) | def test_ollama_model_defaults_model_and_base_url_from_settings(
  function test_ollama_generate_returns_zero_cost (line 86) | def test_ollama_generate_returns_zero_cost(mock_require_dep, settings):

FILE: tests/test_core/test_models/test_openai_extractors.py
  function test_extract_input_parameters_stringifies_multimodal_user_content_list (line 6) | def test_extract_input_parameters_stringifies_multimodal_user_content_li...

FILE: tests/test_core/test_models/test_openai_model.py
  function _make_llm_span (line 20) | def _make_llm_span() -> LlmSpan:
  function _make_usage (line 29) | def _make_usage(**fields):
  function _make_completion (line 33) | def _make_completion(usage, content="hello"):
  class SampleSchema (line 38) | class SampleSchema(BaseModel):
  class TestGPTModelCompletionKwargs (line 45) | class TestGPTModelCompletionKwargs:
    method test_init_without_generation_kwargs (line 48) | def test_init_without_generation_kwargs(self, settings):
    method test_init_with_generation_kwargs (line 56) | def test_init_with_generation_kwargs(self, settings):
    method test_init_with_both_client_and_generation_kwargs (line 71) | def test_init_with_both_client_and_generation_kwargs(self, settings):
    method test_generate_with_generation_kwargs (line 86) | def test_generate_with_generation_kwargs(self, mock_openai_class, sett...
    method test_generate_without_generation_kwargs (line 123) | def test_generate_without_generation_kwargs(
    method test_generate_with_schema_and_generation_kwargs (line 157) | def test_generate_with_schema_and_generation_kwargs(
    method test_async_generate_with_generation_kwargs (line 202) | async def test_async_generate_with_generation_kwargs(
    method test_async_generate_with_schema_and_generation_kwargs (line 254) | async def test_async_generate_with_schema_and_generation_kwargs(
    method test_generate_raw_response_with_generation_kwargs (line 307) | def test_generate_raw_response_with_generation_kwargs(
    method test_generate_samples_with_generation_kwargs (line 353) | def test_generate_samples_with_generation_kwargs(
    method test_backwards_compatibility (line 390) | def test_backwards_compatibility(self, settings):
    method test_gpt5_auto_temperature_adjustment (line 403) | def test_gpt5_auto_temperature_adjustment(self, settings):
    method test_empty_generation_kwargs (line 422) | def test_empty_generation_kwargs(self, settings):
    method test_none_generation_kwargs (line 428) | def test_none_generation_kwargs(self, settings):
  function test_openai_model_accepts_legacy_model_keyword_and_maps_to_model (line 440) | def test_openai_model_accepts_legacy_model_keyword_and_maps_to_model(
  function test_openai_model_accepts_legacy_openai_api_key_keyword_and_uses_it (line 460) | def test_openai_model_accepts_legacy_openai_api_key_keyword_and_uses_it(
  function test_openai_model_uses_explicit_key_over_settings_and_strips_secret (line 506) | def test_openai_model_uses_explicit_key_over_settings_and_strips_secret(
  function test_openai_model_defaults_model_from_settings_when_no_ctor_model (line 544) | def test_openai_model_defaults_model_from_settings_when_no_ctor_model(se...
  function test_openai_model_defaults_to_shared_default_when_no_setting (line 557) | def test_openai_model_defaults_to_shared_default_when_no_setting(settings):
  function test_openai_model_costs_defaults_from_settings_for_missing_pricing (line 566) | def test_openai_model_costs_defaults_from_settings_for_missing_pricing(
  class TestGPTModelUpdateLlmSpanTokenFields (line 592) | class TestGPTModelUpdateLlmSpanTokenFields:
    method test_classic_prompt_tokens_read (line 604) | def test_classic_prompt_tokens_read(self, _mock_span, mock_llm, settin...
    method test_new_input_tokens_read_for_gpt52 (line 619) | def test_new_input_tokens_read_for_gpt52(
    method test_cost_fields_non_none_for_gpt52 (line 636) | def test_cost_fields_non_none_for_gpt52(
    method test_gpt41_no_regression (line 653) | def test_gpt41_no_regression(self, _mock_span, mock_llm, settings):
    method test_zero_prompt_tokens_not_overwritten_by_fallback (line 670) | def test_zero_prompt_tokens_not_overwritten_by_fallback(
  class TestPatchOpenaiClientTokenCounts (line 691) | class TestPatchOpenaiClientTokenCounts:
    method _make_fake_client (line 700) | def _make_fake_client(self, completion):
    method test_new_field_names_in_patcher_for_gpt52 (line 710) | def test_new_field_names_in_patcher_for_gpt52(
    method test_cost_populated_in_patcher_for_gpt52 (line 730) | def test_cost_populated_in_patcher_for_gpt52(
    method test_classic_field_names_no_regression_in_patcher (line 751) | def test_classic_field_names_no_regression_in_patcher(
    method test_patcher_unknown_model_does_not_crash (line 773) | def test_patcher_unknown_model_does_not_crash(
    method test_patcher_zero_prompt_tokens_not_overwritten (line 796) | def test_patcher_zero_prompt_tokens_not_overwritten(
  function test_gpt55_model_data_matches_openai_docs (line 819) | def test_gpt55_model_data_matches_openai_docs():
  function test_gpt54_model_data_matches_openai_docs (line 831) | def test_gpt54_model_data_matches_openai_docs():
  function test_gpt54_snapshot_model_data_matches_alias (line 843) | def test_gpt54_snapshot_model_data_matches_alias():
  function test_gpt55_snapshot_model_data_matches_alias (line 850) | def test_gpt55_snapshot_model_data_matches_alias():
  function test_openai_calculate_cost_returns_correct_value (line 862) | def test_openai_calculate_cost_returns_correct_value(settings):
  function test_openai_calculate_cost_returns_none_when_prices_missing (line 875) | def test_openai_calculate_cost_returns_none_when_prices_missing(settings):
  function test_openai_calculate_cost_with_zero_tokens (line 887) | def test_openai_calculate_cost_with_zero_tokens(settings):

FILE: tests/test_core/test_models/test_openai_retry_policy.py
  class AlwaysLengthLimitClient (line 8) | class AlwaysLengthLimitClient:
    class _Beta (line 11) | class _Beta:
      class _Chat (line 12) | class _Chat:
        class _Completions (line 13) | class _Completions:
          method __init__ (line 14) | def __init__(self, counter):
          method parse (line 17) | def parse(self, *a, **kw):
        method __init__ (line 22) | def __init__(self, counter):
      method __init__ (line 25) | def __init__(self, counter):
    method __init__ (line 28) | def __init__(self, counter):
  class AlwaysRetryableClient (line 32) | class AlwaysRetryableClient:
    method __init__ (line 33) | def __init__(self, counter):
    method _raise (line 39) | def _raise(self, *a, **kw):
  function gpt_model_retryable (line 52) | def gpt_model_retryable(monkeypatch):
  function gpt_model_length_limit (line 63) | def gpt_model_length_limit(monkeypatch, settings):
  function test_retry_respects_max_attempts (line 95) | def test_retry_respects_max_attempts(
  function test_length_limit_is_non_retryable (line 112) | def test_length_limit_is_non_retryable(gpt_model_length_limit):

FILE: tests/test_core/test_models/test_openrouter_model.py
  class SampleSchema (line 10) | class SampleSchema(BaseModel):
  class TestOpenRouterModel (line 17) | class TestOpenRouterModel:
    method test_init_without_generation_kwargs (line 20) | def test_init_without_generation_kwargs(self, settings):
    method test_init_with_generation_kwargs (line 30) | def test_init_with_generation_kwargs(self, settings):
    method test_init_with_custom_pricing (line 44) | def test_init_with_custom_pricing(self, settings):
    method test_generate_with_generation_kwargs (line 58) | def test_generate_with_generation_kwargs(
    method test_async_generate (line 100) | async def test_async_generate(self, mock_async_openai_class, settings):
    method test_generate_with_structured_outputs (line 123) | def test_generate_with_structured_outputs(
    method test_generate_with_structured_outputs_fallback (line 165) | def test_generate_with_structured_outputs_fallback(
    method test_calculate_cost_with_user_pricing (line 212) | def test_calculate_cost_with_user_pricing(self, settings):
    method test_calculate_cost_with_response_pricing (line 227) | def test_calculate_cost_with_response_pricing(self, settings):
    method test_calculate_cost_when_cost_is_unknown_returns_none (line 244) | def test_calculate_cost_when_cost_is_unknown_returns_none(self, settin...
    method test_client_kwargs_includes_custom_headers (line 256) | def test_client_kwargs_includes_custom_headers(
    method test_default_model (line 285) | def test_default_model(self, settings):
    method test_dynamic_model_name (line 293) | def test_dynamic_model_name(self, settings):
    method test_generate_raw_response (line 311) | def test_generate_raw_response(self, mock_openai_class, settings):
    method test_generate_samples (line 339) | def test_generate_samples(self, mock_openai_class, settings):
    method test_base_url_uses_settings_when_not_passed (line 368) | def test_base_url_uses_settings_when_not_passed(self, settings):
    method test_init_rejects_negative_temperature (line 378) | def test_init_rejects_negative_temperature(self, settings):

FILE: tests/test_core/test_models/test_portkey_model.py
  function test_portkey_model_prefers_explicit_params_over_settings (line 12) | def test_portkey_model_prefers_explicit_params_over_settings(settings):
  function test_portkey_model_uses_settings_when_params_missing (line 40) | def test_portkey_model_uses_settings_when_params_missing(settings):
  function test_portkey_model_raises_if_model_missing (line 59) | def test_portkey_model_raises_if_model_missing(settings):
  function test_portkey_model_raises_if_base_url_missing (line 76) | def test_portkey_model_raises_if_base_url_missing(settings):
  function test_portkey_model_raises_if_provider_missing (line 93) | def test_portkey_model_raises_if_provider_missing(settings):
  function test_portkey_generate_sends_request_and_returns_content (line 116) | def test_portkey_generate_sends_request_and_returns_content(
  function test_portkey_a_generate_sends_request_and_returns_content (line 164) | async def test_portkey_a_generate_sends_request_and_returns_content(
  function test_portkey_generate_returns_str_not_cost_tuple (line 228) | def test_portkey_generate_returns_str_not_cost_tuple(mock_post, settings):

FILE: tests/test_core/test_optimization/test_copro/test_configs.py
  function test_copro_defaults (line 8) | def test_copro_defaults() -> None:
  function test_copro_accepts_explicit_random_state (line 17) | def test_copro_accepts_explicit_random_state() -> None:
  function test_copro_int_random_state_sets_seed (line 24) | def test_copro_int_random_state_sets_seed() -> None:
  function test_copro_allows_minimal_hyperparameters (line 30) | def test_copro_allows_minimal_hyperparameters() -> None:

FILE: tests/test_core/test_optimization/test_copro/test_loop.py
  function _goldens (line 18) | def _goldens(n: int = 3) -> list[Golden]:
  function test_copro_sample_minibatch_respects_size (line 22) | def test_copro_sample_minibatch_respects_size() -> None:
  function test_copro_sample_minibatch_returns_all_when_small (line 30) | def test_copro_sample_minibatch_returns_all_when_small() -> None:
  function test_copro_extract_optimized_set_picks_highest_mean (line 36) | def test_copro_extract_optimized_set_picks_highest_mean() -> None:
  function test_copro_execute_smoke (line 49) | def test_copro_execute_smoke(monkeypatch: pytest.MonkeyPatch) -> None:
  function test_copro_a_execute_smoke (line 85) | async def test_copro_a_execute_smoke(monkeypatch: pytest.MonkeyPatch) ->...
  function test_copro_generate_summary_table_renders_iteration_log (line 120) | def test_copro_generate_summary_table_renders_iteration_log() -> None:

FILE: tests/test_core/test_optimization/test_gepa/test_gepa_config.py
  function test_gepa_defaults_sanity (line 7) | def test_gepa_defaults_sanity():
  function test_gepa_random_seed_none_uses_time_based_seed (line 27) | def test_gepa_random_seed_none_uses_time_based_seed():
  function test_gepa_random_seed_preserves_explicit_value (line 44) | def test_gepa_random_seed_preserves_explicit_value():
  function test_gepa_tie_breaker_defaults_and_alias (line 56) | def test_gepa_tie_breaker_defaults_and_alias():
  function test_gepa_accepts_non_default_tie_breaker (line 69) | def test_gepa_accepts_non_default_tie_breaker():
  function test_gepa_field_bounds_validated (line 77) | def test_gepa_field_bounds_validated():

FILE: tests/test_core/test_optimization/test_gepa/test_loop.py
  function test_execute_requires_at_least_two_goldens (line 23) | def test_execute_requires_at_least_two_goldens() -> None:
  function test_a_execute_requires_at_least_two_goldens (line 37) | async def test_a_execute_requires_at_least_two_goldens() -> None:
  function test_execute_raises_without_scorer (line 50) | def test_execute_raises_without_scorer() -> None:
  function test_execute_end_to_end_accepts_improved_child_prompt (line 59) | def test_execute_end_to_end_accepts_improved_child_prompt() -> None:
  function test_a_execute_end_to_end_accepts_improved_child_prompt (line 107) | async def test_a_execute_end_to_end_accepts_improved_child_prompt() -> N...
  function test_draw_minibatch_respects_minibatch_size (line 147) | def test_draw_minibatch_respects_minibatch_size() -> None:
  function test_draw_minibatch_clamps_to_available_data (line 163) | def test_draw_minibatch_clamps_to_available_data() -> None:
  function test_should_accept_child_respects_jitter (line 178) | def test_should_accept_child_respects_jitter() -> None:
  function _make_prompt_config (line 194) | def _make_prompt_config(text: str) -> PromptConfiguration:
  function test_generate_child_prompt_returns_none_when_text_unchanged (line 200) | def test_generate_child_prompt_returns_none_when_text_unchanged() -> None:
  function test_generate_child_prompt_returns_new_prompt_when_text_changes (line 211) | def test_generate_child_prompt_returns_new_prompt_when_text_changes() ->...
  function test_a_generate_child_prompt_async_mirrors_sync_behavior (line 224) | async def test_a_generate_child_prompt_async_mirrors_sync_behavior() -> ...
  function test_make_child_clones_parent_and_sets_parent_id (line 236) | def test_make_child_clones_parent_and_sets_parent_id() -> None:
  function test_accept_child_updates_state_and_returns_accepted_iteration (line 254) | def test_accept_child_updates_state_and_returns_accepted_iteration() -> ...
  function test_a_accept_child_updates_state_and_returns_accepted_iteration (line 293) | async def test_a_accept_child_updates_state_and_returns_accepted_iterati...
  function test_best_by_aggregate_prefers_child_and_emits_tie_status (line 334) | def test_best_by_aggregate_prefers_child_and_emits_tie_status() -> None:
  function test_run_loop_iteration_reports_progress_and_stops_on_false (line 381) | def test_run_loop_iteration_reports_progress_and_stops_on_false() -> None:
  function test_a_run_loop_iteration_reports_error_and_stops (line 415) | async def test_a_run_loop_iteration_reports_error_and_stops() -> None:

FILE: tests/test_core/test_optimization/test_miprov2/test_report_contract.py
  class _DummyTrial (line 10) | class _DummyTrial:
    method __init__ (line 11) | def __init__(self):
    method suggest_categorical (line 14) | def suggest_categorical(self, name, choices):
  class _DummyStudy (line 20) | class _DummyStudy:
    method __init__ (line 21) | def __init__(self):
    method ask (line 24) | def ask(self):
    method tell (line 27) | def tell(self, trial, score):
    method best_trial (line 31) | def best_trial(self):
    method best_trial (line 35) | def best_trial(self, trial):
  class _DummyProposer (line 39) | class _DummyProposer:
    method propose (line 40) | def propose(self, prompt, goldens, num_candidates):
    method a_propose (line 43) | async def a_propose(self, prompt, goldens, num_candidates):
  class _DummyBootstrapper (line 47) | class _DummyBootstrapper:
    method bootstrap (line 48) | def bootstrap(self, prompt, goldens):
    method a_bootstrap (line 51) | async def a_bootstrap(self, prompt, goldens):
  class _DummyScorer (line 55) | class _DummyScorer:
    method score_minibatch (line 56) | def score_minibatch(self, prompt_configuration, minibatch):
    method a_score_minibatch (line 59) | async def a_score_minibatch(self, prompt_configuration, minibatch):
    method score_pareto (line 62) | def score_pareto(self, prompt_configuration, goldens):
    method a_score_pareto (line 65) | async def a_score_pareto(self, prompt_configuration, goldens):
  function _miprov2_with_stubs (line 70) | def _miprov2_with_stubs(monkeypatch):
  function test_miprov2_execute_report_contract (line 96) | def test_miprov2_execute_report_contract(_miprov2_with_stubs):
  function test_miprov2_a_execute_report_contract (line 112) | async def test_miprov2_a_execute_report_contract(_miprov2_with_stubs):

FILE: tests/test_core/test_optimization/test_mutations/test_prompt_rewriter.py
  function test_parse_prompt_text_returns_template (line 8) | def test_parse_prompt_text_returns_template():
  function test_parse_prompt_list_returns_json_string (line 13) | def test_parse_prompt_list_returns_json_string():
  function test_create_prompt_list_accepts_json_array (line 25) | def test_create_prompt_list_accepts_json_array():
  function test_create_prompt_list_rejects_comma_separated_objects_without_array (line 43) | def test_create_prompt_list_rejects_comma_separated_objects_without_arra...
  function test_create_prompt_list_raises_for_invalid_json (line 62) | def test_create_prompt_list_raises_for_invalid_json():

FILE: tests/test_core/test_optimization/test_pareto.py
  function test_pareto_frontier_basic (line 10) | def test_pareto_frontier_basic():
  function test_frequency_weights_counts_matches_alg2_with_global_frontier (line 27) | def test_frequency_weights_counts_matches_alg2_with_global_frontier():
  function test_select_prompt_configuration_deterministic_membership (line 46) | def test_select_prompt_configuration_deterministic_membership():
  function test_frequency_weights_excludes_nonwinners_and_dominated (line 63) | def test_frequency_weights_excludes_nonwinners_and_dominated():

FILE: tests/test_core/test_optimization/test_policies/test_tie_breaker.py
  function test_pick_best_with_ties_single_candidate (line 12) | def test_pick_best_with_ties_single_candidate():
  function test_pick_best_with_ties_raises_on_empty_totals (line 31) | def test_pick_best_with_ties_raises_on_empty_totals():
  function test_pick_best_with_ties_prefers_child_when_tied (line 47) | def test_pick_best_with_ties_prefers_child_when_tied():
  function test_pick_best_with_ties_prefers_root_when_tied (line 75) | def test_pick_best_with_ties_prefers_root_when_tied():
  function test_pick_best_with_ties_random_policy_is_deterministic_with_seed (line 103) | def test_pick_best_with_ties_random_policy_is_deterministic_with_seed():
  function test_pick_best_with_ties_respects_tie_tolerance (line 136) | def test_pick_best_with_ties_respects_tie_tolerance():

FILE: tests/test_core/test_optimization/test_prompt_optimizer.py
  function _dummy_model_callback (line 26) | def _dummy_model_callback(**_kwargs):
  function test_build_default_scorer_requires_metrics (line 30) | def test_build_default_scorer_requires_metrics():
  function test_build_default_scorer_rejects_non_metric_types (line 39) | def test_build_default_scorer_rejects_non_metric_types():
  function test_on_status_error_prints_message_when_indicator_disabled (line 57) | def test_on_status_error_prints_message_when_indicator_disabled(capsys):
  function test_on_status_tie_respects_announce_ties_flag (line 75) | def test_on_status_tie_respects_announce_ties_flag(capsys):
  function test_on_status_progress_updates_progress_when_indicator_enabled (line 107) | def test_on_status_progress_updates_progress_when_indicator_enabled():
  function test_format_iter_description_includes_iteration_and_percent (line 148) | def test_format_iter_description_includes_iteration_and_percent():
  function test_format_step_description_includes_arrow_and_color (line 160) | def test_format_step_description_includes_arrow_and_color():

FILE: tests/test_core/test_optimization/test_simba/test_configs.py
  function test_simba_defaults (line 8) | def test_simba_defaults() -> None:
  function test_simba_accepts_explicit_random_state (line 19) | def test_simba_accepts_explicit_random_state() -> None:
  function test_simba_int_random_state_sets_seed (line 26) | def test_simba_int_random_state_sets_seed() -> None:
  function test_simba_allows_minimal_hyperparameters (line 32) | def test_simba_allows_minimal_hyperparameters() -> None:

FILE: tests/test_core/test_optimization/test_simba/test_loop.py
  function _goldens (line 20) | def _goldens(n: int = 3) -> list[Golden]:
  function test_simba_golden_expected_text (line 24) | def test_simba_golden_expected_text() -> None:
  function test_simba_extract_inputs_golden_and_conversational (line 33) | def test_simba_extract_inputs_golden_and_conversational() -> None:
  function test_simba_sample_minibatch_respects_size (line 48) | def test_simba_sample_minibatch_respects_size() -> None:
  function test_simba_generate_summary_table_renders_iteration_log (line 55) | def test_simba_generate_summary_table_renders_iteration_log() -> None:
  function test_simba_execute_smoke (line 83) | def test_simba_execute_smoke(monkeypatch: pytest.MonkeyPatch) -> None:
  function test_simba_a_execute_smoke (line 138) | async def test_simba_a_execute_smoke(monkeypatch: pytest.MonkeyPatch) ->...

FILE: tests/test_core/test_optimization/test_utils.py
  function test_split_goldens_raises_for_negative_pareto_size (line 18) | def test_split_goldens_raises_for_negative_pareto_size() -> None:
  function test_split_goldens_empty_returns_empty_pairs (line 26) | def test_split_goldens_empty_returns_empty_pairs() -> None:
  function test_split_goldens_single_all_goes_to_pareto (line 38) | def test_split_goldens_single_all_goes_to_pareto() -> None:
  function test_split_goldens_zero_pareto_uses_all_for_feedback (line 52) | def test_split_goldens_zero_pareto_uses_all_for_feedback() -> None:
  function test_split_goldens_large_pareto_leaves_at_least_one_feedback (line 63) | def test_split_goldens_large_pareto_leaves_at_least_one_feedback() -> None:
  function test_split_goldens_deterministic_and_disjoint_with_fixed_seed (line 79) | def test_split_goldens_deterministic_and_disjoint_with_fixed_seed() -> N...
  function test_validate_callback_raises_when_missing (line 111) | def test_validate_callback_raises_when_missing() -> None:
  function test_validate_callback_returns_same_callable (line 122) | def test_validate_callback_returns_same_callable() -> None:
  function test_validate_instance_accepts_expected_type (line 139) | def test_validate_instance_accepts_expected_type():
  function test_validate_instance_accepts_tuple_of_expected_types (line 153) | def test_validate_instance_accepts_tuple_of_expected_types():
  function test_validate_instance_allows_none_when_flag_set (line 172) | def test_validate_instance_allows_none_when_flag_set():
  function test_validate_instance_raises_for_wrong_type (line 184) | def test_validate_instance_raises_for_wrong_type():
  function test_validate_sequence_of_accepts_list_of_expected_type (line 203) | def test_validate_sequence_of_accepts_list_of_expected_type():
  function test_validate_sequence_of_accepts_tuple_when_allowed (line 217) | def test_validate_sequence_of_accepts_tuple_when_allowed():
  function test_validate_sequence_of_allows_none_when_flag_set (line 231) | def test_validate_sequence_of_allows_none_when_flag_set():
  function test_validate_sequence_of_rejects_none_without_allow (line 243) | def test_validate_sequence_of_rejects_none_without_allow():
  function test_validate_sequence_of_rejects_wrong_sequence_type (line 258) | def test_validate_sequence_of_rejects_wrong_sequence_type():
  function test_validate_sequence_of_rejects_wrong_item_type (line 274) | def test_validate_sequence_of_rejects_wrong_item_type():

FILE: tests/test_core/test_prompts/test_interpolation.py
  class TestInterpolateMustache (line 16) | class TestInterpolateMustache:
    method test_simple_variable (line 19) | def test_simple_variable(self):
    method test_multiple_variables (line 24) | def test_multiple_variables(self):
    method test_variable_with_underscore (line 31) | def test_variable_with_underscore(self):
    method test_variable_starting_with_underscore (line 36) | def test_variable_starting_with_underscore(self):
    method test_variable_with_numbers (line 41) | def test_variable_with_numbers(self):
    method test_missing_variable_raises_keyerror (line 46) | def test_missing_variable_raises_keyerror(self):
    method test_missing_one_of_multiple_variables (line 53) | def test_missing_one_of_multiple_variables(self):
    method test_no_placeholders (line 60) | def test_no_placeholders(self):
    method test_empty_string (line 65) | def test_empty_string(self):
    method test_json_braces_not_replaced (line 70) | def test_json_braces_not_replaced(self):
    method test_json_array_not_replaced (line 76) | def test_json_array_not_replaced(self):
    method test_nested_json_structure (line 81) | def test_nested_json_structure(self):
    method test_multiple_json_objects (line 95) | def test_multiple_json_objects(self):
    method test_adjacent_variables (line 100) | def test_adjacent_variables(self):
    method test_variable_at_start (line 105) | def test_variable_at_start(self):
    method test_variable_at_end (line 110) | def test_variable_at_end(self):
    method test_only_variable (line 115) | def test_only_variable(self):
    method test_same_variable_multiple_times (line 120) | def test_same_variable_multiple_times(self):
    method test_integer_value (line 125) | def test_integer_value(self):
    method test_float_value (line 130) | def test_float_value(self):
    method test_boolean_value (line 135) | def test_boolean_value(self):
    method test_none_value (line 140) | def test_none_value(self):
    method test_list_value_converts_to_string (line 145) | def test_list_value_converts_to_string(self):
    method test_dict_value_converts_to_string (line 150) | def test_dict_value_converts_to_string(self):
    method test_unicode_characters (line 155) | def test_unicode_characters(self):
    method test_multiline_text (line 160) | def test_multiline_text(self):
    method test_special_characters_in_text (line 169) | def test_special_characters_in_text(self):
    method test_single_brace_not_replaced (line 174) | def test_single_brace_not_replaced(self):
    method test_triple_braces_not_replaced (line 180) | def test_triple_braces_not_replaced(self):
    method test_invalid_identifier_not_replaced (line 187) | def test_invalid_identifier_not_replaced(self):
    method test_invalid_identifier_with_dash (line 193) | def test_invalid_identifier_with_dash(self):
    method test_invalid_identifier_with_dot (line 199) | def test_invalid_identifier_with_dot(self):
    method test_whitespace_inside_placeholder_not_matched (line 205) | def test_whitespace_inside_placeholder_not_matched(self):
    method test_extra_kwargs_ignored (line 212) | def test_extra_kwargs_ignored(self):
    method test_very_long_variable_name (line 220) | def test_very_long_variable_name(self):
    method test_case_sensitive (line 226) | def test_case_sensitive(self):
    method test_real_world_prompt_template (line 232) | def test_real_world_prompt_template(self):
  class TestInterpolateMustacheWithSpace (line 251) | class TestInterpolateMustacheWithSpace:
    method test_simple_variable (line 254) | def test_simple_variable(self):
    method test_multiple_variables (line 259) | def test_multiple_variables(self):
    method test_json_not_replaced (line 266) | def test_json_not_replaced(self):
    method test_without_spaces_not_matched (line 271) | def test_without_spaces_not_matched(self):
    method test_single_space_only (line 278) | def test_single_space_only(self):
    method test_missing_variable_raises_keyerror (line 285) | def test_missing_variable_raises_keyerror(self):
    method test_extra_kwargs_ignored (line 292) | def test_extra_kwargs_ignored(self):
  class TestInterpolateFString (line 301) | class TestInterpolateFString:
    method test_simple_variable (line 304) | def test_simple_variable(self):
    method test_multiple_variables (line 309) | def test_multiple_variables(self):
    method test_json_braces_not_replaced (line 316) | def test_json_braces_not_replaced(self):
    method test_complex_json_structure (line 322) | def test_complex_json_structure(self):
    method test_missing_variable_raises_keyerror (line 338) | def test_missing_variable_raises_keyerror(self):
    method test_empty_braces_not_replaced (line 345) | def test_empty_braces_not_replaced(self):
    method test_nested_json_with_arrays (line 351) | def test_nested_json_with_arrays(self):
    method test_variable_with_underscore (line 356) | def test_variable_with_underscore(self):
    method test_integer_value (line 361) | def test_integer_value(self):
    method test_same_variable_multiple_times (line 366) | def test_same_variable_multiple_times(self):
    method test_invalid_identifier_with_dot_not_replaced (line 371) | def test_invalid_identifier_with_dot_not_replaced(self):
    method test_invalid_identifier_with_brackets_not_replaced (line 377) | def test_invalid_identifier_with_brackets_not_replaced(self):
    method test_unicode_in_values (line 383) | def test_unicode_in_values(self):
    method test_extra_kwargs_ignored (line 388) | def test_extra_kwargs_ignored(self):
  class TestInterpolateDollarBrackets (line 397) | class TestInterpolateDollarBrackets:
    method test_simple_variable (line 400) | def test_simple_variable(self):
    method test_multiple_variables (line 405) | def test_multiple_variables(self):
    method test_json_not_replaced (line 412) | def test_json_not_replaced(self):
    method test_regular_braces_not_replaced (line 417) | def test_regular_braces_not_replaced(self):
    method test_dollar_without_braces (line 422) | def test_dollar_without_braces(self):
    method test_missing_variable_raises_keyerror (line 427) | def test_missing_variable_raises_keyerror(self):
    method test_variable_with_underscore (line 434) | def test_variable_with_underscore(self):
    method test_shell_style_variable_format (line 439) | def test_shell_style_variable_format(self):
    method test_extra_kwargs_ignored (line 445) | def test_extra_kwargs_ignored(self):
  class TestInterpolateJinja (line 454) | class TestInterpolateJinja:
    method test_simple_variable (line 457) | def test_simple_variable(self):
    method test_multiple_variables (line 462) | def test_multiple_variables(self):
    method test_jinja_if_statement (line 467) | def test_jinja_if_statement(self):
    method test_jinja_if_false (line 473) | def test_jinja_if_false(self):
    method test_jinja_for_loop (line 478) | def test_jinja_for_loop(self):
    method test_jinja_filters (line 484) | def test_jinja_filters(self):
    method test_json_with_jinja (line 490) | def test_json_with_jinja(self):
  class TestEdgeCasesAcrossAllFormats (line 497) | class TestEdgeCasesAcrossAllFormats:
    method test_all_formats_handle_json (line 500) | def test_all_formats_handle_json(self):
    method test_all_formats_handle_empty_string (line 528) | def test_all_formats_handle_empty_string(self):
    method test_all_formats_raise_on_missing_variable (line 536) | def test_all_formats_raise_on_missing_variable(self):
    method test_all_formats_convert_values_to_string (line 555) | def test_all_formats_convert_values_to_string(self):
    method test_variable_in_template_but_not_passed_raises_error (line 565) | def test_variable_in_template_but_not_passed_raises_error(self):
    method test_variable_passed_but_not_in_template_is_ignored (line 602) | def test_variable_passed_but_not_in_template_is_ignored(self):
  class TestRealWorldScenarios (line 635) | class TestRealWorldScenarios:
    method test_product_catalog_with_json_output (line 638) | def test_product_catalog_with_json_output(self):
    method test_api_request_template (line 664) | def test_api_request_template(self):
    method test_markdown_with_code_blocks (line 684) | def test_markdown_with_code_blocks(self):
    method test_sql_template_with_json (line 704) | def test_sql_template_with_json(self):

FILE: tests/test_core/test_prompts/test_load.py
  class TestPromptLoad (line 7) | class TestPromptLoad:
    method test_load_plain_text_file (line 9) | def test_load_plain_text_file(self):
    method test_load_json_list_format (line 26) | def test_load_json_list_format(self):
    method test_load_json_list_format_txt_extension (line 61) | def test_load_json_list_format_txt_extension(self):
    method test_load_json_dict_format_with_correct_key (line 89) | def test_load_json_dict_format_with_correct_key(self):
    method test_load_json_dict_format_without_messages_key_raises_error (line 121) | def test_load_json_dict_format_without_messages_key_raises_error(self):
    method test_load_json_dict_format_with_wrong_key (line 146) | def test_load_json_dict_format_with_wrong_key(self):
    method test_load_unsupported_file_extension (line 168) | def test_load_unsupported_file_extension(self):
    method test_load_invalid_json_falls_back_to_text (line 183) | def test_load_invalid_json_falls_back_to_text(self):
    method test_load_malformed_messages_falls_back_to_text (line 199) | def test_load_malformed_messages_falls_back_to_text(self):
    method test_load_sets_correct_alias_from_filename (line 215) | def test_load_sets_correct_alias_from_filename(self):
    method test_load_dict_with_custom_messages_key (line 231) | def test_load_dict_with_custom_messages_key(self):

FILE: tests/test_core/test_prompts/test_prompt.py
  function test_write_to_cache_flushes_and_syncs (line 13) | def test_write_to_cache_flushes_and_syncs(

FILE: tests/test_core/test_retry_policy.py
  class DummyResponse (line 24) | class DummyResponse:
    method __init__ (line 25) | def __init__(self, payload):
    method json (line 28) | def json(self):
  class RaisingResponse (line 32) | class RaisingResponse:
    method json (line 33) | def json(self):
  class AuthError (line 37) | class AuthError(Exception): ...
  class RateLimitError (line 40) | class RateLimitError(Exception):
    method __init__ (line 41) | def __init__(self, *, response=None, body=None, msg=""):
  class FakeClientError (line 47) | class FakeClientError(Exception):
    method __init__ (line 48) | def __init__(self, response):
  class NetTimeout (line 52) | class NetTimeout(Exception): ...
  class NetConn (line 55) | class NetConn(Exception): ...
  class HTTPStatusError (line 58) | class HTTPStatusError(Exception):
    method __init__ (line 59) | def __init__(self, status_code, *, msg=""):
  function make_policy (line 69) | def make_policy():
  function RL (line 80) | def RL(response=None, body=None, msg=""):
  function policy (line 91) | def policy():
  function pred (line 96) | def pred(policy):
  function test_extract_error_code_variants (line 153) | def test_extract_error_code_variants(response, body, msg, expected):
  function test_extract_code_botocore_shape (line 158) | def test_extract_code_botocore_shape():
  function test_extract_error_code_prefers_response_over_markers (line 166) | def test_extract_error_code_prefers_response_over_markers():
  function test_extract_error_code_grpc_code_lowercased (line 175) | def test_extract_error_code_grpc_code_lowercased():
  function test_extract_error_code_prefers_response_over_body (line 188) | def test_extract_error_code_prefers_response_over_body():
  function test_network_is_retry (line 204) | def test_network_is_retry(pred, exc):
  function test_core_paths (line 216) | def test_core_paths(pred, exc, expected):
  function test_rate_limit_codes (line 227) | def test_rate_limit_codes(policy, code, expected):
  function test_extra_non_retryable_codes (line 233) | def test_extra_non_retryable_codes(policy):
  function test_http_status_non_int_or_missing_means_no_retry (line 241) | def test_http_status_non_int_or_missing_means_no_retry(policy):
  function test_retry_5xx_false_disables_server_retries (line 259) | def test_retry_5xx_false_disables_server_retries(policy):
  function test_message_markers_override_policy_markers (line 273) | def test_message_markers_override_policy_markers(policy):
  function test_dynamic_wait_callable (line 290) | def test_dynamic_wait_callable(monkeypatch):
  function test_dynamic_wait_zeros_with_env (line 296) | def test_dynamic_wait_zeros_with_env(monkeypatch, settings):
  function test_dynamic_stop_callable (line 308) | def test_dynamic_stop_callable():
  function test_retry_respects_max_attempts_env (line 318) | def test_retry_respects_max_attempts_env(monkeypatch, policy, settings):
  function test_create_retry_decorator_no_retry_when_sdk_enabled (line 356) | def test_create_retry_decorator_no_retry_when_sdk_enabled(monkeypatch, p...
  function test_dynamic_retry_no_policy_means_no_retry (line 388) | def test_dynamic_retry_no_policy_means_no_retry(monkeypatch):
  function test_get_retry_policy_for_respects_sdk_retries_for (line 415) | def test_get_retry_policy_for_respects_sdk_retries_for(monkeypatch, poli...
  function test_sdk_retries_for_wildcard (line 432) | def test_sdk_retries_for_wildcard(monkeypatch, settings):
  function test_http_status_string_is_coerced_to_int (line 440) | def test_http_status_string_is_coerced_to_int(policy):
  function test_dynamic_retry_invokes_static_predicate_when_sdk_off (line 460) | def test_dynamic_retry_invokes_static_predicate_when_sdk_off(
  function test_dynamic_retry_does_not_call_static_predicate_when_sdk_on (line 488) | def test_dynamic_retry_does_not_call_static_predicate_when_sdk_on(
  function test_sync_timeout_is_retryable_and_capped (line 520) | def test_sync_timeout_is_retryable_and_capped(monkeypatch, policy, setti...
  function test_dynamic_toggle_sdk_retries_runtime (line 548) | def test_dynamic_toggle_sdk_retries_runtime(monkeypatch, policy, settings):
  function test_retry_logging_levels_change_at_runtime (line 591) | def test_retry_logging_levels_change_at_runtime(

FILE: tests/test_core/test_run/test_file_sync.py
  function _make_dummy_self (line 9) | def _make_dummy_self():
  function test_test_run_save_flushes_and_syncs (line 24) | def test_test_run_save_flushes_and_syncs(monkeypatch):
  function test_cached_test_run_save_flushes_and_syncs (line 57) | def test_cached_test_run_save_flushes_and_syncs(monkeypatch):

FILE: tests/test_core/test_run/test_run_manager.py
  function test_get_test_run_preserves_valid_instance_on_read_lock (line 14) | def test_get_test_run_preserves_valid_instance_on_read_lock(tmp_path):
  function test_get_test_run_preserves_instance_when_file_missing (line 36) | def test_get_test_run_preserves_instance_when_file_missing(
  function test_get_test_run_preserves_instance_on_malformed_json (line 54) | def test_get_test_run_preserves_instance_on_malformed_json(tmp_path):
  function test_update_test_run_falls_back_in_memory_on_read_failure (line 68) | def test_update_test_run_falls_back_in_memory_on_read_failure(tmp_path):
  function test_save_test_run_with_save_under_key_flushes_and_syncs (line 99) | def test_save_test_run_with_save_under_key_flushes_and_syncs(

FILE: tests/test_core/test_run/test_turns_table.py
  function test_turns_table_tools_column_has_no_prefix (line 9) | def test_turns_table_tools_column_has_no_prefix():
  function test_turns_table_no_role_or_tools_duplication_with_format_turn (line 27) | def test_turns_table_no_role_or_tools_duplication_with_format_turn():

FILE: tests/test_core/test_sanitize_nan.py
  class TestMakeJsonSerializable (line 22) | class TestMakeJsonSerializable:
    method test_nan_replaced_with_none (line 25) | def test_nan_replaced_with_none(self):
    method test_inf_replaced_with_none (line 28) | def test_inf_replaced_with_none(self):
    method test_neg_inf_replaced_with_none (line 31) | def test_neg_inf_replaced_with_none(self):
    method test_normal_float_preserved (line 34) | def test_normal_float_preserved(self):
    method test_zero_float_preserved (line 37) | def test_zero_float_preserved(self):
    method test_negative_float_preserved (line 40) | def test_negative_float_preserved(self):
    method test_nan_inside_dict (line 43) | def test_nan_inside_dict(self):
    method test_nan_inside_list (line 48) | def test_nan_inside_list(self):
    method test_deeply_nested (line 52) | def test_deeply_nested(self):
    method test_result_is_valid_json (line 58) | def test_result_is_valid_json(self):
    method test_other_types_unaffected (line 77) | def test_other_types_unaffected(self):
  class TestMakeJsonSerializableForMetadata (line 89) | class TestMakeJsonSerializableForMetadata:
    method test_nan_replaced_with_none (line 98) | def test_nan_replaced_with_none(self):
    method test_inf_replaced_with_none (line 101) | def test_inf_replaced_with_none(self):
    method test_neg_inf_replaced_with_none (line 104) | def test_neg_inf_replaced_with_none(self):
    method test_finite_float_preserved (line 107) | def test_finite_float_preserved(self):
    method test_int_preserved (line 110) | def test_int_preserved(self):
    method test_bool_preserved (line 113) | def test_bool_preserved(self):
    method test_none_preserved (line 117) | def test_none_preserved(self):
    method test_nan_inside_dict (line 120) | def test_nan_inside_dict(self):
    method test_mixed_primitives_inside_dict (line 127) | def test_mixed_primitives_inside_dict(self):
  class TestSanitizeBody (line 153) | class TestSanitizeBody:
    method test_nan (line 156) | def test_nan(self):
    method test_inf (line 159) | def test_inf(self):
    method test_neg_inf (line 162) | def test_neg_inf(self):
    method test_normal_float (line 165) | def test_normal_float(self):
    method test_flat_dict (line 168) | def test_flat_dict(self):
    method test_nested_dict (line 172) | def test_nested_dict(self):
    method test_list (line 176) | def test_list(self):
    method test_tuple_becomes_list (line 180) | def test_tuple_becomes_list(self):
    method test_non_numeric_passthrough (line 184) | def test_non_numeric_passthrough(self):
    method test_full_trace_shaped_payload (line 190) | def test_full_trace_shaped_payload(self):

FILE: tests/test_core/test_simulator/helpers.py
  function sync_callback (line 9) | def sync_callback(
  function async_callback_complete (line 23) | async def async_callback_complete(
  function static_callback (line 37) | def static_callback(input: str) -> Turn:
  function async_static_callback (line 41) | async def async_static_callback(input: str) -> Turn:
  class StaticSimulatorModel (line 45) | class StaticSimulatorModel(DeepEvalBaseLLM):
    method __init__ (line 46) | def __init__(self, expected_outcome_complete: bool = False):
    method load_model (line 53) | def load_model(self):
    method generate (line 56) | def generate(self, prompt: str, schema=None):
    method a_generate (line 74) | async def a_generate(self, prompt: str, schema=None):
    method get_model_name (line 77) | def get_model_name(self):

FILE: tests/test_core/test_simulator/test_conversation_simulator.py
  function test_no_existing_turns (line 22) | def test_no_existing_turns():
  function test_existing_turns (line 46) | def test_existing_turns():
  function test_stop_early (line 70) | def test_stop_early():
  function test_invalid_max_user_simulations (line 91) | def test_invalid_max_user_simulations():
  function test_custom_simulation_template_is_used (line 109) | def test_custom_simulation_template_is_used():
  function test_custom_simulation_template_must_inherit_base_template (line 138) | def test_custom_simulation_template_must_inherit_base_template():
  function test_custom_simulation_template_validates_first_turn_signature (line 151) | def test_custom_simulation_template_validates_first_turn_signature():
  function test_custom_simulation_template_validates_next_turn_signature (line 166) | def test_custom_simulation_template_validates_next_turn_signature():
  function test_turn_alternation (line 181) | def test_turn_alternation():
  function test_max_simulations_ignores_existing_turns (line 204) | def test_max_simulations_ignores_existing_turns():
  function test_on_simulation_complete_hook_single_conversation (line 236) | def test_on_simulation_complete_hook_single_conversation():
  function test_on_simulation_complete_hook_multiple_conversations (line 266) | def test_on_simulation_complete_hook_multiple_conversations():

FILE: tests/test_core/test_simulator/test_conversation_simulator_controller.py
  function test_sync_controller_can_end_simulation (line 11) | def test_sync_controller_can_end_simulation():
  function test_async_controller_can_run_in_sync_mode (line 45) | def test_async_controller_can_run_in_sync_mode():
  function test_sync_controller_can_run_in_async_mode (line 70) | def test_sync_controller_can_run_in_async_mode():
  function test_controller_replaces_expected_outcome_completion (line 95) | def test_controller_replaces_expected_outcome_completion():
  function test_max_user_simulations_is_checked_before_controller (line 119) | def test_max_user_simulations_is_checked_before_controller():
  function test_async_controller_none_defaults_to_proceed (line 145) | def test_async_controller_none_defaults_to_proceed():
  function test_controller_unknown_return_value_defaults_to_proceed (line 169) | def test_controller_unknown_return_value_defaults_to_proceed():

FILE: tests/test_core/test_simulator/test_conversation_simulator_json_mode.py
  function assistant_ok_callback (line 12) | def assistant_ok_callback(input: str, turns, thread_id=None) -> Turn:
  function patch_initialize_model (line 17) | def patch_initialize_model(monkeypatch):
  function _golden (line 26) | def _golden():
  function test_simulator_handles_example_json_from_prompt (line 38) | def test_simulator_handles_example_json_from_prompt(

FILE: tests/test_core/test_stubs_contract.py
  function test_make_trace_api_like_shape (line 17) | def test_make_trace_api_like_shape():
  function test_make_span_api_like_shape (line 49) | def test_make_span_api_like_shape():
  function test_dummy_metric_behaviour_and_surface (line 56) | def test_dummy_metric_behaviour_and_surface():
  function test_dummy_task_completion_metric_behaviour_and_surface (line 72) | def test_dummy_task_completion_metric_behaviour_and_surface():
  function test_fake_span_shape_and_defaults (line 82) | def test_fake_span_shape_and_defaults():
  function test_fake_trace_shape_and_defaults (line 100) | def test_fake_trace_shape_and_defaults():
  function test_api_test_case_like_protocol_conformance (line 131) | def test_api_test_case_like_protocol_conformance():

FILE: tests/test_core/test_synthesizer/example_simulator.py
  function chatbot_callback (line 50) | async def chatbot_callback(input, turns: List[Turn]):

FILE: tests/test_core/test_synthesizer/test_context_generator.py
  class _FakeTextLoader (line 16) | class _FakeTextLoader:
    method __init__ (line 17) | def __init__(self, path, encoding=None, autodetect_encoding=True):
    method load (line 20) | def load(self):
    method aload (line 28) | async def aload(self):
  class _FakeSplitter (line 32) | class _FakeSplitter:
    method __init__ (line 33) | def __init__(self, chunk_size, chunk_overlap):
    method split_documents (line 37) | def split_documents(self, docs):
  function _make_stub_embedder (line 46) | def _make_stub_embedder():
  class _CapturingCollection (line 67) | class _CapturingCollection:
    method __init__ (line 68) | def __init__(self, name, count_value=10):
    method count (line 73) | def count(self):
    method get (line 76) | def get(self, ids):
    method query (line 80) | def query(self, _embedding, n_results):
    method add (line 86) | def add(self, *args, **kwargs):
  class _CapturingClient (line 90) | class _CapturingClient:
    method __init__ (line 91) | def __init__(self, count_value=10):
    method get_collection (line 96) | def get_collection(self, name):
    method create_collection (line 101) | def create_collection(self, name):
    method delete_collection (line 108) | def delete_collection(self, name):
  class _CapturingChromaMod (line 113) | class _CapturingChromaMod:
    method __init__ (line 114) | def __init__(self, client: "_CapturingClient" = None):
    method PersistentClient (line 118) | def PersistentClient(self, path, **kwargs):
  function _patch_langchain (line 124) | def _patch_langchain(monkeypatch):
  function context_generator_fixture (line 141) | def context_generator_fixture():
  function ensure_synthesizer_data (line 152) | def ensure_synthesizer_data():
  function test_generate_contexts (line 161) | def test_generate_contexts(
  function test_multiple_context_generations (line 183) | def test_multiple_context_generations(
  function test_many_docs_should_spawn_a_single_chroma_client (line 213) | def test_many_docs_should_spawn_a_single_chroma_client(monkeypatch, tmp_...
  function test_async_many_docs_uses_single_chroma_client (line 287) | async def test_async_many_docs_uses_single_chroma_client(monkeypatch, tm...
  function test_sync_min_context_size_validation (line 337) | def test_sync_min_context_size_validation(monkeypatch, tmp_path, caplog):
  function test_async_per_doc_failure_is_logged_and_others_continue (line 388) | async def test_async_per_doc_failure_is_logged_and_others_continue(
  function test_sync_deletes_one_collection_per_doc (line 460) | def test_sync_deletes_one_collection_per_doc(monkeypatch, tmp_path):

FILE: tests/test_core/test_synthesizer/test_doc_chunker.py
  class StubEmbedder (line 10) | class StubEmbedder:
    method embed_texts (line 18) | def embed_texts(self, xs):
    method a_embed_texts (line 21) | def a_embed_texts(self, xs):
    method embed_text (line 24) | def embed_text(self, x):
    method a_embed_text (line 27) | async def a_embed_text(self, x):
  class StubAsyncEmbedder (line 31) | class StubAsyncEmbedder(StubEmbedder):
    method a_embed_texts (line 39) | async def a_embed_texts(self, xs):
  class FakeCollection (line 43) | class FakeCollection:
    method __init__ (line 51) | def __init__(self):
    method add (line 54) | def add(self, documents, embeddings, metadatas, ids):
  class FakeClient (line 58) | class FakeClient:
    method __init__ (line 66) | def __init__(self):
    method get_collection (line 69) | def get_collection(self, name):
    method create_collection (line 74) | def create_collection(self, name):
  class FakeChromaMod (line 80) | class FakeChromaMod:
    method __init__ (line 87) | def __init__(self, client):
    method PersistentClient (line 90) | def PersistentClient(self, path, **kwargs):
  function test_markdown_family_preserves_table (line 100) | def test_markdown_family_preserves_table(tmp_path, ext):
  function test_unsupported_extension (line 113) | def test_unsupported_extension(tmp_path):
  function test_textloader_autodetect_encoding (line 124) | def test_textloader_autodetect_encoding(tmp_path):
  function test_count_tokens_runs (line 137) | def test_count_tokens_runs(tmp_path):
  function test_lazy_imports_langchain_required_on_loader (line 154) | def test_lazy_imports_langchain_required_on_loader(monkeypatch):
  function test_lazy_imports_chromadb_required_on_chunk (line 172) | def test_lazy_imports_chromadb_required_on_chunk(monkeypatch, tmp_path):
  function test_chunk_doc_raises_if_not_loaded (line 200) | def test_chunk_doc_raises_if_not_loaded():
  function test_chunk_doc_batches_and_metadata (line 211) | def test_chunk_doc_batches_and_metadata(monkeypatch, tmp_path):
  function test_a_chunk_doc_works (line 244) | async def test_a_chunk_doc_works(monkeypatch, tmp_path):
  function test_chunk_doc_uses_existing_collection (line 268) | def test_chunk_doc_uses_existing_collection(monkeypatch, tmp_path):
  function test_persistent_path_and_collection_name (line 296) | def test_persistent_path_and_collection_name(monkeypatch, tmp_path):

FILE: tests/test_core/test_synthesizer/test_generate_from_goldens.py
  function synthesizer (line 30) | def synthesizer():
  function test_expand_dataset_from_inputs (line 34) | def test_expand_dataset_from_inputs(synthesizer: Synthesizer):
  function test_expand_dataset_from_scenarios (line 46) | def test_expand_dataset_from_scenarios(synthesizer: Synthesizer):

FILE: tests/test_core/test_synthesizer/test_synthesizer.py
  function evolution_config (line 83) | def evolution_config():
  function styling_config (line 91) | def styling_config():
  function conversational_styling_config (line 102) | def conversational_styling_config():
  function context_config (line 113) | def context_config():
  function sync_synthesizer (line 124) | def sync_synthesizer(evolution_config, styling_config):
  function async_synthesizer (line 134) | def async_synthesizer(evolution_config, styling_config):
  function sync_conversational_synthesizer (line 144) | def sync_conversational_synthesizer(
  function async_conversational_synthesizer (line 156) | def async_conversational_synthesizer(
  function test_generate_goldens_from_contexts (line 167) | def test_generate_goldens_from_contexts(sync_synthesizer: Synthesizer):
  function test_generate_goldens_from_docs (line 188) | def test_generate_goldens_from_docs(
  function test_generate_goldens_from_scratch (line 208) | def test_generate_goldens_from_scratch(sync_synthesizer: Synthesizer):
  function test_async_generate_goldens_from_contexts (line 222) | async def test_async_generate_goldens_from_contexts(
  function test_async_generate_goldens_from_docs (line 237) | async def test_async_generate_goldens_from_docs(
  function test_async_generate_goldens_from_scratch (line 253) | async def test_async_generate_goldens_from_scratch(
  function test_generate_conversational_goldens_from_contexts (line 267) | def test_generate_conversational_goldens_from_contexts(
  function test_generate_conversational_goldens_from_docs (line 292) | def test_generate_conversational_goldens_from_docs(
  function test_generate_conversational_goldens_from_scratch (line 309) | def test_generate_conversational_goldens_from_scratch(
  function test_async_generate_conversational_goldens_from_contexts (line 325) | async def test_async_generate_conversational_goldens_from_contexts(
  function test_async_generate_conversational_goldens_from_docs (line 340) | async def test_async_generate_conversational_goldens_from_docs(
  function test_async_generate_conversational_goldens_from_scratch (line 356) | async def test_async_generate_conversational_goldens_from_scratch(
  function test_a_generate_goldens_from_contexts_no_deadlock_when_max_concurrent_lt_contexts (line 371) | async def test_a_generate_goldens_from_contexts_no_deadlock_when_max_con...
  function test_a_generate_conversational_goldens_from_contexts_no_deadlock_when_max_concurrent_lt_contexts (line 420) | async def test_a_generate_conversational_goldens_from_contexts_no_deadlo...

FILE: tests/test_core/test_synthesizer_bugs.py
  function _make_synthesizer (line 31) | def _make_synthesizer(**overrides):
  class TestTextToSqlNoneExpectedOutput (line 59) | class TestTextToSqlNoneExpectedOutput:
    method test_no_crash_when_expected_output_disabled (line 65) | async def test_no_crash_when_expected_output_disabled(self):
    method test_expected_output_populated_when_enabled (line 87) | async def test_expected_output_populated_when_enabled(self):
  class TestFromScratchEvolutionsMetadata (line 116) | class TestFromScratchEvolutionsMetadata:
    method test_each_golden_has_own_evolutions (line 120) | def test_each_golden_has_own_evolutions(self):
  class TestRewriteInputsZeroRetries (line 185) | class TestRewriteInputsZeroRetries:
    method test_sync_no_crash_with_zero_retries (line 189) | def test_sync_no_crash_with_zero_retries(self):
    method test_async_no_crash_with_zero_retries (line 215) | async def test_async_no_crash_with_zero_retries(self):

FILE: tests/test_core/test_telemetry.py
  function _no_hidden_store_dir (line 10) | def _no_hidden_store_dir(base: Path):
  function test_telemetry_writes_create_dir_when_missing (line 15) | def test_telemetry_writes_create_dir_when_missing(tmp_path, monkeypatch):

FILE: tests/test_core/test_test_case/test_deprecated_params.py
  function test_llm_test_case_params_alias_is_single_turn_params (line 4) | def test_llm_test_case_params_alias_is_single_turn_params():
  function test_turn_params_alias_is_multi_turn_params (line 18) | def test_turn_params_alias_is_multi_turn_params():
  function test_llm_test_case_params_alias_from_submodule (line 32) | def test_llm_test_case_params_alias_from_submodule():
  function test_turn_params_alias_from_submodule (line 43) | def test_turn_params_alias_from_submodule():

FILE: tests/test_core/test_test_case/test_multi_turn/test_conversational_test_case.py
  class TestConversationalTestCaseInitialization (line 7) | class TestConversationalTestCaseInitialization:
    method test_minimal_initialization (line 9) | def test_minimal_initialization(self):
    method test_full_initialization (line 28) | def test_full_initialization(self):
    method test_turns_deep_copy (line 74) | def test_turns_deep_copy(self):
  class TestConversationalTestCaseValidation (line 83) | class TestConversationalTestCaseValidation:
    method test_empty_turns_raises_error (line 85) | def test_empty_turns_raises_error(self):
    method test_non_turn_objects_raises_error (line 89) | def test_non_turn_objects_raises_error(self):
    method test_dict_turn_is_accepted (line 93) | def test_dict_turn_is_accepted(self):
    method test_invalid_context_type_raises_error (line 97) | def test_invalid_context_type_raises_error(self):
    method test_invalid_context_items_raises_error (line 104) | def test_invalid_context_items_raises_error(self):
    method test_valid_context_list (line 113) | def test_valid_context_list(self):
    method test_none_context_is_valid (line 119) | def test_none_context_is_valid(self):
  class TestConversationalTestCaseComplexScenarios (line 125) | class TestConversationalTestCaseComplexScenarios:
    method test_single_user_turn (line 127) | def test_single_user_turn(self):
    method test_single_assistant_turn (line 133) | def test_single_assistant_turn(self):
    method test_alternating_conversation (line 139) | def test_alternating_conversation(self):
    method test_consecutive_user_turns (line 153) | def test_consecutive_user_turns(self):
    method test_consecutive_assistant_turns (line 165) | def test_consecutive_assistant_turns(self):
    method test_long_conversation (line 177) | def test_long_conversation(self):
  class TestConversationalTestCaseTurnProperties (line 188) | class TestConversationalTestCaseTurnProperties:
    method test_turns_with_metadata (line 190) | def test_turns_with_metadata(self):
    method test_turns_with_retrieval_context (line 213) | def test_turns_with_retrieval_context(self):
  class TestConversationalTestCaseEdgeCases (line 235) | class TestConversationalTestCaseEdgeCases:
    method test_empty_content_turns (line 237) | def test_empty_content_turns(self):
    method test_very_long_content (line 246) | def test_very_long_content(self):
    method test_special_characters_in_content (line 252) | def test_special_characters_in_content(self):
    method test_multiline_content (line 258) | def test_multiline_content(self):
    method test_empty_tags_list (line 267) | def test_empty_tags_list(self):
    method test_empty_additional_metadata (line 272) | def test_empty_additional_metadata(self):
    method test_metadata_input_compatibility (line 278) | def test_metadata_input_compatibility(self):
    method test_api_test_case_uses_metadata (line 285) | def test_api_test_case_uses_metadata(self):
  class TestConversationalTestCaseEquality (line 299) | class TestConversationalTestCaseEquality:
    method test_identical_test_cases_are_equal (line 301) | def test_identical_test_cases_are_equal(self):
    method test_different_turns_are_not_equal (line 310) | def test_different_turns_are_not_equal(self):
    method test_different_scenarios_are_not_equal (line 319) | def test_different_scenarios_are_not_equal(self):
  class TestConversationalTestCaseSerialization (line 328) | class TestConversationalTestCaseSerialization:
    method test_model_dump_includes_all_fields (line 330) | def test_model_dump_includes_all_fields(self):
    method test_serialization_aliases (line 350) | def test_serialization_aliases(self):
  class TestConversationalTestCaseCamelCaseInitialization (line 369) | class TestConversationalTestCaseCamelCaseInitialization:
    method test_camelcase_field_initialization (line 371) | def test_camelcase_field_initialization(self):
    method test_mixed_case_initialization (line 417) | def test_mixed_case_initialization(self):

FILE: tests/test_core/test_test_case/test_multi_turn/test_turn.py
  class TestTurnInitialization (line 6) | class TestTurnInitialization:
    method test_minimal_initialization (line 8) | def test_minimal_initialization(self):
    method test_user_role_initialization (line 22) | def test_user_role_initialization(self):
    method test_assistant_role_initialization (line 27) | def test_assistant_role_initialization(self):
    method test_full_initialization (line 32) | def test_full_initialization(self):
  class TestTurnValidation (line 62) | class TestTurnValidation:
    method test_invalid_role_raises_error (line 64) | def test_invalid_role_raises_error(self):
    method test_empty_content_is_valid (line 68) | def test_empty_content_is_valid(self):
    method test_none_content_raises_error (line 72) | def test_none_content_raises_error(self):
    method test_non_string_content_raises_error (line 76) | def test_non_string_content_raises_error(self):
  class TestTurnWithRetrievalContext (line 81) | class TestTurnWithRetrievalContext:
    method test_single_context_item (line 83) | def test_single_context_item(self):
    method test_multiple_context_items (line 94) | def test_multiple_context_items(self):
    method test_empty_retrieval_context_list (line 108) | def test_empty_retrieval_context_list(self):
    method test_none_retrieval_context (line 112) | def test_none_retrieval_context(self):
  class TestTurnWithTools (line 117) | class TestTurnWithTools:
    method test_single_tool_call (line 119) | def test_single_tool_call(self):
    method test_multiple_tool_calls (line 136) | def test_multiple_tool_calls(self):
    method test_empty_tools_called_list (line 160) | def test_empty_tools_called_list(self):
  class TestTurnWithMetadata (line 165) | class TestTurnWithMetadata:
    method test_simple_metadata (line 167) | def test_simple_metadata(self):
    method test_additional_metadata_input_compatibility (line 173) | def test_additional_metadata_input_compatibility(self):
    method test_additional_metadata_camelcase_input_compatibility (line 181) | def test_additional_metadata_camelcase_input_compatibility(self):
    method test_complex_metadata (line 189) | def test_complex_metadata(self):
    method test_empty_metadata_dict (line 205) | def test_empty_metadata_dict(self):
    method test_none_metadata (line 209) | def test_none_metadata(self):
  class TestTurnWithUserID (line 214) | class TestTurnWithUserID:
    method test_simple_user_id (line 216) | def test_simple_user_id(self):
    method test_uuid_user_id (line 220) | def test_uuid_user_id(self):
    method test_none_user_id (line 225) | def test_none_user_id(self):
  class TestTurnRepresentation (line 230) | class TestTurnRepresentation:
    method test_repr_minimal (line 232) | def test_repr_minimal(self):
    method test_repr_with_optional_fields (line 238) | def test_repr_with_optional_fields(self):
    method test_repr_with_tools (line 251) | def test_repr_with_tools(self):
  class TestTurnEdgeCases (line 262) | class TestTurnEdgeCases:
    method test_very_long_content (line 264) | def test_very_long_content(self):
    method test_special_characters_in_content (line 269) | def test_special_characters_in_content(self):
    method test_multiline_content (line 274) | def test_multiline_content(self):
    method test_json_like_content (line 282) | def test_json_like_content(self):
    method test_code_content (line 287) | def test_code_content(self):
  class TestTurnEquality (line 298) | class TestTurnEquality:
    method test_identical_turns_are_equal (line 300) | def test_identical_turns_are_equal(self):
    method test_different_content_not_equal (line 305) | def test_different_content_not_equal(self):
    method test_different_roles_not_equal (line 310) | def test_different_roles_not_equal(self):
    method test_different_metadata_not_equal (line 315) | def test_different_metadata_not_equal(self):
  class TestTurnSerialization (line 321) | class TestTurnSerialization:
    method test_model_dump_basic (line 323) | def test_model_dump_basic(self):
    method test_model_dump_with_all_fields (line 332) | def test_model_dump_with_all_fields(self):
    method test_model_dump_exclude_none (line 354) | def test_model_dump_exclude_none(self):
  class TestTurnCamelCaseInitialization (line 364) | class TestTurnCamelCaseInitialization:
    method test_camelcase_field_initialization (line 366) | def test_camelcase_field_initialization(self):
    method test_mixed_case_initialization (line 403) | def test_mixed_case_initialization(self):
    method test_turn_with_camelcase_tools (line 425) | def test_turn_with_camelcase_tools(self):

FILE: tests/test_core/test_test_case/test_multi_turn/test_utils.py
  function make_turns (line 5) | def make_turns(seq):
  class TestGetUnitInteractions (line 10) | class TestGetUnitInteractions:
    method test_consecutive_users (line 11) | def test_consecutive_users(self):
    method test_consecutive_assistants (line 19) | def test_consecutive_assistants(self):
    method test_user_assistant_user (line 27) | def test_user_assistant_user(self):
    method test_starts_with_assistant (line 35) | def test_starts_with_assistant(self):
    method test_assistant_only_start_then_end_with_user (line 43) | def test_assistant_only_start_then_end_with_user(self):
    method test_multiple_units (line 51) | def test_multiple_units(self):
    method test_empty_input (line 67) | def test_empty_input(self):

FILE: tests/test_core/test_test_case/test_single_turn.py
  class TestLLMTestCaseInitialization (line 16) | class TestLLMTestCaseInitialization:
    method test_minimal_initialization (line 18) | def test_minimal_initialization(self):
    method test_full_initialization (line 46) | def test_full_initialization(self):
  class TestLLMTestCaseCamelCaseInitialization (line 99) | class TestLLMTestCaseCamelCaseInitialization:
    method test_camelcase_field_initialization (line 101) | def test_camelcase_field_initialization(self):
    method test_mixed_case_initialization (line 160) | def test_mixed_case_initialization(self):
    method test_tool_call_camelcase_initialization (line 190) | def test_tool_call_camelcase_initialization(self):
  class TestLLMTestCaseTypeValidation (line 238) | class TestLLMTestCaseTypeValidation:
    method test_input_must_be_string (line 240) | def test_input_must_be_string(self):
    method test_actual_output_must_be_string_or_none (line 250) | def test_actual_output_must_be_string_or_none(self):
    method test_context_must_be_list_of_strings_or_none (line 263) | def test_context_must_be_list_of_strings_or_none(self):
    method test_retrieval_context_must_be_list_of_strings_or_none (line 287) | def test_retrieval_context_must_be_list_of_strings_or_none(self):
    method test_tools_called_must_be_list_of_toolcall_or_none (line 307) | def test_tools_called_must_be_list_of_toolcall_or_none(self):
    method test_expected_tools_must_be_list_of_toolcall_or_none (line 338) | def test_expected_tools_must_be_list_of_toolcall_or_none(self):
  class TestToolCallFunctionality (line 357) | class TestToolCallFunctionality:
    method test_tool_call_minimal_initialization (line 360) | def test_tool_call_minimal_initialization(self):
    method test_tool_call_full_initialization (line 370) | def test_tool_call_full_initialization(self):
    method test_tool_call_equality (line 392) | def test_tool_call_equality(self):
    method test_tool_call_hashing (line 414) | def test_tool_call_hashing(self):
    method test_tool_call_hashing_with_complex_types (line 438) | def test_tool_call_hashing_with_complex_types(self):
    method test_tool_call_repr (line 453) | def test_tool_call_repr(self):
    method test_tool_call_repr_minimal (line 470) | def test_tool_call_repr_minimal(self):
  class TestEdgeCases (line 480) | class TestEdgeCases:
    method test_empty_strings (line 482) | def test_empty_strings(self):
    method test_empty_lists (line 492) | def test_empty_lists(self):
    method test_very_long_strings (line 508) | def test_very_long_strings(self):
    method test_special_characters (line 521) | def test_special_characters(self):
    method test_numeric_edge_values (line 534) | def test_numeric_edge_values(self):
    method test_large_lists (line 547) | def test_large_lists(self):
    method test_deeply_nested_structures (line 560) | def test_deeply_nested_structures(self):
  class TestSerialization (line 583) | class TestSerialization:
    method test_serialization_aliases (line 585) | def test_serialization_aliases(self):
    method test_metadata_serialization (line 610) | def test_metadata_serialization(self):
    method test_additional_metadata_input_compatibility (line 628) | def test_additional_metadata_input_compatibility(self):
    method test_api_test_case_uses_metadata (line 639) | def test_api_test_case_uses_metadata(self):
  class TestLLMTestCaseParams (line 650) | class TestLLMTestCaseParams:
    method test_enum_values (line 651) | def test_enum_values(self):
  class TestToolCallParams (line 670) | class TestToolCallParams:
    method test_enum_values (line 671) | def test_enum_values(self):
  class TestPrivateAttributes (line 676) | class TestPrivateAttributes:
    method test_private_attributes_not_in_model_dump (line 677) | def test_private_attributes_not_in_model_dump(self):
    method test_private_attributes_accessible (line 688) | def test_private_attributes_accessible(self):
    method test_identifier_is_unique (line 707) | def test_identifier_is_unique(self):
  class TestIntegrationScenarios (line 718) | class TestIntegrationScenarios:
    method test_rag_evaluation_scenario (line 719) | def test_rag_evaluation_scenario(self):
    method test_tool_calling_scenario (line 750) | def test_tool_calling_scenario(self):

FILE: tests/test_core/test_threadpool_tracing.py
  function child_function (line 10) | def child_function():
  function parent_with_plain_executor (line 15) | def parent_with_plain_executor():
  function parent_with_copy_context (line 24) | def parent_with_copy_context():
  function clean_trace_state (line 34) | def clean_trace_state():
  function test_threadpool_without_copy_context_creates_two_traces (line 46) | def test_threadpool_without_copy_context_creates_two_traces(completed_tr...
  function test_threadpool_with_copy_context_creates_one_trace (line 57) | def test_threadpool_with_copy_context_creates_one_trace(completed_traces):

FILE: tests/test_core/test_trace_memory_leak.py
  function simple_agent (line 10) | def simple_agent():
  function agent_with_child (line 15) | def agent_with_child():
  function clean_state (line 25) | def clean_state():
  class TestTraceCleanup (line 35) | class TestTraceCleanup:
    method test_single_trace_removed_after_completion (line 38) | def test_single_trace_removed_after_completion(self):
    method test_nested_spans_trace_removed_after_completion (line 44) | def test_nested_spans_trace_removed_after_completion(self):
    method test_many_traces_do_not_accumulate (line 51) | def test_many_traces_do_not_accumulate(self):
    method test_active_traces_cleaned_up (line 59) | def test_active_traces_cleaned_up(self):
  class TestTraceRetentionDuringEvaluation (line 67) | class TestTraceRetentionDuringEvaluation:
    method _evaluation_mode (line 71) | def _evaluation_mode(self):
    method test_traces_retained_during_evaluation (line 78) | def test_traces_retained_during_evaluation(self):

FILE: tests/test_core/test_tracing/apps/async_app.py
  function generate_text (line 12) | async def generate_text(prompt: str):
  function retrieve_documents (line 23) | async def retrieve_documents(query: str, top_k: int = 3):
  function custom_embed (line 37) | async def custom_embed(text: str, model: str = "custom-model"):
  function custom_retrieve (line 43) | async def custom_retrieve(query: str, embedding_model: str = "custom-mod...
  function custom_generate (line 53) | async def custom_generate(prompt: str, model: str = "custom-model"):
  function custom_research_agent (line 59) | async def custom_research_agent(query: str):
  function weather_agent (line 69) | async def weather_agent(query: str):
  function research_agent (line 78) | async def research_agent(query: str):
  function meta_agent (line 90) | async def meta_agent(input: str):

FILE: tests/test_core/test_tracing/apps/sync_app.py
  function generate_text (line 19) | def generate_text(prompt: str):
  function retrieve_documents (line 50) | def retrieve_documents(query: str, top_k: int = 3):
  function custom_embed (line 79) | def custom_embed(text: str, model: str = "text-embedding-ada-002"):
  function custom_retrieve (line 90) | def custom_retrieve(query: str):
  function custom_generate (line 114) | def custom_generate(prompt: str, model: str = "gpt-3.5-turbo"):
  function custom_research_agent (line 146) | def custom_research_agent(query: str):
  function weather_agent (line 168) | def weather_agent(query: str):
  function research_agent (line 209) | def research_agent(query: str):
  function meta_agent (line 238) | def meta_agent(input: str):

FILE: tests/test_core/test_tracing/conftest.py
  function get_schema_path (line 35) | def get_schema_path(schema_name: str) -> str:
  function trace_test (line 40) | def trace_test(schema_name: str):
  function ensure_event_loop (line 56) | def ensure_event_loop():
  function silence_confident_trace (line 71) | def silence_confident_trace(monkeypatch):
  function reset_trace_state (line 82) | def reset_trace_state():
  function get_active_trace_and_span (line 106) | def get_active_trace_and_span():

FILE: tests/test_core/test_tracing/example_e2e_trace_evals.py
  function llm_app (line 26) | def llm_app(input):

FILE: tests/test_core/test_tracing/test_configuration/test_configure_multiple.py
  function dummy_mask (line 5) | def dummy_mask(data):
  class TestConfigureMultiple (line 10) | class TestConfigureMultiple:
    method test_configure_all_options (line 13) | def test_configure_all_options(self, monkeypatch):
    method test_configure_subset_of_options (line 32) | def test_configure_subset_of_options(self, monkeypatch):
    method test_configure_with_invalid_option_fails_atomically (line 60) | def test_configure_with_invalid_option_fails_atomically(self, monkeypa...
    method test_configure_empty_call_does_nothing (line 80) | def test_configure_empty_call_does_nothing(self, monkeypatch):

FILE: tests/test_core/test_tracing/test_configuration/test_environment.py
  class TestEnvironmentInit (line 6) | class TestEnvironmentInit:
    method test_default_environment_is_development (line 9) | def test_default_environment_is_development(self, monkeypatch):
    method test_init_with_production_env_var (line 15) | def test_init_with_production_env_var(self, monkeypatch):
    method test_init_with_staging_env_var (line 21) | def test_init_with_staging_env_var(self, monkeypatch):
    method test_init_with_testing_env_var (line 27) | def test_init_with_testing_env_var(self, monkeypatch):
    method test_init_with_invalid_env_var_raises (line 33) | def test_init_with_invalid_env_var_raises(self, monkeypatch):
  class TestEnvironmentConfigure (line 40) | class TestEnvironmentConfigure:
    method test_configure_production (line 43) | def test_configure_production(self, monkeypatch):
    method test_configure_staging (line 52) | def test_configure_staging(self, monkeypatch):
    method test_configure_testing (line 60) | def test_configure_testing(self, monkeypatch):
    method test_configure_development (line 68) | def test_configure_development(self, monkeypatch):
    method test_configure_invalid_environment_raises (line 77) | def test_configure_invalid_environment_raises(self, monkeypatch):
    method test_configure_none_does_not_change (line 85) | def test_configure_none_does_not_change(self, monkeypatch):
    method test_configure_environment_case_sensitive (line 94) | def test_configure_environment_case_sensitive(self, monkeypatch):
  class TestAllEnvironmentValues (line 106) | class TestAllEnvironmentValues:
    method test_all_valid_environments_init (line 118) | def test_all_valid_environments_init(self, monkeypatch, env_value):
    method test_all_valid_environments_configure (line 133) | def test_all_valid_environments_configure(self, monkeypatch, env_value):

FILE: tests/test_core/test_tracing/test_configuration/test_masking_config.py
  function simple_mask (line 7) | def simple_mask(data):
  function credit_card_mask (line 18) | def credit_card_mask(data):
  function email_mask (line 30) | def email_mask(data):
  class TestMaskingConfiguration (line 42) | class TestMaskingConfiguration:
    method test_no_mask_by_default (line 45) | def test_no_mask_by_default(self, monkeypatch):
    method test_configure_mask_function (line 51) | def test_configure_mask_function(self, monkeypatch):
    method test_configure_mask_to_none_removes_mask (line 59) | def test_configure_mask_to_none_removes_mask(self, monkeypatch):
    method test_mask_function_is_called (line 72) | def test_mask_function_is_called(self, monkeypatch):
    method test_mask_returns_original_when_no_mask (line 81) | def test_mask_returns_original_when_no_mask(self, monkeypatch):
  class TestMaskFunctionBehavior (line 91) | class TestMaskFunctionBehavior:
    method test_credit_card_mask_function (line 94) | def test_credit_card_mask_function(self, monkeypatch):
    method test_email_mask_function (line 103) | def test_email_mask_function(self, monkeypatch):
    method test_mask_nested_dict (line 112) | def test_mask_nested_dict(self, monkeypatch):
    method test_mask_list (line 125) | def test_mask_list(self, monkeypatch):
    method test_mask_non_string_unchanged (line 135) | def test_mask_non_string_unchanged(self, monkeypatch):
  class TestMaskingWithTraces (line 147) | class TestMaskingWithTraces:
    method test_mask_applied_to_input (line 150) | def test_mask_applied_to_input(self, monkeypatch):
    method test_mask_applied_to_output (line 171) | def test_mask_applied_to_output(self, monkeypatch):

FILE: tests/test_core/test_tracing/test_configuration/test_sampling_rate.py
  class TestSamplingRateInit (line 5) | class TestSamplingRateInit:
    method test_default_sampling_rate_is_one (line 8) | def test_default_sampling_rate_is_one(self, monkeypatch):
    method test_init_with_sampling_rate_env_var (line 15) | def test_init_with_sampling_rate_env_var(self, monkeypatch):
    method test_init_with_zero_sampling_rate (line 22) | def test_init_with_zero_sampling_rate(self, monkeypatch):
    method test_init_with_one_sampling_rate (line 29) | def test_init_with_one_sampling_rate(self, monkeypatch):
  class TestSamplingRateConfigure (line 37) | class TestSamplingRateConfigure:
    method test_configure_sampling_rate (line 40) | def test_configure_sampling_rate(self, monkeypatch):
    method test_configure_sampling_rate_zero (line 50) | def test_configure_sampling_rate_zero(self, monkeypatch):
    method test_configure_sampling_rate_one (line 59) | def test_configure_sampling_rate_one(self, monkeypatch):
    method test_configure_invalid_sampling_rate_negative_raises (line 69) | def test_configure_invalid_sampling_rate_negative_raises(self, monkeyp...
    method test_configure_invalid_sampling_rate_above_one_raises (line 78) | def test_configure_invalid_sampling_rate_above_one_raises(
    method test_configure_none_does_not_change (line 89) | def test_configure_none_does_not_change(self, monkeypatch):
  class TestSamplingRateEdgeCases (line 100) | class TestSamplingRateEdgeCases:
    method test_valid_sampling_rates (line 104) | def test_valid_sampling_rates(self, monkeypatch, rate):
    method test_invalid_sampling_rates (line 113) | def test_invalid_sampling_rates(self, monkeypatch, rate):

FILE: tests/test_core/test_tracing/test_generators/test_async_generator.py
  function async_streaming_llm (line 8) | async def async_streaming_llm(prompt: str):
  function async_streaming_processor (line 20) | async def async_streaming_processor(data: str):
  function async_streaming_with_nested (line 28) | async def async_streaming_with_nested(data: str):
  function async_helper (line 36) | async def async_helper(data: str) -> str:
  function async_streaming_with_updates (line 42) | async def async_streaming_with_updates(prompt: str):
  function async_streaming_with_error (line 56) | async def async_streaming_with_error(data: str):
  function async_streaming_concurrent (line 66) | async def async_streaming_concurrent(data: str):
  class TestAsyncGenerator (line 77) | class TestAsyncGenerator:
    method test_async_streaming_llm (line 81) | async def test_async_streaming_llm(self):
    method test_async_streaming_processor (line 88) | async def test_async_streaming_processor(self):
    method test_async_streaming_with_nested (line 95) | async def test_async_streaming_with_nested(self):
    method test_async_streaming_with_updates (line 102) | async def test_async_streaming_with_updates(self):
    method test_async_streaming_error_handling (line 108) | async def test_async_streaming_error_handling(self):
    method test_async_streaming_concurrent (line 118) | async def test_async_streaming_concurrent(self):

FILE: tests/test_core/test_tracing/test_generators/test_fastapi_streaming_repro.py
  function retrieve_documents (line 34) | def retrieve_documents(query: str) -> str:
  function stream_llm_tokens (line 42) | def stream_llm_tokens(query: str, context: str):
  function summarize (line 59) | def summarize(text: str) -> str:
  function transform_text (line 65) | def transform_text(text: str) -> str:
  function streamed_tokens (line 73) | def streamed_tokens(prompt: str):
  function streamed_with_child (line 81) | def streamed_with_child(prompt: str):
  function streaming_rag_pipeline (line 89) | def streaming_rag_pipeline(query: str):
  function multi_step_pipeline (line 103) | def multi_step_pipeline(query: str):
  function streaming_rag_pipeline_plain (line 124) | def streaming_rag_pipeline_plain(query: str):
  function observed_endpoint (line 137) | def observed_endpoint(query: str):
  function trace_wrapped_endpoint (line 141) | def trace_wrapped_endpoint(query: str):
  function trace_wrapped_plain_endpoint (line 146) | def trace_wrapped_plain_endpoint(query: str):
  function _next_or_sentinel (line 164) | def _next_or_sentinel(gen):
  function _iterate_in_threadpool (line 171) | async def _iterate_in_threadpool(gen):
  function run_in_threadpool (line 188) | def run_in_threadpool(gen):
  function _call_then_iterate (line 193) | async def _call_then_iterate(endpoint_fn, prompt):
  function _assert_all_traces_valid (line 207) | def _assert_all_traces_valid(traces):
  function _assert_all_spans_closed (line 218) | def _assert_all_spans_closed(span, trace_idx=0, depth=0):
  class TestFastAPIStreamingRepro (line 231) | class TestFastAPIStreamingRepro:
    method test_simple_generator_across_threadpool (line 234) | def test_simple_generator_across_threadpool(self):
    method test_generator_with_child_span_across_threadpool (line 241) | def test_generator_with_child_span_across_threadpool(self):
    method test_rag_pipeline_across_threadpool (line 248) | def test_rag_pipeline_across_threadpool(self):
    method test_deep_nesting_across_threadpool (line 260) | def test_deep_nesting_across_threadpool(self):
    method test_same_thread_rag_pipeline (line 273) | def test_same_thread_rag_pipeline(self):
  class TestFastAPIStreamingMultiTrace (line 282) | class TestFastAPIStreamingMultiTrace:
    method test_observe_on_both_endpoint_and_generator (line 289) | def test_observe_on_both_endpoint_and_generator(self, completed_traces):
    method test_trace_context_wrapping_observed_generator (line 300) | def test_trace_context_wrapping_observed_generator(self, completed_tra...
    method test_trace_context_with_plain_generator (line 313) | def test_trace_context_with_plain_generator(self, completed_traces):
    method test_single_trace_with_nested_hierarchy (line 326) | def test_single_trace_with_nested_hierarchy(self, completed_traces):

FILE: tests/test_core/test_tracing/test_generators/test_generator_context_safety.py
  function stream_chunks (line 13) | def stream_chunks(message: str):
  function stream_simple (line 22) | def stream_simple(data: str):
  function stream_with_error (line 28) | def stream_with_error(data: str):
  function process_item (line 37) | def process_item(item) -> dict:
  function async_stream_chunks (line 42) | async def async_stream_chunks(message: str):
  function async_stream_simple (line 52) | async def async_stream_simple(data: str):
  function outer_observe_consumes_inner_gen (line 62) | def outer_observe_consumes_inner_gen(message: str):
  function outer_observe_breaks_inner_gen (line 71) | def outer_observe_breaks_inner_gen(message: str):
  function outer_gen_yields_from_inner_gen (line 80) | def outer_gen_yields_from_inner_gen(message: str):
  function outer_gen_breaks_inner_gen (line 89) | def outer_gen_breaks_inner_gen(message: str):
  function outer_gen_calls_regular_observe (line 100) | def outer_gen_calls_regular_observe(data: str):
  function three_level_gen (line 108) | def three_level_gen(data: str):
  function mid_level_gen (line 117) | def mid_level_gen(data: str):
  function sibling_generators (line 126) | def sibling_generators(data: str):
  function async_outer_observe_consumes_inner_gen (line 134) | async def async_outer_observe_consumes_inner_gen(message: str):
  function async_outer_gen_yields_from_inner_gen (line 143) | async def async_outer_gen_yields_from_inner_gen(message: str):
  function async_outer_gen_breaks_inner (line 152) | async def async_outer_gen_breaks_inner(message: str):
  function async_three_level_gen (line 163) | async def async_three_level_gen(data: str):
  function async_mid_level_gen (line 172) | async def async_mid_level_gen(data: str):
  class TestSyncGeneratorContextSafety (line 179) | class TestSyncGeneratorContextSafety:
    method test_thread_boundary (line 181) | def test_thread_boundary(self):
    method test_interleaved_creation (line 203) | def test_interleaved_creation(self):
    method test_run_in_executor (line 217) | def test_run_in_executor(self):
    method test_break_after_final_response (line 234) | def test_break_after_final_response(self):
    method test_next_then_abandon (line 246) | def test_next_then_abandon(self):
    method test_consumer_calls_observe_between_yields (line 257) | def test_consumer_calls_observe_between_yields(self):
    method test_error_still_propagates (line 269) | def test_error_still_propagates(self):
    method test_full_consumption_still_works (line 280) | def test_full_consumption_still_works(self):
    method test_sequential_calls_no_context_leak (line 288) | def test_sequential_calls_no_context_leak(self):
    method test_many_interleaved_generators (line 298) | def test_many_interleaved_generators(self):
  class TestSyncNestedGeneratorContext (line 312) | class TestSyncNestedGeneratorContext:
    method test_observe_consumes_inner_gen (line 315) | def test_observe_consumes_inner_gen(self):
    method test_observe_breaks_inner_gen (line 323) | def test_observe_breaks_inner_gen(self):
    method test_gen_yields_from_inner_gen (line 332) | def test_gen_yields_from_inner_gen(self):
    method test_gen_breaks_inner_gen (line 342) | def test_gen_breaks_inner_gen(self):
    method test_gen_calls_regular_observe_between_yields (line 355) | def test_gen_calls_regular_observe_between_yields(self):
    method test_three_level_nesting (line 365) | def test_three_level_nesting(self):
    method test_sibling_generators (line 380) | def test_sibling_generators(self):
    method test_nested_gen_with_break_at_every_level (line 389) | def test_nested_gen_with_break_at_every_level(self):
    method test_nested_gen_inner_error_propagates (line 400) | def test_nested_gen_inner_error_propagates(self):
  class TestAsyncGeneratorContextSafety (line 419) | class TestAsyncGeneratorContextSafety:
    method test_async_full_consumption (line 423) | async def test_async_full_consumption(self):
    method test_async_break_after_final_response (line 434) | async def test_async_break_after_final_response(self):
    method test_async_interleaved_creation (line 447) | async def test_async_interleaved_creation(self):
    method test_async_sequential_no_leak (line 460) | async def test_async_sequential_no_leak(self):
  class TestAsyncNestedGeneratorContext (line 477) | class TestAsyncNestedGeneratorContext:
    method test_async_observe_consumes_inner_gen (line 481) | async def test_async_observe_consumes_inner_gen(self):
    method test_async_gen_yields_from_inner_gen (line 491) | async def test_async_gen_yields_from_inner_gen(self):
    method test_async_gen_breaks_inner (line 504) | async def test_async_gen_breaks_inner(self):
    method test_async_three_level_nesting (line 518) | async def test_async_three_level_nesting(self):
    method test_async_nested_break_at_every_level (line 535) | async def test_async_nested_break_at_every_level(self):

FILE: tests/test_core/test_tracing/test_generators/test_sync_generator.py
  function streaming_llm (line 7) | def streaming_llm(prompt: str):
  function streaming_processor (line 18) | def streaming_processor(data: str):
  function streaming_with_nested_call (line 25) | def streaming_with_nested_call(data: str):
  function non_streaming_helper (line 33) | def non_streaming_helper(data: str) -> str:
  function streaming_with_updates (line 38) | def streaming_with_updates(prompt: str):
  function streaming_with_error (line 51) | def streaming_with_error(data: str):
  class TestSyncGenerator (line 59) | class TestSyncGenerator:
    method test_streaming_llm (line 62) | def test_streaming_llm(self):
    method test_streaming_processor (line 66) | def test_streaming_processor(self):
    method test_streaming_with_nested (line 70) | def test_streaming_with_nested(self):
    method test_streaming_with_updates (line 74) | def test_streaming_with_updates(self):
    method test_streaming_error_handling (line 77) | def test_streaming_error_handling(self):

FILE: tests/test_core/test_tracing/test_integration/test_current_golden_context.py
  class GoldenStub (line 12) | class GoldenStub:
    method __init__ (line 13) | def __init__(self, name, expected_output=None):
  function test_current_golden_is_task_local_and_resets (line 19) | async def test_current_golden_is_task_local_and_resets():
  function test_task_creation_captures_context (line 56) | async def test_task_creation_captures_context():
  function test_each_task_captures_value_at_creation_time (line 81) | async def test_each_task_captures_value_at_creation_time():
  function test_set_none_restores_previous_on_reset (line 108) | def test_set_none_restores_previous_on_reset():
  function test_gather_sees_per_task_snapshots (line 123) | async def test_gather_sees_per_task_snapshots():

FILE: tests/test_core/test_tracing/test_integration/test_dataset_iterator.py
  function test_async_run_async (line 29) | def test_async_run_async():
  function test_sync_run_async (line 38) | def test_sync_run_async():
  function test_sync_run_sync (line 47) | def test_sync_run_sync():
  function test_no_leftovers_runs_trace_eval (line 56) | def test_no_leftovers_runs_trace_eval(monkeypatch):
  function test_snapshot_tasks_runtimeerror_still_runs_trace_eval (line 117) | def test_snapshot_tasks_runtimeerror_still_runs_trace_eval(monkeypatch):
  function test_closed_loop_skips_trace_eval (line 187) | def test_closed_loop_skips_trace_eval(monkeypatch):
  function _bare_agent (line 268) | def _bare_agent(question: str) -> str:
  function test_no_metrics_error_async_iterator (line 273) | def test_no_metrics_error_async_iterator():
  function test_no_metrics_error_sync_iterator (line 292) | def test_no_metrics_error_sync_iterator():
  function test_no_metrics_error_not_raised_when_top_level_metrics_provided (line 307) | def test_no_metrics_error_not_raised_when_top_level_metrics_provided(
  function test_no_metrics_error_raised_when_span_has_only_metric_collection (line 338) | def test_no_metrics_error_raised_when_span_has_only_metric_collection():

FILE: tests/test_core/test_tracing/test_integration/test_execute_integration.py
  function _bypass_no_metrics_guard (line 27) | def _bypass_no_metrics_guard(monkeypatch):
  function _silence_confident_trace (line 41) | def _silence_confident_trace(monkeypatch):
  function _reset_eval_state (line 52) | def _reset_eval_state():
  function test_execute_propagates_expected_output (line 59) | def test_execute_propagates_expected_output(
  function test_trace_uses_test_case_expected_output_when_present (line 129) | def test_trace_uses_test_case_expected_output_when_present():
  function test_trace_kwarg_expected_output_overrides_test_case (line 140) | def test_trace_kwarg_expected_output_overrides_test_case():
  function test_trace_expected_output_remains_none_when_unset (line 154) | def test_trace_expected_output_remains_none_when_unset():
  function test_span_kwarg_expected_output_overrides_test_case (line 165) | def test_span_kwarg_expected_output_overrides_test_case():
  function test_span_expected_output_remains_none_when_unset (line 183) | def test_span_expected_output_remains_none_when_unset():
  function test_noop_when_no_active_trace_or_span (line 194) | def test_noop_when_no_active_trace_or_span():
  function test_async_evaluator_skips_empty_traces_without_crash (line 202) | def test_async_evaluator_skips_empty_traces_without_crash(
  function test_async_evaluator_handles_extra_traces_with_spans (line 245) | def test_async_evaluator_handles_extra_traces_with_spans(

FILE: tests/test_core/test_tracing/test_integration/test_tools_called.py
  function level_1 (line 6) | def level_1():
  function test_tools_called_propogation (line 26) | async def test_tools_called_propogation():

FILE: tests/test_core/test_tracing/test_masking/test_masking.py
  function mask_credit_cards (line 7) | def mask_credit_cards(data):
  function mask_emails (line 18) | def mask_emails(data):
  function comprehensive_mask (line 29) | def comprehensive_mask(data):
  function process_with_credit_card (line 36) | def process_with_credit_card(user_input: str) -> str:
  function process_with_email (line 41) | def process_with_email(user_input: str) -> str:
  function process_sensitive_data (line 46) | def process_sensitive_data(data: dict) -> dict:
  function process_unmasked (line 51) | def process_unmasked(data: str) -> str:
  class TestMasking (line 55) | class TestMasking:
    method test_credit_card_masking (line 58) | def test_credit_card_masking(self):
    method test_email_masking (line 67) | def test_email_masking(self):
    method test_comprehensive_masking (line 76) | def test_comprehensive_masking(self):
    method test_no_masking_by_default (line 90) | def test_no_masking_by_default(self):
    method test_masking_function_helpers (line 95) | def test_masking_function_helpers(self):

FILE: tests/test_core/test_tracing/test_metadata/test_span_metadata.py
  function span_with_metadata (line 6) | def span_with_metadata(data: str) -> str:
  function span_with_complex_metadata (line 18) | def span_with_complex_metadata(data: str) -> str:
  function llm_with_metadata (line 37) | def llm_with_metadata(prompt: str) -> str:
  function agent_with_metadata (line 48) | def agent_with_metadata(query: str) -> str:
  class TestSpanMetadata (line 59) | class TestSpanMetadata:
    method test_basic_metadata (line 62) | def test_basic_metadata(self):
    method test_complex_metadata (line 66) | def test_complex_metadata(self):
    method test_llm_with_metadata (line 70) | def test_llm_with_metadata(self):
    method test_agent_with_metadata (line 74) | def test_agent_with_metadata(self):

FILE: tests/test_core/test_tracing/test_metadata/test_trace_metadata.py
  function trace_with_metadata (line 6) | def trace_with_metadata(data: str) -> str:
  function trace_with_user_info (line 18) | def trace_with_user_info(data: str) -> str:
  function trace_with_full_context (line 31) | def trace_with_full_context(query: str) -> str:
  function outer_function (line 46) | def outer_function(data: str) -> str:
  function inner_function (line 52) | def inner_function(data: str) -> str:
  class TestTraceMetadata (line 56) | class TestTraceMetadata:
    method test_basic_trace_metadata (line 59) | def test_basic_trace_metadata(self):
    method test_trace_with_user_info (line 63) | def test_trace_with_user_info(self):
    method test_trace_full_context (line 67) | def test_trace_full_context(self):
    method test_trace_metadata_persists (line 71) | def test_trace_metadata_persists(self):

FILE: tests/test_core/test_tracing/test_nested_spans/test_nested_spans.py
  function parent_function (line 8) | def parent_function(data: str) -> str:
  function child_function (line 14) | def child_function(data: str) -> str:
  function agent_workflow (line 19) | def agent_workflow(query: str) -> str:
  function retriever_step (line 26) | def retriever_step(query: str) -> str:
  function llm_step (line 31) | def llm_step(context: str) -> str:
  function deep_nesting_level_1 (line 36) | def deep_nesting_level_1(data: str) -> str:
  function deep_nesting_level_2 (line 41) | def deep_nesting_level_2(data: str) -> str:
  function deep_nesting_level_3 (line 46) | def deep_nesting_level_3(data: str) -> str:
  function parent_with_multiple_children (line 51) | def parent_with_multiple_children(data: str) -> str:
  function first_child (line 59) | def first_child(data: str) -> str:
  function second_child (line 64) | def second_child(data: str) -> str:
  function third_child (line 69) | def third_child(data: str) -> str:
  function async_agent (line 74) | async def async_agent(query: str) -> str:
  function async_retrieve (line 81) | async def async_retrieve(query: str) -> str:
  function async_generate (line 87) | async def async_generate(context: str) -> str:
  class TestNestedSpans (line 92) | class TestNestedSpans:
    method test_simple_parent_child (line 95) | def test_simple_parent_child(self):
    method test_agent_workflow_nesting (line 99) | def test_agent_workflow_nesting(self):
    method test_deep_nesting (line 103) | def test_deep_nesting(self):
    method test_multiple_children (line 107) | def test_multiple_children(self):
    method test_async_nesting (line 112) | async def test_async_nesting(self):

FILE: tests/test_core/test_tracing/test_span_types/test_agent_span.py
  function simple_agent (line 6) | def simple_agent(query: str) -> str:
  function agent_with_handoffs (line 15) | def agent_with_handoffs(query: str) -> str:
  function minimal_agent (line 20) | def minimal_agent(query: str) -> str:
  function agent_with_custom_name (line 25) | def agent_with_custom_name(query: str) -> str:
  function agent_multiple_handoffs (line 30) | def agent_multiple_handoffs(query: str) -> str:
  function agent_full_attributes (line 39) | def agent_full_attributes(query: str) -> str:
  class TestAgentSpan (line 43) | class TestAgentSpan:
    method test_agent_with_tools (line 46) | def test_agent_with_tools(self):
    method test_agent_with_handoffs (line 50) | def test_agent_with_handoffs(self):
    method test_minimal_agent (line 54) | def test_minimal_agent(self):
    method test_agent_with_custom_name (line 58) | def test_agent_with_custom_name(self):
    method test_agent_multiple_handoffs (line 62) | def test_agent_multiple_handoffs(self):
    method test_agent_full_attributes (line 66) | def test_agent_full_attributes(self):

FILE: tests/test_core/test_tracing/test_span_types/test_custom_span.py
  function custom_processor (line 6) | def custom_processor(data: str) -> str:
  function custom_with_name (line 11) | def custom_with_name(data: dict) -> dict:
  function custom_validator (line 16) | def custom_validator(value: str) -> bool:
  function default_span (line 21) | def default_span(input_data: str) -> str:
  function span_with_only_name (line 26) | def span_with_only_name(data: str) -> str:
  class TestCustomSpan (line 30) | class TestCustomSpan:
    method test_custom_processor (line 33) | def test_custom_processor(self):
    method test_custom_with_name (line 37) | def test_custom_with_name(self):
    method test_custom_validator (line 41) | def test_custom_validator(self):
    method test_default_span (line 45) | def test_default_span(self):
    method test_span_with_only_name (line 49) | def test_span_with_only_name(self):

FILE: tests/test_core/test_tracing/test_span_types/test_llm_span.py
  function llm_generation (line 6) | def llm_generation(prompt: str) -> str:
  function llm_with_costs (line 21) | def llm_with_costs(prompt: str) -> str:
  function llm_minimal (line 31) | def llm_minimal(prompt: str) -> str:
  function llm_with_custom_name (line 36) | def llm_with_custom_name(prompt: str) -> str:
  function llm_with_full_attributes (line 41) | def llm_with_full_attributes(prompt: str) -> str:
  function llm_set_model_at_runtime (line 54) | def llm_set_model_at_runtime(prompt: str) -> str:
  class TestLlmSpan (line 64) | class TestLlmSpan:
    method test_llm_generation (line 67) | def test_llm_generation(self):
    method test_llm_with_costs (line 71) | def test_llm_with_costs(self):
    method test_llm_minimal (line 75) | def test_llm_minimal(self):
    method test_llm_with_custom_name (line 79) | def test_llm_with_custom_name(self):
    method test_llm_full_attributes (line 83) | def test_llm_full_attributes(self):
    method test_llm_set_model_at_runtime (line 87) | def test_llm_set_model_at_runtime(self):

FILE: tests/test_core/test_tracing/test_span_types/test_retriever_span.py
  function retrieve_documents (line 6) | def retrieve_documents(query: str, top_k: int = 5) -> list:
  function retrieve_with_custom_embedder (line 16) | def retrieve_with_custom_embedder(query: str) -> list:
  function retrieve_minimal (line 26) | def retrieve_minimal(query: str) -> list:
  function retriever_with_custom_name (line 31) | def retriever_with_custom_name(query: str) -> list:
  function retriever_full_attributes (line 36) | def retriever_full_attributes(query: str) -> list:
  function retriever_override_embedder (line 47) | def retriever_override_embedder(query: str) -> list:
  class TestRetrieverSpan (line 53) | class TestRetrieverSpan:
    method test_retriever_with_embedder (line 56) | def test_retriever_with_embedder(self):
    method test_retriever_custom_embedder (line 60) | def test_retriever_custom_embedder(self):
    method test_retriever_minimal (line 64) | def test_retriever_minimal(self):
    method test_retriever_with_custom_name (line 68) | def test_retriever_with_custom_name(self):
    method test_retriever_full_attributes (line 72) | def test_retriever_full_attributes(self):
    method test_retriever_override_embedder (line 76) | def test_retriever_override_embedder(self):

FILE: tests/test_core/test_tracing/test_span_types/test_tool_span.py
  function web_search (line 6) | def web_search(query: str) -> str:
  function calculator (line 11) | def calculator(expression: str) -> float:
  function tool_with_custom_name (line 16) | def tool_with_custom_name(data: str) -> str:
  function minimal_tool (line 21) | def minimal_tool(input_data: str) -> str:
  function tool_with_description_and_name (line 26) | def tool_with_description_and_name(url: str) -> dict:
  function tool_with_long_description (line 34) | def tool_with_long_description(data: str) -> str:
  class TestToolSpan (line 38) | class TestToolSpan:
    method test_tool_with_description (line 41) | def test_tool_with_description(self):
    method test_calculator_tool (line 45) | def test_calculator_tool(self):
    method test_tool_with_custom_name (line 49) | def test_tool_with_custom_name(self):
    method test_minimal_tool (line 53) | def test_minimal_tool(self):
    method test_tool_with_description_and_name (line 57) | def test_tool_with_description_and_name(self):
    method test_tool_with_long_description (line 61) | def test_tool_with_long_description(self):

FILE: tests/test_core/test_tracing/test_tags/test_trace_tags.py
  function trace_with_tags (line 6) | def trace_with_tags(data: str) -> str:
  function trace_with_env_tags (line 12) | def trace_with_env_tags(data: str, env: str = "dev") -> str:
  function trace_with_feature_tags (line 18) | def trace_with_feature_tags(query: str, features: list = None) -> str:
  function trace_with_name_and_tags (line 27) | def trace_with_name_and_tags(data: str) -> str:
  class TestTraceTags (line 34) | class TestTraceTags:
    method test_basic_tags (line 37) | def test_basic_tags(self):
    method test_environment_tags (line 41) | def test_environment_tags(self):
    method test_feature_tags (line 45) | def test_feature_tags(self):
    method test_name_and_tags (line 49) | def test_name_and_tags(self):

FILE: tests/test_core/test_tracing/test_update_functions/test_update_current_span.py
  function span_update_input_output (line 7) | def span_update_input_output(data: str) -> str:
  function span_update_context (line 16) | def span_update_context(query: str) -> str:
  function span_update_expected_output (line 25) | def span_update_expected_output(query: str) -> str:
  function span_update_tools (line 31) | def span_update_tools(query: str) -> str:
  function span_update_name (line 43) | def span_update_name(data: str) -> str:
  function span_from_test_case (line 49) | def span_from_test_case(data: str) -> str:
  function span_override_test_case (line 61) | def span_override_test_case(data: str) -> str:
  class TestUpdateCurrentSpan (line 72) | class TestUpdateCurrentSpan:
    method test_update_input_output (line 75) | def test_update_input_output(self):
    method test_update_context (line 79) | def test_update_context(self):
    method test_update_expected_output (line 83) | def test_update_expected_output(self):
    method test_update_tools (line 87) | def test_update_tools(self):
    method test_update_name (line 91) | def test_update_name(self):
    method test_from_test_case (line 95) | def test_from_test_case(self):
    method test_override_test_case (line 99) | def test_override_test_case(self):

FILE: tests/test_core/test_tracing/test_update_functions/test_update_current_trace.py
  function trace_update_name (line 7) | def trace_update_name(data: str) -> str:
  function trace_update_identifiers (line 13) | def trace_update_identifiers(data: str) -> str:
  function trace_update_all_context (line 22) | def trace_update_all_context(query: str) -> str:
  function trace_update_context_info (line 36) | def trace_update_context_info(query: str) -> str:
  function trace_update_tools (line 46) | def trace_update_tools(query: str) -> str:
  function trace_from_test_case (line 55) | def trace_from_test_case(data: str) -> str:
  function outer_sets_trace_context (line 67) | def outer_sets_trace_context(data: str) -> str:
  function inner_reads_context (line 73) | def inner_reads_context(data: str) -> str:
  class TestUpdateCurrentTrace (line 78) | class TestUpdateCurrentTrace:
    method test_update_name (line 81) | def test_update_name(self):
    method test_update_identifiers (line 85) | def test_update_identifiers(self):
    method test_update_all_context (line 89) | def test_update_all_context(self):
    method test_update_context_info (line 93) | def test_update_context_info(self):
    method test_update_tools (line 97) | def test_update_tools(self):
    method test_from_test_case (line 101) | def test_from_test_case(self):
    method test_nested_trace_updates (line 105) | def test_nested_trace_updates(self):

FILE: tests/test_core/test_utils.py
  function test_read_env_int_valid (line 12) | def test_read_env_int_valid(monkeypatch):
  function test_read_env_int_invalid (line 17) | def test_read_env_int_invalid(monkeypatch):
  function test_read_env_int_min (line 22) | def test_read_env_int_min(monkeypatch):
  function test_read_env_float_valid (line 27) | def test_read_env_float_valid(monkeypatch):
  function test_read_env_float_invalid (line 32) | def test_read_env_float_invalid(monkeypatch):
  function test_read_env_float_min (line 37) | def test_read_env_float_min(monkeypatch):
  function test_dynamic_stop_env_override (line 42) | def test_dynamic_stop_env_override(monkeypatch, settings):
  function test_dynamic_wait_env_override (line 72) | def test_dynamic_wait_env_override(monkeypatch, settings):
  function test_shorten_basic (line 131) | def test_shorten_basic(text, max_len, expected):
  function test_shorten_zero_len (line 135) | def test_shorten_zero_len():
  function test_shorten_suffix_longer_than_max (line 139) | def test_shorten_suffix_longer_than_max():
  function test_shorten_non_string_input (line 144) | def test_shorten_non_string_input():
  function md (line 153) | def md(**kw):
  function test_is_metric_successful_priority_error_over_success (line 157) | def test_is_metric_successful_priority_error_over_success():
  function test_is_metric_successful_bool (line 161) | def test_is_metric_successful_bool():
  function test_is_metric_successful_none_and_missing (line 166) | def test_is_metric_successful_none_and_missing():
  function test_is_metric_successful_numeric_and_string (line 171) | def test_is_metric_successful_numeric_and_string():
  function _task (line 185) | def _task(*, task_id, finished, remaining=0):
  function test_update_pbar_noops_when_task_id_missing (line 189) | def test_update_pbar_noops_when_task_id_missing():
  function test_update_pbar_noops_when_task_removed_between_callbacks (line 204) | def test_update_pbar_noops_when_task_removed_between_callbacks():

FILE: tests/test_docs/test_confident/test_integrations/test_anthropic.py
  function generate_response (line 10) | def generate_response(input: str) -> str:
  function generate_response2 (line 34) | def generate_response2(input: str) -> str:

FILE: tests/test_docs/test_confident/test_integrations/test_langchain.py
  function multiply (line 6) | def multiply(a: int, b: int) -> int:

FILE: tests/test_docs/test_confident/test_integrations/test_langgraph.py
  function get_weather (line 12) | def get_weather(city: str) -> str:

FILE: tests/test_docs/test_confident/test_integrations/test_litellm.py
  function llm (line 22) | def llm(input: str) -> str:
  function my_app (line 44) | def my_app(input: str):

FILE: tests/test_docs/test_confident/test_integrations/test_openai.py
  function generate_response (line 10) | def generate_response(input: str) -> str:
  function generate_response (line 30) | def generate_response(input: str) -> str:

FILE: tests/test_docs/test_confident/test_tracing_features/test_attributes.py
  function generate_response (line 5) | def generate_response(prompt):
  function retrieve_documents (line 24) | def retrieve_documents(query):

FILE: tests/test_docs/test_confident/test_tracing_features/test_environment.py
  function llm_app (line 9) | def llm_app(query: str):

FILE: tests/test_docs/test_confident/test_tracing_features/test_input_output.py
  function llm_app (line 8) | def llm_app(query: str):
  function llm_app (line 32) | def llm_app(query: str):

FILE: tests/test_docs/test_confident/test_tracing_features/test_masking.py
  function masking_function (line 5) | def masking_function(data):
  function llm_app (line 16) | def llm_app(query: str):

FILE: tests/test_docs/test_confident/test_tracing_features/test_metadata.py
  function llm_app (line 5) | async def llm_app(query: str):

FILE: tests/test_docs/test_confident/test_tracing_features/test_name.py
  function llm_app (line 8) | def llm_app(query: str):
  function llm_app (line 30) | def llm_app(query: str):

FILE: tests/test_docs/test_confident/test_tracing_features/test_sampling.py
  function llm_app (line 9) | def llm_app(query: str):

FILE: tests/test_docs/test_confident/test_tracing_features/test_span_types.py
  function rag_pipeline (line 5) | def rag_pipeline(query: str) -> str:
  function generate_response (line 15) | def generate_response(prompt: str) -> str:
  function retrieve_documents (line 26) | def retrieve_documents(query: str) -> List[str]:
  function web_search (line 36) | def web_search(query: str) -> str:
  function supervisor_agent (line 52) | def supervisor_agent(query: str) -> str:
  function web_search (line 67) | def web_search(query: str) -> str:
  function retrieve_documents (line 74) | def retrieve_documents(query: str) -> List[str]:
  function generate_response (line 91) | def generate_response(input: str) -> str:
  function rag_pipeline (line 100) | def rag_pipeline(query: str) -> str:
  function research_agent (line 112) | def research_agent(query: str) -> str:

FILE: tests/test_docs/test_confident/test_tracing_features/test_tags.py
  function llm_app (line 8) | def llm_app(query: str):

FILE: tests/test_docs/test_confident/test_tracing_features/test_test_case.py
  function llm_app (line 6) | def llm_app(query: str):
  function retriever_component (line 28) | def retriever_component(query: str):
  function llm_app (line 35) | def llm_app(query: str):

FILE: tests/test_docs/test_confident/test_tracing_features/test_threads.py
  function llm_app (line 8) | def llm_app(query: str):
  function llm_app (line 32) | def llm_app(query: str):

FILE: tests/test_docs/test_confident/test_tracing_features/test_users.py
  function llm_app (line 8) | def llm_app(query: str):

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_crewai/test_crewai_e2e.py
  function get_weather (line 16) | def get_weather(city: str) -> str:
  function run_crewai_e2e_async (line 55) | async def run_crewai_e2e_async(input: str):

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_e2e_langchain.py
  function multiply (line 8) | def multiply(a: int, b: int) -> int:

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_e2e_langraph.py
  function get_weather (line 8) | def get_weather(city: str) -> str:

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_langchain/test_langchain_e2e_async.py
  function multiply (line 16) | def multiply(a: int, b: int) -> int:

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_langchain/test_langchain_e2e_sync.py
  function multiply (line 15) | def multiply(a: int, b: int) -> int:
  function llm_agent_eval (line 49) | def llm_agent_eval(golden: Golden):

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_langchain/test_langchain_online_evals.py
  function multiply (line 11) | def multiply(a: int, b: int) -> int:

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_langchain/test_langchain_setup.py
  function multiply (line 15) | def multiply(a: int, b: int) -> int:

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_langgraph/test_e2e_langgraph_async.py
  function get_weather (line 8) | def get_weather(city: str) -> str:

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_langgraph/test_e2e_langgraph_sync.py
  function get_weather (line 7) | def get_weather(city: str) -> str:

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_langgraph/test_langgraph_component.py
  function get_weather (line 15) | def get_weather(location: str) -> str:
  class State (line 31) | class State(TypedDict):
  function call_llm (line 45) | def call_llm(state: State):
  function call_tools (line 53) | def call_tools(state: State):
  function route_messages (line 70) | def route_messages(state: State):

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_langgraph/test_langgraph_online.py
  function get_weather (line 7) | def get_weather(city: str) -> str:

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_langgraph/test_langgraph_setup.py
  function get_weather (line 7) | def get_weather(city: str) -> str:

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_llamaindex/test_llamaindex_e2e_async.py
  function multiply (line 15) | def multiply(a: float, b: float) -> float:
  function llm_app (line 29) | async def llm_app(input: str):

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_overall.py
  function web_search_tool (line 29) | def web_search_tool(web_query):
  function llm_component (line 35) | def llm_component(query):
  function your_ai_agent (line 60) | def your_ai_agent(query: str) -> str:

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_pydanticai/test_pydanticai_component_level.py
  class Deps (line 18) | class Deps:
  class LatLng (line 30) | class LatLng(BaseModel):
  function get_lat_lng (line 36) | async def get_lat_lng(
  function get_weather (line 62) | async def get_weather(
  function run_agent (line 91) | async def run_agent(input_query: str):

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_setup_end_to_end_python.py
  function your_ai_agent_tool (line 10) | def your_ai_agent_tool():
  function your_ai_agent (line 16) | def your_ai_agent(input):

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_setup_langchain.py
  function multiply (line 5) | def multiply(a: int, b: int) -> int:

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_setup_langraph.py
  function get_weather (line 5) | def get_weather(city: str) -> str:

FILE: tests/test_docs/test_deepeval/test_ai_agent_evals/test_setup_llm_tracing.py
  function your_ai_agent_tool (line 5) | def your_ai_agent_tool():
  function your_ai_agent (line 10) | def your_ai_agent(input):

FILE: tests/test_docs/test_deepeval/test_llm_evals/test_component_level_evals.py
  function your_llm_app (line 11) | def your_llm_app(input: str):

FILE: tests/test_docs/test_deepeval/test_llm_evals/test_setup_tracing.py
  function complete (line 10) | def complete(query: str):
  function web_search (line 36) | def web_search(query: str) -> str:
  function retrieve_documents (line 40) | def retrieve_documents(query: str) -> List[str]:
  function generate_response (line 45) | def generate_response(input: str) -> str:
  function rag_pipeline (line 55) | def rag_pipeline(query: str) -> str:
  function research_agent (line 70) | def research_agent(query: str) -> str:

FILE: tests/test_integrations/test_agentcore/apps/agentcore_eval_app.py
  function special_tool (line 28) | def special_tool(query: str) -> str:
  function init_evals_agentcore (line 40) | def init_evals_agentcore(
  function invoke_evals_agent (line 83) | def invoke_evals_agent(prompt: str, invoke_func=None) -> str:
  function ainvoke_evals_agent (line 90) | async def ainvoke_evals_agent(prompt: str, invoke_func=None) -> str:

FILE: tests/test_integrations/test_agentcore/apps/agentcore_multiple_tools_app.py
  function get_weather (line 8) | def get_weather(city: str) -> str:
  function get_time (line 21) | def get_time(city: str) -> str:
  function init_multiple_tools_agentcore (line 31) | def init_multiple_tools_agentcore(
  function invoke_multiple_tools_agent (line 81) | def invoke_multiple_tools_agent(prompt: str, invoke_func=None) -> str:
  function ainvoke_multiple_tools_agent (line 88) | async def ainvoke_multiple_tools_agent(prompt: str, invoke_func=None) ->...

FILE: tests/test_integrations/test_agentcore/apps/agentcore_simple_app.py
  function init_simple_agentcore (line 6) | def init_simple_agentcore(
  function invoke_simple_agent (line 51) | def invoke_simple_agent(prompt: str, invoke_func=None) -> str:
  function ainvoke_simple_agent (line 58) | async def ainvoke_simple_agent(prompt: str, invoke_func=None) -> str:

FILE: tests/test_integrations/test_agentcore/apps/agentcore_tool_app.py
  function calculate (line 8) | def calculate(operation: str, a: float, b: float) -> float:
  function init_tool_agentcore (line 22) | def init_tool_agentcore(
  function invoke_tool_agent (line 64) | def invoke_tool_agent(prompt: str, invoke_func=None) -> str:
  function ainvoke_tool_agent (line 71) | async def ainvoke_tool_agent(prompt: str, invoke_func=None) -> str:

FILE: tests/test_integrations/test_agentcore/conftest.py
  function deepeval_isolated_no_disk (line 7) | def deepeval_isolated_no_disk(tmp_path, monkeypatch):

FILE: tests/test_integrations/test_agentcore/test_async.py
  function trace_test (line 39) | def trace_test(schema_name: str):
  class TestAsyncSimpleApp (line 47) | class TestAsyncSimpleApp:
    method test_async_simple_greeting (line 51) | async def test_async_simple_greeting(self):
  class TestAsyncToolApp (line 69) | class TestAsyncToolApp:
    method test_async_tool_calculation (line 73) | async def test_async_tool_calculation(self):
  class TestAsyncMultipleToolsApp (line 91) | class TestAsyncMultipleToolsApp:
    method test_async_parallel_tool_calls (line 95) | async def test_async_parallel_tool_calls(self):
  class TestDeepEvalFeaturesAsync (line 115) | class TestDeepEvalFeaturesAsync:
    method test_full_features_async (line 124) | async def test_full_features_async(self):

FILE: tests/test_integrations/test_agentcore/test_evaluate_agent.py
  function test_evaluate_agent (line 51) | def test_evaluate_agent():

FILE: tests/test_integrations/test_agentcore/test_span_interceptor.py
  function _make_mock_span (line 56) | def _make_mock_span(
  function _make_settings (line 102) | def _make_settings(**kwargs):
  function _make_agent_span_mock (line 127) | def _make_agent_span_mock(agent_name: str = "agent_x"):
  class TestTraceContextReads (line 138) | class TestTraceContextReads:
    method test_uses_settings_when_no_trace_context (line 139) | def test_uses_settings_when_no_trace_context(self):
    method test_prefers_trace_context_over_settings_for_scalars (line 167) | def test_prefers_trace_context_over_settings_for_scalars(self):
    method test_metadata_is_merged_with_context_winning (line 182) | def test_metadata_is_merged_with_context_winning(self):
    method test_update_current_trace_after_on_start_lands_on_otel_attrs (line 198) | def test_update_current_trace_after_on_start_lands_on_otel_attrs(self):
    method test_trace_metric_collection_resolution_order (line 228) | def test_trace_metric_collection_resolution_order(self):
  class TestSpanContextPushPop (line 247) | class TestSpanContextPushPop:
    method test_current_span_context_set_during_span_lifetime (line 248) | def test_current_span_context_set_during_span_lifetime(self):
    method test_update_current_span_metadata_lands_in_otel_attrs (line 264) | def test_update_current_span_metadata_lands_in_otel_attrs(self):
    method test_update_current_span_metric_collection_lands_in_otel_attrs (line 287) | def test_update_current_span_metric_collection_lands_in_otel_attrs(self):
    method test_nested_spans_lifo_pop_restores_parent_placeholder (line 305) | def test_nested_spans_lifo_pop_restores_parent_placeholder(self):
  class TestImplicitTraceContext (line 329) | class TestImplicitTraceContext:
    method test_root_span_pushes_implicit_trace_when_no_user_context (line 339) | def test_root_span_pushes_implicit_trace_when_no_user_context(self):
    method test_does_not_overwrite_user_pushed_trace_context (line 357) | def test_does_not_overwrite_user_pushed_trace_context(self):
    method test_child_span_does_not_push_its_own_placeholder (line 375) | def test_child_span_does_not_push_its_own_placeholder(self):
    method test_update_current_trace_in_implicit_context_lands_on_otel_attrs (line 398) | def test_update_current_trace_in_implicit_context_lands_on_otel_attrs(
  class TestParentBridge (line 433) | class TestParentBridge:
    method test_stamps_parent_uuid_when_enclosed_in_deepeval_span (line 434) | def test_stamps_parent_uuid_when_enclosed_in_deepeval_span(self):
    method test_no_parent_uuid_when_otel_span_has_native_parent (line 465) | def test_no_parent_uuid_when_otel_span_has_native_parent(self):
  class TestNextSpanInterceptorIntegration (line 495) | class TestNextSpanInterceptorIntegration:
    method test_next_agent_span_metric_collection_lands_on_otel_attrs (line 496) | def test_next_agent_span_metric_collection_lands_on_otel_attrs(self):
    method test_next_agent_span_consumed_only_by_first_agent_span (line 510) | def test_next_agent_span_consumed_only_by_first_agent_span(self):
    method test_next_agent_span_does_not_affect_non_agent_span (line 531) | def test_next_agent_span_does_not_affect_non_agent_span(self):
    method test_next_tool_span_metric_collection_lands_on_tool_otel_attrs (line 555) | def test_next_tool_span_metric_collection_lands_on_tool_otel_attrs(self):
    method test_update_current_span_overrides_next_agent_span_after_creation (line 574) | def test_update_current_span_overrides_next_agent_span_after_creation(
    method test_next_agent_span_metrics_stashed_when_evaluating (line 594) | def test_next_agent_span_metrics_stashed_when_evaluating(self):
    method test_next_agent_span_metrics_not_stashed_outside_eval_mode (line 628) | def test_next_agent_span_metrics_not_stashed_outside_eval_mode(self):
  function test_removed_kwargs_raise_typeerror_on_settings (line 669) | def test_removed_kwargs_raise_typeerror_on_settings(kwarg):
  function test_removed_kwargs_raise_typeerror_on_instrument_agentcore (line 693) | def test_removed_kwargs_raise_typeerror_on_instrument_agentcore(kwarg):
  function test_settings_no_api_key_does_not_raise (line 709) | def test_settings_no_api_key_does_not_raise(monkeypatch):

FILE: tests/test_integrations/test_agentcore/test_sync.py
  function trace_test (line 39) | def trace_test(schema_name: str):
  class TestSimpleApp (line 47) | class TestSimpleApp:
    method test_simple_greeting (line 50) | def test_simple_greeting(self):
  class TestToolApp (line 68) | class TestToolApp:
    method test_tool_calculation (line 71) | def test_tool_calculation(self):
    method test_tool_metric_collection (line 89) | def test_tool_metric_collection(self):
  class TestMultipleToolsApp (line 116) | class TestMultipleToolsApp:
    method test_multiple_tools_weather_only (line 119) | def test_multiple_tools_weather_only(self):
    method test_multiple_tools_time_only (line 137) | def test_multiple_tools_time_only(self):
    method test_parallel_tool_calls (line 155) | def test_parallel_tool_calls(self):
  class TestDeepEvalFeatures (line 175) | class TestDeepEvalFeatures:
    method test_full_features_sync (line 189) | def test_full_features_sync(self):

FILE: tests/test_integrations/test_anthropic/conftest.py
  function _setup_anthropic_instrumentation (line 5) | def _setup_anthropic_instrumentation():

FILE: tests/test_integrations/test_anthropic/simple_anthropic.py
  function run_async_anthropic (line 29) | async def run_async_anthropic():

FILE: tests/test_integrations/test_anthropic/test_async_anthropic.py
  function test_async_messages_create_without_trace (line 18) | async def test_async_messages_create_without_trace():
  function test_async_messages_create_with_trace (line 32) | async def test_async_messages_create_with_trace():
  function generate_all_json_dumps (line 52) | async def generate_all_json_dumps():

FILE: tests/test_integrations/test_anthropic/test_sync_anthropic.py
  function test_sync_messages_create_without_trace (line 22) | def test_sync_messages_create_without_trace():
  function test_sync_messages_create_with_trace (line 32) | def test_sync_messages_create_with_trace():
  function generate_all_json_dumps (line 52) | def generate_all_json_dumps():

FILE: tests/test_integrations/test_crewai/apps/async_app.py
  function get_async_app (line 13) | def get_async_app():

FILE: tests/test_integrations/test_crewai/apps/evals_app.py
  function special_metric_tool (line 12) | def special_metric_tool(query: str) -> str:
  function get_evals_crew (line 17) | def get_evals_crew():

FILE: tests/test_integrations/test_crewai/apps/hierarchical_app.py
  function get_hierarchical_app (line 9) | def get_hierarchical_app():

FILE: tests/test_integrations/test_crewai/apps/knowledge_retriever_app.py
  function get_knowledge_app (line 12) | def get_knowledge_app():

FILE: tests/test_integrations/test_crewai/apps/multi_agent_app.py
  function get_multi_agent_app (line 9) | def get_multi_agent_app():

FILE: tests/test_integrations/test_crewai/apps/simple_app.py
  function get_simple_app (line 12) | def get_simple_app(id_suffix: str = ""):

FILE: tests/test_integrations/test_crewai/apps/tool_usage_app.py
  function get_weather (line 11) | def get_weather(city: str) -> str:
  function get_tool_usage_app (line 22) | def get_tool_usage_app():

FILE: tests/test_integrations/test_crewai/conftest.py
  function _setup_crewai_instrumentation (line 13) | def _setup_crewai_instrumentation():
  function _clear_traces_between_tests (line 20) | def _clear_traces_between_tests():

FILE: tests/test_integrations/test_crewai/test_async.py
  function trace_test (line 49) | def trace_test(schema_name: str):
  class TestCrewAIAsync (line 60) | class TestCrewAIAsync:
    method test_async_kickoff (line 65) | async def test_async_kickoff(self):
    method test_async_tool_usage (line 73) | async def test_async_tool_usage(self):
    method test_kickoff_for_each_async (line 81) | async def test_kickoff_for_each_async(self):
    method test_akickoff_alias (line 90) | async def test_akickoff_alias(self):
    method test_features_async (line 105) | async def test_features_async(self):
    method reset_instrumentation (line 125) | def reset_instrumentation(self):

FILE: tests/test_integrations/test_crewai/test_crewai.py
  function get_weather (line 23) | def get_weather(city: str) -> str:
  function test_crewai (line 68) | def test_crewai():

FILE: tests/test_integrations/test_crewai/test_crewai_component.py
  function get_weather (line 23) | def get_weather(city: str) -> str:
  function test_crewai_component (line 68) | def test_crewai_component():

FILE: tests/test_integrations/test_crewai/test_knowledge_retriever.py
  function test_knowledge_retriever (line 17) | def test_knowledge_retriever():

FILE: tests/test_integrations/test_crewai/test_stress.py
  function test_concurrent_crews_isolation (line 17) | async def test_concurrent_crews_isolation():

FILE: tests/test_integrations/test_crewai/test_sync.py
  function trace_test (line 58) | def trace_test(schema_name: str):
  class TestCrewAISync (line 69) | class TestCrewAISync:
    method test_simple_kickoff (line 73) | def test_simple_kickoff(self):
    method test_multi_agent_flow (line 80) | def test_multi_agent_flow(self):
    method test_tool_usage (line 88) | def test_tool_usage(self):
    method test_knowledge_retrieval (line 95) | def test_knowledge_retrieval(self):
    method test_hierarchical_process (line 102) | def test_hierarchical_process(self):
    method test_kickoff_for_each (line 111) | def test_kickoff_for_each(self):
    method test_features_sync (line 119) | def test_features_sync(self):
    method reset_instrumentation (line 143) | def reset_instrumentation(self):

FILE: tests/test_integrations/test_exporter/test_pydantic_ai.py
  function test_pydantic_ai_trace (line 13) | async def test_pydantic_ai_trace():
  function test_multi_turn_trace (line 34) | async def test_multi_turn_trace():
  function test_llm_trace (line 60) | async def test_llm_trace():

FILE: tests/test_integrations/test_googleadk/apps/googleadk_eval_app.py
  function special_tool (line 34) | def special_tool(query: str) -> dict:
  function _build_agent (line 53) | def _build_agent() -> LlmAgent:
  function init_evals_googleadk (line 62) | def init_evals_googleadk(
  function invoke_evals_agent (line 110) | def invoke_evals_agent(prompt: str, invoke_func=None) -> str:
  function ainvoke_evals_agent (line 117) | async def ainvoke_evals_agent(prompt: str, invoke_func=None) -> str:

FILE: tests/test_integrations/test_googleadk/apps/googleadk_multiple_tools_app.py
  function get_weather (line 23) | def get_weather(city: str) -> dict:
  function get_time (line 44) | def get_time(city: str) -> dict:
  function _build_agent (line 65) | def _build_agent() -> LlmAgent:
  function init_multiple_tools_googleadk (line 78) | def init_multiple_tools_googleadk(
  function invoke_multiple_tools_agent (line 125) | def invoke_multiple_tools_agent(prompt: str, invoke_func=None) -> str:
  function ainvoke_multiple_tools_agent (line 132) | async def ainvoke_multiple_tools_agent(prompt: str, invoke_func=None) ->...

FILE: tests/test_integrations/test_googleadk/apps/googleadk_simple_app.py
  function _build_agent (line 22) | def _build_agent() -> LlmAgent:
  function init_simple_googleadk (line 32) | def init_simple_googleadk(
  function invoke_simple_agent (line 82) | def invoke_simple_agent(prompt: str, invoke_func=None) -> str:
  function ainvoke_simple_agent (line 89) | async def ainvoke_simple_agent(prompt: str, invoke_func=None) -> str:

FILE: tests/test_integrations/test_googleadk/apps/googleadk_tool_app.py
  function calculate (line 22) | def calculate(operation: str, a: float, b: float) -> dict:
  function _build_agent (line 45) | def _build_agent() -> LlmAgent:
  function init_tool_googleadk (line 57) | def init_tool_googleadk(
  function invoke_tool_agent (line 104) | def invoke_tool_agent(prompt: str, invoke_func=None) -> str:
  function ainvoke_tool_agent (line 111) | async def ainvoke_tool_agent(prompt: str, invoke_func=None) -> str:

FILE: tests/test_integrations/test_googleadk/conftest.py
  function trace_test (line 37) | def trace_test(schema_name: str):

FILE: tests/test_integrations/test_googleadk/test_async.py
  class TestAsyncSimpleApp (line 47) | class TestAsyncSimpleApp:
    method test_async_simple_greeting (line 51) | async def test_async_simple_greeting(self):
  class TestAsyncToolApp (line 69) | class TestAsyncToolApp:
    method test_async_tool_calculation (line 73) | async def test_async_tool_calculation(self):
  class TestAsyncMultipleToolsApp (line 91) | class TestAsyncMultipleToolsApp:
    method test_async_parallel_tool_calls (line 95) | async def test_async_parallel_tool_calls(self):
  class TestDeepEvalFeaturesAsync (line 115) | class TestDeepEvalFeaturesAsync:
    method test_full_features_async (line 124) | async def test_full_features_async(self):

FILE: tests/test_integrations/test_googleadk/test_evaluate_agent.py
  function test_evaluate_agent (line 51) | def test_evaluate_agent():

FILE: tests/test_integrations/test_googleadk/test_span_interceptor.py
  function _make_mock_span (line 74) | def _make_mock_span(
  function _make_settings (line 133) | def _make_settings(**kwargs):
  function _make_agent_span_mock (line 157) | def _make_agent_span_mock(agent_name: str = "agent_x"):
  function _make_tool_span_mock (line 162) | def _make_tool_span_mock(tool_name: str = "calculate"):
  function _make_llm_span_mock (line 167) | def _make_llm_span_mock():
  class TestTraceContextReads (line 177) | class TestTraceContextReads:
    method test_uses_settings_when_no_trace_context (line 178) | def test_uses_settings_when_no_trace_context(self):
    method test_prefers_trace_context_over_settings_for_scalars (line 206) | def test_prefers_trace_context_over_settings_for_scalars(self):
    method test_metadata_is_merged_with_context_winning (line 221) | def test_metadata_is_merged_with_context_winning(self):
    method test_update_current_trace_after_on_start_lands_on_otel_attrs (line 237) | def test_update_current_trace_after_on_start_lands_on_otel_attrs(self):
    method test_trace_metric_collection_resolution_order (line 267) | def test_trace_metric_collection_resolution_order(self):
  class TestSpanContextPushPop (line 286) | class TestSpanContextPushPop:
    method test_current_span_context_set_during_span_lifetime (line 287) | def test_current_span_context_set_during_span_lifetime(self):
    method test_update_current_span_metadata_lands_in_otel_attrs (line 303) | def test_update_current_span_metadata_lands_in_otel_attrs(self):
    method test_update_current_span_metric_collection_lands_in_otel_attrs (line 326) | def test_update_current_span_metric_collection_lands_in_otel_attrs(self):
    method test_nested_spans_lifo_pop_restores_parent_placeholder (line 343) | def test_nested_spans_lifo_pop_restores_parent_placeholder(self):
  class TestImplicitTraceContext (line 367) | class TestImplicitTraceContext:
    method test_root_span_pushes_implicit_trace_when_no_user_context (line 377) | def test_root_span_pushes_implicit_trace_when_no_user_context(self):
    method test_does_not_overwrite_user_pushed_trace_context (line 395) | def test_does_not_overwrite_user_pushed_trace_context(self):
    method test_child_span_does_not_push_its_own_placeholder (line 413) | def test_child_span_does_not_push_its_own_placeholder(self):
    method test_update_current_trace_in_implicit_context_lands_on_otel_attrs (line 436) | def test_update_current_trace_in_implicit_context_lands_on_otel_attrs(
  class TestParentBridge (line 471) | class TestParentBridge:
    method test_stamps_parent_uuid_when_enclosed_in_deepeval_span (line 472) | def test_stamps_parent_uuid_when_enclosed_in_deepeval_span(self):
    method test_no_parent_uuid_when_otel_span_has_native_parent (line 503) | def test_no_parent_uuid_when_otel_span_has_native_parent(self):
  class TestNextSpanInterceptorIntegration (line 533) | class TestNextSpanInterceptorIntegration:
    method test_next_agent_span_metric_collection_lands_on_otel_attrs (line 534) | def test_next_agent_span_metric_collection_lands_on_otel_attrs(self):
    method test_next_agent_span_consumed_only_by_first_agent_span (line 548) | def test_next_agent_span_consumed_only_by_first_agent_span(self):
    method test_next_agent_span_does_not_affect_non_agent_span (line 569) | def test_next_agent_span_does_not_affect_non_agent_span(self):
    method test_next_tool_span_metric_collection_lands_on_tool_otel_attrs (line 593) | def test_next_tool_span_metric_collection_lands_on_tool_otel_attrs(self):
    method test_next_llm_span_metric_collection_lands_on_llm_otel_attrs (line 610) | def test_next_llm_span_metric_collection_lands_on_llm_otel_attrs(self):
    method test_update_current_span_overrides_next_agent_span_after_creation (line 624) | def test_update_current_span_overrides_next_agent_span_after_creation(
    method test_next_agent_span_metrics_stashed_when_evaluating (line 644) | def test_next_agent_span_metrics_stashed_when_evaluating(self):
    method test_next_agent_span_metrics_not_stashed_outside_eval_mode (line 678) | def test_next_agent_span_metrics_not_stashed_outside_eval_mode(self):
  class TestFrameworkAttrExtraction (line 708) | class TestFrameworkAttrExtraction:
    method test_agent_span_kind_lands_as_confident_span_type_agent (line 716) | def test_agent_span_kind_lands_as_confident_span_type_agent(self):
    method test_chain_span_kind_classified_as_agent (line 727) | def test_chain_span_kind_classified_as_agent(self):
    method test_llm_span_kind_lands_as_confident_span_type_llm (line 739) | def test_llm_span_kind_lands_as_confident_span_type_llm(self):
    method test_tool_span_kind_lands_as_confident_span_type_tool (line 749) | def test_tool_span_kind_lands_as_confident_span_type_tool(self):
    method test_unknown_span_kind_classified_as_custom (line 760) | def test_unknown_span_kind_classified_as_custom(self):
    method test_missing_span_kind_leaves_type_unset (line 773) | def test_missing_span_kind_leaves_type_unset(self):
    method test_llm_span_extracts_flattened_input_output_messages (line 786) | def test_llm_span_extracts_flattened_input_output_messages(self):
    method test_llm_span_extracts_token_counts_and_model_name (line 814) | def test_llm_span_extracts_token_counts_and_model_name(self):
    method test_llm_span_extracts_tool_calls_from_output_messages (line 833) | def test_llm_span_extracts_tool_calls_from_output_messages(self):
    method test_tool_span_extracts_self_as_single_tool_call (line 871) | def test_tool_span_extracts_self_as_single_tool_call(self):
    method test_agent_span_input_output_also_lands_on_trace_attrs (line 902) | def test_agent_span_input_output_also_lands_on_trace_attrs(self):
    method test_update_current_span_input_wins_over_framework_input (line 931) | def test_update_current_span_input_wins_over_framework_input(self):
  function test_removed_kwargs_raise_typeerror_on_settings (line 973) | def test_removed_kwargs_raise_typeerror_on_settings(kwarg):
  function test_removed_kwargs_raise_typeerror_on_instrument_google_adk (line 988) | def test_removed_kwargs_raise_typeerror_on_instrument_google_adk(kwarg):
  function test_settings_no_api_key_does_not_raise (line 1007) | def test_settings_no_api_key_does_not_raise(monkeypatch):

FILE: tests/test_integrations/test_googleadk/test_sync.py
  class TestSimpleApp (line 50) | class TestSimpleApp:
    method test_simple_greeting (line 53) | def test_simple_greeting(self):
  class TestToolApp (line 71) | class TestToolApp:
    method test_tool_calculation (line 74) | def test_tool_calculation(self):
    method test_tool_metric_collection (line 92) | def test_tool_metric_collection(self):
  class TestMultipleToolsApp (line 119) | class TestMultipleToolsApp:
    method test_multiple_tools_weather_only (line 122) | def test_multiple_tools_weather_only(self):
    method test_multiple_tools_time_only (line 140) | def test_multiple_tools_time_only(self):
    method test_parallel_tool_calls (line 158) | def test_parallel_tool_calls(self):
  class TestDeepEvalFeatures (line 178) | class TestDeepEvalFeatures:
    method test_full_features_sync (line 192) | def test_full_features_sync(self):

FILE: tests/test_integrations/test_langchain/apps/langchain_agent_app.py
  function search_web (line 16) | def search_web(query: str) -> str:
  function calculator (line 31) | def calculator(expression: str) -> str:
  function get_current_time (line 44) | def get_current_time() -> str:
  function _run_agent_loop (line 66) | def _run_agent_loop(
  function _arun_agent_loop (line 115) | async def _arun_agent_loop(
  function _simple_agent_sync (line 165) | def _simple_agent_sync(inputs: dict, config: RunnableConfig = None):
  function _simple_agent_async (line 171) | async def _simple_agent_async(inputs: dict, config: RunnableConfig = None):
  function _multi_step_agent_sync (line 177) | def _multi_step_agent_sync(inputs: dict, config: RunnableConfig = None):
  function _multi_step_agent_async (line 183) | async def _multi_step_agent_async(inputs: dict, config: RunnableConfig =...
  function _complex_agent_sync (line 189) | def _complex_agent_sync(inputs: dict, config: RunnableConfig = None):
  function _complex_agent_async (line 195) | async def _complex_agent_async(inputs: dict, config: RunnableConfig = No...
  function invoke_simple_agent (line 223) | def invoke_simple_agent(inputs: dict, config: RunnableConfig = None):
  function ainvoke_simple_agent (line 228) | async def ainvoke_simple_agent(inputs: dict, config: RunnableConfig = No...
  function invoke_multi_step_agent (line 234) | def invoke_multi_step_agent(inputs: dict, config: RunnableConfig = None):
  function ainvoke_multi_step_agent (line 239) | async def ainvoke_multi_step_agent(inputs: dict, config: RunnableConfig ...
  function invoke_complex_agent (line 245) | def invoke_complex_agent(inputs: dict, config: RunnableConfig = None):
  function ainvoke_complex_agent (line 250) | async def ainvoke_complex_agent(inputs: dict, config: RunnableConfig = N...

FILE: tests/test_integrations/test_langchain/apps/langchain_conditional_app.py
  function research_topic (line 15) | def research_topic(topic: str) -> str:
  function summarize_text (line 30) | def summarize_text(text: str) -> str:
  function fact_check (line 38) | def fact_check(claim: str) -> str:
  function _run_conditional_chain (line 68) | def _run_conditional_chain(
  function _arun_conditional_chain (line 113) | async def _arun_conditional_chain(
  function _research_sync (line 159) | def _research_sync(inputs: dict, config: RunnableConfig = None):
  function _research_async (line 165) | async def _research_async(inputs: dict, config: RunnableConfig = None):
  function _summarize_sync (line 171) | def _summarize_sync(inputs: dict, config: RunnableConfig = None):
  function _summarize_async (line 177) | async def _summarize_async(inputs: dict, config: RunnableConfig = None):
  function _fact_check_sync (line 183) | def _fact_check_sync(inputs: dict, config: RunnableConfig = None):
  function _fact_check_async (line 189) | async def _fact_check_async(inputs: dict, config: RunnableConfig = None):
  function _general_sync (line 195) | def _general_sync(inputs: dict, config: RunnableConfig = None):
  function _general_async (line 202) | async def _general_async(inputs: dict, config: RunnableConfig = None):
  function invoke_research (line 237) | def invoke_research(inputs: dict, config: RunnableConfig = None):
  function ainvoke_research (line 242) | async def ainvoke_research(inputs: dict, config: RunnableConfig = None):
  function invoke_summarize (line 248) | def invoke_summarize(inputs: dict, config: RunnableConfig = None):
  function ainvoke_summarize (line 253) | async def ainvoke_summarize(inputs: dict, config: RunnableConfig = None):
  function invoke_fact_check (line 259) | def invoke_fact_check(inputs: dict, config: RunnableConfig = None):
  function ainvoke_fact_check (line 264) | async def ainvoke_fact_check(inputs: dict, config: RunnableConfig = None):
  function invoke_general (line 270) | def invoke_general(inputs: dict, config: RunnableConfig = None):
  function ainvoke_general (line 275) | async def ainvoke_general(inputs: dict, config: RunnableConfig = None):

FILE: tests/test_integrations/test_langchain/apps/langchain_metric_collection_app.py
  function calculate (line 24) | def calculate(expression: str) -> str:
  function invoke_metric_collection_app (line 57) | def invoke_metric_collection_app(inputs: dict, config: RunnableConfig = ...
  function ainvoke_metric_collection_app (line 62) | async def ainvoke_metric_collection_app(

FILE: tests/test_integrations/test_langchain/apps/langchain_multiple_tools_app.py
  function get_weather (line 15) | def get_weather(city: str) -> str:
  function get_population (line 30) | def get_population(city: str) -> str:
  function get_timezone (line 45) | def get_timezone(city: str) -> str:
  function calculate (line 60) | def calculate(expression: str) -> str:
  function _run_multi_tool_chain (line 86) | def _run_multi_tool_chain(
  function _arun_multi_tool_chain (line 131) | async def _arun_multi_tool_chain(
  function _city_info_sync (line 177) | def _city_info_sync(inputs: dict, config: RunnableConfig = None):
  function _city_info_async (line 183) | async def _city_info_async(inputs: dict, config: RunnableConfig = None):
  function _mixed_tools_sync (line 189) | def _mixed_tools_sync(inputs: dict, config: RunnableConfig = None):
  function _mixed_tools_async (line 195) | async def _mixed_tools_async(inputs: dict, config: RunnableConfig = None):
  function invoke_city_info (line 216) | def invoke_city_info(inputs: dict, config: RunnableConfig = None):
  function ainvoke_city_info (line 221) | async def ainvoke_city_info(inputs: dict, config: RunnableConfig = None):
  function invoke_mixed_tools (line 226) | def invoke_mixed_tools(inputs: dict, config: RunnableConfig = None):
  function ainvoke_mixed_tools (line 231) | async def ainvoke_mixed_tools(inputs: dict, config: RunnableConfig = None):

FILE: tests/test_integrations/test_langchain/apps/langchain_next_span_app.py
  function square (line 33) | def square(n: int) -> int:
  function invoke_with_next_llm_span (line 51) | def invoke_with_next_llm_span(
  function ainvoke_with_next_llm_span (line 72) | async def ainvoke_with_next_llm_span(

FILE: tests/test_integrations/test_langchain/apps/langchain_parallel_tools_app.py
  function get_weather (line 15) | def get_weather(city: str) -> str:
  function get_stock_price (line 28) | def get_stock_price(symbol: str) -> str:
  function get_exchange_rate (line 41) | def get_exchange_rate(from_currency: str, to_currency: str) -> str:
  function calculate (line 56) | def calculate(expression: str) -> str:
  function _run_parallel_chain (line 86) | def _run_parallel_chain(
  function _arun_parallel_chain (line 131) | async def _arun_parallel_chain(
  function _parallel_weather_sync (line 177) | def _parallel_weather_sync(inputs: dict, config: RunnableConfig = None):
  function _parallel_weather_async (line 183) | async def _parallel_weather_async(inputs: dict, config: RunnableConfig =...
  function _parallel_mixed_sync (line 189) | def _parallel_mixed_sync(inputs: dict, config: RunnableConfig = None):
  function _parallel_mixed_async (line 195) | async def _parallel_mixed_async(inputs: dict, config: RunnableConfig = N...
  function _parallel_stocks_sync (line 201) | def _parallel_stocks_sync(inputs: dict, config: RunnableConfig = None):
  function _parallel_stocks_async (line 207) | async def _parallel_stocks_async(inputs: dict, config: RunnableConfig = ...
  function invoke_parallel_weather (line 235) | def invoke_parallel_weather(inputs: dict, config: RunnableConfig = None):
  function ainvoke_parallel_weather (line 240) | async def ainvoke_parallel_weather(inputs: dict, config: RunnableConfig ...
  function invoke_parallel_mixed (line 246) | def invoke_parallel_mixed(inputs: dict, config: RunnableConfig = None):
  function ainvoke_parallel_mixed (line 251) | async def ainvoke_parallel_mixed(inputs: dict, config: RunnableConfig = ...
  function invoke_parallel_stocks (line 257) | def invoke_parallel_stocks(inputs: dict, config: RunnableConfig = None):
  function ainvoke_parallel_stocks (line 262) | async def ainvoke_parallel_stocks(inputs: dict, config: RunnableConfig =...

FILE: tests/test_integrations/test_langchain/apps/langchain_retriever_app.py
  class DeterministicRetriever (line 19) | class DeterministicRetriever(BaseRetriever):
    method _get_relevant_documents (line 55) | def _get_relevant_documents(
  function _run_rag_chain (line 77) | def _run_rag_chain(inputs: dict, config: RunnableConfig = None):
  function _arun_rag_chain (line 122) | async def _arun_rag_chain(inputs: dict, config: RunnableConfig = None):
  function invoke_rag_app (line 174) | def invoke_rag_app(inputs: dict, config: RunnableConfig = None):
  function ainvoke_rag_app (line 179) | async def ainvoke_rag_app(inputs: dict, config: RunnableConfig = None):
  function _run_rag_chain_with_metric_collection (line 184) | def _run_rag_chain_with_metric_collection(
  function invoke_rag_app_with_metric_collection (line 236) | def invoke_rag_app_with_metric_collection(

FILE: tests/test_integrations/test_langchain/apps/langchain_simple_app.py
  function _run_simple_chain (line 15) | def _run_simple_chain(messages: list, config: RunnableConfig = None):
  function _arun_simple_chain (line 21) | async def _arun_simple_chain(messages: list, config: RunnableConfig = No...
  function invoke_simple_app (line 36) | def invoke_simple_app(messages: list, config: RunnableConfig = None):
  function ainvoke_simple_app (line 41) | async def ainvoke_simple_app(messages: list, config: RunnableConfig = No...

FILE: tests/test_integrations/test_langchain/apps/langchain_single_tool_app.py
  function get_weather (line 15) | def get_weather(city: str) -> str:
  function _run_tool_chain (line 35) | def _run_tool_chain(inputs: dict, config: RunnableConfig = None):
  function _arun_tool_chain (line 84) | async def _arun_tool_chain(inputs: dict, config: RunnableConfig = None):
  function invoke_single_tool_app (line 137) | def invoke_single_tool_app(inputs: dict, config: RunnableConfig = None):
  function ainvoke_single_tool_app (line 142) | async def ainvoke_single_tool_app(inputs: dict, config: RunnableConfig =...

FILE: tests/test_integrations/test_langchain/apps/langchain_streaming_app.py
  function get_stock_price (line 15) | def get_stock_price(symbol: str) -> str:
  function get_company_info (line 28) | def get_company_info(symbol: str) -> str:
  function _run_streaming_chain (line 56) | def _run_streaming_chain(
  function _arun_streaming_chain (line 101) | async def _arun_streaming_chain(
  function stream_chain (line 146) | def stream_chain(
  function astream_chain (line 191) | async def astream_chain(
  function _streaming_single_sync (line 237) | def _streaming_single_sync(inputs: dict, config: RunnableConfig = None):
  function _streaming_single_async (line 243) | async def _streaming_single_async(inputs: dict, config: RunnableConfig =...
  function _streaming_multi_sync (line 249) | def _streaming_multi_sync(inputs: dict, config: RunnableConfig = None):
  function _streaming_multi_async (line 255) | async def _streaming_multi_async(inputs: dict, config: RunnableConfig = ...
  function invoke_streaming_single (line 277) | def invoke_streaming_single(inputs: dict, config: RunnableConfig = None):
  function ainvoke_streaming_single (line 282) | async def ainvoke_streaming_single(inputs: dict, config: RunnableConfig ...
  function stream_streaming_single (line 287) | def stream_streaming_single(inputs: dict, config: RunnableConfig = None):
  function astream_streaming_single (line 295) | async def astream_streaming_single(inputs: dict, config: RunnableConfig ...
  function invoke_streaming_multi (line 305) | def invoke_streaming_multi(inputs: dict, config: RunnableConfig = None):
  function ainvoke_streaming_multi (line 310) | async def ainvoke_streaming_multi(inputs: dict, config: RunnableConfig =...
  function stream_streaming_multi (line 315) | def stream_streaming_multi(inputs: dict, config: RunnableConfig = None):
  function astream_streaming_multi (line 321) | async def astream_streaming_multi(inputs: dict, config: RunnableConfig =...

FILE: tests/test_integrations/test_langchain/conftest.py
  function _upload_enabled (line 26) | def _upload_enabled() -> bool:
  function pytest_configure (line 38) | def pytest_configure(config):
  function pytest_sessionstart (line 44) | def pytest_sessionstart(session: pytest.Session):
  function pytest_runtest_makereport (line 71) | def pytest_runtest_makereport(item: pytest.Item, call):
  function _upload_trace_to_observatory (line 105) | def _upload_trace_to_observatory(trace_dict: dict) -> str:
  function _truncate (line 130) | def _truncate(s: str, max_len: int = MAX_FIELD_LENGTH) -> str:
  function _extract_input_from_trace (line 137) | def _extract_input_from_trace(trace_dict: Dict[str, Any]) -> str:
  function _extract_output_from_trace (line 153) | def _extract_output_from_trace(trace_dict: Dict[str, Any]) -> str:
  function _extract_tools_called_from_trace (line 176) | def _extract_tools_called_from_trace(
  function _extract_token_cost (line 230) | def _extract_token_cost(trace_dict: Dict[str, Any]) -> Optional[float]:
  function _extract_completion_time (line 256) | def _extract_completion_time(trace_dict: Dict[str, Any]) -> Optional[flo...
  function _extract_tags (line 273) | def _extract_tags(
  function _get_environment_info (line 299) | def _get_environment_info() -> Dict[str, str]:
  function _add_test_case_to_run (line 322) | def _add_test_case_to_run(
  function pytest_sessionfinish (line 415) | def pytest_sessionfinish(session: pytest.Session, exitstatus):

FILE: tests/test_integrations/test_langchain/test_async.py
  function trace_test (line 64) | def trace_test(schema_name: str):
  class TestAsyncSimpleApp (line 83) | class TestAsyncSimpleApp:
    method test_async_simple_greeting (line 88) | async def test_async_simple_greeting(self):
  class TestAsyncSingleToolApp (line 112) | class TestAsyncSingleToolApp:
    method test_async_weather_query (line 117) | async def test_async_weather_query(self):
  class TestAsyncMultipleToolsApp (line 147) | class TestAsyncMultipleToolsApp:
    method test_async_city_info (line 152) | async def test_async_city_info(self):
    method test_async_mixed_query (line 178) | async def test_async_mixed_query(self):
  class TestAsyncStreamingApp (line 205) | class TestAsyncStreamingApp:
    method test_async_streaming_single (line 210) | async def test_async_streaming_single(self):
    method test_async_streaming_multi (line 234) | async def test_async_streaming_multi(self):
  class TestAsyncConditionalApp (line 261) | class TestAsyncConditionalApp:
    method test_async_research_route (line 266) | async def test_async_research_route(self):
    method test_async_summarize_route (line 290) | async def test_async_summarize_route(self):
    method test_async_fact_check_route (line 313) | async def test_async_fact_check_route(self):
    method test_async_general_route (line 336) | async def test_async_general_route(self):
  class TestAsyncParallelToolsApp (line 361) | class TestAsyncParallelToolsApp:
    method test_async_parallel_weather (line 366) | async def test_async_parallel_weather(self):
    method test_async_parallel_mixed (line 390) | async def test_async_parallel_mixed(self):
    method test_async_parallel_stocks (line 413) | async def test_async_parallel_stocks(self):
  class TestAsyncRetrieverApp (line 440) | class TestAsyncRetrieverApp:
    method test_async_retrieve_python_docs (line 445) | async def test_async_retrieve_python_docs(self):
    method test_async_retrieve_langchain_docs (line 469) | async def test_async_retrieve_langchain_docs(self):
  class TestAsyncAgentApp (line 494) | class TestAsyncAgentApp:
    method test_async_simple_agent (line 499) | async def test_async_simple_agent(self):
    method test_async_multi_step_agent (line 523) | async def test_async_multi_step_agent(self):
    method test_async_complex_agent (line 546) | async def test_async_complex_agent(self):
  class TestAsyncNextSpanApp (line 573) | class TestAsyncNextSpanApp:
    method test_async_next_llm_span_only (line 581) | async def test_async_next_llm_span_only(self):

FILE: tests/test_integrations/test_langchain/test_langchain.py
  function multiply (line 18) | def multiply(a: int, b: int) -> int:
  function test_execute_agent (line 41) | def test_execute_agent():

FILE: tests/test_integrations/test_langchain/test_next_span.py
  class _RecordingCallbackHandler (line 50) | class _RecordingCallbackHandler(CallbackHandler):
    method __init__ (line 60) | def __init__(self, *args, **kwargs):
    method on_chat_model_start (line 66) | def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs):
    method on_llm_start (line 75) | def on_llm_start(self, serialized, prompts, *, run_id, **kwargs):
    method on_tool_start (line 82) | def on_tool_start(self, serialized, input_str, *, run_id, **kwargs):
    method on_retriever_start (line 91) | def on_retriever_start(self, serialized, query, *, run_id, **kwargs):
  class _EchoToolInput (line 101) | class _EchoToolInput(BaseModel):
  class _EchoTool (line 105) | class _EchoTool(BaseTool):
    method _run (line 113) | def _run(self, text: str, **_kwargs: Any) -> str:
  class _StaticRetriever (line 117) | class _StaticRetriever(BaseRetriever):
    method _get_relevant_documents (line 128) | def _get_relevant_documents(
    method _aget_relevant_documents (line 133) | async def _aget_relevant_documents(
  function _fake_metric (line 142) | def _fake_metric(name: str = "fake") -> BaseMetric:
  class TestNextLlmSpanWiring (line 156) | class TestNextLlmSpanWiring:
    method test_metric_collection_lands_on_llm_span (line 163) | def test_metric_collection_lands_on_llm_span(self):
    method test_metrics_list_lands_on_llm_span (line 173) | def test_metrics_list_lands_on_llm_span(self):
    method test_metadata_lands_on_llm_span (line 183) | def test_metadata_lands_on_llm_span(self):
    method test_one_shot_only_first_llm_span_consumes (line 192) | def test_one_shot_only_first_llm_span_consumes(self):
    method test_unconsumed_payload_does_not_leak_to_next_with (line 210) | def test_unconsumed_payload_does_not_leak_to_next_with(self):
    method test_outside_with_block_no_staging (line 224) | def test_outside_with_block_no_staging(self):
    method test_overrides_with_config_metadata_metric_collection (line 240) | def test_overrides_with_config_metadata_metric_collection(self):
    method test_does_not_override_metadata_when_only_metric_collection_staged (line 256) | def test_does_not_override_metadata_when_only_metric_collection_staged(
  class TestNextToolSpanWiring (line 286) | class TestNextToolSpanWiring:
    method test_metric_collection_lands_on_tool_span (line 287) | def test_metric_collection_lands_on_tool_span(self):
    method test_metadata_lands_on_tool_span (line 297) | def test_metadata_lands_on_tool_span(self):
    method test_one_shot_only_first_tool_span_consumes (line 306) | def test_one_shot_only_first_tool_span_consumes(self):
  class TestNextRetrieverSpanWiring (line 324) | class TestNextRetrieverSpanWiring:
    method test_metric_collection_lands_on_retriever_span (line 325) | def test_metric_collection_lands_on_retriever_span(self):
    method test_top_k_and_embedder_land_on_retriever_span (line 338) | def test_top_k_and_embedder_land_on_retriever_span(self):
  class TestCrossTypeIsolation (line 359) | class TestCrossTypeIsolation:
    method test_next_tool_span_does_not_leak_to_llm_span (line 364) | def test_next_tool_span_does_not_leak_to_llm_span(self):
    method test_next_llm_span_does_not_leak_to_tool_span (line 373) | def test_next_llm_span_does_not_leak_to_tool_span(self):
  class TestNextSpanBaseSlotWiring (line 388) | class TestNextSpanBaseSlotWiring:
    method test_base_slot_lands_on_first_llm_span (line 394) | def test_base_slot_lands_on_first_llm_span(self):
    method test_typed_slot_overrides_base_slot_on_overlap (line 403) | def test_typed_slot_overrides_base_slot_on_overlap(self):
  function test_next_llm_span_lands_on_async_llm_call (line 427) | async def test_next_llm_span_lands_on_async_llm_call():
  function test_next_llm_span_lands_inside_runnable_lambda_async (line 445) | async def test_next_llm_span_lands_inside_runnable_lambda_async():

FILE: tests/test_integrations/test_langchain/test_sync.py
  function trace_test (line 68) | def trace_test(schema_name: str):
  class TestSimpleApp (line 87) | class TestSimpleApp:
    method test_simple_greeting (line 91) | def test_simple_greeting(self):
  class TestSingleToolApp (line 115) | class TestSingleToolApp:
    method test_weather_query (line 119) | def test_weather_query(self):
  class TestMultipleToolsApp (line 149) | class TestMultipleToolsApp:
    method test_city_info (line 153) | def test_city_info(self):
    method test_mixed_query (line 178) | def test_mixed_query(self):
  class TestStreamingApp (line 206) | class TestStreamingApp:
    method test_sync_streaming (line 210) | def test_sync_streaming(self):
    method test_sync_streaming_multiple_tools (line 233) | def test_sync_streaming_multiple_tools(self):
  class TestConditionalApp (line 260) | class TestConditionalApp:
    method test_research_route (line 264) | def test_research_route(self):
    method test_summarize_route (line 287) | def test_summarize_route(self):
    method test_fact_check_route (line 309) | def test_fact_check_route(self):
    method test_general_route (line 331) | def test_general_route(self):
  class TestParallelToolsApp (line 356) | class TestParallelToolsApp:
    method test_parallel_weather_queries (line 360) | def test_parallel_weather_queries(self):
    method test_parallel_mixed_tools (line 383) | def test_parallel_mixed_tools(self):
    method test_parallel_stock_queries (line 405) | def test_parallel_stock_queries(self):
  class TestRetrieverApp (line 432) | class TestRetrieverApp:
    method test_retrieve_python_docs (line 436) | def test_retrieve_python_docs(self):
    method test_retrieve_langchain_docs (line 459) | def test_retrieve_langchain_docs(self):
    method test_retriever_metric_collection (line 479) | def test_retriever_metric_collection(self):
  class TestAgentApp (line 507) | class TestAgentApp:
    method test_simple_agent (line 511) | def test_simple_agent(self):
    method test_multi_step_agent (line 534) | def test_multi_step_agent(self):
    method test_complex_agent (line 556) | def test_complex_agent(self):
  class TestMetricCollectionApp (line 583) | class TestMetricCollectionApp:
    method test_metric_collection (line 587) | def test_metric_collection(self):
  class TestNextSpanApp (line 616) | class TestNextSpanApp:
    method test_next_llm_span_only (line 627) | def test_next_llm_span_only(self):

FILE: tests/test_integrations/test_langgraph/apps/langgraph_async_app.py
  function search_database (line 16) | def search_database(query: str) -> str:
  function translate (line 32) | def translate(text: str, target_language: str) -> str:
  function agent_node (line 51) | async def agent_node(state: dict, config: RunnableConfig) -> dict:
  function should_continue (line 58) | def should_continue(state: dict) -> Literal["tools", "__end__"]:
  function build_app (line 68) | def build_app():

FILE: tests/test_integrations/test_langgraph/apps/langgraph_conditional_app.py
  class ConditionalState (line 17) | class ConditionalState(TypedDict):
  function research_topic (line 25) | def research_topic(topic: str) -> str:
  function summarize_text (line 40) | def summarize_text(text: str) -> str:
  function fact_check (line 48) | def fact_check(claim: str) -> str:
  function classify_intent (line 64) | def classify_intent(state: dict) -> dict:
  function research_node (line 83) | def research_node(state: dict, config: RunnableConfig) -> dict:
  function summarize_node (line 93) | def summarize_node(state: dict, config: RunnableConfig) -> dict:
  function fact_check_node (line 103) | def fact_check_node(state: dict, config: RunnableConfig) -> dict:
  function general_node (line 113) | def general_node(state: dict, config: RunnableConfig) -> dict:
  function route_by_intent (line 120) | def route_by_intent(
  function should_continue (line 127) | def should_continue(state: dict) -> Literal["tools", "__end__"]:
  function route_after_tools (line 137) | def route_after_tools(state: dict) -> str:
  function build_app (line 143) | def build_app():

FILE: tests/test_integrations/test_langgraph/apps/langgraph_metric_collection_app.py
  function convert_temperature (line 27) | def convert_temperature(celsius: float) -> str:
  function agent_node (line 48) | def agent_node(state: dict, config: RunnableConfig) -> dict:
  function should_continue (line 55) | def should_continue(state: dict) -> Literal["tools", "__end__"]:
  function build_app (line 65) | def build_app():

FILE: tests/test_integrations/test_langgraph/apps/langgraph_multi_turn_app.py
  function add_to_cart (line 18) | def add_to_cart(item: str, quantity: int = 1) -> str:
  function remove_from_cart (line 24) | def remove_from_cart(item: str) -> str:
  function view_cart (line 30) | def view_cart() -> str:
  function apply_coupon (line 36) | def apply_coupon(code: str) -> str:
  function checkout (line 47) | def checkout() -> str:
  function confirm_order (line 53) | def confirm_order() -> str:
  function agent_node (line 71) | def agent_node(state: dict, config: RunnableConfig) -> dict:
  function async_agent_node (line 86) | async def async_agent_node(state: dict, config: RunnableConfig) -> dict:
  function should_continue (line 98) | def should_continue(state: dict) -> Literal["tools", "__end__"]:
  function build_app_with_memory (line 108) | def build_app_with_memory():
  function build_async_app_with_memory (line 125) | def build_async_app_with_memory():
  function build_stateless_app (line 142) | def build_stateless_app():
  function get_app_with_memory (line 165) | def get_app_with_memory():
  function get_async_app_with_memory (line 170) | def get_async_app_with_memory():

FILE: tests/test_integrations/test_langgraph/apps/langgraph_multiple_tools_app.py
  function get_weather (line 16) | def get_weather(city: str) -> str:
  function get_population (line 31) | def get_population(city: str) -> str:
  function get_timezone (line 46) | def get_timezone(city: str) -> str:
  function calculate (line 61) | def calculate(expression: str) -> str:
  function agent_node (line 79) | def agent_node(state: dict, config: RunnableConfig) -> dict:
  function should_continue (line 86) | def should_continue(state: dict) -> Literal["tools", "__end__"]:
  function build_app (line 96) | def build_app():

FILE: tests/test_integrations/test_langgraph/apps/langgraph_next_span_app.py
  function square (line 29) | def square(n: int) -> int:
  function _agent_node (line 39) | def _agent_node(state: dict, config: RunnableConfig) -> dict:
  function _should_continue (line 44) | def _should_continue(state: dict) -> Literal["tools", "__end__"]:
  function _build_app (line 51) | def _build_app():
  function invoke_with_next_llm_span (line 66) | def invoke_with_next_llm_span(
  function ainvoke_with_next_llm_span (line 87) | async def ainvoke_with_next_llm_span(

FILE: tests/test_integrations/test_langgraph/apps/langgraph_parallel_tools_app.py
  function get_weather (line 17) | def get_weather(city: str) -> str:
  function get_stock_price (line 30) | def get_stock_price(symbol: str) -> str:
  function get_exchange_rate (line 43) | def get_exchange_rate(from_currency: str, to_currency: str) -> str:
  function search_news (line 58) | def search_news(topic: str) -> str:
  function calculate (line 73) | def calculate(expression: str) -> str:
  function agent_node (line 96) | def agent_node(state: dict, config: RunnableConfig) -> dict:
  function async_agent_node (line 108) | async def async_agent_node(state: dict, config: RunnableConfig) -> dict:
  function should_continue (line 121) | def should_continue(state: dict) -> Literal["tools", "__end__"]:
  function build_sync_app (line 131) | def build_sync_app():
  function build_async_app (line 147) | def build_async_app():

FILE: tests/test_integrations/test_langgraph/apps/langgraph_retriever_app.py
  class DeterministicRetriever (line 20) | class DeterministicRetriever(BaseRetriever):
    method _get_relevant_documents (line 56) | def _get_relevant_documents(
  class RAGState (line 70) | class RAGState(TypedDict):
  function retrieve_node (line 86) | def retrieve_node(state: RAGState, config: RunnableConfig) -> RAGState:
  function generate_node (line 106) | def generate_node(state: RAGState, config: RunnableConfig) -> RAGState:
  function retrieve_node_with_metric_collection (line 128) | def retrieve_node_with_metric_collection(
  function build_app (line 150) | def build_app():
  function build_app_with_metric_collection (line 164) | def build_app_with_metric_collection():

FILE: tests/test_integrations/test_langgraph/apps/langgraph_simple_app.py
  function get_weather (line 16) | def get_weather(city: str) -> str:
  function agent_node (line 35) | def agent_node(state: dict, config: RunnableConfig) -> dict:
  function should_continue (line 42) | def should_continue(state: dict) -> Literal["tools", "__end__"]:
  function build_app (line 52) | def build_app():

FILE: tests/test_integrations/test_langgraph/apps/langgraph_streaming_app.py
  function get_stock_price (line 16) | def get_stock_price(symbol: str) -> str:
  function get_company_info (line 29) | def get_company_info(symbol: str) -> str:
  function agent_node (line 48) | def agent_node(state: dict, config: RunnableConfig) -> dict:
  function async_agent_node (line 55) | async def async_agent_node(state: dict, config: RunnableConfig) -> dict:
  function should_continue (line 62) | def should_continue(state: dict) -> Literal["tools", "__end__"]:
  function build_sync_app (line 72) | def build_sync_app():
  function build_async_app (line 88) | def build_async_app():

FILE: tests/test_integrations/test_langgraph/apps/main.py
  function separator (line 11) | def separator(title: str):
  function main (line 17) | def main():

FILE: tests/test_integrations/test_langgraph/conftest.py
  function _upload_enabled (line 46) | def _upload_enabled() -> bool:
  function pytest_configure (line 58) | def pytest_configure(config):
  function pytest_sessionstart (line 64) | def pytest_sessionstart(session: pytest.Session):
  function pytest_runtest_makereport (line 91) | def pytest_runtest_makereport(item: pytest.Item, call):
  function _upload_trace_to_observatory (line 125) | def _upload_trace_to_observatory(trace_dict: dict) -> str:
  function _truncate (line 153) | def _truncate(s: str, max_len: int = MAX_FIELD_LENGTH) -> str:
  function _extract_input_from_trace (line 160) | def _extract_input_from_trace(trace_dict: Dict[str, Any]) -> str:
  function _extract_output_from_trace (line 182) | def _extract_output_from_trace(trace_dict: Dict[str, Any]) -> str:
  function _extract_tools_called_from_trace (line 213) | def _extract_tools_called_from_trace(
  function _extract_expected_output (line 273) | def _extract_expected_output(
  function _extract_expected_tools (line 297) | def _extract_expected_tools(
  function _extract_context (line 322) | def _extract_context(
  function _extract_retrieval_context (line 347) | def _extract_retrieval_context(
  function _extract_token_cost (line 398) | def _extract_token_cost(trace_dict: Dict[str, Any]) -> Optional[float]:
  function _extract_completion_time (line 431) | def _extract_completion_time(trace_dict: Dict[str, Any]) -> Optional[flo...
  function _extract_tags (line 456) | def _extract_tags(
  function _get_environment_info (line 491) | def _get_environment_info() -> Dict[str, str]:
  function _add_test_case_to_run (line 531) | def _add_test_case_to_run(
  function pytest_sessionfinish (line 650) | def pytest_sessionfinish(session: pytest.Session, exitstatus):

FILE: tests/test_integrations/test_langgraph/test_async.py
  function trace_test (line 46) | def trace_test(schema_name: str):
  class TestAsyncApp (line 65) | class TestAsyncApp:
    method test_single_tool (line 70) | async def test_single_tool(self):
    method test_multiple_tools (line 98) | async def test_multiple_tools(self):
    method test_no_tool_needed (line 127) | async def test_no_tool_needed(self):
  class TestAsyncStreamingApp (line 147) | class TestAsyncStreamingApp:
    method test_async_streaming (line 152) | async def test_async_streaming(self):
    method test_async_streaming_multiple_tools (line 175) | async def test_async_streaming_multiple_tools(self):
  class TestAsyncConditionalApp (line 203) | class TestAsyncConditionalApp:
    method test_async_conditional_routing (line 208) | async def test_async_conditional_routing(self):
  class TestAsyncParallelToolsApp (line 238) | class TestAsyncParallelToolsApp:
    method test_async_parallel_tools (line 243) | async def test_async_parallel_tools(self):
    method test_async_heavy_parallel (line 271) | async def test_async_heavy_parallel(self):
  class TestAsyncMultiTurnApp (line 312) | class TestAsyncMultiTurnApp:
    method test_async_multi_turn (line 317) | async def test_async_multi_turn(self):
  class TestAsyncNextSpanApp (line 359) | class TestAsyncNextSpanApp:
    method test_async_next_llm_span_only (line 368) | async def test_async_next_llm_span_only(self):

FILE: tests/test_integrations/test_langgraph/test_create_task.py
  class RaisingLLM (line 20) | class RaisingLLM(LLM):
    method _llm_type (line 24) | def _llm_type(self) -> str:
    method _call (line 27) | def _call(
    method _acall (line 36) | async def _acall(
  class RecordingCallbackHandler (line 46) | class RecordingCallbackHandler(CallbackHandler):
    method __init__ (line 47) | def __init__(self, *args, **kwargs):
    method _record_parent_if_present (line 58) | def _record_parent_if_present(self, run_id: str, target: dict):
    method on_chain_start (line 63) | def on_chain_start(
    method on_chain_end (line 82) | def on_chain_end(self, outputs, *, run_id, parent_run_id=None, **kwargs):
    method on_chain_error (line 97) | def on_chain_error(self, error, *, run_id, parent_run_id=None, **kwargs):
    method on_llm_start (line 110) | def on_llm_start(
    method on_llm_end (line 129) | def on_llm_end(self, response, *, run_id, parent_run_id=None, **kwargs):
    method on_llm_error (line 141) | def on_llm_error(self, error, *, run_id, parent_run_id=None, **kwargs):
  class State (line 154) | class State(TypedDict, total=False):
  function test_langgraph_async_callback_does_not_print_span_mismatch (line 163) | async def test_langgraph_async_callback_does_not_print_span_mismatch(cap...
  function test_nested_async_calls_are_parented_correctly_by_ids (line 201) | async def test_nested_async_calls_are_parented_correctly_by_ids(capsys):
  function test_llm_error_path_tracks_correct_ids_and_cleans_up (line 251) | async def test_llm_error_path_tracks_correct_ids_and_cleans_up(capsys):
  function test_chain_error_path_cleans_up_and_no_mismatch (line 286) | async def test_chain_error_path_cleans_up_and_no_mismatch(capsys):
  function test_parallel_llm_calls_under_same_parent_are_parented_correctly (line 319) | async def test_parallel_llm_calls_under_same_parent_are_parented_correctly(
  function test_chain_inside_chain_then_llm_is_parented_correctly (line 370) | async def test_chain_inside_chain_then_llm_is_parented_correctly(capsys):
  function test_nested_chain_chain_llm_end_order_and_parentage (line 437) | async def test_nested_chain_chain_llm_end_order_and_parentage(capsys):
  function test_observe_wrapped_async_langgraph_callback_no_span_stack_mismatch (line 497) | async def test_observe_wrapped_async_langgraph_callback_no_span_stack_mi...

FILE: tests/test_integrations/test_langgraph/test_next_span.py
  class _RecordingCallbackHandler (line 39) | class _RecordingCallbackHandler(CallbackHandler):
    method __init__ (line 45) | def __init__(self, *args, **kwargs):
    method on_chat_model_start (line 50) | def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs):
    method on_llm_start (line 59) | def on_llm_start(self, serialized, prompts, *, run_id, **kwargs):
  class _State (line 67) | class _State(TypedDict, total=False):
  function _fake_metric (line 72) | def _fake_metric(name: str = "fake") -> BaseMetric:
  function _build_single_llm_graph (line 78) | def _build_single_llm_graph(llm: FakeListLLM):
  function _build_two_llm_graph (line 94) | def _build_two_llm_graph(llm: FakeListLLM):
  class TestNextLlmSpanInStateGraph (line 125) | class TestNextLlmSpanInStateGraph:
    method test_metric_collection_lands_on_llm_span (line 126) | async def test_metric_collection_lands_on_llm_span(self):
    method test_metrics_lands_on_llm_span (line 139) | async def test_metrics_lands_on_llm_span(self):
    method test_metadata_lands_on_llm_span (line 152) | async def test_metadata_lands_on_llm_span(self):
    method test_only_first_llm_span_in_multi_node_graph (line 164) | async def test_only_first_llm_span_in_multi_node_graph(self):
    method test_unconsumed_payload_does_not_leak_across_invocations (line 183) | async def test_unconsumed_payload_does_not_leak_across_invocations(
  function test_next_tool_span_does_not_leak_to_llm_span_in_graph (line 213) | async def test_next_tool_span_does_not_leak_to_llm_span_in_graph():
  function test_base_next_span_lands_on_first_llm_span_in_graph (line 238) | async def test_base_next_span_lands_on_first_llm_span_in_graph():
  function test_next_llm_span_in_sync_state_graph (line 264) | def test_next_llm_span_in_sync_state_graph():

FILE: tests/test_integrations/test_langgraph/test_sync.py
  function trace_test (line 58) | def trace_test(schema_name: str):
  class TestSimpleApp (line 77) | class TestSimpleApp:
    method test_weather_query (line 81) | def test_weather_query(self):
  class TestMultipleToolsApp (line 110) | class TestMultipleToolsApp:
    method test_city_info (line 114) | def test_city_info(self):
    method test_mixed_query (line 138) | def test_mixed_query(self):
  class TestStreamingApp (line 165) | class TestStreamingApp:
    method test_sync_streaming (line 169) | def test_sync_streaming(self):
    method test_sync_streaming_multiple_tools (line 191) | def test_sync_streaming_multiple_tools(self):
  class TestConditionalApp (line 219) | class TestConditionalApp:
    method test_research_route (line 223) | def test_research_route(self):
    method test_summarize_route (line 249) | def test_summarize_route(self):
    method test_fact_check_route (line 270) | def test_fact_check_route(self):
    method test_general_route (line 295) | def test_general_route(self):
  class TestParallelToolsApp (line 315) | class TestParallelToolsApp:
    method test_parallel_weather_queries (line 319) | def test_parallel_weather_queries(self):
    method test_parallel_mixed_tools (line 341) | def test_parallel_mixed_tools(self):
    method test_parallel_stock_queries (line 370) | def test_parallel_stock_queries(self):
  class TestMultiTurnApp (line 396) | class TestMultiTurnApp:
    method test_multi_turn_shopping (line 400) | def test_multi_turn_shopping(self):
    method test_stateless_single_turn (line 459) | def test_stateless_single_turn(self):
    method test_full_shopping_flow (line 474) | def test_full_shopping_flow(self):
  class TestRetrieverApp (line 558) | class TestRetrieverApp:
    method test_retrieve_python_docs (line 562) | def test_retrieve_python_docs(self):
    method test_retrieve_langchain_docs (line 584) | def test_retrieve_langchain_docs(self):
    method test_retriever_metric_collection (line 603) | def test_retriever_metric_collection(self):
  class TestMetricCollectionApp (line 631) | class TestMetricCollectionApp:
    method test_metric_collection (line 635) | def test_metric_collection(self):
  class TestNextSpanApp (line 663) | class TestNextSpanApp:
    method test_next_llm_span_only (line 672) | def test_next_llm_span_only(self):

FILE: tests/test_integrations/test_llamaindex/apps/agent_app.py
  function get_weather (line 11) | def get_weather(city: str) -> str:
  function multiply (line 23) | def multiply(a: float, b: float) -> float:
  function get_agent (line 28) | def get_agent():

FILE: tests/test_integrations/test_llamaindex/apps/eval_app.py
  function multiply (line 13) | def multiply(a: float, b: float) -> float:
  function get_evals_agent (line 18) | def get_evals_agent():

FILE: tests/test_integrations/test_llamaindex/apps/rag_app.py
  class DeterministicRetriever (line 17) | class DeterministicRetriever(BaseRetriever):
    method _retrieve (line 22) | def _retrieve(self, query_bundle: QueryBundle) -> List[NodeWithScore]:
  function get_rag_engine (line 50) | def get_rag_engine():

FILE: tests/test_integrations/test_llamaindex/apps/router_app.py
  class MockMathEngine (line 17) | class MockMathEngine:
    method query (line 20) | def query(self, query_str: str):
    method aquery (line 24) | async def aquery(self, query_str: str):
  function get_router_engine (line 29) | def get_router_engine():

FILE: tests/test_integrations/test_llamaindex/apps/simple_app.py
  function get_simple_engine (line 14) | def get_simple_engine():

FILE: tests/test_integrations/test_llamaindex/conftest.py
  function _setup_llama_index_instrumentation (line 12) | def _setup_llama_index_instrumentation():
  function reset_trace_state (line 22) | def reset_trace_state():

FILE: tests/test_integrations/test_llamaindex/test_async.py
  function trace_test (line 46) | def trace_test(schema_name: str):
  class TestAsyncSimpleApp (line 63) | class TestAsyncSimpleApp:
    method test_async_simple_query (line 68) | async def test_async_simple_query(self):
  class TestAsyncRAGApp (line 88) | class TestAsyncRAGApp:
    method test_async_rag_query (line 93) | async def test_async_rag_query(self):
  class TestAsyncAgentApp (line 113) | class TestAsyncAgentApp:
    method test_async_agent_tool (line 118) | async def test_async_agent_tool(self):
    method test_async_agent_math (line 135) | async def test_async_agent_math(self):
  class TestAsyncRouterApp (line 155) | class TestAsyncRouterApp:
    method test_async_router_selection (line 160) | async def test_async_router_selection(self):
  class TestDeepEvalFeaturesAsync (line 180) | class TestDeepEvalFeaturesAsync:
    method reset_instrumentation (line 184) | def reset_instrumentation(self):
    method test_features_async (line 195) | async def test_features_async(self):

FILE: tests/test_integrations/test_llamaindex/test_sync.py
  function trace_test (line 30) | def trace_test(schema_name: str):
  class TestSimpleApp (line 47) | class TestSimpleApp:
    method test_simple_query (line 51) | def test_simple_query(self):
  class TestRAGApp (line 71) | class TestRAGApp:
    method test_rag_python_query (line 75) | def test_rag_python_query(self):
    method test_rag_llama_query (line 90) | def test_rag_llama_query(self):
  class TestRouterApp (line 110) | class TestRouterApp:
    method test_router_math_selection (line 114) | def test_router_math_selection(self):

FILE: tests/test_integrations/test_openai/conftest.py
  function _setup_openai_instrumentation (line 5) | def _setup_openai_instrumentation():

FILE: tests/test_integrations/test_openai/simple_openai.py
  function run_async_openai (line 31) | async def run_async_openai():

FILE: tests/test_integrations/test_openai/test_async_openai.py
  function test_async_openai_without_trace (line 23) | async def test_async_openai_without_trace():
  function test_async_openai_with_trace (line 31) | async def test_async_openai_with_trace():
  function test_async_response_create_without_trace (line 50) | async def test_async_response_create_without_trace():
  function test_async_response_create_with_trace (line 63) | async def test_async_response_create_with_trace():
  function generate_all_json_dumps (line 82) | async def generate_all_json_dumps():

FILE: tests/test_integrations/test_openai/test_sync_openai.py
  function test_sync_openai_without_trace (line 19) | def test_sync_openai_without_trace():
  function test_sync_openai_with_trace (line 29) | def test_sync_openai_with_trace():
  function test_sync_response_create_without_trace (line 53) | def test_sync_response_create_without_trace():
  function test_sync_response_create_with_trace (line 62) | def test_sync_response_create_with_trace():
  function generate_all_json_dumps (line 81) | def generate_all_json_dumps():

FILE: tests/test_integrations/test_openai/test_tool_call_flow_completion.py
  function get_weather (line 12) | def get_weather(location: str, unit: str = "c") -> Dict[str, Any]:
  function run_main (line 66) | def run_main():
  function test_tool_call_flow_completion (line 140) | def test_tool_call_flow_completion():

FILE: tests/test_integrations/test_openai/test_tool_call_flow_response.py
  function get_weather (line 11) | def get_weather(location: str, unit: str = "c") -> Dict[str, Any]:
  function run_main (line 63) | def run_main():
  function test_tool_call_flow_response (line 125) | def test_tool_call_flow_response():

FILE: tests/test_integrations/test_openai_agents/apps/session_agent.py
  function get_agent (line 9) | def get_agent():
  function get_session (line 18) | def get_session(session_id: str):

FILE: tests/test_integrations/test_openai_agents/apps/streaming_agent.py
  function get_company_info (line 10) | def get_company_info(symbol: str) -> str:

FILE: tests/test_integrations/test_openai_agents/apps/tool_agent.py
  function get_weather (line 12) | def get_weather(city: str) -> str:
  function calculate (line 27) | def calculate(expression: str) -> str:

FILE: tests/test_integrations/test_openai_agents/conftest.py
  function _install_deepeval_tracer (line 7) | def _install_deepeval_tracer():

FILE: tests/test_integrations/test_openai_agents/test_async.py
  function trace_test (line 30) | def trace_test(schema_name: str):
  class TestAsyncSimpleAgent (line 39) | class TestAsyncSimpleAgent:
    method test_async_greeting (line 42) | async def test_async_greeting(self):
  class TestAsyncToolAgent (line 51) | class TestAsyncToolAgent:
    method test_async_tool (line 54) | async def test_async_tool(self):
  class TestAsyncHandoffAgent (line 63) | class TestAsyncHandoffAgent:
    method test_async_handoff (line 66) | async def test_async_handoff(self):

FILE: tests/test_integrations/test_openai_agents/test_scenerios/test_multi_agents.py
  function run (line 31) | async def run():
  function test_json_schema (line 41) | async def test_json_schema():
  function generate_actual_json_dump (line 60) | async def generate_actual_json_dump():

FILE: tests/test_integrations/test_openai_agents/test_scenerios/test_run.py
  function get_current_weather (line 15) | def get_current_weather(latitude: float, longitude: float) -> dict:
  function get_location_coordinates (line 40) | def get_location_coordinates(city_name: str) -> dict:
  function run (line 87) | async def run():
  function test_json_schema (line 100) | async def test_json_schema():
  function generate_actual_json_dump (line 119) | async def generate_actual_json_dump():

FILE: tests/test_integrations/test_openai_agents/test_scenerios/test_run_streamed.py
  function get_current_weather (line 15) | def get_current_weather(latitude: float, longitude: float) -> dict:
  function get_location_coordinates (line 40) | def get_location_coordinates(city_name: str) -> dict:
  function run_weather_agent_streamed (line 87) | async def run_weather_agent_streamed():
  function test_json_schema (line 103) | async def test_json_schema():
  function generate_actual_json_dump (line 122) | async def generate_actual_json_dump():

FILE: tests/test_integrations/test_openai_agents/test_scenerios/test_run_sync.py
  function get_current_weather (line 14) | def get_current_weather(latitude: float, longitude: float) -> dict:
  function get_location_coordinates (line 39) | def get_location_coordinates(city_name: str) -> dict:
  function run_sync (line 86) | def run_sync():
  function test_json_schema (line 100) | async def test_json_schema():
  function generate_actual_json_dump (line 119) | async def generate_actual_json_dump():

FILE: tests/test_integrations/test_openai_agents/test_scenerios/test_weather_agent_patched.py
  function get_current_weather (line 30) | def get_current_weather(latitude: float, longitude: float) -> dict:
  function get_location_coordinates (line 55) | def get_location_coordinates(city_name: str) -> dict:
  function run_weather_agent (line 105) | async def run_weather_agent(input: str):
  function test_json_schema (line 118) | async def test_json_schema():
  function generate_actual_json_dump (line 137) | async def generate_actual_json_dump():

FILE: tests/test_integrations/test_openai_agents/test_scenerios/test_with_trace.py
  function get_current_weather (line 17) | def get_current_weather(latitude: float, longitude: float) -> dict:
  function get_location_coordinates (line 42) | def get_location_coordinates(city_name: str) -> dict:
  function run (line 89) | async def run():
  function test_json_schema (line 109) | async def test_json_schema():
  function generate_actual_json_dump (line 128) | async def generate_actual_json_dump():

FILE: tests/test_integrations/test_openai_agents/test_scenerios/test_with_trace_and_wrapped.py
  function get_current_weather (line 34) | def get_current_weather(latitude: float, longitude: float) -> dict:
  function get_location_coordinates (line 59) | def get_location_coordinates(city_name: str) -> dict:
  function run_weather_agent (line 109) | async def run_weather_agent(input: str):
  function test_json_schema (line 129) | async def test_json_schema():
  function generate_actual_json_dump (line 148) | async def generate_actual_json_dump():

FILE: tests/test_integrations/test_openai_agents/test_sync.py
  function trace_test (line 40) | def trace_test(schema_name: str):
  class TestSimpleAgent (line 50) | class TestSimpleAgent:
    method test_simple_greeting (line 52) | def test_simple_greeting(self):
  class TestToolAgent (line 65) | class TestToolAgent:
    method test_tool_weather (line 67) | def test_tool_weather(self):
    method test_tool_calculation (line 78) | def test_tool_calculation(self):
  class TestEvalAgent (line 88) | class TestEvalAgent:
    method test_eval_agent_metrics (line 90) | def test_eval_agent_metrics(self):
  class TestHandoffAgent (line 100) | class TestHandoffAgent:
    method test_handoff_spanish (line 102) | def test_handoff_spanish(self):
  class TestSessionAgent (line 112) | class TestSessionAgent:
    method test_session_memory (line 114) | def test_session_memory(self):

FILE: tests/test_integrations/test_openrouter/test_openrouter_generation.py
  class SampleSchema (line 10) | class SampleSchema(BaseModel):
  class TestOpenRouterModelIntegration (line 22) | class TestOpenRouterModelIntegration:
    method test_basic_generation (line 25) | def test_basic_generation(self):
    method test_async_generation (line 39) | async def test_async_generation(self):
    method test_structured_outputs (line 52) | def test_structured_outputs(self):
    method test_different_models (line 69) | def test_different_models(self):

FILE: tests/test_integrations/test_pydanticai/apps/eval_app.py
  function create_evals_agent (line 23) | def create_evals_agent(
  function invoke_evals_agent (line 57) | def invoke_evals_agent(
  function ainvoke_evals_agent (line 76) | async def ainvoke_evals_agent(

FILE: tests/test_integrations/test_pydanticai/apps/pydanticai_isolation_app.py
  function create_isolation_agent (line 45) | def create_isolation_agent(
  function _build_prompt (line 97) | def _build_prompt(key: str) -> str:
  function make_distinct_requests (line 101) | def make_distinct_requests() -> List[RequestSpec]:
  function concurrent_isolation_run (line 111) | async def concurrent_isolation_run(
  function threaded_isolation_run (line 137) | def threaded_isolation_run(

FILE: tests/test_integrations/test_pydanticai/apps/pydanticai_metric_collection_app.py
  function create_trace_metric_collection_agent (line 24) | def create_trace_metric_collection_agent(
  function invoke_metric_collection_agent (line 50) | def invoke_metric_collection_agent(prompt: str, agent: Agent) -> str:
  function ainvoke_metric_collection_agent (line 55) | async def ainvoke_metric_collection_agent(prompt: str, agent: Agent) -> ...

FILE: tests/test_integrations/test_pydanticai/apps/pydanticai_modes_app.py
  function create_modes_agent (line 33) | def create_modes_agent(
  function invoke_in_observe_mode (line 61) | def invoke_in_observe_mode(
  function invoke_in_with_trace_mode (line 96) | def invoke_in_with_trace_mode(
  function create_enrichment_agent (line 121) | def create_enrichment_agent(
  function invoke_with_tool_enrichment (line 177) | def invoke_with_tool_enrichment(

FILE: tests/test_integrations/test_pydanticai/apps/pydanticai_multiple_tools_app.py
  function create_multiple_tools_agent (line 12) | def create_multiple_tools_agent(
  function invoke_multiple_tools_agent (line 85) | def invoke_multiple_tools_agent(prompt: str, agent: Agent = None) -> str:
  function ainvoke_multiple_tools_agent (line 93) | async def ainvoke_multiple_tools_agent(prompt: str, agent: Agent = None)...

FILE: tests/test_integrations/test_pydanticai/apps/pydanticai_next_span_app.py
  function create_next_span_agent (line 27) | def create_next_span_agent(
  function invoke_with_next_llm_span (line 53) | def invoke_with_next_llm_span(
  function invoke_with_stacked_next_spans (line 71) | def invoke_with_stacked_next_spans(

FILE: tests/test_integrations/test_pydanticai/apps/pydanticai_simple_app.py
  function create_simple_agent (line 12) | def create_simple_agent(
  function invoke_simple_agent (line 36) | def invoke_simple_agent(prompt: str, agent: Agent = None) -> str:
  function ainvoke_simple_agent (line 44) | async def ainvoke_simple_agent(prompt: str, agent: Agent = None) -> str:

FILE: tests/test_integrations/test_pydanticai/apps/pydanticai_streaming_app.py
  function create_streaming_agent (line 12) | def create_streaming_agent(
  function stream_agent (line 36) | async def stream_agent(prompt: str, agent: Agent = None) -> str:

FILE: tests/test_integrations/test_pydanticai/apps/pydanticai_tool_app.py
  function create_tool_agent (line 12) | def create_tool_agent(
  function invoke_tool_agent (line 67) | def invoke_tool_agent(prompt: str, agent: Agent = None) -> str:
  function ainvoke_tool_agent (line 75) | async def ainvoke_tool_agent(prompt: str, agent: Agent = None) -> str:

FILE: tests/test_integrations/test_pydanticai/conftest.py
  function deepeval_isolated_no_disk (line 7) | def deepeval_isolated_no_disk(tmp_path, monkeypatch):

FILE: tests/test_integrations/test_pydanticai/test_async.py
  function trace_test (line 49) | def trace_test(schema_name: str):
  class TestAsyncSimpleApp (line 68) | class TestAsyncSimpleApp:
    method test_async_simple_greeting (line 73) | async def test_async_simple_greeting(self):
  class TestAsyncMultipleToolsApp (line 97) | class TestAsyncMultipleToolsApp:
    method test_async_parallel_tool_calls (line 102) | async def test_async_parallel_tool_calls(self):
  class TestAsyncToolApp (line 131) | class TestAsyncToolApp:
    method test_async_tool_calculation (line 136) | async def test_async_tool_calculation(self):
  class TestStreamingApp (line 160) | class TestStreamingApp:
    method test_streaming_response (line 165) | async def test_streaming_response(self):
  class TestDeepEvalFeaturesAsync (line 189) | class TestDeepEvalFeaturesAsync:
    method test_full_features_async (line 194) | async def test_full_features_async(self):
  class TestConcurrentIsolation (line 219) | class TestConcurrentIsolation:
    method test_concurrent_isolation (line 231) | async def test_concurrent_isolation(self):

FILE: tests/test_integrations/test_pydanticai/test_evaluate_agent.py
  function run_agent (line 24) | async def run_agent(input: str):
  function run_eval (line 29) | def run_eval():
  function test_evaluate_agent (line 46) | def test_evaluate_agent():

FILE: tests/test_integrations/test_pydanticai/test_span_interceptor.py
  function _make_mock_span (line 52) | def _make_mock_span(operation_name=None, agent_name=None, tool_name=None):
  function _make_settings (line 81) | def _make_settings(**kwargs):
  class TestSpanInterceptorTraceContextReads (line 112) | class TestSpanInterceptorTraceContextReads:
    method test_uses_settings_when_no_trace_context (line 113) | def test_uses_settings_when_no_trace_context(self):
    method test_prefers_trace_context_over_settings_for_scalars (line 141) | def test_prefers_trace_context_over_settings_for_scalars(self):
    method test_metadata_is_merged_with_context_winning (line 162) | def test_metadata_is_merged_with_context_winning(self):
    method test_no_attributes_set_when_all_none (line 179) | def test_no_attributes_set_when_all_none(self):
  class TestSpanInterceptorNewTraceContextReads (line 203) | class TestSpanInterceptorNewTraceContextReads:
    method test_user_id_from_trace_context_overrides_settings (line 204) | def test_user_id_from_trace_context_overrides_settings(self):
    method test_tags_from_trace_context_overrides_settings (line 215) | def test_tags_from_trace_context_overrides_settings(self):
    method test_test_case_id_and_turn_id_from_trace_context_override_settings (line 229) | def test_test_case_id_and_turn_id_from_trace_context_override_settings(
    method test_trace_metric_collection_resolution_order (line 247) | def test_trace_metric_collection_resolution_order(self):
    method test_update_current_trace_after_on_start_lands_on_otel_attrs (line 263) | def test_update_current_trace_after_on_start_lands_on_otel_attrs(self):
    method test_trace_metric_collection_falls_back_to_settings (line 298) | def test_trace_metric_collection_falls_back_to_settings(self):
    method test_trace_metric_collection_omitted_when_neither_set (line 318) | def test_trace_metric_collection_omitted_when_neither_set(self):
  class TestSpanInterceptorSpanContextPushPop (line 342) | class TestSpanInterceptorSpanContextPushPop:
    method test_current_span_context_set_during_span_lifetime (line 343) | def test_current_span_context_set_during_span_lifetime(self):
    method test_update_current_span_metadata_lands_in_otel_attrs (line 361) | def test_update_current_span_metadata_lands_in_otel_attrs(self):
    method test_update_current_span_metric_collection_lands_in_otel_attrs (line 384) | def test_update_current_span_metric_collection_lands_in_otel_attrs(self):
    method test_nested_spans_lifo_pop_restores_parent_placeholder (line 399) | def test_nested_spans_lifo_pop_restores_parent_placeholder(self):
  class TestSpanInterceptorImplicitTraceContext (line 425) | class TestSpanInterceptorImplicitTraceContext:
    method test_root_span_pushes_implicit_trace_when_no_user_context (line 434) | def test_root_span_pushes_implicit_trace_when_no_user_context(self):
    method test_does_not_overwrite_user_pushed_trace_context (line 452) | def test_does_not_overwrite_user_pushed_trace_context(self):
    method test_child_span_does_not_push_its_own_placeholder (line 475) | def test_child_span_does_not_push_its_own_placeholder(self):
    method test_update_current_trace_in_implicit_context_lands_on_otel_attrs (line 505) | def test_update_current_trace_in_implicit_context_lands_on_otel_attrs(
  class _FakeSpan (line 543) | class _FakeSpan:
  class TestContextAwareSpanProcessorRouting (line 547) | class TestContextAwareSpanProcessorRouting:
    method _make_processor (line 549) | def _make_processor():
    method test_routes_to_rest_when_trace_context_active (line 559) | def test_routes_to_rest_when_trace_context_active(self):
    method test_routes_to_otlp_when_no_context (line 569) | def test_routes_to_otlp_when_no_context(self):
    method test_routes_to_rest_when_evaluating (line 586) | def test_routes_to_rest_when_evaluating(self):
    method test_routes_to_otlp_when_only_implicit_trace_in_context (line 603) | def test_routes_to_otlp_when_only_implicit_trace_in_context(self):
    method test_routes_to_rest_when_evaluating_even_with_implicit_trace (line 632) | def test_routes_to_rest_when_evaluating_even_with_implicit_trace(self):
    method test_routes_to_rest_when_test_name_is_set (line 661) | def test_routes_to_rest_when_test_name_is_set(self):
    method test_routes_to_rest_when_test_name_set_with_implicit_trace (line 696) | def test_routes_to_rest_when_test_name_set_with_implicit_trace(self):
    method test_routes_to_otlp_when_test_name_is_none (line 735) | def test_routes_to_otlp_when_test_name_is_none(self):
    method test_on_start_forwarded_to_both (line 765) | def test_on_start_forwarded_to_both(self):
    method test_shutdown_and_force_flush_forwarded_to_both (line 774) | def test_shutdown_and_force_flush_forwarded_to_both(self):
  function test_is_test_mode_kwarg_is_removed_from_settings (line 794) | def test_is_test_mode_kwarg_is_removed_from_settings():
  function test_span_related_kwargs_are_removed_from_settings (line 828) | def test_span_related_kwargs_are_removed_from_settings(kwarg):
  function test_no_api_key_does_not_raise (line 851) | def test_no_api_key_does_not_raise(monkeypatch):
  function test_confident_alias_emits_deprecation_warning (line 875) | def test_confident_alias_emits_deprecation_warning(monkeypatch):
  class TestNextSpanPureContextAPI (line 907) | class TestNextSpanPureContextAPI:
    method test_pop_outside_with_returns_empty (line 908) | def test_pop_outside_with_returns_empty(self):
    method test_next_agent_span_one_shot_consumption (line 913) | def test_next_agent_span_one_shot_consumption(self):
    method test_scope_exit_restores_prior_value (line 925) | def test_scope_exit_restores_prior_value(self):
    method test_stacked_typed_slots_are_independent (line 934) | def test_stacked_typed_slots_are_independent(self):
    method test_base_slot_consumed_by_first_typed_pop (line 946) | def test_base_slot_consumed_by_first_typed_pop(self):
    method test_typed_overrides_base_on_key_overlap (line 956) | def test_typed_overrides_base_on_key_overlap(self):
    method test_pop_for_mismatched_type_drains_only_base (line 965) | def test_pop_for_mismatched_type_drains_only_base(self):
    method test_nested_same_type_innermost_wins (line 979) | def test_nested_same_type_innermost_wins(self):
    method test_drops_none_kwargs (line 991) | def test_drops_none_kwargs(self):
    method test_unconsumed_payload_does_not_leak_across_scopes (line 1000) | def test_unconsumed_payload_does_not_leak_across_scopes(self):
    method test_drain_visible_across_asyncio_sub_context (line 1010) | def test_drain_visible_across_asyncio_sub_context(self):
    method test_other_typed_helpers_each_use_their_own_slot (line 1039) | def test_other_typed_helpers_each_use_their_own_slot(self):
  function _make_placeholder (line 1059) | def _make_placeholder(cls=BaseSpan, **kw) -> BaseSpan:
  class TestApplyPendingToSpan (line 1073) | class TestApplyPendingToSpan:
    method test_empty_payload_is_noop (line 1074) | def test_empty_payload_is_noop(self):
    method test_base_field_setattr (line 1081) | def test_base_field_setattr(self):
    method test_agent_specific_fields_apply_only_to_agent_span (line 1091) | def test_agent_specific_fields_apply_only_to_agent_span(self):
    method test_cross_type_keys_silently_dropped (line 1100) | def test_cross_type_keys_silently_dropped(self):
    method test_test_case_unpacking_then_individual_fields_override (line 1119) | def test_test_case_unpacking_then_individual_fields_override(self):
  function _make_agent_span_mock (line 1156) | def _make_agent_span_mock(agent_name="agent_x"):
  class TestNextSpanInterceptorIntegration (line 1162) | class TestNextSpanInterceptorIntegration:
    method test_next_agent_span_metric_collection_lands_on_otel_attrs (line 1163) | def test_next_agent_span_metric_collection_lands_on_otel_attrs(self):
    method test_next_agent_span_consumed_only_by_first_agent_span (line 1180) | def test_next_agent_span_consumed_only_by_first_agent_span(self):
    method test_next_agent_span_does_not_affect_non_agent_span (line 1201) | def test_next_agent_span_does_not_affect_non_agent_span(self):
    method test_next_agent_span_metadata_lands_on_agent_placeholder (line 1226) | def test_next_agent_span_metadata_lands_on_agent_placeholder(self):
    method test_update_current_span_overrides_next_agent_span_after_creation (line 1250) | def test_update_current_span_overrides_next_agent_span_after_creation(

FILE: tests/test_integrations/test_pydanticai/test_sync.py
  function trace_test (line 61) | def trace_test(schema_name: str):
  class TestSimpleApp (line 80) | class TestSimpleApp:
    method test_simple_greeting (line 84) | def test_simple_greeting(self):
  class TestToolApp (line 108) | class TestToolApp:
    method test_tool_calculation (line 112) | def test_tool_calculation(self):
  class TestMetricCollectionApp (line 136) | class TestMetricCollectionApp:
    method test_trace_metric_collection (line 145) | def test_trace_metric_collection(self):
  class TestMultipleToolsApp (line 170) | class TestMultipleToolsApp:
    method test_multiple_tools_weather_only (line 174) | def test_multiple_tools_weather_only(self):
    method test_multiple_tools_time_only (line 194) | def test_multiple_tools_time_only(self):
    method test_parallel_tool_calls (line 214) | def test_parallel_tool_calls(self):
  class TestDeepEvalFeatures (line 247) | class TestDeepEvalFeatures:
    method test_full_features_sync (line 251) | def test_full_features_sync(self):
  class TestNextSpanApp (line 280) | class TestNextSpanApp:
    method test_next_llm_span_only (line 288) | def test_next_llm_span_only(self):
    method test_next_stacked_agent_and_llm (line 314) | def test_next_stacked_agent_and_llm(self):
  class TestExecutionModes (line 345) | class TestExecutionModes:
    method test_observe_mode (line 352) | def test_observe_mode(self):
    method test_with_trace_mode (line 379) | def test_with_trace_mode(self):
    method test_bare_trace_enrichment_from_tool (line 405) | def test_bare_trace_enrichment_from_tool(self):
  class TestThreadIsolation (line 432) | class TestThreadIsolation:
    method test_thread_isolation (line 443) | def test_thread_isolation(self):

FILE: tests/test_integrations/test_strands/apps/strands_eval_app.py
  function _build_openai_model (line 31) | def _build_openai_model() -> OpenAIModel:
  function special_tool (line 40) | def special_tool(query: str) -> str:
  function init_evals_strands (line 52) | def init_evals_strands(
  function invoke_evals_agent (line 93) | def invoke_evals_agent(prompt: str, invoke_func=None) -> str:
  function ainvoke_evals_agent (line 100) | async def ainvoke_evals_agent(prompt: str, invoke_func=None) -> str:

FILE: tests/test_integrations/test_strands/apps/strands_multiple_tools_app.py
  function _build_openai_model (line 12) | def _build_openai_model() -> OpenAIModel:
  function get_weather (line 21) | def get_weather(city: str) -> str:
  function get_time (line 34) | def get_time(city: str) -> str:
  function init_multiple_tools_strands (line 44) | def init_multiple_tools_strands(
  function invoke_multiple_tools_agent (line 92) | def invoke_multiple_tools_agent(prompt: str, invoke_func=None) -> str:
  function ainvoke_multiple_tools_agent (line 99) | async def ainvoke_multiple_tools_agent(prompt: str, invoke_func=None) ->...

FILE: tests/test_integrations/test_strands/apps/strands_simple_app.py
  function _build_openai_model (line 12) | def _build_openai_model() -> OpenAIModel:
  function init_simple_strands (line 23) | def init_simple_strands(
  function invoke_simple_agent (line 66) | def invoke_simple_agent(prompt: str, invoke_func=None) -> str:
  function ainvoke_simple_agent (line 73) | async def ainvoke_simple_agent(prompt: str, invoke_func=None) -> str:

FILE: tests/test_integrations/test_strands/apps/strands_tool_app.py
  function _build_openai_model (line 12) | def _build_openai_model() -> OpenAIModel:
  function calculate (line 21) | def calculate(operation: str, a: float, b: float) -> float:
  function init_tool_strands (line 35) | def init_tool_strands(
  function invoke_tool_agent (line 81) | def invoke_tool_agent(prompt: str, invoke_func=None) -> str:
  function ainvoke_tool_agent (line 88) | async def ainvoke_tool_agent(prompt: str, invoke_func=None) -> str:

FILE: tests/test_integrations/test_strands/conftest.py
  function deepeval_isolated_no_disk (line 7) | def deepeval_isolated_no_disk(tmp_path, monkeypatch):

FILE: tests/test_integrations/test_strands/test_async.py
  function trace_test (line 40) | def trace_test(schema_name: str):
  class TestAsyncSimpleApp (line 48) | class TestAsyncSimpleApp:
    method test_async_simple_greeting (line 52) | async def test_async_simple_greeting(self):
  class TestAsyncToolApp (line 70) | class TestAsyncToolApp:
    method test_async_tool_calculation (line 74) | async def test_async_tool_calculation(self):
  class TestAsyncMultipleToolsApp (line 92) | class TestAsyncMultipleToolsApp:
    method test_async_parallel_tool_calls (line 96) | async def test_async_parallel_tool_calls(self):
  class TestDeepEvalFeaturesAsync (line 116) | class TestDeepEvalFeaturesAsync:
    method test_full_features_async (line 125) | async def test_full_features_async(self):

FILE: tests/test_integrations/test_strands/test_evaluate_agent.py
  function test_evaluate_agent (line 51) | def test_evaluate_agent():

FILE: tests/test_integrations/test_strands/test_span_interceptor.py
  function _make_mock_span (line 56) | def _make_mock_span(
  function _make_settings (line 105) | def _make_settings(**kwargs):
  function _make_agent_span_mock (line 130) | def _make_agent_span_mock(agent_name: str = "agent_x"):
  class TestTraceContextReads (line 141) | class TestTraceContextReads:
    method test_uses_settings_when_no_trace_context (line 142) | def test_uses_settings_when_no_trace_context(self):
    method test_prefers_trace_context_over_settings_for_scalars (line 170) | def test_prefers_trace_context_over_settings_for_scalars(self):
    method test_metadata_is_merged_with_context_winning (line 185) | def test_metadata_is_merged_with_context_winning(self):
    method test_update_current_trace_after_on_start_lands_on_otel_attrs (line 201) | def test_update_current_trace_after_on_start_lands_on_otel_attrs(self):
    method test_trace_metric_collection_resolution_order (line 231) | def test_trace_metric_collection_resolution_order(self):
    method test_strands_session_id_falls_through_to_thread_id (line 244) | def test_strands_session_id_falls_through_to_thread_id(self):
    method test_explicit_thread_id_wins_over_session_id (line 267) | def test_explicit_thread_id_wins_over_session_id(self):
  class TestSpanContextPushPop (line 289) | class TestSpanContextPushPop:
    method test_current_span_context_set_during_span_lifetime (line 290) | def test_current_span_context_set_during_span_lifetime(self):
    method test_update_current_span_metadata_lands_in_otel_attrs (line 306) | def test_update_current_span_metadata_lands_in_otel_attrs(self):
    method test_update_current_span_metric_collection_lands_in_otel_attrs (line 329) | def test_update_current_span_metric_collection_lands_in_otel_attrs(self):
    method test_nested_spans_lifo_pop_restores_parent_placeholder (line 347) | def test_nested_spans_lifo_pop_restores_parent_placeholder(self):
  class TestImplicitTraceContext (line 371) | class TestImplicitTraceContext:
    method test_root_span_pushes_implicit_trace_when_no_user_context (line 381) | def test_root_span_pushes_implicit_trace_when_no_user_context(self):
    method test_does_not_overwrite_user_pushed_trace_context (line 399) | def test_does_not_overwrite_user_pushed_trace_context(self):
    method test_child_span_does_not_push_its_own_placeholder (line 417) | def test_child_span_does_not_push_its_own_placeholder(self):
    method test_update_current_trace_in_implicit_context_lands_on_otel_attrs (line 440) | def test_update_current_trace_in_implicit_context_lands_on_otel_attrs(
  class TestParentBridge (line 475) | class TestParentBridge:
    method test_stamps_parent_uuid_when_enclosed_in_deepeval_span (line 476) | def test_stamps_parent_uuid_when_enclosed_in_deepeval_span(self):
    method test_no_parent_uuid_when_otel_span_has_native_parent (line 507) | def test_no_parent_uuid_when_otel_span_has_native_parent(self):
  class TestNextSpanInterceptorIntegration (line 537) | class TestNextSpanInterceptorIntegration:
    method test_next_agent_span_metric_collection_lands_on_otel_attrs (line 538) | def test_next_agent_span_metric_collection_lands_on_otel_attrs(self):
    method test_next_agent_span_consumed_only_by_first_agent_span (line 552) | def test_next_agent_span_consumed_only_by_first_agent_span(self):
    method test_next_agent_span_does_not_affect_non_agent_span (line 573) | def test_next_agent_span_does_not_affect_non_agent_span(self):
    method test_next_tool_span_metric_collection_lands_on_tool_otel_attrs (line 597) | def test_next_tool_span_metric_collection_lands_on_tool_otel_attrs(self):
    method test_update_current_span_overrides_next_agent_span_after_creation (line 616) | def test_update_current_span_overrides_next_agent_span_after_creation(
    method test_next_agent_span_metrics_stashed_when_evaluating (line 636) | def test_next_agent_span_metrics_stashed_when_evaluating(self):
    method test_next_agent_span_metrics_not_stashed_outside_eval_mode (line 670) | def test_next_agent_span_metrics_not_stashed_outside_eval_mode(self):
  function test_removed_kwargs_raise_typeerror_on_settings (line 711) | def test_removed_kwargs_raise_typeerror_on_settings(kwarg):
  function test_removed_kwargs_raise_typeerror_on_instrument_strands (line 735) | def test_removed_kwargs_raise_typeerror_on_instrument_strands(kwarg):
  function test_settings_no_api_key_does_not_raise (line 751) | def test_settings_no_api_key_does_not_raise(monkeypatch):

FILE: tests/test_integrations/test_strands/test_sync.py
  function trace_test (line 40) | def trace_test(schema_name: str):
  class TestSimpleApp (line 48) | class TestSimpleApp:
    method test_simple_greeting (line 51) | def test_simple_greeting(self):
  class TestToolApp (line 69) | class TestToolApp:
    method test_tool_calculation (line 72) | def test_tool_calculation(self):
    method test_tool_metric_collection (line 90) | def test_tool_metric_collection(self):
  class TestMultipleToolsApp (line 117) | class TestMultipleToolsApp:
    method test_multiple_tools_weather_only (line 120) | def test_multiple_tools_weather_only(self):
    method test_multiple_tools_time_only (line 138) | def test_multiple_tools_time_only(self):
    method test_parallel_tool_calls (line 156) | def test_parallel_tool_calls(self):
  class TestDeepEvalFeatures (line 176) | class TestDeepEvalFeatures:
    method test_full_features_sync (line 186) | def test_full_features_sync(self):

FILE: tests/test_integrations/utils.py
  function is_generate_mode (line 12) | def is_generate_mode() -> bool:
  function _compute_tools_used (line 28) | def _compute_tools_used(obj: Dict[str, Any]) -> bool:
  function assert_json_object_structure (line 87) | def assert_json_object_structure(
  function load_trace_data (line 405) | def load_trace_data(file_path: str):
  function _store_trace_for_upload (line 414) | def _store_trace_for_upload(trace_dict: Dict[str, Any]):
  function get_stored_trace (line 430) | def get_stored_trace(nodeid: str) -> Dict[str, Any]:
  function generate_trace_json (line 435) | def generate_trace_json(json_path: str):
  function _assert_trace_capture_succeeded (line 500) | def _assert_trace_capture_succeeded(
  function assert_trace_json (line 527) | def assert_trace_json(json_path: str):

FILE: tests/test_metrics/test_answer_relevancy_metric.py
  class TestAnswerRelevancyMetric (line 17) | class TestAnswerRelevancyMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 44) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 68) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 89) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 110) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 130) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 154) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_answer_relevancy_metric_empty_output.py
  function make_metric (line 24) | def make_metric(*, async_mode: bool = False) -> AnswerRelevancyMetric:
  function test_answer_relevancy_none_actual_output_raises_sync (line 36) | def test_answer_relevancy_none_actual_output_raises_sync():
  function test_answer_relevancy_empty_actual_output_raises_sync (line 49) | def test_answer_relevancy_empty_actual_output_raises_sync():
  function test_answer_relevancy_whitespace_actual_output_does_not_raise_validation (line 63) | def test_answer_relevancy_whitespace_actual_output_does_not_raise_valida...

FILE: tests/test_metrics/test_arena_geval_metric.py
  class TestArenaGEval (line 23) | class TestArenaGEval:
    method test_normal_sync_metric_measure (line 26) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 71) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 115) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 160) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 206) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_compare_method (line 249) | def test_normal_compare_method(self):
    method test_multimodal_compare_method (line 293) | def test_multimodal_compare_method(self):

FILE: tests/test_metrics/test_bias_metric.py
  class TestBiasMetric (line 17) | class TestBiasMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 44) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 68) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 89) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 110) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 128) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 152) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_contextual_precision_metric.py
  class TestContextualPrecisionMetric (line 17) | class TestContextualPrecisionMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 44) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 68) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 89) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 110) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 130) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 154) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_contextual_recall_metric.py
  class TestContextualRecallMetric (line 17) | class TestContextualRecallMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 44) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 68) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 89) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 110) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 130) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 154) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_contextual_relevancy_metric.py
  class TestContextualRelevancyMetric (line 17) | class TestContextualRelevancyMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 44) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 68) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 89) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 110) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 130) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 154) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_conversation_completeness_metric.py
  class TestConversationCompletenessMetric (line 17) | class TestConversationCompletenessMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 42) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 64) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 88) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 112) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 133) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 154) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_conversational_dag.py
  class TestConversationalDeepAcyclicGraph (line 20) | class TestConversationalDeepAcyclicGraph:
    method test_is_valid_dag_true (line 21) | def test_is_valid_dag_true(self):
    method test_is_acyclic_dag (line 35) | def test_is_acyclic_dag(self):
    method test_is_valid_dag_deep_nested_mixed_nodes (line 45) | def test_is_valid_dag_deep_nested_mixed_nodes(self):
    method test_binary_judge_2_values (line 65) | def test_binary_judge_2_values(self):
    method test_valid_non_binary (line 75) | def test_valid_non_binary(self):
    method test_invalid_non_binary (line 85) | def test_invalid_non_binary(self):
    method test_invalid_verdicts (line 94) | def test_invalid_verdicts(self):
    method test_extract_required_params (line 102) | def test_extract_required_params(self):
    method test_invalid_child_type (line 121) | def test_invalid_child_type(self):
    method test_extract_required_params_non_binary (line 131) | def test_extract_required_params_non_binary(self):
    method test_disallow_multiple_judgement_roots (line 150) | def test_disallow_multiple_judgement_roots(self):
    method test_only_score_or_child (line 164) | def test_only_score_or_child(self):
    method test_allow_multiple_tasknode_roots (line 171) | def test_allow_multiple_tasknode_roots(self):
    method test_copy_graph_isolated_and_deep (line 177) | def test_copy_graph_isolated_and_deep(self):
    method test_non_binary_node_in_dag (line 234) | def test_non_binary_node_in_dag(self):
    method test_task_node_leaf (line 252) | def test_task_node_leaf(self):
    method test_verdict_node_with_child (line 262) | def test_verdict_node_with_child(self):

FILE: tests/test_metrics/test_conversational_g_eval.py
  class TestConversationalGEval (line 22) | class TestConversationalGEval:
    method test_normal_sync_metric_measure (line 25) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 52) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 78) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 106) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 135) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 161) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 186) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_dag.py
  class TestDeepAcyclicGraph (line 18) | class TestDeepAcyclicGraph:
    method test_is_valid_dag_true (line 21) | def test_is_valid_dag_true(self):
    method test_is_acyclic_dag (line 35) | def test_is_acyclic_dag(self):
    method test_is_valid_dag_deep_nested_mixed_nodes (line 45) | def test_is_valid_dag_deep_nested_mixed_nodes(self):
    method test_binary_judge_2_values (line 63) | def test_binary_judge_2_values(self):
    method test_valid_non_binary (line 73) | def test_valid_non_binary(self):
    method test_invalid_non_binary (line 84) | def test_invalid_non_binary(self):
    method test_invalid_verdicts (line 93) | def test_invalid_verdicts(self):
    method test_extract_required_params (line 99) | def test_extract_required_params(self):
    method test_invalid_child_type (line 122) | def test_invalid_child_type(self):
    method test_extract_required_params_non_binary (line 132) | def test_extract_required_params_non_binary(self):
    method test_disallow_multiple_judgement_roots (line 151) | def test_disallow_multiple_judgement_roots(self):
    method test_only_score_or_child (line 165) | def test_only_score_or_child(self):
    method test_allow_multiple_tasknode_roots (line 170) | def test_allow_multiple_tasknode_roots(self):
    method test_copy_graph_isolated_and_deep (line 176) | def test_copy_graph_isolated_and_deep(self):
    method test_non_binary_node_in_dag (line 234) | def test_non_binary_node_in_dag(self):
    method test_task_node_leaf (line 252) | def test_task_node_leaf(self):
    method test_verdict_node_with_child (line 262) | def test_verdict_node_with_child(self):

FILE: tests/test_metrics/test_dag_serialization.py
  function _build_simple_single_turn_dag (line 36) | def _build_simple_single_turn_dag() -> DeepAcyclicGraph:
  class TestSingleTurnRoundTrip (line 57) | class TestSingleTurnRoundTrip:
    method test_dag_to_dict_shape (line 58) | def test_dag_to_dict_shape(self):
    method test_dag_to_dict_ids_are_unique_uuids (line 66) | def test_dag_to_dict_ids_are_unique_uuids(self):
    method test_dag_to_dict_node_types_use_enum_values (line 76) | def test_dag_to_dict_node_types_use_enum_values(self):
    method test_dag_to_dict_evaluation_params_serialized_as_strings (line 84) | def test_dag_to_dict_evaluation_params_serialized_as_strings(self):
    method test_dag_to_dict_verdict_with_score_only (line 97) | def test_dag_to_dict_verdict_with_score_only(self):
    method test_round_trip_via_dict_preserves_structure (line 110) | def test_round_trip_via_dict_preserves_structure(self):
    method test_round_trip_via_json_string (line 133) | def test_round_trip_via_json_string(self):
    method test_round_trip_via_graph_methods (line 141) | def test_round_trip_via_graph_methods(self):
  class TestNonBinaryJudgement (line 149) | class TestNonBinaryJudgement:
    method test_non_binary_round_trip (line 150) | def test_non_binary_round_trip(self):
  class TestSharedChildDAG (line 169) | class TestSharedChildDAG:
    method test_shared_judgement_node_is_one_object (line 172) | def test_shared_judgement_node_is_one_object(self):
  function _build_simple_multiturn_dag (line 229) | def _build_simple_multiturn_dag() -> DeepAcyclicGraph:
  class TestMultiturnRoundTrip (line 240) | class TestMultiturnRoundTrip:
    method test_multiturn_round_trip (line 241) | def test_multiturn_round_trip(self):
    method test_multiturn_node_type_strings_are_mode_agnostic (line 256) | def test_multiturn_node_type_strings_are_mode_agnostic(self):
    method test_multiturn_task_node_turn_window_round_trip (line 265) | def test_multiturn_task_node_turn_window_round_trip(self):
  class TestNegative (line 292) | class TestNegative:
    method test_missing_nodes_key (line 293) | def test_missing_nodes_key(self):
    method test_empty_nodes (line 297) | def test_empty_nodes(self):
    method test_unknown_node_type (line 301) | def test_unknown_node_type(self):
    method test_unknown_child_type_on_verdict (line 310) | def test_unknown_child_type_on_verdict(self):
    method test_unknown_metric_class (line 323) | def test_unknown_metric_class(self):
    method test_cycle_in_json_refs (line 350) | def test_cycle_in_json_refs(self):
    method test_invalid_evaluation_param_value (line 375) | def test_invalid_evaluation_param_value(self):
  class TestSmoke (line 401) | class TestSmoke:
    method test_deserialized_dag_passes_validation (line 402) | def test_deserialized_dag_passes_validation(self):

FILE: tests/test_metrics/test_exact_match_metric.py
  class TestExactMatchMetric (line 17) | class TestExactMatchMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_evaluate_method (line 44) | def test_normal_evaluate_method(self):

FILE: tests/test_metrics/test_faithfulness_metric.py
  class TestFaithfulnessMetric (line 17) | class TestFaithfulnessMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 44) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 68) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 89) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 110) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 128) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 152) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_g_eval_metric.py
  class TestGEval (line 22) | class TestGEval:
    method test_normal_sync_metric_measure (line 25) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 57) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 88) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 116) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 145) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 171) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 200) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_g_eval_utils.py
  class DummyMetric (line 25) | class DummyMetric:
  class DummyConversationalMetric (line 30) | class DummyConversationalMetric:
  function test_geval_accepts_metadata_and_tags (line 35) | def test_geval_accepts_metadata_and_tags():
  function test_geval_requires_metadata_when_selected (line 58) | def test_geval_requires_metadata_when_selected():
  function test_conversational_geval_accepts_metadata_and_tags (line 71) | def test_conversational_geval_accepts_metadata_and_tags():
  function test_conversational_geval_requires_metadata_when_selected (line 110) | def test_conversational_geval_requires_metadata_when_selected():
  function test_conversational_geval_requires_tags_when_selected (line 124) | def test_conversational_geval_requires_tags_when_selected():

FILE: tests/test_metrics/test_goal_accuracy_metric.py
  class TestGoalAccuracyMetric (line 17) | class TestGoalAccuracyMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 42) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 64) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 88) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 112) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 133) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 154) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_hallucination_metric.py
  class TestHallucinationMetric (line 17) | class TestHallucinationMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 44) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 68) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 89) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 110) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 130) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 154) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_image_coherence_metric.py
  class TestImageCoherenceMetric (line 17) | class TestImageCoherenceMetric:
    method test_multimodal_async_metric_measure (line 20) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 41) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 62) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_multimodal_evaluate_method (line 82) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_image_editing_metric.py
  class TestImageEditingMetric (line 17) | class TestImageEditingMetric:
    method test_multimodal_async_metric_measure (line 20) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 41) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 62) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_multimodal_evaluate_method (line 80) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_image_helpfulness_metric.py
  class TestImageHelpfulnessMetric (line 17) | class TestImageHelpfulnessMetric:
    method test_multimodal_async_metric_measure (line 20) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 41) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 62) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_multimodal_evaluate_method (line 82) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_image_reference_metric.py
  class TestImageReferenceMetric (line 17) | class TestImageReferenceMetric:
    method test_multimodal_async_metric_measure (line 20) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 41) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 62) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_multimodal_evaluate_method (line 82) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_json_correctness_metric.py
  class ExampleSchema (line 9) | class ExampleSchema(BaseModel):
  class TestJsonCorrectnessMetric (line 23) | class TestJsonCorrectnessMetric:
    method test_normal_sync_metric_measure (line 26) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 52) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 78) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 101) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 124) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 146) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 172) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_knowledge_retention_metric.py
  class TestTurnRelevancyMetric (line 18) | class TestTurnRelevancyMetric:
    method test_normal_sync_metric_measure (line 21) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 43) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 65) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 89) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 113) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 134) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 155) | def test_multimodal_evaluate_method(self):
  class TestKnowledgeRetentionMetric (line 179) | class TestKnowledgeRetentionMetric:
    method test_knowledge_schema_unpacking (line 182) | def test_knowledge_schema_unpacking(self):
    method test_knowledge_schema_rejects_double_wrap (line 189) | def test_knowledge_schema_rejects_double_wrap(self):
    method test_normal_sync_metric_measure (line 196) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 221) | def test_normal_async_metric_measure(self):

FILE: tests/test_metrics/test_mcp_task_completetion_metric.py
  class TestMCPTaskCompletionMetric (line 22) | class TestMCPTaskCompletionMetric:
    method test_normal_sync_metric_measure (line 25) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 48) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 71) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 96) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 121) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 143) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 165) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_mcp_use_metric.py
  class TestMCPUseMetric (line 18) | class TestMCPUseMetric:
    method test_normal_sync_metric_measure (line 21) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 41) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 61) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 78) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 95) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 109) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 129) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_misuse_metric.py
  class TestMisuseMetric (line 17) | class TestMisuseMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 44) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 70) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 93) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 114) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 134) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 160) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_multi_turn_mcp_use_metric.py
  class TestMultiTurnMCPUseMetric (line 22) | class TestMultiTurnMCPUseMetric:
    method test_normal_sync_metric_measure (line 25) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 48) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 71) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 96) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 121) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 143) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 165) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_non_advice_metric.py
  class TestNonAdviceMetric (line 17) | class TestNonAdviceMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 46) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 72) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 95) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 118) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 140) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 166) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_pattern_match_metric.py
  class TestPatternMatchMetric (line 17) | class TestPatternMatchMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_evaluate_method (line 44) | def test_normal_evaluate_method(self):

FILE: tests/test_metrics/test_pii_lekage_metric.py
  class TestPIILeakageMetric (line 17) | class TestPIILeakageMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 44) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 68) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 89) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 110) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 128) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 152) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_plan_adherence_metric.py
  class TestPlanAdherenceMetric (line 18) | class TestPlanAdherenceMetric:
    method test_normal_sync_metric_measure (line 21) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 52) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 83) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 118) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 153) | def test_invalid_model_throws_error_for_multimodal(self):

FILE: tests/test_metrics/test_plan_quality_metric.py
  class TestPlanQualityMetric (line 18) | class TestPlanQualityMetric:
    method test_normal_sync_metric_measure (line 21) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 52) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 83) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 118) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 153) | def test_invalid_model_throws_error_for_multimodal(self):

FILE: tests/test_metrics/test_prompt_alignment_metric.py
  class TestPromptAlignmentMetric (line 17) | class TestPromptAlignmentMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 46) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 72) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 95) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 118) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 140) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 166) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_role_adherence_metric.py
  class TestRoleAdherenceMetric (line 17) | class TestRoleAdherenceMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 42) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 64) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 88) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 112) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 133) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 154) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_role_violation_metric.py
  class TestRoleViolationMetric (line 17) | class TestRoleViolationMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 44) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 70) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 93) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 114) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 136) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 162) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_step_efficiency_metric.py
  class TestStepEfficiencyMetric (line 18) | class TestStepEfficiencyMetric:
    method test_normal_sync_metric_measure (line 21) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 52) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 83) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 118) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 153) | def test_invalid_model_throws_error_for_multimodal(self):

FILE: tests/test_metrics/test_summarization_metric.py
  class TestSummarizationMetric (line 17) | class TestSummarizationMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 44) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 68) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 89) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 110) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 130) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 154) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_task_completetion_metric.py
  class TestTaskCompletionMetric (line 18) | class TestTaskCompletionMetric:
    method test_normal_sync_metric_measure (line 21) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 52) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 83) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 118) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 153) | def test_invalid_model_throws_error_for_multimodal(self):

FILE: tests/test_metrics/test_text_to_image_metric.py
  class TestTextToImageMetric (line 17) | class TestTextToImageMetric:
    method test_multimodal_async_metric_measure (line 20) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 41) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 62) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_multimodal_evaluate_method (line 80) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_tool_use_metric.py
  class TestToolUseMetric (line 17) | class TestToolUseMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 48) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 75) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 104) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 134) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 161) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 187) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_topic_adherence_metric.py
  class TestTopicAdherenceMetric (line 17) | class TestTopicAdherenceMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 44) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 66) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 90) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 116) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 139) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 160) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_toxicity_metric.py
  class TestToxicityMetric (line 17) | class TestToxicityMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 44) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 68) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 89) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 110) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 128) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 152) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_turn_contextual_precision_metric.py
  class TestTurnContextualPrecisionMetric (line 17) | class TestTurnContextualPrecisionMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 42) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 64) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 88) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 112) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 133) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 154) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_turn_contextual_recall_metric.py
  class TestTurnContextualRecallMetric (line 22) | class TestTurnContextualRecallMetric:
    method test_normal_sync_metric_measure (line 25) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 47) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 69) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 93) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 117) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 138) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 159) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_turn_faithfulness_metric.py
  class TestTurnFaithfulnessMetric (line 17) | class TestTurnFaithfulnessMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 42) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 64) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 88) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 112) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 133) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 154) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/test_turn_relevancy_metric.py
  class TestTurnRelevancyMetric (line 17) | class TestTurnRelevancyMetric:
    method test_normal_sync_metric_measure (line 20) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 42) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 64) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 88) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 112) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 133) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 154) | def test_multimodal_evaluate_method(self):

FILE: tests/test_metrics/turn_contextual_relevancy_metric.py
  class TestTurnContextualRelevancyMetric (line 22) | class TestTurnContextualRelevancyMetric:
    method test_normal_sync_metric_measure (line 25) | def test_normal_sync_metric_measure(self):
    method test_normal_async_metric_measure (line 47) | def test_normal_async_metric_measure(self):
    method test_multimodal_async_metric_measure (line 69) | def test_multimodal_async_metric_measure(self):
    method test_multimodal_sync_metric_measure (line 93) | def test_multimodal_sync_metric_measure(self):
    method test_invalid_model_throws_error_for_multimodal (line 117) | def test_invalid_model_throws_error_for_multimodal(self):
    method test_normal_evaluate_method (line 138) | def test_normal_evaluate_method(self):
    method test_multimodal_evaluate_method (line 159) | def test_multimodal_evaluate_method(self):