Showing preview only (5,204K chars total). Download the full file or copy to clipboard to get everything.
Repository: vibrantlabsai/ragas
Branch: main
Commit: 298b68274234
Files: 638
Total size: 4.9 MB
Directory structure:
gitextract_i6tp0pjv/
├── .cursor/
│ ├── commands/
│ │ ├── git-pr.md
│ │ └── update-howto-guide.md
│ ├── rules/
│ │ ├── docs-diataxis-guidelines.mdc
│ │ ├── docs-structure.mdc
│ │ ├── project-structure.mdc
│ │ ├── update-guide.mdc
│ │ └── use-uv-cli.mdc
│ └── worktrees.json
├── .dockerignore
├── .gitattributes
├── .github/
│ ├── ISSUE_TEMPLATE/
│ │ ├── bug_report.md
│ │ ├── feature_request.md
│ │ └── question.md
│ ├── pull_request_template.md
│ └── workflows/
│ ├── ci.yaml
│ ├── claude-code-review.yml
│ ├── claude-docs-apply.yml
│ ├── claude-docs-check.yml
│ ├── claude.yml
│ ├── issue-manager.yaml
│ ├── publish-examples.yml
│ └── python-publish.yml
├── .gitignore
├── .pre-commit-config.yaml
├── .readthedocs.yml
├── CLAUDE.md
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE
├── Makefile
├── README.md
├── SECURITY.md
├── docs/
│ ├── INSTALL
│ ├── Makefile
│ ├── _static/
│ │ ├── annotated_data.json
│ │ ├── css/
│ │ │ ├── highlight_ipython3.css
│ │ │ ├── highlight_ipython3_dark.css
│ │ │ ├── highlight_ipython3_light.css
│ │ │ ├── highlight_python.css
│ │ │ ├── highlight_python_dark.css
│ │ │ ├── highlight_python_light.css
│ │ │ └── ragas.css
│ │ ├── edited_chain_runs.json
│ │ ├── js/
│ │ │ ├── commonroom.js
│ │ │ ├── header_border.js
│ │ │ ├── mathjax.js
│ │ │ ├── mendable_chat_bubble.js
│ │ │ └── toggle.js
│ │ └── sample_annotated_summary.json
│ ├── alfred.py
│ ├── community/
│ │ ├── index.md
│ │ └── pdf_export.md
│ ├── concepts/
│ │ ├── components/
│ │ │ ├── eval_dataset.md
│ │ │ ├── eval_sample.md
│ │ │ ├── index.md
│ │ │ └── prompt.md
│ │ ├── datasets.md
│ │ ├── experimentation.md
│ │ ├── feedback/
│ │ │ └── index.md
│ │ ├── index.md
│ │ ├── metrics/
│ │ │ ├── available_metrics/
│ │ │ │ ├── agents.md
│ │ │ │ ├── answer_correctness.md
│ │ │ │ ├── answer_relevance.md
│ │ │ │ ├── aspect_critic.md
│ │ │ │ ├── context_entities_recall.md
│ │ │ │ ├── context_precision.md
│ │ │ │ ├── context_recall.md
│ │ │ │ ├── factual_correctness.md
│ │ │ │ ├── faithfulness.md
│ │ │ │ ├── general_purpose.md
│ │ │ │ ├── index.md
│ │ │ │ ├── multi_modal_faithfulness.md
│ │ │ │ ├── multi_modal_relevance.md
│ │ │ │ ├── noise_sensitivity.md
│ │ │ │ ├── nvidia_metrics.md
│ │ │ │ ├── rubrics_based.md
│ │ │ │ ├── semantic_similarity.md
│ │ │ │ ├── sql.md
│ │ │ │ ├── summarization_score.md
│ │ │ │ └── traditional.md
│ │ │ ├── index.md
│ │ │ └── overview/
│ │ │ └── index.md
│ │ └── test_data_generation/
│ │ ├── agents.md
│ │ ├── index.md
│ │ └── rag.md
│ ├── extra/
│ │ ├── components/
│ │ │ ├── choose_evaluator_llm.md
│ │ │ └── choose_generator_llm.md
│ │ ├── overrides/
│ │ │ └── main.html
│ │ ├── ragas-modern.css
│ │ └── style.css
│ ├── getstarted/
│ │ ├── evals.md
│ │ ├── experiments_quickstart.md
│ │ ├── index.md
│ │ ├── install.md
│ │ ├── quickstart.md
│ │ ├── rag_eval.md
│ │ └── rag_testset_generation.md
│ ├── howtos/
│ │ ├── applications/
│ │ │ ├── _cost.md
│ │ │ ├── add_to_ci.md
│ │ │ ├── align-llm-as-judge.md
│ │ │ ├── benchmark_llm.md
│ │ │ ├── compare_embeddings.md
│ │ │ ├── compare_llms.md
│ │ │ ├── cost.ipynb
│ │ │ ├── evaluate-and-improve-rag.md
│ │ │ ├── evaluating_multi_turn_conversations.md
│ │ │ ├── index.md
│ │ │ ├── iterate_prompt.md
│ │ │ ├── prompt_optimization.md
│ │ │ ├── singlehop_testset_gen.md
│ │ │ ├── text2sql.md
│ │ │ ├── vertexai_alignment.md
│ │ │ ├── vertexai_model_comparision.md
│ │ │ └── vertexai_x_ragas.md
│ │ ├── cli/
│ │ │ ├── agent_evals.md
│ │ │ ├── benchmark_llm.md
│ │ │ ├── improve_rag.md
│ │ │ ├── index.md
│ │ │ ├── judge_alignment.md
│ │ │ ├── llamaIndex_agent_evals.md
│ │ │ ├── prompt_evals.md
│ │ │ ├── rag_eval.md
│ │ │ ├── text2sql.md
│ │ │ └── workflow_eval.md
│ │ ├── customizations/
│ │ │ ├── _caching.md
│ │ │ ├── caching.ipynb
│ │ │ ├── cancellation.md
│ │ │ ├── customize_models.md
│ │ │ ├── index.md
│ │ │ ├── metrics/
│ │ │ │ ├── _cost.md
│ │ │ │ ├── cost.ipynb
│ │ │ │ ├── metrics_language_adaptation.md
│ │ │ │ ├── modifying-prompts-metrics.md
│ │ │ │ └── tracing.md
│ │ │ ├── optimizers/
│ │ │ │ └── index.md
│ │ │ ├── run_config.md
│ │ │ └── testgenerator/
│ │ │ ├── _language_adaptation.md
│ │ │ ├── _persona_generator.md
│ │ │ ├── _testgen-custom-single-hop.md
│ │ │ ├── _testgen-customisation.md
│ │ │ ├── index.md
│ │ │ ├── language_adaptation.ipynb
│ │ │ ├── persona_generator.ipynb
│ │ │ ├── prechunked_data.md
│ │ │ ├── testgen-custom-single-hop.ipynb
│ │ │ └── testgen-customisation.ipynb
│ │ ├── index.md
│ │ ├── integrations/
│ │ │ ├── _ag_ui.md
│ │ │ ├── _arize.md
│ │ │ ├── _athina.md
│ │ │ ├── _haystack.md
│ │ │ ├── _helicone.md
│ │ │ ├── _langchain.md
│ │ │ ├── _langfuse.md
│ │ │ ├── _langgraph_agent_evaluation.md
│ │ │ ├── _langsmith.md
│ │ │ ├── _llamaindex.md
│ │ │ ├── _openlayer.md
│ │ │ ├── _opik.md
│ │ │ ├── _tonic-validate.md
│ │ │ ├── _zeno.md
│ │ │ ├── ag_ui.ipynb
│ │ │ ├── ag_ui.md
│ │ │ ├── amazon_bedrock.md
│ │ │ ├── arize.ipynb
│ │ │ ├── athina.ipynb
│ │ │ ├── gemini.md
│ │ │ ├── griptape.md
│ │ │ ├── haystack.ipynb
│ │ │ ├── haystack.md
│ │ │ ├── helicone.ipynb
│ │ │ ├── index.md
│ │ │ ├── langchain.ipynb
│ │ │ ├── langchain.md
│ │ │ ├── langfuse.ipynb
│ │ │ ├── langgraph_agent_evaluation.ipynb
│ │ │ ├── langsmith.ipynb
│ │ │ ├── langsmith.md
│ │ │ ├── llama_stack.md
│ │ │ ├── llamaindex.ipynb
│ │ │ ├── llamaindex_agents.md
│ │ │ ├── nyc_wikipedia/
│ │ │ │ └── nyc_text.txt
│ │ │ ├── oci_genai.md
│ │ │ ├── openlayer.ipynb
│ │ │ ├── opik.ipynb
│ │ │ ├── r2r.md
│ │ │ ├── swarm_agent_evaluation.md
│ │ │ ├── tonic-validate.ipynb
│ │ │ └── zeno.ipynb
│ │ ├── llm-adapters.md
│ │ ├── migrations/
│ │ │ ├── migrate_from_v01_to_v02.md
│ │ │ └── migrate_from_v03_to_v04.md
│ │ └── observability.md
│ ├── index.md
│ ├── ipynb_to_md.py
│ ├── make.bat
│ ├── quoted_spans_metric.md
│ ├── references/
│ │ ├── aevaluate.md
│ │ ├── cache.md
│ │ ├── embeddings.md
│ │ ├── evaluate.md
│ │ ├── evaluation_schema.md
│ │ ├── executor.md
│ │ ├── generate.md
│ │ ├── graph.md
│ │ ├── index.md
│ │ ├── integrations.md
│ │ ├── llms.md
│ │ ├── metrics.md
│ │ ├── optimizers.md
│ │ ├── prompt.md
│ │ ├── run_config.md
│ │ ├── synthesizers.md
│ │ ├── testset_schema.md
│ │ ├── tokenizers.md
│ │ └── transforms.md
│ └── tutorials/
│ ├── agent.md
│ ├── index.md
│ ├── prompt.md
│ ├── rag.md
│ └── workflow.md
├── examples/
│ ├── LICENSE
│ ├── README.md
│ ├── gdrive_append_example.py
│ ├── gdrive_backend_example.py
│ ├── iterate_prompt/
│ │ ├── __init__.py
│ │ ├── datasets/
│ │ │ └── support_triage.csv
│ │ ├── evals.py
│ │ ├── promptv1.txt
│ │ ├── promptv2_fewshot.txt
│ │ └── run_prompt.py
│ ├── oci_genai_example.py
│ ├── pyproject.toml
│ └── ragas_examples/
│ ├── __init__.py
│ ├── ag_ui_agent_experiments/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── experiments.py
│ │ └── test_data/
│ │ └── datasets/
│ │ ├── scientist_biographies.csv
│ │ └── weather_tool_calls.csv
│ ├── agent_evals/
│ │ ├── __init__.py
│ │ ├── agent.py
│ │ └── evals.py
│ ├── benchmark_llm/
│ │ ├── __init__.py
│ │ ├── datasets/
│ │ │ └── discount_benchmark.csv
│ │ ├── evals.py
│ │ └── prompt.py
│ ├── improve_rag/
│ │ ├── __init__.py
│ │ ├── evals/
│ │ │ └── datasets/
│ │ │ └── hf_doc_qa_eval.csv
│ │ ├── evals.py
│ │ ├── pyproject.toml
│ │ └── rag.py
│ ├── judge_alignment/
│ │ ├── __init__.py
│ │ └── evals.py
│ ├── llamaIndex_agent_evals/
│ │ ├── __init__.py
│ │ ├── contexts/
│ │ │ ├── ambiguous_removal_request.json
│ │ │ ├── duplicate_addition.json
│ │ │ └── repeated_removal.json
│ │ ├── evals.py
│ │ └── llamaindex_agent.py
│ ├── prompt_evals/
│ │ ├── __init__.py
│ │ ├── evals.py
│ │ └── prompt.py
│ ├── rag_eval/
│ │ ├── __init__.py
│ │ ├── evals.py
│ │ ├── pyproject.toml
│ │ └── rag.py
│ ├── text2sql/
│ │ ├── __init__.py
│ │ ├── analyze_errors.py
│ │ ├── data_utils.py
│ │ ├── datasets/
│ │ │ └── booksql_sample.csv
│ │ ├── db_utils.py
│ │ ├── evals.py
│ │ ├── prompt.txt
│ │ ├── prompt_v2.txt
│ │ ├── prompt_v3.txt
│ │ ├── text2sql_agent.py
│ │ └── validate_sql_dataset.py
│ └── workflow_eval/
│ ├── __init__.py
│ ├── evals.py
│ └── workflow.py
├── mkdocs-pdf.yml
├── mkdocs.yml
├── pyproject.toml
├── scripts/
│ └── dev_docs.sh
├── src/
│ └── ragas/
│ ├── __init__.py
│ ├── _analytics.py
│ ├── async_utils.py
│ ├── backends/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── gdrive_backend.md
│ │ ├── gdrive_backend.py
│ │ ├── inmemory.py
│ │ ├── local_csv.py
│ │ ├── local_jsonl.py
│ │ ├── registry.py
│ │ └── utils.py
│ ├── cache.py
│ ├── callbacks.py
│ ├── cli.py
│ ├── config.py
│ ├── cost.py
│ ├── dataset.py
│ ├── dataset_schema.py
│ ├── embeddings/
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── google_provider.py
│ │ ├── haystack_wrapper.py
│ │ ├── huggingface_provider.py
│ │ ├── litellm_provider.py
│ │ ├── openai_provider.py
│ │ └── utils.py
│ ├── evaluation.py
│ ├── exceptions.py
│ ├── executor.py
│ ├── experiment.py
│ ├── integrations/
│ │ ├── __init__.py
│ │ ├── ag_ui.py
│ │ ├── amazon_bedrock.py
│ │ ├── griptape.py
│ │ ├── helicone.py
│ │ ├── langchain.py
│ │ ├── langgraph.py
│ │ ├── langsmith.py
│ │ ├── llama_index.py
│ │ ├── opik.py
│ │ ├── r2r.py
│ │ ├── swarm.py
│ │ └── tracing/
│ │ ├── __init__.py
│ │ ├── langfuse.py
│ │ └── mlflow.py
│ ├── llms/
│ │ ├── __init__.py
│ │ ├── adapters/
│ │ │ ├── __init__.py
│ │ │ ├── base.py
│ │ │ ├── instructor.py
│ │ │ └── litellm.py
│ │ ├── base.py
│ │ ├── haystack_wrapper.py
│ │ ├── litellm_llm.py
│ │ └── oci_genai_wrapper.py
│ ├── losses.py
│ ├── messages.py
│ ├── metrics/
│ │ ├── __init__.py
│ │ ├── _answer_correctness.py
│ │ ├── _answer_relevance.py
│ │ ├── _answer_similarity.py
│ │ ├── _aspect_critic.py
│ │ ├── _bleu_score.py
│ │ ├── _chrf_score.py
│ │ ├── _context_entities_recall.py
│ │ ├── _context_precision.py
│ │ ├── _context_recall.py
│ │ ├── _datacompy_score.py
│ │ ├── _domain_specific_rubrics.py
│ │ ├── _factual_correctness.py
│ │ ├── _faithfulness.py
│ │ ├── _goal_accuracy.py
│ │ ├── _instance_specific_rubrics.py
│ │ ├── _multi_modal_faithfulness.py
│ │ ├── _multi_modal_relevance.py
│ │ ├── _noise_sensitivity.py
│ │ ├── _nv_metrics.py
│ │ ├── _rouge_score.py
│ │ ├── _simple_criteria.py
│ │ ├── _sql_semantic_equivalence.py
│ │ ├── _string.py
│ │ ├── _summarization.py
│ │ ├── _tool_call_accuracy.py
│ │ ├── _tool_call_f1.py
│ │ ├── _topic_adherence.py
│ │ ├── base.py
│ │ ├── collections/
│ │ │ ├── __init__.py
│ │ │ ├── _bleu_score.py
│ │ │ ├── _rouge_score.py
│ │ │ ├── _semantic_similarity.py
│ │ │ ├── _string.py
│ │ │ ├── agent_goal_accuracy/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── answer_accuracy/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── answer_correctness/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── answer_relevancy/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── base.py
│ │ │ ├── chrf_score/
│ │ │ │ ├── __init__.py
│ │ │ │ └── metric.py
│ │ │ ├── context_entity_recall/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── context_precision/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── context_recall/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── context_relevance/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── datacompy_score/
│ │ │ │ ├── __init__.py
│ │ │ │ └── metric.py
│ │ │ ├── domain_specific_rubrics/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── example_metric.py
│ │ │ ├── factual_correctness/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── faithfulness/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── instance_specific_rubrics/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── multi_modal_faithfulness/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── multi_modal_relevance/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── noise_sensitivity/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── quoted_spans/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── response_groundedness/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── sql_semantic_equivalence/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── summary_score/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── tool_call_accuracy/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── tool_call_f1/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ └── topic_adherence/
│ │ │ ├── __init__.py
│ │ │ ├── metric.py
│ │ │ └── util.py
│ │ ├── decorator.py
│ │ ├── discrete.py
│ │ ├── numeric.py
│ │ ├── quoted_spans.py
│ │ ├── ranking.py
│ │ ├── result.py
│ │ ├── utils.py
│ │ └── validators.py
│ ├── optimizers/
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── dspy_adapter.py
│ │ ├── dspy_llm_wrapper.py
│ │ ├── dspy_optimizer.py
│ │ ├── genetic.py
│ │ └── utils.py
│ ├── prompt/
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── dynamic_few_shot.py
│ │ ├── few_shot_pydantic_prompt.py
│ │ ├── metrics/
│ │ │ ├── __init__.py
│ │ │ ├── answer_accuracy.py
│ │ │ ├── answer_correctness.py
│ │ │ ├── answer_relevance.py
│ │ │ ├── base_prompt.py
│ │ │ ├── common.py
│ │ │ ├── context_entity_recall.py
│ │ │ ├── context_recall.py
│ │ │ ├── context_relevance.py
│ │ │ ├── factual_correctness.py
│ │ │ ├── noise_sensitivity.py
│ │ │ ├── response_groundedness.py
│ │ │ └── summary_score.py
│ │ ├── mixin.py
│ │ ├── multi_modal_prompt.py
│ │ ├── prompt-formats.md
│ │ ├── pydantic_prompt.py
│ │ ├── simple_prompt.py
│ │ └── utils.py
│ ├── py.typed
│ ├── run_config.py
│ ├── sdk.py
│ ├── testset/
│ │ ├── __init__.py
│ │ ├── graph.py
│ │ ├── graph_queries.py
│ │ ├── persona.py
│ │ ├── synthesizers/
│ │ │ ├── __init__.py
│ │ │ ├── base.py
│ │ │ ├── generate.py
│ │ │ ├── multi_hop/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── abstract.py
│ │ │ │ ├── base.py
│ │ │ │ ├── prompts.py
│ │ │ │ └── specific.py
│ │ │ ├── prompts.py
│ │ │ ├── single_hop/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── base.py
│ │ │ │ ├── prompts.py
│ │ │ │ └── specific.py
│ │ │ ├── testset_schema.py
│ │ │ └── utils.py
│ │ └── transforms/
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── default.py
│ │ ├── engine.py
│ │ ├── extractors/
│ │ │ ├── __init__.py
│ │ │ ├── embeddings.py
│ │ │ ├── llm_based.py
│ │ │ └── regex_based.py
│ │ ├── filters.py
│ │ ├── relationship_builders/
│ │ │ ├── __init__.py
│ │ │ ├── cosine.py
│ │ │ └── traditional.py
│ │ └── splitters/
│ │ ├── __init__.py
│ │ └── headline.py
│ ├── tokenizers.py
│ ├── utils.py
│ └── validation.py
└── tests/
├── __init__.py
├── benchmarks/
│ ├── Dockerfile
│ ├── benchmark_eval.py
│ ├── benchmark_testsetgen.py
│ └── utils.py
├── conftest.py
├── docs/
│ ├── __init__.py
│ └── test_run_config.py
├── e2e/
│ ├── __init__.py
│ ├── metrics_migration/
│ │ ├── __init__.py
│ │ ├── base_migration_test.py
│ │ ├── conftest.py
│ │ ├── metric_score_diff.ipynb
│ │ ├── plan-for-metrics-migration.md
│ │ ├── test_answer_accuracy_migration.py
│ │ ├── test_answer_correctness_migration.py
│ │ ├── test_answer_relevancy_migration.py
│ │ ├── test_bleu_migration.py
│ │ ├── test_context_entity_recall_migration.py
│ │ ├── test_context_precision_migration.py
│ │ ├── test_context_recall_migration.py
│ │ ├── test_context_relevance_migration.py
│ │ ├── test_factual_correctness_migration.py
│ │ ├── test_faithfulness_migration.py
│ │ ├── test_noise_sensitivity_migration.py
│ │ ├── test_response_groundedness_migration.py
│ │ ├── test_rouge_migration.py
│ │ ├── test_semantic_similarity_migration.py
│ │ ├── test_string_migration.py
│ │ ├── test_summary_score_migration.py
│ │ └── test_utils.py
│ ├── test_adaptation.py
│ ├── test_amnesty_in_ci.py
│ ├── test_dataset_utils.py
│ ├── test_dspy_integration.py
│ ├── test_fullflow.py
│ ├── test_langchain_llm_attributes.py
│ └── test_testset_generation.py
├── test_quoted_spans.py
├── unit/
│ ├── backends/
│ │ ├── test_gdrive_backend.py
│ │ ├── test_inmemory.py
│ │ ├── test_local_csv.py
│ │ └── test_local_jsonl.py
│ ├── integrations/
│ │ ├── test_ag_ui.py
│ │ ├── test_tracing.py
│ │ └── test_tracing_simple.py
│ ├── llms/
│ │ ├── test_adapters.py
│ │ ├── test_instructor_factory.py
│ │ ├── test_llm.py
│ │ └── test_system_prompt.py
│ ├── prompt/
│ │ ├── test_base_prompt.py
│ │ ├── test_dynamic_few_shot_prompt.py
│ │ ├── test_prompt_mixin.py
│ │ ├── test_prompt_save_load.py
│ │ └── test_prompt_utils.py
│ ├── test_analytics.py
│ ├── test_async_evaluation.py
│ ├── test_async_utils.py
│ ├── test_average_precision_algorithm.py
│ ├── test_cache.py
│ ├── test_cancellation.py
│ ├── test_chrf_score.py
│ ├── test_chrf_score_collections.py
│ ├── test_cli.py
│ ├── test_cosine_relationship_builders.py
│ ├── test_cost.py
│ ├── test_datacompy_score_collections.py
│ ├── test_dataset_schema.py
│ ├── test_datatable_inheritance.py
│ ├── test_domain_specific_rubrics_collections.py
│ ├── test_dspy_adapter.py
│ ├── test_dspy_optimizer.py
│ ├── test_embeddings.py
│ ├── test_embeddings_caching.py
│ ├── test_engine.py
│ ├── test_executor.py
│ ├── test_executor_in_jupyter.ipynb
│ ├── test_experiment.py
│ ├── test_graph.py
│ ├── test_import.py
│ ├── test_instance_specific_rubrics_collections.py
│ ├── test_knowledge_graph_clusters.py
│ ├── test_knowledge_graph_save.py
│ ├── test_langgraph.py
│ ├── test_llm_context.py
│ ├── test_metric.py
│ ├── test_metric_decorators.py
│ ├── test_multi_hop_query_synthesizer.py
│ ├── test_multi_modal_faithfulness_collections.py
│ ├── test_multi_modal_relevance_collections.py
│ ├── test_oci_genai_wrapper.py
│ ├── test_optimizer_config.py
│ ├── test_prechunked_generation.py
│ ├── test_prompt.py
│ ├── test_quoted_spans_collections.py
│ ├── test_run_config.py
│ ├── test_simple.py
│ ├── test_simple_llm_metric_persistence.py
│ ├── test_single_hop_query_synthesizer.py
│ ├── test_sql_semantic_equivalence_collections.py
│ ├── test_testset_schema.py
│ ├── test_tokenizers.py
│ ├── test_tool_call_accuracy.py
│ ├── test_tool_call_accuracy_collections.py
│ ├── test_tool_call_f1.py
│ ├── test_tool_call_f1_collections.py
│ ├── test_traditional_relationship_builders.py
│ ├── test_utils.py
│ ├── test_uvloop_compatibility.py
│ └── test_validation.py
└── utils/
├── __init__.py
├── llm_setup.py
└── metric_comparison.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .cursor/commands/git-pr.md
================================================
Make sure you are on a branch other than main.
Commit changes.
Run make format.
Create PR using gh cli following .github/pull_request_template.md template
================================================
FILE: .cursor/commands/update-howto-guide.md
================================================
# Update How-to Guide
Updates the mentioned how-to guide to use src/ragas/metrics/collections API instead of the legacy ragas/metrics API and LLM factory pattern instead of langchainwrapper.
## File Format Decision
If the source is an `.ipynb` file (or if the `.md` filename starts with `_`, indicating it's derived from a notebook via `docs/ipynb_to_md.py`):
1. **Delete** the `.ipynb` file
2. **Delete** the corresponding `_xxx.md` file (if it exists)
3. **Create** a new `.md` file directly (without the `_` prefix)
This simplifies maintenance by having pure markdown docs instead of notebooks.
## Process
### Phase 1: Research (do NOT make changes yet)
Refer pr-description-customizations.md for the list of guides that are already updated. And finally update the doc after you're done.
#### 1.1 Understand the Guide's Purpose
- Read the target file thoroughly
- Identify **what the guide is trying to achieve** (e.g., caching, run config, retry handling)
- Note the specific use case or need the guide addresses
- Understand what underlying tools/libraries are being used (e.g., instructor, liteLLM, httpx)
#### 1.2 Feasibility Check
Before doing anything else, check if the feature works with the new API:
1. **Check `src/ragas/experiment.py`** - Does experiment() support this feature?
2. **Check `src/ragas/evaluation.py`** - Is this an evaluate()-only feature?
3. **Check `src/ragas/metrics/collections/`** - Do collections metrics support this?
4. **Check if simpler alternatives exist** - Does a newer, simpler API make this guide obsolete? (e.g., decorator-based metrics vs subclassing, built-in features vs manual workarounds). Check concept docs and `src/ragas/metrics/` for modern patterns.
**If a simpler approach exists → recommend deletion** instead of migration. See "When to Recommend Deletion" section.
**If not supported in new API → STOP immediately:**
- Keep guide as-is
- Output this Slack message for the team:
```
📋 *Doc Update Skipped*: `<guide_path>`
*Link*: https://docs.ragas.io/en/latest/<guide_path_without_extension>/
*Reason*: <feature> only works with legacy `evaluate()` API, not yet supported in `experiment()`/collections
*Action*: Keep as-is until collections API adds support
```
**If supported → continue to 1.3**
#### 1.3 Present Plan & Wait for Approval
**⏸️ STOP HERE - Do NOT proceed to Phase 2 without explicit user approval.**
Present a clear summary:
1. **Current state**: What the guide currently does and how
2. **Proposed changes**:
- Imports to change (from old → new)
- LLM/embeddings setup patterns to update
- **How the specific use case/feature will be achieved** with the new API
- Any restructuring or content changes
3. **Potential concerns**: Anything uncertain or risky
4. **Ask**: "Does this plan look good? Should I proceed?"
**Wait for user to say "yes", "proceed", "go ahead", or similar before continuing.**
---
### Phase 2: Execute (only after approval)
#### 2.1 Apply Updates
**Keep it Concise**:
- Remove unnecessary explanations and verbose text
- Focus on the essential information needed to achieve the goal
- Use clear, direct language
- Avoid redundant examples - one good example is better than multiple similar ones
**Import Updates**:
```python
# Change from:
from ragas.metrics import MetricName
# To:
from ragas.metrics.collections import MetricName
```
**LLM Setup**:
```python
# Change from:
from langchain_openai import ChatOpenAI
from ragas.llms import LangchainLLMWrapper
llm = LangchainLLMWrapper(ChatOpenAI(model="gpt-4o"))
# To:
from openai import OpenAI
from ragas.llms import llm_factory
client = OpenAI(api_key="sk-...")
llm = llm_factory("gpt-4o", client=client)
```
**Embeddings Setup**:
```python
# Change from:
from langchain_openai import OpenAIEmbeddings
from ragas.embeddings import LangchainEmbeddingsWrapper
embeddings = LangchainEmbeddingsWrapper(OpenAIEmbeddings())
# To:
from openai import OpenAI
from ragas.embeddings.base import embedding_factory # Use .base to avoid deprecation warning
client = OpenAI(api_key="sk-...")
embeddings = embedding_factory("openai", model="text-embedding-3-small", client=client)
```
**What to Fix**:
- Update imports and LLM/embeddings patterns
- Use `ragas.embeddings.base` import to avoid deprecation warnings
- **Replace all legacy code with modern approaches** (no need to keep legacy sections)
- Fix minor issues automatically
- Don't restructure content unless fixing issues
#### 2.2 Verify Accuracy & Test Code
**Verify with Web Search**:
- Search for official documentation of any libraries/tools mentioned (instructor, liteLLM, httpx, etc.)
- Confirm API signatures, parameter names, and usage patterns are correct
- Verify any claims about library behavior are accurate
**Run the Code**:
- Install any missing packages first: `uv pip install <package>`
- Extract ALL Python code blocks from the guide
- Save as `tests/docs/test_<guide_name>.py` (e.g., `test_run_config.py`)
- Use `.env` from root for API keys. .env only has openai keys, if you need anything else, let me know
- Run: `uv run python tests/docs/test_<guide_name>.py`
- **Verify the original use case/goal is achieved** with the new approach
**If tests fail**:
1. Check the underlying implementation in `src/` to understand correct usage
2. Fix the code in the guide based on what you learn from `src/`
3. Re-run the test
4. Repeat until tests pass
5. If stuck after multiple attempts, report the issue with details
**Keep the test file** - excluded from default `pytest` runs via `norecursedirs` in `pyproject.toml`.
**Both verification methods are required** - web search for accuracy, code execution for functionality.
#### 2.3 Check Navigation
- Verify file is in `mkdocs.yml`
- Note if location seems wrong or can be put in a more appropriate section
#### 2.4 Summarize Changes
- List all changes made
- Mention if anything is not tested due to any reasons (like missing packages, missing API keys, etc.)
## Notes
- **Two-phase workflow**: Research first, get approval, then execute
- **Never skip approval**: Always present the plan and wait for explicit "go ahead" before making changes
- **This is not just a straightforward migration** - understand if the original goal is achievable first
- **Keep guides concise** - remove fluff, focus on essential information
- **Verify accuracy** - use web search to confirm library APIs and behavior before writing
- **Test everything** - run all code examples before finalizing
- Only fix what's broken or outdated
- Check `src/` before updating to verify APIs exist
- Don't add legacy sections
- Use root `.env` for testing
- **Keep test files** in `tests/docs/` - excluded from default pytest runs
## When to Recommend Deletion
If a guide teaches **writing custom metrics by subclassing** (`MetricWithLLM`, `SingleTurnMetric`, etc.), it's likely obsolete. The decorator-based approach is simpler:
```python
from ragas.metrics import discrete_metric, numeric_metric, ranking_metric
@discrete_metric(name="my_metric", allowed_values=["pass", "fail"])
def my_metric(response: str, context: str) -> str:
return "pass" if condition else "fail"
```
See `docs/concepts/metrics/overview/index.md` for details. Recommend deletion if decorators cover the use case.
## Reporting Gaps
If you identify a gap, use the Slack message template from section 1.2.
================================================
FILE: .cursor/rules/docs-diataxis-guidelines.mdc
================================================
---
globs: docs/**
---
# Diátaxis Documentation Guidelines
When writing or editing documentation, categorise each page as **one** of the four Diátaxis modes and follow its specific guidance. *Do not mix modes in a single page.*
## 1. Tutorials 🧑🏫 (`docs/getstarted/`, `docs/experimental/tutorials/`)
• Purpose: provide a structured **learning experience** – “Can you teach me to…?”
• Form: narrative lesson that leads the reader from zero to a working result.
• Include: context, motivation, complete working example.
• Avoid: deep technical detail; troubleshooting; exhaustive options.
## 2. How-to Guides 🍳 (`docs/howtos/`)
• Purpose: help the user **achieve a specific goal** – “How do I…?”
• Form: concise series of **step-by-step instructions** focused on the user’s project.
• Write from the **user’s perspective**, not the tool’s operations.
• Link to reference and explanation pages for background; keep prose minimal.
• Add any code run outputs as expandable click blocks. Readers should be able to understand the guide without running the code.
## 3. Reference 📑 (`docs/references/`)
• Purpose: provide **neutral, complete, accurate description** of APIs, commands, options – “What is…?”
• Maintain consistent patterns (parameter tables, return types, examples).
• Avoid instruction or opinion; instead *link* to how-to or explanation pages.
• Examples are welcome if they illustrate usage without drifting into tutorial style.
## 4. Explanation 💡 (`docs/concepts/`, `docs/experimental/core_concepts/`)
• Purpose: **clarify concepts and rationale** – “Why…?”
• Form: discursive article that illuminates design decisions, theory, background.
• May link out to tutorials, how-tos, and reference, but does not instruct step-by-step.
### Keep the Borders Sharp
• Do **not** let content blur between modes (e.g., no instructions inside reference; no lengthy theory in how-tos).
• If a page starts serving two modes, split it.
### Filing & Navigation
• Place the file in the folder matching its mode (above).
• Update `mkdocs.yml` `nav:` under the corresponding section.
### Incremental Improvement Cycle (per Diátaxis)
Choose → Assess → Decide → Do. Focus on small, atomic upgrades rather than grand rewrites.
### Writing Style
• Use second-person ("you") and active voice.
• Ensure code blocks are **copy-pasteable** and include necessary context (imports, environment).
• Prefer short sentences; use Markdown admonitions (`!!! note`, `!!! warning`) sparingly for important side-information.
• Use `??? "Click to expand"` collapsible admonitions to contain outputs, long prompts, verbose logs, or any content that would clutter the main article flow. This keeps the primary content scannable while preserving detailed information for readers who need it.
• **Always add a blank line after text ending with a colon before starting a list.** This ensures proper Markdown rendering in MkDocs. Without the blank line, list items may render as continuation text instead of a proper bulleted/numbered list.
### Cross-linking Between Modes
• End tutorials with pointers to relevant how-to guides for further exploration.
• How-to guides should include links to reference/API pages for deeper details.
• Explanations can reference tutorials and how-tos to illustrate concepts in action.
### Page Metadata & Prerequisites
• Start each page with a one-sentence purpose statement and a brief list of prerequisites (libraries, data, environment variables).
• Highlight any external services/configuration required before the reader begins.
### Keep Pages Atomic
• One page = one task, concept, or API surface. If content grows, **split** rather than creating a mega-guide.
================================================
FILE: .cursor/rules/docs-structure.mdc
================================================
---
globs: docs/**
---
# Documentation Structure & Workflow
Follow these conventions when creating or editing documentation:
1. **Docs live in [docs/](mdc:docs/)**
• Use Markdown (`.md`) files.
• Images and other assets go in [docs/_static/](mdc:docs/_static/).
2. **Section Folders mirror MkDocs navigation** (see [mkdocs.yml](mdc:mkdocs.yml)):
• 🚀 Get Started → [docs/getstarted/](mdc:docs/getstarted/)
• 📚 Core Concepts → [docs/concepts/](mdc:docs/concepts/)
• 🧪 Experimental → [docs/experimental/](mdc:docs/experimental/)
• 🛠️ How-to Guides → [docs/howtos/](mdc:docs/howtos/)
• 📖 References → [docs/references/](mdc:docs/references/)
• Community → [docs/community/](mdc:docs/community/)
Place new pages in the appropriate folder **and** update `mkdocs.yml` `nav:` so the page appears in navigation.
3. **Notebook-to-Markdown**
• Convert notebooks to Markdown with [docs/ipynb_to_md.py](mdc:docs/ipynb_to_md.py).
• Commit the generated `.md`; notebooks themselves should not live in `docs/`.
4. **Local preview / build**
• Run `make build-docs` to build HTML, `make serve-docs` to preview locally (defined in [DEVELOPMENT.md](mdc:DEVELOPMENT.md)).
5. **Style & Assets**
• Use relative links (`../`) within docs.
• Reference images via `_static/…` paths so they work in both dev and hosted docs.
• Custom templates/CSS live in [docs/extra/](mdc:docs/extra/) — avoid editing `material` theme defaults directly.
6. **API References (mkdocstrings)**
• Always use public API paths in `[ClassName][ragas.module.ClassName]` references.
• Check what's exported in `__init__.py` — if a class isn't in `__all__`, mkdocstrings can't link to it.
• Example: Use `[BasePrompt][ragas.prompt.BasePrompt]` not `[BasePrompt][ragas.prompt.base.BasePrompt]` or internal module paths.
7. **Do not modify generated or third-party files** in `_static/`, `extra/overrides/`, or `extra/components/` without good reason.
---
# Formatting Guidelines
- When introducing a list with text ending in a colon (e.g., "This will:"), always add a blank line before the first list item.
- In a numbered list, do not add any new line between the items.
================================================
FILE: .cursor/rules/project-structure.mdc
================================================
---
alwaysApply: true
---
# Monorepo Project Structure
The repository is a monorepo consisting of two primary components:
1. [/](./) – Core evaluation toolkit
• Source code lives in [src/](src/)
• Tests live in [tests/](tests/)
• Build configuration is in [pyproject.toml](pyproject.toml)
2. [examples/](examples/) – Installable examples package
• Package: ragas_examples/ containing agent_evals, prompt_evals, rag_eval, workflow_eval, benchmark_llm
• Build configuration in [examples/pyproject.toml](examples/pyproject.toml)
• Shipped as `ragas-examples` package on PyPI via `ragas[examples]` extra
• Local development: `uv pip install -e . -e ./examples`
• Usage: `python -m ragas_examples.benchmark_llm.prompt`
Shared documentation for all projects is located under [docs/](docs/).
Root-level [Makefile](Makefile) provides combined commands, while each project directory also contains its own Makefile for project-specific tasks.
================================================
FILE: .cursor/rules/update-guide.mdc
================================================
---
alwaysApply: false
---
We are writing a how to guide for Ragas docs and Ragas users.
So after any coding step we complete, or after any succesful runs, always update the guide with what was done. Make sure the content is concise and to the point.
Current guide: docs/howtos/applications/evaluate-and-improve-rag.md
================================================
FILE: .cursor/rules/use-uv-cli.mdc
================================================
---
alwaysApply: true
---
# Use `uv run` for Python CLI commands
This repository manages its virtual environment and dependencies with **uv**. Therefore, always execute Python or Python-related CLI tools through `uv run`.
Examples:
- `uv run pytest`
- `uv run ruff check .`
- `uv run isort .`
- `uv run pyright`
================================================
FILE: .cursor/worktrees.json
================================================
{
"setup-worktree": [
"cp $ROOT_WORKTREE_PATH/.env .env",
"make install-minimal",
"make check"
]
}
================================================
FILE: .dockerignore
================================================
Dockerfile
test_resources
================================================
FILE: .gitattributes
================================================
================================================
FILE: .github/ISSUE_TEMPLATE/bug_report.md
================================================
---
name: Bug Report
about: Report any bugs your encounter and we will try our best to fix it for you 🙂
title: ''
labels: 'bug'
assignees: ''
---
[ ] I have checked the [documentation](https://docs.ragas.io/) and related resources and couldn't resolve my bug.
**Describe the bug**
A clear and concise description of what the bug is.
Ragas version:
Python version:
**Code to Reproduce**
Share code to reproduce the issue
**Error trace**
**Expected behavior**
A clear and concise description of what you expected to happen.
**Additional context**
Add any other context about the problem here.
<!-- PS: bugs suck but is also part of the process. We sincerely apologies for breaking your flow because of it, but don't worry, we got your back ❤️. We will get this fixed as fast as we can and thanks for helping us out by reporting it 🙏. -->
================================================
FILE: .github/ISSUE_TEMPLATE/feature_request.md
================================================
---
name: Feature Request
about: Feel like something is mising? Let us know!
title: ''
labels: 'enhancement'
assignees: ''
---
**Describe the Feature**
A clear and concise description of what the what you want to be added.
**Why is the feature important for you?**
Share code to reproduce the issue
**Additional context**
Add any other context about the feature you want to share with us.
<!-- PS: Thanks for your valuable feedback. Really! Its feedback from valuable community members like you that help us make Ragas event better for the whole community. So thanks again for taking the time to improve our community 🙂 -->
================================================
FILE: .github/ISSUE_TEMPLATE/question.md
================================================
---
name: Questions
about: Any questions or doubts? Ask us here!
title: ''
labels: 'question'
assignees: ''
---
[ ] I checked the [documentation](https://docs.ragas.io/) and related resources and couldn't find an answer to my question.
**Your Question**
what is unclear to you? What would you like to know?
**Code Examples**
This community speaks code. Share your code snippets to help us understand your question better.
**Additional context**
Anything else you want to share with us?
================================================
FILE: .github/pull_request_template.md
================================================
## Issue Link / Problem Description
<!-- Link to related issue or describe the problem this PR solves -->
- Fixes #[issue_number]
- OR describe the issue: What problem does this solve? How can it be replicated?
## Changes Made
<!-- Describe what you changed and why -->
-
-
-
## Testing
<!-- Describe how this should be tested -->
### How to Test
- [ ] Automated tests added/updated
- [ ] Manual testing steps:
1.
2.
3.
## References
<!-- Link to related issues, discussions, forums, or external resources -->
- Related issues:
- Documentation:
- External references:
## Screenshots/Examples (if applicable)
<!-- Add screenshots or code examples showing the change -->
---
<!--
Thank you for contributing to Ragas!
Please fill out the sections above as completely as possible.
The more information you provide, the faster your PR can be reviewed and merged.
-->
================================================
FILE: .github/workflows/ci.yaml
================================================
name: CI
on:
pull_request:
permissions:
contents: read
env:
LINES: 120
COLUMNS: 120
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#defaultsrun
defaults:
run:
shell: bash --noprofile --norc -exo pipefail {0}
jobs:
diff:
runs-on: ubuntu-latest
outputs:
related: ${{ steps.filter.outputs.related }}
ragas: ${{ steps.filter.outputs.ragas }}
docs: ${{ steps.filter.outputs.docs }}
steps:
- uses: actions/checkout@v4
- uses: dorny/paths-filter@v3
id: filter
with:
base: "main"
token: ${{ github.token }}
filters: |
related: &related
- .github/workflows/ci.yaml
- codecov.yml
- pyproject.toml
- Makefile
ragas:
- *related
- "src/ragas/**"
- "tests/**"
- "examples/**"
docs:
- *related
- "docs/**"
unit_tests:
needs:
- diff
strategy:
fail-fast: false
matrix:
include:
# Critical path: Latest + oldest Python on Ubuntu (full test suite)
- os: ubuntu-latest
python-version: "3.9"
test-type: "full"
- os: ubuntu-latest
python-version: "3.12"
test-type: "full"
- os: ubuntu-latest
python-version: "3.13"
test-type: "full"
# Cross-platform validation (essential tests only)
- os: macos-latest
python-version: "3.11"
test-type: "essential"
- os: windows-latest
python-version: "3.10"
test-type: "essential"
if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.ragas == 'true') || github.event_name == 'push' }}
name: python${{ matrix.python-version }}_unit_tests (${{ matrix.os }}, ${{ matrix.test-type }})
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # fetch all tags and branches
- name: Setup python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
architecture: ${{ matrix.os == 'macos-latest' && 'arm64' || 'x64' }}
- name: Install uv
uses: astral-sh/setup-uv@v4
- name: Get pip cache dir
id: cache-dir
run: |
echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
- name: Cache dependencies (UV cache)
uses: actions/cache@v4
id: cache-deps
with:
path: |
${{ steps.cache-dir.outputs.dir }}
~/.cache/uv
key: deps-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
restore-keys: |
deps-${{ runner.os }}-py${{ matrix.python-version }}-
deps-${{ runner.os }}-py3.11-
deps-${{ runner.os }}-
- name: Install dependencies
run: |
# Use minimal install for fast CI runs (79 packages vs 383)
# This uses make install-minimal for consistency with local development
make install-minimal
- name: Run unit tests
run: |
# Configure test options based on OS and test type
if [ "${{ matrix.os }}" != 'windows-latest' ]; then
# Use pytest-xdist to improve test run-time on Linux/macOS
OPTS=(--dist loadfile -n auto)
fi
# Run different test suites based on test type
if [ "${{ matrix.test-type }}" = "full" ]; then
# Full test suite with notebook tests
uv run pytest --nbmake tests/unit "${OPTS[@]}"
else
# Essential tests only (faster for cross-platform validation)
uv run pytest tests/unit -k "not slow" "${OPTS[@]}"
fi
env:
__RAGAS_DEBUG_TRACKING: true
RAGAS_DO_NOT_TRACK: true
code_quality_check:
runs-on: ubuntu-latest
needs:
- diff
if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.ragas == 'true') || github.event_name == 'push' }}
steps:
- uses: actions/checkout@v4
- name: Setup python
uses: actions/setup-python@v5
with:
python-version: "3.11"
architecture: x64
- name: Install uv
uses: astral-sh/setup-uv@v4
- name: Get pip cache dir
id: cache-dir
run: |
echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
- name: Cache dependencies (UV cache)
uses: actions/cache@v4
id: cache-deps
with:
path: |
${{ steps.cache-dir.outputs.dir }}
~/.cache/uv
key: deps-ubuntu-py3.11-codestyle-${{ hashFiles('pyproject.toml') }}
restore-keys: |
deps-ubuntu-py3.11-codestyle-
deps-ubuntu-py3.11-
deps-ubuntu-
- name: Install dependencies
run: |
# Use minimal install for fast CI runs (79 packages vs 383)
# This uses make install-minimal for consistency with local development
make install-minimal
- name: Format check (dry run)
run: |
# Check if code is properly formatted (without making changes)
# Note: We use direct commands here instead of the standalone Makefiles
# to have precise control over CI-specific options like --check for dry-run
echo "Checking ragas formatting..."
uv run ruff format --check src tests docs --exclude src/ragas/_version.py --config pyproject.toml
uv run ruff check src docs tests --exclude src/ragas/_version.py --config pyproject.toml
- name: Type check
run: make type
================================================
FILE: .github/workflows/claude-code-review.yml
================================================
name: Claude Code Review
on:
issue_comment:
types: [created]
jobs:
claude-review:
if: |
github.event.issue.pull_request &&
contains(github.event.comment.body, '/claude-review')
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
issues: write
id-token: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Run Claude Code Review
id: claude-review
uses: anthropics/claude-code-action@beta
with:
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
# Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4.1)
# model: "claude-opus-4-1-20250805"
# Customize the trigger phrase to use /claude-review
trigger_phrase: "/claude-review"
# Custom instructions for the review
custom_instructions: |
When triggered with /claude-review, please analyze this pull request and provide:
## Change Type Classification
First, identify the primary type of change based on the files modified and changes made:
- **🐛 Bug Fix**: Fixes existing functionality
- **✨ New Feature**: Adds new functionality
- **📚 Documentation**: Updates or adds documentation (README, docs/, comments)
- **🔧 Refactor**: Code restructuring without changing functionality
- **🧪 Tests**: Adds or modifies tests
- **🏗️ Build/CI**: Changes to build process, CI/CD, dependencies
- **🎨 Style**: Code formatting, linting fixes
- **⚡ Performance**: Improves performance
- **🔒 Security**: Security-related improvements
- **🗑️ Cleanup**: Removes deprecated code, unused files
- **🔀 Merge**: Merge commits or branch management
- **📦 Dependencies**: Updates dependencies or package versions
## Code Review
Then provide feedback on:
- Code quality and best practices
- Potential bugs or issues
- Performance considerations
- Security concerns
- Test coverage
Be constructive and helpful in your feedback.
# Optional: Use sticky comments to make Claude reuse the same comment on subsequent pushes to the same PR
# use_sticky_comment: true
# Optional: Customize review based on file types
# direct_prompt: |
# Review this PR focusing on:
# - For TypeScript files: Type safety and proper interface usage
# - For API endpoints: Security, input validation, and error handling
# - For React components: Performance, accessibility, and best practices
# - For tests: Coverage, edge cases, and test quality
# Optional: Different prompts for different authors
# direct_prompt: |
# ${{ github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' &&
# 'Welcome! Please review this PR from a first-time contributor. Be encouraging and provide detailed explanations for any suggestions.' ||
# 'Please provide a thorough code review focusing on our coding standards and best practices.' }}
# Optional: Add specific tools for running tests or linting
# allowed_tools: "Bash(npm run test),Bash(npm run lint),Bash(npm run typecheck)"
# Optional: Skip review for certain conditions
# if: |
# !contains(github.event.pull_request.title, '[skip-review]') &&
# !contains(github.event.pull_request.title, '[WIP]')
================================================
FILE: .github/workflows/claude-docs-apply.yml
================================================
name: Claude Docs Apply
on:
pull_request_target:
types: [labeled]
jobs:
apply-docs:
if: github.event.label.name == 'update-docs'
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
issues: write
id-token: write
steps:
- name: Checkout PR branch
uses: actions/checkout@v4
with:
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}
# Use PAT for fork PRs (requires CLAUDE_CODE_PAT secret), GITHUB_TOKEN for same-repo PRs
token: ${{ secrets.CLAUDE_CODE_PAT || secrets.GITHUB_TOKEN }}
fetch-depth: 0
- name: Configure git
run: |
git config --global user.name "Claude Code Bot"
git config --global user.email "noreply@anthropic.com"
- name: Apply documentation updates
uses: anthropics/claude-code-action@v1
with:
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
github_token: ${{ secrets.CLAUDE_CODE_PAT || secrets.GITHUB_TOKEN }}
prompt: |
REPO: ${{ github.repository }}
PR NUMBER: ${{ github.event.pull_request.number }}
PR TITLE: ${{ github.event.pull_request.title }}
You are a documentation assistant for the Ragas project. Update the documentation based on the code changes in this PR.
## Quick Action Plan
1. Run `gh pr diff` to review changes
2. Identify what docs need updating (see structure below)
3. Make focused updates efficiently
4. Commit with clear message
## Documentation Structure (Diátaxis Framework)
**Where to update:**
- `docs/howtos/` - How-to guides (step-by-step instructions)
- `docs/concepts/` - Concept docs (explanations and rationale)
- `docs/getstarted/` - Tutorials (learning experiences)
- Source code docstrings - API documentation (feeds auto-generated reference)
**DO NOT edit:**
- `docs/references/**` - AUTO-GENERATED by mkdocstrings
## Writing Guidelines
- Use second-person ("you") and active voice
- Code blocks must be copy-pasteable with imports
- Use `??? "Click to expand"` for verbose outputs
- Add blank line after text ending with colon before lists
- Update `mkdocs.yml` nav if adding new pages
- Keep modes separate: no theory in how-tos, no instructions in concepts
## Documentation Modes Reference
1. **Tutorials** (`docs/getstarted/`) - "Can you teach me to...?"
- Narrative learning experience with complete working examples
2. **How-to Guides** (`docs/howtos/`) - "How do I...?"
- Concise step-by-step from user's perspective
3. **Reference** (`docs/references/`) - "What is...?"
- AUTO-GENERATED - edit source docstrings instead
4. **Explanation** (`docs/concepts/`) - "Why...?"
- Discursive articles on design decisions and theory
## Completion
After making changes, commit to this PR branch with a concise, descriptive message.
claude_args: |
--max-turns 30
--allowedTools "Read,Write,Edit,Glob,Grep,Bash(git:*),Bash(gh pr diff:*),Bash(gh pr view:*)"
- name: Remove labels after completion
if: always()
run: |
# Remove both labels
gh pr edit ${{ github.event.pull_request.number }} --remove-label "update-docs" || true
gh pr edit ${{ github.event.pull_request.number }} --remove-label "needs-doc-update" || true
# Comment that docs were updated
gh pr comment ${{ github.event.pull_request.number }} --body "✅ Documentation update completed."
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
================================================
FILE: .github/workflows/claude-docs-check.yml
================================================
name: Claude Docs Check
on:
pull_request_target:
types: [opened, synchronize, reopened]
paths:
- "src/**/*.py"
jobs:
check-docs:
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
issues: write
id-token: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.sha }}
token: ${{ secrets.GITHUB_TOKEN }}
fetch-depth: 0
- name: Analyze PR for documentation needs
id: analyze
uses: anthropics/claude-code-action@v1
with:
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
github_token: ${{ secrets.GITHUB_TOKEN }}
allowed_non_write_users: "*"
prompt: |
REPO: ${{ github.repository }}
PR NUMBER: ${{ github.event.pull_request.number }}
PR TITLE: ${{ github.event.pull_request.title }}
You are a documentation analyst for the Ragas project. Analyze this PR to determine if documentation updates are needed.
## Quick Decision Rules
**needs_update: false** (most common):
- Docstrings already updated in code → no action needed (API docs are auto-generated)
- Internal refactoring with no API changes → no action needed
- Bug fixes with no user-facing changes → no action needed
- Infrastructure/build changes → no action needed
**needs_update: true** (only when necessary):
- New user-facing features WITHOUT docstrings → need docs
- Changed usage patterns in how-to guides → need updates
- New core concepts without explanation → need concept docs
- Modified getting started flow → need tutorial updates
## Your Task
1. Run `gh pr diff` to review code changes
2. Check if docstrings are present for API changes
3. Return JSON immediately with your decision
Return format:
- `needs_update`: boolean
- `reason`: brief explanation (1-2 sentences max)
## Documentation Structure Reference
- `docs/howtos/` - Step-by-step guides
- `docs/concepts/` - Conceptual explanations
- `docs/getstarted/` - Tutorials
- `docs/references/` - AUTO-GENERATED (never edit directly)
IMPORTANT: Be decisive. Default to needs_update: false if docstrings are present. Return JSON within 3 turns.
claude_args: |
--max-turns 20
--json-schema '{"type":"object","properties":{"needs_update":{"type":"boolean"},"reason":{"type":"string"}},"required":["needs_update","reason"]}'
--allowedTools "Bash(gh pr diff:*),Bash(gh pr view:*),Read,Glob,Grep"
- name: Parse analysis result
id: parse
run: |
# Use heredoc to safely handle JSON with special characters
cat <<'EOF' > /tmp/output.json
${{ steps.analyze.outputs.structured_output }}
EOF
echo "structured_output=$(cat /tmp/output.json)"
NEEDS_UPDATE=$(jq -r '.needs_update' /tmp/output.json)
REASON=$(jq -r '.reason' /tmp/output.json)
echo "needs_update=$NEEDS_UPDATE" >> $GITHUB_OUTPUT
# Use multiline string format for reason to handle special characters
{
echo 'reason<<EOF'
jq -r '.reason' /tmp/output.json
echo 'EOF'
} >> $GITHUB_OUTPUT
- name: Add label and comment if docs needed
if: steps.parse.outputs.needs_update == 'true'
run: |
# Add the needs-doc-update label
gh pr edit ${{ github.event.pull_request.number }} --add-label "needs-doc-update"
# Comment with instructions
gh pr comment ${{ github.event.pull_request.number }} --body "📝 **Documentation update may be needed**
${{ steps.parse.outputs.reason }}
**To apply documentation updates:** Add the \`update-docs\` label to this PR."
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Comment if no docs needed
if: steps.parse.outputs.needs_update == 'false'
run: |
gh pr comment ${{ github.event.pull_request.number }} --body "✅ No documentation update needed — ${{ steps.parse.outputs.reason }}"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
================================================
FILE: .github/workflows/claude.yml
================================================
name: Claude Code
on:
issue_comment:
types: [created]
pull_request_review_comment:
types: [created]
issues:
types: [opened, assigned]
pull_request_review:
types: [submitted]
jobs:
claude:
if: |
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
issues: write
id-token: write
actions: read # Required for Claude to read CI results on PRs
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Run Claude Code
id: claude
uses: anthropics/claude-code-action@beta
with:
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
# This is an optional setting that allows Claude to read CI results on PRs
additional_permissions: |
actions: read
# Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4.1)
# model: "claude-opus-4-1-20250805"
# Optional: Customize the trigger phrase (default: @claude)
# trigger_phrase: "/claude"
# Optional: Trigger when specific user is assigned to an issue
# assignee_trigger: "claude-bot"
# Optional: Allow Claude to run specific commands
# allowed_tools: "Bash(npm install),Bash(npm run build),Bash(npm run test:*),Bash(npm run lint:*)"
# Optional: Add custom instructions for Claude to customize its behavior for your project
# custom_instructions: |
# Follow our coding standards
# Ensure all new code has tests
# Use TypeScript for new files
# Optional: Custom environment variables for Claude
# claude_env: |
# NODE_ENV: test
================================================
FILE: .github/workflows/issue-manager.yaml
================================================
name: Issue Manager
on:
schedule:
- cron: "0 0 * * *"
issue_comment:
types:
- created
- edited
issues:
types:
- labeled
pull_request_target:
types:
- labeled
workflow_dispatch:
jobs:
issue-manager:
runs-on: ubuntu-latest
permissions:
issues: write
pull-requests: write
steps:
- uses: tiangolo/issue-manager@0.4.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
config: >
{
"$schema": "https://raw.githubusercontent.com/tiangolo/issue-manager/master/schema.json",
"answered": {
"delay": "P3DT12H30M5S",
"message": "It seems the issue was answered, closing this now.",
"remove_label_on_comment": false,
"remove_label_on_close": false
},
"validated": {
"delay": 300,
"message": "The issue could not be validated after 5 minutes. Closing now.",
"remove_label_on_comment": true,
"remove_label_on_close": false
},
"waiting": {
"delay": 691200,
"message": "Closing after 8 days of waiting for the additional info requested.",
"remove_label_on_comment": true,
"remove_label_on_close": true
}
}
================================================
FILE: .github/workflows/publish-examples.yml
================================================
name: Upload ragas-examples Package
on:
release:
types: [published]
permissions:
contents: read
jobs:
deploy:
runs-on: ubuntu-latest
environment: pypi-release
strategy:
matrix:
package:
- name: ragas-examples
directory: examples
token: PYPI_API_TOKEN
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: '3.x'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install --upgrade setuptools setuptools_scm[toml] build
- name: get setuptools-scm version
run: python -m setuptools_scm
working-directory: ${{ matrix.package.directory }}
- name: Build package
run: python -m build
working-directory: ${{ matrix.package.directory }}
- name: Publish package
uses: pypa/gh-action-pypi-publish@release/v1
with:
password: ${{ secrets[matrix.package.token] }}
packages-dir: ${{ matrix.package.directory }}/dist/
attestations: false
================================================
FILE: .github/workflows/python-publish.yml
================================================
# This workflow will upload Python Packages using Twine when a release is created
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.
name: Upload Python Packages
on:
release:
types: [published]
permissions:
contents: read
jobs:
deploy:
runs-on: ubuntu-latest
environment: pypi-release
strategy:
matrix:
package:
- name: ragas
directory: .
token: PYPI_API_TOKEN
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: '3.x'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install --upgrade setuptools setuptools_scm[toml] build
- name: get setuptools-scm version
run: python -m setuptools_scm
working-directory: ${{ matrix.package.directory }}
- name: Build package
run: python -m build
working-directory: ${{ matrix.package.directory }}
- name: Publish package
uses: pypa/gh-action-pypi-publish@release/v1
with:
password: ${{ secrets[matrix.package.token] }}
packages-dir: ${{ matrix.package.directory }}/dist/
attestations: false
================================================
FILE: .gitignore
================================================
# General
.DS_Store
.AppleDouble
.LSOverride
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
# Cursor
.cursorignore
.cursor/plans
# Ragas specific
_experiments/
**/fil-result/
src/ragas/_version.py
experimental/ragas_experimental/_version.py
examples/ragas_examples/_version.py
.vscode
.envrc
uv.lock
.cache/
.claude/*
!.claude/commands/
node_modules
# Ragas examples
experimental/ragas_examples/benchmark_llm/experiments/*.csv
examples/*/logs/*run*json
examples/*/experiments/*csv
examples/ragas_examples/_version.py
**/test_dataset.csv
# Ragas examples package build artifacts
examples/dist/
examples/build/
examples/*.egg-info/
examples/ragas_examples/_version.py
examples/ragas_examples/text2sql/experiments/*
examples/ragas_examples/benchmark_llm/experiments/*
BookSQL-files
text2sql_logs
# MLflow artifacts
mlartifacts
mlflow.db
plan
================================================
FILE: .pre-commit-config.yaml
================================================
# Pre-commit configuration for entire ragas monorepo
# Install with: make install && pre-commit install
repos:
- repo: local
hooks:
- id: monorepo-ci
name: Run complete monorepo CI pipeline
entry: make run-ci-format-check
language: system
pass_filenames: false
always_run: true
stages: [pre-commit]
verbose: true
================================================
FILE: .readthedocs.yml
================================================
version: 2
mkdocs:
configuration: mkdocs.yml
build:
os: ubuntu-22.04
tools:
python: "3.12"
commands:
- pip install uv
- uv pip install --system -e "." --group docs
- if [ -n "$GH_TOKEN" ]; then pip install git+https://${GH_TOKEN}@github.com/squidfunk/mkdocs-material-insiders.git; fi
- mkdocs build --site-dir $READTHEDOCS_OUTPUT/html
================================================
FILE: CLAUDE.md
================================================
# CLAUDE.md
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Project Overview
Ragas is an evaluation toolkit for Large Language Model (LLM) applications. It provides objective metrics for evaluating LLM applications, test data generation capabilities, and integrations with popular LLM frameworks.
The repository contains:
1. **Ragas Library** - The main evaluation toolkit including experimental features (in `src/ragas/` directory)
- Core evaluation metrics and test generation
- Experimental features available at `ragas.experimental`
## Development Environment Setup
### Installation
Choose the appropriate installation based on your needs:
```bash
# RECOMMENDED: Minimal dev setup (79 packages - fast)
make install-minimal
# FULL: Complete dev environment (383 packages - comprehensive)
make install
# OR manual installation:
# Create a virtual environment
python -m venv venv
source venv/bin/activate # On Windows, use `venv\Scripts\activate`
# Minimal dev setup (uses [project.optional-dependencies].dev-minimal)
uv pip install -e ".[dev-minimal]"
# Full dev setup (uses [dependency-groups].dev)
uv sync --group dev
```
### Installation Methods Explained
- **Minimal setup**: Uses `uv pip install` with optional dependencies for selective installation
- **Full setup**: Uses `uv sync` with dependency groups for comprehensive environment management
- **No naming conflicts**: `dev-minimal` vs `dev` clearly distinguish the two approaches
### Workspace Structure
The project uses a UV workspace configuration for managing multiple packages:
```bash
# Install
uv sync
# Install examples separately
uv sync --package ragas-examples
# Build specific workspace package
uv build --package ragas-examples
```
**Workspace Members:**
- `ragas` (main package) - Located in `src/ragas/`
- `ragas-examples` (examples package) - Located in `examples/`
The workspace ensures consistent dependency versions across packages and enables editable installs of workspace members.
## Common Commands
### Commands (from root directory)
```bash
# Setup and installation
make install-minimal # Minimal dev setup (79 packages - recommended)
make install # Full dev environment (383 packages - complete)
# Code quality
make format # Format and lint all code
make type # Type check all code
make check # Quick health check (format + type, no tests)
# Testing
make test # Run all unit tests
make test-e2e # Run end-to-end tests
# CI/Build
make run-ci # Run complete CI pipeline
make clean # Clean all generated files
# Documentation
make build-docs # Build all documentation
make serve-docs # Serve documentation locally
# Benchmarks
make benchmarks # Run performance benchmarks
make benchmarks-docker # Run benchmarks in Docker
```
### Testing
```bash
# Run all tests (from root)
make test
# Run specific test (using pytest -k flag)
make test k="test_name"
# Run end-to-end tests
make test-e2e
# Direct pytest commands for more control
uv run pytest tests/unit -k "test_name"
uv run pytest tests/unit -v
```
### Documentation
```bash
# Build all documentation (from root)
make build-docs
# Serve documentation locally
make serve-docs
```
### Benchmarks
```bash
# Run all benchmarks locally
make benchmarks
# Run benchmarks in Docker
make benchmarks-docker
```
## Project Architecture
The repository has the following structure:
```sh
/ # Main ragas project
├── src/ragas/ # Source code including experimental features
│ └── experimental/ # Experimental features
├── tests/ # All tests (core + experimental)
│ └── experimental/ # Experimental tests
├── examples/ # Example code
├── pyproject.toml # Build config
├── docs/ # Documentation
├── scripts/ # Build/CI scripts
├── Makefile # Build commands
└── README.md # Repository overview
```
### Ragas Core Components
The Ragas core library provides metrics, test data generation and evaluation functionality for LLM applications:
1. **Metrics** - Various metrics for evaluating LLM applications including:
- AspectCritic
- AnswerCorrectness
- ContextPrecision
- ContextRecall
- Faithfulness
- and many more
2. **Test Data Generation** - Automatic creation of test datasets for LLM applications
3. **Integrations** - Integrations with popular LLM frameworks like LangChain, LlamaIndex, and observability tools
### Experimental Components
The experimental features are now integrated into the main ragas package:
1. **Experimental features** are available at `ragas.experimental`
2. **Dataset and Experiment management** - Enhanced data handling for experiments
3. **Advanced metrics** - Extended metric capabilities
4. **Backend support** - Multiple storage backends (CSV, JSONL, Google Drive, in-memory)
To use experimental features:
```python
from ragas import Dataset
from ragas import experiment
from ragas.backends import get_registry
```
## Debugging Logs
To view debug logs for any module:
```python
import logging
# Configure logging for a specific module (example with analytics)
analytics_logger = logging.getLogger('ragas._analytics')
analytics_logger.setLevel(logging.DEBUG)
# Create a console handler and set its level
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.DEBUG)
# Create a formatter and add it to the handler
formatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
console_handler.setFormatter(formatter)
# Add the handler to the logger
analytics_logger.addHandler(console_handler)
```
## Memories
- whenever you create such docs put in in /\_experiments because that is gitignored and you can use it as a scratchpad or tmp directory for storing these
- always use uv to run python and python related commandline tools like isort, ruff, pyright etc. This is because we are using uv to manage the .venv and dependencies.
- The project uses two distinct dependency management approaches:
- **Minimal setup**: `[project.optional-dependencies].dev-minimal` for fast development (79 packages)
- **Full setup**: `[dependency-groups].dev` for comprehensive development (383 packages)
- Use `make install-minimal` for most development tasks, `make install` for full ML stack work
- if the user asks you to save a plan, save it into the plan/ directory with an appropriate file name.
================================================
FILE: CODE_OF_CONDUCT.md
================================================
# Code of Conduct
## Our Commitment
We are committed to providing a welcoming and inclusive environment for all people, regardless of age, body size, caste, disability, ethnicity, gender identity and expression, level of experience, family status, gender, immigration status, level of expertise, national origin, personal appearance, political belief, race, religion, sexual identity and orientation, socioeconomic status, tribe, and veteran status.
We expect all participants in the Ragas community—whether contributing code, providing feedback, reporting issues, participating in discussions, attending events, or engaging in any other capacity—to embody the values of respect, inclusion, and professionalism.
## Our Standards
Examples of behaviour that contributes to creating a positive environment include:
- Using welcoming and inclusive language
- Being respectful of differing opinions, viewpoints, and experiences
- Gracefully accepting constructive criticism
- Focusing on what is best for the community
- Showing empathy towards other community members
- Being patient and understanding with newcomers
- Giving credit to others' work and contributions
- Asking clarifying questions rather than making assumptions
Examples of unacceptable behaviour include:
- Harassment, intimidation, or discrimination of any kind
- Unwelcome sexual attention or advances
- Trolling, insulting/derogatory comments, and personal or political attacks
- Publishing others' private information without explicit permission (doxing)
- Gatekeeping—deliberately excluding or discouraging participation
- Deliberate disinformation or misinformation
- Other conduct which could reasonably be considered inappropriate in a professional setting
- Sustained disruption of discussions or project activities, including:
- Spam, off-topic posts, or repeated low-effort comments in issues or discussions
- Duplicate issues or discussions that have already been reported
- Cross-posting the same issue or question across multiple channels without justification
- Deliberately posting controversial or unrelated content to distract from ongoing discussions
- Threats of violence or violent language directed at another person
## Scope
This Code of Conduct applies to all spaces managed by the Ragas project, including:
- GitHub repositories (issues, pull requests, discussions, and code reviews)
- Official communication channels (Discord, Slack, mailing lists, forums)
- Official events and conferences organised by Ragas maintainers
- Any official online or offline event, conference, or gathering representing Ragas
This Code of Conduct also applies to conduct outside of these spaces if it demonstrates a pattern of harassment or is reasonably perceived as affecting the safety or well-being of community members.
The Code of Conduct applies equally to all participants, including maintainers, contributors, sponsors, and community members.
## Reporting Violations
If you experience or witness behaviour that violates this Code of Conduct, please report it by emailing **support@ragas.io**. Include as much detail as you're comfortable sharing, including:
- What happened
- Who was involved
- When it occurred
- Any relevant links or context
- Any witnesses (optional)
All reports will be treated confidentially. We will not disclose the identity of the reporter without their consent, except as necessary for investigation and response.
If the violation involves a member of the Code of Conduct committee, or if you're not comfortable reporting directly to that address, please reach out to a project maintainer directly through alternative means.
## Enforcement
The Ragas project maintainers are responsible for clarifying standards of acceptable behaviour and will take appropriate action in response to violations of this Code of Conduct.
### Our Commitment to Enforcement
We recognise that:
- Not all violations are equally severe
- Context matters
- People can learn and grow
- The goal is to maintain a healthy, inclusive community
### Enforcement Guidelines
The following are examples of how we may respond to violations. Responses will be proportionate to the severity and pattern of behaviour:
1. **Warning**: For minor or first-time violations, a private message explaining the issue and its impact, with an expectation to change behaviour.
2. **Temporary Suspension**: For more serious or repeated violations, temporary removal from community spaces (ranging from hours to weeks) to allow for reflection and de-escalation.
3. **Permanent Removal**: For severe, repeated, or unresolved violations, permanent removal from the project and its community spaces.
4. **Law Enforcement**: In cases involving illegal activity or threats of violence, we may involve law enforcement.
The maintainers may also take action to address behaviour even if no formal complaint has been filed, if they reasonably believe it violates this Code of Conduct.
## Consequences for Violations
Anyone who violates this Code of Conduct may face consequences determined by the Ragas maintainers, including:
- Editing or deletion of comments or contributions
- Removal from the project repository or community spaces
- Temporary or permanent ban from participating in Ragas spaces
- Public acknowledgment of the violation (at the discretion of the reporter and maintainers)
## Appeal Process
If you believe you have been unfairly sanctioned under this Code of Conduct, you may appeal by sending a detailed explanation to **support@ragas.io**. The appeal will be reviewed by a different set of maintainers when possible, and a decision will be communicated to you within a reasonable timeframe.
## Attribution
This Code of Conduct is adapted from the Contributor Covenant (https://www.contributor-covenant.org/), and incorporates best practices from codes of conduct in the Python community and other leading open source projects.
## Questions?
If you have questions about this Code of Conduct or how it applies to a specific situation, please reach out to the maintainers at **support@ragas.io** or through a project maintainer you trust.
---
**Last Updated**: November 2024
We appreciate your participation in making Ragas a welcoming and inclusive community for everyone.
================================================
FILE: CONTRIBUTING.md
================================================
# Development Guide for Ragas Monorepo
This comprehensive guide covers development workflows for the Ragas monorepo, designed for both human developers and AI agents.
## Quick Start (for Developers)
```bash
# 1. Clone and enter the repository
git clone https://github.com/vibrantlabsai/ragas.git
# 2. Install uv (if not already installed)
curl -LsSf https://astral.sh/uv/install.sh | sh
# 3. Choose your installation type:
# RECOMMENDED: Minimal dev setup (fast)
make install-minimal
# FULL: Complete dev environment (comprehensive)
make install
# 4. Verify everything works
make check
# 5. Start developing!
make help # See all available commands
```
## Quick Start (for AI Agents)
AI agents working with this codebase should use these standardized commands:
```bash
# Essential commands for AI development
make help # See all available targets
make install-minimal # Minimal dev setup (fast)
make install # Full environment (modern uv sync)
make check # Quick health check (format + type)
make test # Run all tests
make run-ci # Full CI pipeline locally
# Individual development tasks
make format # Format and lint all code
make type # Type check all code
make clean # Clean generated files
```
**Key Points for AI Agents:**
- Always use `make` commands rather than direct tool invocation
- Use `uv run` prefix for any direct Python tool usage
- Check `make help` for the complete command reference
- The CI pipeline uses the same commands as local development
## Monorepo Architecture
This repository is organized as a single project with integrated experimental features:
```sh
/ # Main ragas project
├── src/ragas/ # Main source code
│ └── experimental/ # Experimental features
├── tests/ # Tests (unit, e2e, benchmarks)
│ └── experimental/ # Experimental tests
├── examples/ # Example code
├── pyproject.toml # Dependencies and configuration
├── docs/ # Documentation
├── .github/workflows/ # CI/CD pipeline
├── Makefile # Build commands
└── CLAUDE.md # AI assistant instructions
```
### Project Components
- **Ragas Core**: The main evaluation toolkit for LLM applications (in `src/ragas/`)
- **Ragas Experimental**: Advanced features integrated at `src/ragas/experimental/`
- **Infrastructure**: Single CI/CD, documentation, and build system
### Examples Package (ragas-examples)
- Lives under `examples/` as an installable package `ragas-examples`
- Published independently to PyPI via GitHub Actions workflow `publish-examples.yml`
- Versioning via Git tags with prefix `examples-v` (e.g., `examples-v0.1.0`)
- Local development: `uv pip install -e . -e ./examples`
- Run examples: `python -m ragas_examples.benchmark_llm.prompt`
## Development Environment Setup
### Prerequisites
- Python 3.9+
- [uv](https://docs.astral.sh/uv/) (recommended) or pip
- Git
### Setup Process
#### Option 1: Using Make (Recommended)
```bash
# Recommended: Minimal dev setup
make install-minimal
# Full: Complete environment
make install
```
#### Option 2: Manual Setup
```bash
# Install uv if not available
curl -LsSf https://astral.sh/uv/install.sh | sh
# Minimal dev: Core + essential dev tools
uv pip install -e ".[dev-minimal]"
# Full dev: Everything (uses modern uv sync)
uv sync --group dev
```
#### Which Option to Choose?
**Use `make install-minimal` if you're:**
- Contributing to ragas development
- Need testing and linting tools
- Want fast CI/CD builds
- Working on code quality, docs, or basic features
**Use `make install` if you're:**
- Working on ML features requiring the full stack
- Need observability tools (Phoenix, MLflow)
- Developing with notebooks and advanced integrations
- Want the complete development environment
#### Installation Methods Explained
- **`install-minimal`**: Uses `uv pip install -e ".[dev-minimal]"` for selective minimal dev dependencies
- **`install`**: Uses `uv sync --group dev` for complete modern dependency management
### Verification
```bash
make check # Runs format + type checking
make test # Runs all tests
```
## Available Commands Reference
Run `make help` to see all targets. Here are the essential commands:
### Setup & Installation
- `make install-minimal` - Install minimal dev setup (recommended)
- `make install` - Install full environment with uv sync (complete)
### Code Quality
- `make format` - Format and lint all code (includes unused import cleanup)
- `make type` - Type check all code
- `make check` - Quick health check (format + type, no tests)
### Testing
- `make test` - Run all unit tests
- `make test-e2e` - Run end-to-end tests
- `make benchmarks` - Run performance benchmarks
- `make benchmarks-docker` - Run benchmarks in Docker
### CI/Build
- `make run-ci` - Run complete CI pipeline locally
- `make clean` - Clean all generated files
### Documentation
- `make build-docs` - Build all documentation
- `make build-docs-pdf` - Build documentation with PDF export (requires WeasyPrint)
- `make serve-docs` - Serve documentation locally
- See `docs/community/pdf_export.md` for PDF export details and limitations
## Development Workflows
### Daily Development
```bash
# 1. Start your work
git checkout -b feature/your-feature
# 2. Make changes to code
# 3. Check your work
make check # Format and type check
make test # Run tests
# 4. Commit and push
git add .
git commit -m "feat: your feature description"
git push origin feature/your-feature
```
### Before Submitting PR
```bash
make run-ci # Run full CI pipeline
# Ensure all checks pass before creating PR
```
#### Development Workflow
```bash
# Use the Makefile for all development
make help # See available commands
make format # Format all code (core + experimental)
make type # Type check all code
make test # Run all tests (core + experimental)
make check # Quick format + type check
make run-ci # Run full CI pipeline
# Or use direct commands for specific tasks
uv run pytest tests/unit # Run core unit tests
uv run pytest tests/unit # Run unit tests
uv run pyright src # Type check source code
```
## Testing Strategy
### Test Types
1. **Unit Tests**: Fast, isolated tests for individual components
2. **End-to-End Tests**: Integration tests for complete workflows
3. **Benchmarks**: Performance tests for evaluation metrics
### Running Tests
```bash
# All tests
make test
# Specific test categories
uv run pytest tests/unit
uv run pytest tests/e2e
# With coverage or specific options
uv run pytest tests/unit -k "test_name"
```
### Test Organization
- **Unit Tests**: `tests/unit/`
- **End-to-End Tests**: `tests/e2e/`
- **Benchmarks**: `tests/benchmarks/`
## Code Quality & CI/CD
### Code Quality Pipeline
The `make format` command runs:
1. **isort**: Import sorting
2. **ruff format**: Code formatting
3. **ruff --fix-only**: Auto-fix issues (including unused imports)
4. **ruff check**: Final linting validation
### Type Checking
```bash
make type # Type check all code with pyright
```
### CI/CD Pipeline
Our GitHub Actions CI runs:
1. **Dependency Installation**: Using uv for consistent environments
2. **Code Quality Checks**: Format and type validation
3. **Testing**: Unit and integration tests across Python 3.9-3.12
4. **Multi-OS Testing**: Ubuntu, macOS, Windows
### Local CI Simulation
```bash
make run-ci # Runs: format + type + test
```
## Project Guidelines
### Ragas Project
- **Language**: Python with type hints
- **Testing**: pytest with nbmake for notebook tests
- **Style**: Google-style docstrings
- **Architecture**: Modular metrics and evaluation framework with experimental features
- **Dependencies**: All defined in `pyproject.toml`
### Adding Dependencies
- **All features**: Add to `pyproject.toml`
- **Always**: Test with `make install` and `make test`
## Troubleshooting
### Common Issues
#### Import Errors
```bash
# Reinstall in development mode
make install
```
#### Test Failures
```bash
# Run specific failing test
uv run pytest tests/unit/test_specific.py -v
# Check experimental test dependencies
uv run pytest tests/unit --collect-only
```
#### Formatting Issues
```bash
# Fix formatting
make format
# Check specific files
uv run ruff check path/to/file.py --fix
```
#### CI Failures
```bash
# Run the same checks locally
make run-ci
# Individual checks
make format # Must pass
make type # Must pass
make test # Must pass
```
### Development Environment Issues
#### uv Not Found
```bash
# Install uv
curl -LsSf https://astral.sh/uv/install.sh | sh
# or use pip: pip install uv
```
#### Dependency Conflicts
```bash
# Clean install
make clean
make install
```
### Getting Help
- **Documentation**: Check `CLAUDE.md` for AI assistant guidance
- **Commands**: Run `make help` for all available targets
- **Issues**: Check existing GitHub issues or create a new one
## Contributing Guidelines
### Pull Request Process
1. **Fork** the repository
2. **Create** a feature branch: `git checkout -b feature/amazing-feature`
3. **Develop** using the workflows above
4. **Test** thoroughly: `make run-ci`
5. **Submit** a pull request with clear description
### Commit Message Format
```
feat: add new evaluation metric
fix: resolve import error in experimental
docs: update development guide
test: add unit tests for metric base
```
### Code Review Checklist
- [ ] All tests pass (`make test`)
- [ ] Code is formatted (`make format`)
- [ ] Type checking passes (`make type`)
- [ ] Documentation is updated
- [ ] Appropriate tests are included
## AI Agent Best Practices
### Recommended Workflow for AI Agents
1. **Understand the task**: Read relevant documentation and code
2. **Plan the approach**: Identify which project(s) need changes
3. **Use standardized commands**: Always prefer `make` targets
4. **Test incrementally**: Use `make check` frequently during development
5. **Validate thoroughly**: Run `make run-ci` before completing
### Command Patterns for AI Agents
```bash
# Always start with understanding the current state
make help
ls -la # Check current directory structure
# For code changes
make format # After making changes
make test # Verify functionality
# For project-specific work
make help # See available commands
# For investigation
uv run pytest --collect-only # See available tests
uv run ruff check --no-fix # Check issues without fixing
```
### File Modification Guidelines
- **Prefer editing** existing files over creating new ones
- **Use project conventions** (check similar files for patterns)
- **Update tests** when modifying functionality
- **Follow existing code style** (enforced by `make format`)
---
#### Python 3.13 on macOS ARM: NumPy fails to install (builds from source)
- Symptom: `make install` attempts to build `numpy==2.0.x` from source on Python 3.13 (no prebuilt wheel), failing with C/C++ errors.
- Status: Ragas CI supports Python 3.9–3.12. Python 3.13 is not officially supported yet.
Workarounds:
1) Recommended: use Python 3.12
```bash
uv python install 3.12
rm -rf .venv
uv venv -p 3.12
make install
```
2) Stay on 3.13 (best effort):
- Install minimal first, then add extras as needed:
```bash
rm -rf .venv
uv venv -p 3.13
make install-minimal
uv pip install "ragas[tracing,gdrive,ai-frameworks]"
```
- Or force a newer NumPy wheel:
```bash
uv pip install "numpy>=2.1" --only-binary=:all:
```
If conflicts pin NumPy to 2.0.x, temporarily set `numpy>=2.1` in `pyproject.toml` and run `uv sync --group dev`.
**Happy coding! 🚀**
For additional context and instructions specific to AI assistants, see [CLAUDE.md](./CLAUDE.md).
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [2023] [Vibrant Labs]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: Makefile
================================================
GIT_ROOT ?= $(shell git rev-parse --show-toplevel)
# Optionally show commands being executed with V=1
Q := $(if $(V),,@)
# Common paths
RAGAS_PATHS := src tests docs
help: ## Show all Makefile targets
$(Q)grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'
# =============================================================================
# SETUP & INSTALLATION
# =============================================================================
setup-venv: ## Set up uv virtual environment
@echo "Setting up uv virtual environment..."
$(Q)VIRTUAL_ENV= uv venv
@echo "Virtual environment created at .venv"
@echo "To activate: source .venv/bin/activate"
install-minimal: ## Install minimal dev dependencies (fast setup - 79 packages)
@echo "Installing minimal development dependencies (fast setup)..."
@if [ ! -d ".venv" ]; then \
echo "Virtual environment not found, creating one..."; \
$(MAKE) setup-venv; \
fi
@echo "Installing core ragas + essential dev tools..."
$(Q)uv pip install -e ".[dev-minimal]"
@echo "Setting up pre-commit hooks..."
$(Q)uv run pre-commit install
@echo "Minimal installation complete! (79 packages)"
@echo "Note: For full features including ML packages, use 'make install'"
install: ## Install full dependencies with uv sync (backward compatible - modern approach)
@echo "Installing full development dependencies with uv sync..."
@if [ ! -d ".venv" ]; then \
echo "Virtual environment not found, creating one..."; \
$(MAKE) setup-venv; \
fi
@echo "Installing ragas with full dev environment..."
$(Q)VIRTUAL_ENV= uv sync --group dev
@echo "Setting up pre-commit hooks..."
$(Q)uv run pre-commit install
@echo "Full installation complete! (Modern uv sync approach)"
# =============================================================================
# CODE QUALITY
# =============================================================================
.PHONY: help setup-venv install-minimal install format type check clean test test-e2e benchmarks benchmarks-docker run-ci run-ci-fast run-ci-format-check run-ci-type run-ci-tests build-docs serve-docs
format: ## Format and lint all code
@echo "Formatting and linting all code..."
@echo "(ruff format) Formatting ragas..."
$(Q)uv run --active ruff format src tests docs --exclude src/ragas/_version.py --config pyproject.toml
@echo "(ruff) Auto-fixing ragas (includes import sorting and unused imports)..."
$(Q)uv run --active ruff check src tests docs --exclude src/ragas/_version.py --fix-only --config pyproject.toml
@echo "(ruff) Final linting check for ragas..."
$(Q)uv run --active ruff check src tests docs --exclude src/ragas/_version.py --config pyproject.toml
type: ## Type check all code
@echo "Type checking all code..."
@echo "(pyright) Typechecking ragas..."
$(Q)PYRIGHT_PYTHON_FORCE_VERSION=latest uv run --active pyright -p pyproject.toml src
check: format type ## Quick health check (format + type, no tests)
@echo "Code quality check complete!"
# =============================================================================
# BENCHMARKS
# =============================================================================
benchmarks: ## Run all benchmarks locally
@echo "Running all benchmarks..."
@echo "Running evaluation benchmarks..."
$(Q)cd $(GIT_ROOT)/tests/benchmarks && uv run python benchmark_eval.py
@echo "Running testset generation benchmarks..."
$(Q)cd $(GIT_ROOT)/tests/benchmarks && uv run python benchmark_testsetgen.py
benchmarks-docker: ## Run benchmarks in docker
@echo "Running benchmarks in docker..."
$(Q)cd $(GIT_ROOT) || exit 1
docker buildx build --build-arg OPENAI_API_KEY=$(OPENAI_API_KEY) -t ragas-benchmark -f $(GIT_ROOT)/tests/benchmarks/Dockerfile .
docker inspect ragas-benchmark:latest | jq ".[0].Size" | numfmt --to=si
benchmarks-test: ## Run benchmarks for ragas unit tests
@echo "Running ragas unit tests with timing benchmarks..."
$(Q)uv run --active pytest --nbmake tests/unit --durations=0 -v $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)
# =============================================================================
# CI/BUILD
# =============================================================================
run-ci: ## Run complete CI pipeline (mirrors GitHub CI exactly)
@echo "Running complete CI pipeline..."
@echo "Format check..."
$(Q)uv run --active ruff format --check src tests docs --exclude src/ragas/_version.py --config pyproject.toml
$(Q)uv run --active ruff check src tests docs --exclude src/ragas/_version.py --config pyproject.toml
@echo "Type check..."
$(Q)$(MAKE) type
@echo "Unit tests..."
$(Q)__RAGAS_DEBUG_TRACKING=true RAGAS_DO_NOT_TRACK=true uv run --active pytest --nbmake tests/unit --dist loadfile -n auto
@echo "All CI checks passed!"
run-ci-format-check: ## Run format check in dry-run mode (like GitHub CI)
@echo "Running format check (dry-run, like GitHub CI)..."
@echo "Checking ragas formatting..."
$(Q)uv run --active ruff format --check src tests docs --exclude src/ragas/_version.py --config pyproject.toml
$(Q)uv run --active ruff check src docs tests --exclude src/ragas/_version.py --config pyproject.toml
run-ci-type: ## Run type checking (matches GitHub CI)
@echo "Running type checking (matches GitHub CI)..."
$(Q)$(MAKE) type
run-ci-tests: ## Run all tests with CI options
@echo "Running all tests with CI options..."
$(Q)__RAGAS_DEBUG_TRACKING=true RAGAS_DO_NOT_TRACK=true pytest --nbmake tests/unit --dist loadfile -n auto
run-ci-fast: ## Fast CI check for quick local validation (2-3 minutes)
@echo "Running fast CI check for quick feedback..."
@echo "Format check..."
$(Q)uv run --active ruff format --check src tests docs --exclude src/ragas/_version.py --config pyproject.toml
$(Q)uv run --active ruff check src docs tests --exclude src/ragas/_version.py --config pyproject.toml
@echo "Core unit tests (no nbmake for speed)..."
$(Q)uv run --active pytest tests/unit --dist loadfile -n auto -x
@echo "Fast CI check completed!"
clean: ## Clean all generated files
@echo "Cleaning all generated files..."
$(Q)find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
$(Q)rm -rf site/ docs/site/ .mypy_cache .pytest_cache .ruff_cache
$(Q)rm -rf dist/ build/ *.egg-info/ src/*.egg-info/
$(Q)rm -rf .coverage htmlcov/ .tox/ .venv/
$(Q)find . -name '*.log' -delete
$(Q)find . -name '.DS_Store' -delete
$(Q)find . -name 'temp*' -type d -exec rm -rf {} + 2>/dev/null || true
$(Q)find . -name '.tmp*' -type d -exec rm -rf {} + 2>/dev/null || true
@echo "Cleanup complete!"
# =============================================================================
# TESTING
# =============================================================================
test: ## Run all unit tests
@echo "Running all unit tests..."
$(Q)uv run --active pytest tests/unit $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)
test-all: ## Run all unit tests (including notebooks)
@echo "Running all unit tests (including notebooks)..."
$(Q)uv run --active pytest --nbmake tests/unit $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)
test-e2e: ## Run all end-to-end tests
@echo "Running all end-to-end tests..."
$(Q)uv run --active pytest --nbmake tests/e2e -s
# =============================================================================
# DOCUMENTATION
# =============================================================================
build-docs: ## Build all documentation
@echo "Building all documentation..."
@echo "Converting ipynb notebooks to md files..."
$(Q)MKDOCS_CI=true uv run python $(GIT_ROOT)/docs/ipynb_to_md.py
@echo "Building ragas documentation..."
$(Q)MKDOCS_CI=false uv run --group docs mkdocs build
check-pdf-deps: ## Check if WeasyPrint is properly installed with all dependencies
@echo "Checking if WeasyPrint is properly installed..."
@uv run --group docs-pdf python -c "import weasyprint; weasyprint.HTML(string='<h1>Test</h1>').write_pdf(target=None)" 2>/dev/null && \
echo "WeasyPrint is installed and all dependencies are available" || \
(echo ""; \
echo "WeasyPrint is not installed or has missing system dependencies"; \
echo ""; \
echo "Setup Instructions: https://doc.courtbouillon.org/weasyprint/stable/first_steps.html"; \
echo "Troubleshooting: https://doc.courtbouillon.org/weasyprint/stable/first_steps.html#troubleshooting"; \
echo ""; \
exit 1)
check-mermaid-deps: ## Check Mermaid CLI is available and can render a diagram
@command -v node >/dev/null || (echo "Node.js is required for Mermaid PDF rendering"; exit 1)
@command -v mmdc >/dev/null || (echo "Missing 'mmdc' (Mermaid CLI). Mermaid diagrams in PDF depend on Mermaid CLI."; exit 1)
@tmp_dir="$$(mktemp -d)"; \
printf "graph TD\n A-->B\n" > "$$tmp_dir/diag.mmd"; \
mmdc -i "$$tmp_dir/diag.mmd" -o "$$tmp_dir/diag.svg" >/dev/null 2>&1 || \
(echo "Mermaid CLI found, but rendering failed (mmdc couldn't produce SVG)."; rm -rf "$$tmp_dir"; exit 1); \
test -s "$$tmp_dir/diag.svg" || \
(echo "Mermaid CLI ran but produced an empty SVG."; rm -rf "$$tmp_dir"; exit 1); \
rm -rf "$$tmp_dir"; \
echo "Mermaid CLI is installed and can render diagrams"
build-docs-pdf: check-pdf-deps check-mermaid-deps ## Build documentation with PDF export (requires WeasyPrint)
@echo "Building documentation with PDF export..."
$(Q)MKDOCS_CI=false ENABLE_PDF_EXPORT=1 uv run --group docs --group docs-pdf mkdocs build -f mkdocs-pdf.yml
@echo "PDF generated at: site/pdf/document.pdf"
serve-docs: ## Build and serve documentation locally
$(Q)MKDOCS_CI=false uv run --group docs mkdocs serve --dirtyreload
================================================
FILE: README.md
================================================
<h1 align="center">
<img style="vertical-align:middle" height="200"
src="https://raw.githubusercontent.com/vibrantlabsai/ragas/main/docs/_static/imgs/logo.png">
</h1>
<p align="center">
<i>Supercharge Your LLM Application Evaluations 🚀</i>
</p>
<p align="center">
<a href="https://github.com/vibrantlabsai/ragas/releases">
<img alt="Latest release" src="https://img.shields.io/github/release/vibrantlabsai/ragas.svg">
</a>
<a href="https://www.python.org/">
<img alt="Made with Python" src="https://img.shields.io/badge/Made%20with-Python-1f425f.svg?color=purple">
</a>
<a href="https://github.com/vibrantlabsai/ragas/blob/master/LICENSE">
<img alt="License Apache-2.0" src="https://img.shields.io/github/license/vibrantlabsai/ragas.svg?color=green">
</a>
<a href="https://pypi.org/project/ragas/">
<img alt="Ragas Downloads per month" src="https://static.pepy.tech/badge/ragas/month">
</a>
<a href="https://discord.gg/5djav8GGNZ">
<img alt="Join Ragas community on Discord" src="https://img.shields.io/discord/1119637219561451644">
</a>
<a target="_blank" href="https://deepwiki.com/vibrantlabsai/ragas">
<img
src="https://devin.ai/assets/deepwiki-badge.png"
alt="Ask DeepWiki.com"
height="20"
/>
</a>
</p>
<h4 align="center">
<p>
<a href="https://docs.ragas.io/">Documentation</a> |
<a href="#fire-quickstart">Quick start</a> |
<a href="https://discord.gg/5djav8GGNZ">Join Discord</a> |
<a href="https://blog.ragas.io/">Blog</a> |
<a href="https://newsletter.ragas.io/">NewsLetter</a> |
<a href="https://www.ragas.io/careers">Careers</a>
<p>
</h4>
Objective metrics, intelligent test generation, and data-driven insights for LLM apps
Ragas is your ultimate toolkit for evaluating and optimizing Large Language Model (LLM) applications. Say goodbye to time-consuming, subjective assessments and hello to data-driven, efficient evaluation workflows.
Don't have a test dataset ready? We also do production-aligned test set generation.
## Key Features
- 🎯 Objective Metrics: Evaluate your LLM applications with precision using both LLM-based and traditional metrics.
- 🧪 Test Data Generation: Automatically create comprehensive test datasets covering a wide range of scenarios.
- 🔗 Seamless Integrations: Works flawlessly with popular LLM frameworks like LangChain and major observability tools.
- 📊 Build feedback loops: Leverage production data to continually improve your LLM applications.
## :shield: Installation
Pypi:
```bash
pip install ragas
```
Alternatively, from source:
```bash
pip install git+https://github.com/vibrantlabsai/ragas
```
## :fire: Quickstart
### Clone a Complete Example Project
The fastest way to get started is to use the `ragas quickstart` command:
```bash
# List available templates
ragas quickstart
# Create a RAG evaluation project
ragas quickstart rag_eval
# Specify where you want to create it.
ragas quickstart rag_eval -o ./my-project
```
Available templates:
- `rag_eval` - Evaluate RAG systems
Coming Soon:
- `agent_evals` - Evaluate AI agents
- `benchmark_llm` - Benchmark and compare LLMs
- `prompt_evals` - Evaluate prompt variations
- `workflow_eval` - Evaluate complex workflows
### Evaluate your LLM App
`ragas` comes with pre-built metrics for common evaluation tasks. For example, Aspect Critique evaluates any aspect of your output using `DiscreteMetric`:
```python
import asyncio
from openai import AsyncOpenAI
from ragas.metrics import DiscreteMetric
from ragas.llms import llm_factory
# Setup your LLM
client = AsyncOpenAI()
llm = llm_factory("gpt-4o", client=client)
# Create a custom aspect evaluator
metric = DiscreteMetric(
name="summary_accuracy",
allowed_values=["accurate", "inaccurate"],
prompt="""Evaluate if the summary is accurate and captures key information.
Response: {response}
Answer with only 'accurate' or 'inaccurate'."""
)
# Score your application's output
async def main():
score = await metric.ascore(
llm=llm,
response="The summary of the text is..."
)
print(f"Score: {score.value}") # 'accurate' or 'inaccurate'
print(f"Reason: {score.reason}")
if __name__ == "__main__":
asyncio.run(main())
```
> **Note**: Make sure your `OPENAI_API_KEY` environment variable is set.
Find the complete [Quickstart Guide](https://docs.ragas.io/en/latest/getstarted/quickstart)
## Want help in improving your AI application using evals?
In the past 2 years, we have seen and helped improve many AI applications using evals. If you want help with improving and scaling up your AI application using evals.
🔗 Book a [slot](https://cal.com/team/vibrantlabs/app) or drop us a line: [founders@vibrantlabs.com](mailto:founders@vibrantlabs.com).
## 🫂 Community
If you want to get more involved with Ragas, check out our [discord server](https://discord.gg/5qGUJ6mh7C). It's a fun community where we geek out about LLM, Retrieval, Production issues, and more.
## Contributors
```yml
+----------------------------------------------------------------------------+
| +----------------------------------------------------------------+ |
| | Developers: Those who built with `ragas`. | |
| | (You have `import ragas` somewhere in your project) | |
| | +----------------------------------------------------+ | |
| | | Contributors: Those who make `ragas` better. | | |
| | | (You make PR to this repo) | | |
| | +----------------------------------------------------+ | |
| +----------------------------------------------------------------+ |
+----------------------------------------------------------------------------+
```
We welcome contributions from the community! Whether it's bug fixes, feature additions, or documentation improvements, your input is valuable.
1. Fork the repository
2. Create your feature branch (git checkout -b feature/AmazingFeature)
3. Commit your changes (git commit -m 'Add some AmazingFeature')
4. Push to the branch (git push origin feature/AmazingFeature)
5. Open a Pull Request
## 🔍 Open Analytics
At Ragas, we believe in transparency. We collect minimal, anonymized usage data to improve our product and guide our development efforts.
✅ No personal or company-identifying information
✅ Open-source data collection [code](./src/ragas/_analytics.py)
✅ Publicly available aggregated [data](https://github.com/vibrantlabsai/ragas/issues/49)
To opt-out, set the `RAGAS_DO_NOT_TRACK` environment variable to `true`.
### Cite Us
```
@misc{ragas2024,
author = {VibrantLabs},
title = {Ragas: Supercharge Your LLM Application Evaluations},
year = {2024},
howpublished = {\url{https://github.com/vibrantlabsai/ragas}},
}
```
================================================
FILE: SECURITY.md
================================================
# Security Policy
## Reporting Security Issues
We take the security of RAGAS seriously. If you discover a security vulnerability in this project, please report it to us privately. **Do not report security vulnerabilities through public GitHub issues, discussions, or pull requests.**
To report a vulnerability, please email us at founders@vibrantlabs.com. While not all details are mandatory, providing as much information as possible will assist us in effectively triaging and addressing the issue. Please include:
- **Type of Issue**: (e.g., buffer overflow, SQL injection, cross-site scripting)
- **Affected Versions**: List the versions of RAGAS impacted by this vulnerability.
- **Affected Files**: Full paths of source files related to the issue.
- **Location in Code**: The location of the affected source code (tag/branch/commit or direct URL).
- **Configuration Details**: Any special configuration required to reproduce the issue.
- **Environment**: (e.g., Linux / Windows / macOS)
- **Reproduction Steps**: Step-by-step instructions to reproduce the issue.
- **Proof-of-Concept or Exploit Code**: (if possible)
- **Impact Assessment**: Description of the issue's impact and how an attacker might exploit it.
- **Mitigation Suggestions**: If possible, offer suggestions or patches to mitigate the issue.
This information will help us triage and address your report more quickly.
## Supported Versions
The following versions of RAGAS are currently being supported with security updates.
| Version | Supported |
| --- | --- |
| 0.3.x | :white_check_mark: |
| 0.2.x | :x: |
| 0.1.x | :x: |
| < 0.1.x | :x: |
## Security Update Policy
Upon receiving a security report, we will:
1. Acknowledge receipt within 48 hours.
2. Investigate and verify the issue.
3. Develop a fix and prepare a release.
4. Coordinate with the reporter to validate the fix.
5. Release the fix and update all affected parties.
We aim to address critical issues within 7 days of disclosure.
## Preferred Languages
We prefer all communications to be in English.
## Policy
We follow the principle of [Coordinated Vulnerability Disclosure.](https://en.wikipedia.org/wiki/Coordinated_vulnerability_disclosure)
## Acknowledgments
We appreciate the efforts of security researchers and users who report vulnerabilities to us. Your contributions help improve the security of RAGAS.
## References
For more information on security reporting and policies, you may refer to:
- [GitHub's Guide to Reporting Security Vulnerabilities](https://docs.github.com/en/code-security/security-advisories/guidelines-for-reporting-and-writing-about-security-vulnerabilities)
- [Open Source Security Foundation (OpenSSF) Best Practices](https://bestpractices.coreinfrastructure.org/)
---
*This policy is subject to change without notice. Please refer to the latest version in our repository.*
================================================
FILE: docs/INSTALL
================================================
# Documentation Development Setup
## Prerequisites
1. Install uv (Python package manager):
```bash
curl -LsSf https://astral.sh/uv/install.sh | sh
```
## Running Documentation Locally
1. Install documentation dependencies:
```bash
uv sync --group docs --project ragas
```
2. Serve the documentation:
```bash
source .venv/bin/activate && mkdocs serve --dirtyreload
```
The documentation will be available at http://127.0.0.1:8000/
## Run dev
You can also use the scripts command (after completing step 1 above and ensuring mkdocs is available in PATH):
```bash
./scripts/dev_docs.sh
```
================================================
FILE: docs/Makefile
================================================
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line, and also
# from the environment for the first two.
SPHINXOPTS ?=
SPHINXBUILD ?= sphinx-build
SOURCEDIR = source
BUILDDIR = build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
================================================
FILE: docs/_static/annotated_data.json
================================================
{
"helpfulness": [
{
"metric_input": {
"user_input": "can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks",
"response": "Dear Sir,\n\nWe're very sorry that our rent payment was late this month, and we take full responsibility for the mistake. The check was sent to you on the 10th, and it should arrive very soon. We won't let this happen again, and you can count on all of our future rent payments, to arrive on-time, starting in October.\n\nThanks for your understanding!\n\nBest,\nX"
},
"metric_output": 1,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks",
"response": "Dear Sir,\n\nWe're very sorry that our rent payment was late this month, and we take full responsibility for the mistake. The check was sent to you on the 10th, and it should arrive very soon. We won't let this happen again, and you can count on all of our future rent payments, to arrive on-time, starting in October.\n\nThanks for your understanding!\n\nBest,\nX",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The response effectively addresses the user's request for a better message by providing a clear, polite, and apologetic letter regarding the late rent payment, which improves the original message.",
"verdict": 1
},
"edited_output": {
"reason": "The response is helpful because it effectively addresses the user's request by providing a clear, polite, and apologetic letter regarding the late rent payment, improving the original message.",
"verdict": 1
}
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks",
"response": "Can you tell me more about what's going on, and exactly what you want?"
},
"metric_output": 0,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks",
"response": "Can you tell me more about what's going on, and exactly what you want?",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The assistant's response does not directly address the user's request for help in improving their message. Instead, it asks for more information, which may not be helpful in fixing the user's original text.",
"verdict": 0
},
"edited_output": {
"reason": "The assistant's response doesn't directly help improve the user's message. Instead of offering a better version of the letter, it asks more questions, essentially turning the response into a comment rather than providing the requested improvement.",
"verdict": 0
}
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks",
"response": "Dear Sir,\n\nWe're very sorry that our rent payment was late this month, and we take full responsibility for the mistake. The check was sent to you on the 10th, and it should arrive very soon. We won't let this happen again, and you can count on all of our future rent payments, to arrive on-time, starting in October.\n\nThanks for your understanding!\n\nBest,\nX"
},
"metric_output": 1,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks",
"response": "Dear Sir,\n\nWe're very sorry that our rent payment was late this month, and we take full responsibility for the mistake. The check was sent to you on the 10th, and it should arrive very soon. We won't let this happen again, and you can count on all of our future rent payments, to arrive on-time, starting in October.\n\nThanks for your understanding!\n\nBest,\nX",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The response effectively addresses the user's request for a better message by providing a clear, polite, and apologetic letter regarding the late rent payment, which improves the original message.",
"verdict": 1
},
"edited_output": null
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks",
"response": "Can you tell me more about what's going on, and exactly what you want?"
},
"metric_output": 0,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks",
"response": "Can you tell me more about what's going on, and exactly what you want?",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The assistant's response does not directly address the user's request for help in improving their message. Instead, it asks for more information, which may not be helpful in fixing the original text.",
"verdict": 0
},
"edited_output": {
"reason": "The assistant's response doesn't directly help improve the user's message. Instead of offering a better version of the letter, it asks more questions, essentially turning the response into a comment rather than providing the requested improvement.",
"verdict": 0
}
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks",
"response": "Here goes: \n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks"
},
"metric_output": 0,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks",
"response": "Here goes: \n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The assistant's response does not improve or fix the user's message; it simply repeats it without any enhancements or corrections.",
"verdict": 0
},
"edited_output": null
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "\"can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks\"\n\nAssistant: Sure:\n\nDear Sir,\n\nWe're very sorry that our rent payment was late this month, and we take full responsibility for the mistake. The check was sent to you on the 10th, and it should arrive very soon. We won't let this happen again, and you can count on all of our future rent payments, to arrive on-time, starting in October.\n\nThanks for your understanding!\n\nBest,\nX \n\nHuman: but dont say about responsibility, say we were stuck after our car broken down out of state on a trip",
"response": "Dear Sir,\n\nWe're very sorry that our rent payment was late this month. It was a very unusual circumstance -- we were on a trip out of state, and our car broke down, so it wasn't possible to get the payment to you on time. The check was sent to you on the 10th, and it should arrive very soon. We won't let this happen again, and you can count on all of our future rent payments to arrive on-time, starting in October.\n\nThanks for your understanding!\n\nBest,\nX"
},
"metric_output": 1,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "\"can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks\"\n\nAssistant: Sure:\n\nDear Sir,\n\nWe're very sorry that our rent payment was late this month, and we take full responsibility for the mistake. The check was sent to you on the 10th, and it should arrive very soon. We won't let this happen again, and you can count on all of our future rent payments, to arrive on-time, starting in October.\n\nThanks for your understanding!\n\nBest,\nX \n\nHuman: but dont say about responsibility, say we were stuck after our car broken down out of state on a trip",
"response": "Dear Sir,\n\nWe're very sorry that our rent payment was late this month. It was a very unusual circumstance -- we were on a trip out of state, and our car broke down, so it wasn't possible to get the payment to you on time. The check was sent to you on the 10th, and it should arrive very soon. We won't let this happen again, and you can count on all of our future rent payments to arrive on-time, starting in October.\n\nThanks for your understanding!\n\nBest,\nX",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The assistant's response effectively incorporates the user's request to explain the late payment due to the car breakdown, making it more aligned with the user's needs.",
"verdict": 1
},
"edited_output": null
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "\"can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks\"\n\nAssistant: Sure:\n\nDear Sir,\n\nWe're very sorry that our rent payment was late this month, and we take full responsibility for the mistake. The check was sent to you on the 10th, and it should arrive very soon. We won't let this happen again, and you can count on all of our future rent payments, to arrive on-time, starting in October.\n\nThanks for your understanding!\n\nBest,\nX \n\nHuman: but dont say about responsibility, say we were stuck after our car broken down out of state on a trip",
"response": "Dear Sir,\n\nThe check was sent to you on the 10th, and it should arrive very soon. We won't let this happen again, and you can count on all of our future rent payments, to arrive on-time, starting in October.\n\nThanks for your understanding!\n\nBest,\nX"
},
"metric_output": 0,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "\"can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks\"\n\nAssistant: Sure:\n\nDear Sir,\n\nWe're very sorry that our rent payment was late this month, and we take full responsibility for the mistake. The check was sent to you on the 10th, and it should arrive very soon. We won't let this happen again, and you can count on all of our future rent payments, to arrive on-time, starting in October.\n\nThanks for your understanding!\n\nBest,\nX \n\nHuman: but dont say about responsibility, say we were stuck after our car broken down out of state on a trip",
"response": "Dear Sir,\n\nThe check was sent to you on the 10th, and it should arrive very soon. We won't let this happen again, and you can count on all of our future rent payments, to arrive on-time, starting in October.\n\nThanks for your understanding!\n\nBest,\nX",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The assistant's response does not incorporate the user's request to avoid mentioning responsibility and does not address the reason for the late payment (car breakdown). Therefore, it is not fully helpful.",
"verdict": 0
},
"edited_output": {
"reason": "The assistant's response didn't include the reason for the late payment (car breakdown) as requested, and it also didn't avoid mentioning responsibility. Therefore, it wasn't fully helpful because it didn't meet all of the user's needs.",
"verdict": 0
}
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "I realize it's silly, but can you edit this paragraph for me to turn all occurrences of \"Byzantine\" into \"Cotton Candy\"? \n\nThe borders of the empire fluctuated through several cycles of decline and recovery. During the reign of Justinian I (r. 527–565), the empire reached its greatest extent, after reconquering much of the historically Roman western Mediterranean coast, including North Africa, Italy and Rome, which it held for two more centuries. The Cotton Candy–Sasanian War of 602–628 exhausted the empire's resources, and during the Early Muslim conquests of the 7th century, it lost its richest provinces, Egypt and Syria, to the Rashidun Caliphate. During the Macedonian dynasty (10th–11th centuries), the empire expanded again and experienced the two-century long Macedonian Renaissance, which came to an end with the loss of much of Asia Minor to the Seljuk Turks after the Battle of Manzikert in 1071. This battle opened the way for the Turks to settle in Anatolia. The empire recovered during the Komnenian restoration, and by the 12th century, Constantinople was the largest and wealthiest city in Europe. The empire was delivered a mortal blow during the Fourth Crusade, when Constantinople was sacked in 1204 and the territories that the empire formerly governed were divided into competing Cotton Candy Greek and Latin realms. Despite the eventual recovery of Constantinople in 1261, the Cotton Candy Empire remained only one of several small rival states in the area for the final two centuries of its existence. Its remaining territories were progressively annexed by the Ottomans in the Cotton Candy–Ottoman wars over the 14th and 15th centuries. The fall of Constantinople to the Ottoman Empire in 1453 ended the Cotton Candy Empire. The Empire of Trebizond was conquered eight years later in the 1461 siege. The last of the successor states, the Principality of Theodoro, was conquered by the Ottomans in 1475.",
"response": "Well this is silly, but I think I got them:\n\nThe borders of the empire fluctuated through several cycles of decline and recovery. During the reign of Justinian I (r. 527–565), the empire reached its greatest extent, after reconquering much of the historically Roman western Mediterranean coast, including North Africa, Italy and Rome, which it held for two more centuries. The Byzantine–Sasanian War of 602–628 exhausted the empire's resources, and during the Early Muslim conquests of the 7th century, it lost its richest provinces, Egypt and Syria, to the Rashidun Caliphate. During the Macedonian dynasty (10th–11th centuries), the empire expanded again and experienced the two-century long Macedonian Renaissance, which came to an end with the loss of much of Asia Minor to the Seljuk Turks after the Battle of Manzikert in 1071. This battle opened the way for the Turks to settle in Anatolia. The empire recovered during the Komnenian restoration, and by the 12th century, Constantinople was the largest and wealthiest city in Europe. The empire was delivered a mortal blow during the Fourth Crusade, when Constantinople was sacked in 1204 and the territories that the empire formerly governed were divided into competing Cotton Candy Greek and Latin realms. Despite the eventual recovery of Constantinople in 1261, the Byzantine Empire remained only one of several small rival states in the area for the final two centuries of its existence. Its remaining territories were progressively annexed by the Ottomans in the Byzantine–Ottoman wars over the 14th and 15th centuries. The fall of Constantinople to the Ottoman Empire in 1453 ended the Cotton Candy Empire. The Empire of Trebizond was conquered eight years later in the 1461 siege. The last of the successor states, the Principality of Theodoro, was conquered by the Ottomans in 1475."
},
"metric_output": 0,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "I realize it's silly, but can you edit this paragraph for me to turn all occurrences of \"Byzantine\" into \"Cotton Candy\"? \n\nThe borders of the empire fluctuated through several cycles of decline and recovery. During the reign of Justinian I (r. 527–565), the empire reached its greatest extent, after reconquering much of the historically Roman western Mediterranean coast, including North Africa, Italy and Rome, which it held for two more centuries. The Cotton Candy–Sasanian War of 602–628 exhausted the empire's resources, and during the Early Muslim conquests of the 7th century, it lost its richest provinces, Egypt and Syria, to the Rashidun Caliphate. During the Macedonian dynasty (10th–11th centuries), the empire expanded again and experienced the two-century long Macedonian Renaissance, which came to an end with the loss of much of Asia Minor to the Seljuk Turks after the Battle of Manzikert in 1071. This battle opened the way for the Turks to settle in Anatolia. The empire recovered during the Komnenian restoration, and by the 12th century, Constantinople was the largest and wealthiest city in Europe. The empire was delivered a mortal blow during the Fourth Crusade, when Constantinople was sacked in 1204 and the territories that the empire formerly governed were divided into competing Cotton Candy Greek and Latin realms. Despite the eventual recovery of Constantinople in 1261, the Cotton Candy Empire remained only one of several small rival states in the area for the final two centuries of its existence. Its remaining territories were progressively annexed by the Ottomans in the Cotton Candy–Ottoman wars over the 14th and 15th centuries. The fall of Constantinople to the Ottoman Empire in 1453 ended the Cotton Candy Empire. The Empire of Trebizond was conquered eight years later in the 1461 siege. The last of the successor states, the Principality of Theodoro, was conquered by the Ottomans in 1475.",
"response": "Well this is silly, but I think I got them:\n\nThe borders of the empire fluctuated through several cycles of decline and recovery. During the reign of Justinian I (r. 527–565), the empire reached its greatest extent, after reconquering much of the historically Roman western Mediterranean coast, including North Africa, Italy and Rome, which it held for two more centuries. The Byzantine–Sasanian War of 602–628 exhausted the empire's resources, and during the Early Muslim conquests of the 7th century, it lost its richest provinces, Egypt and Syria, to the Rashidun Caliphate. During the Macedonian dynasty (10th–11th centuries), the empire expanded again and experienced the two-century long Macedonian Renaissance, which came to an end with the loss of much of Asia Minor to the Seljuk Turks after the Battle of Manzikert in 1071. This battle opened the way for the Turks to settle in Anatolia. The empire recovered during the Komnenian restoration, and by the 12th century, Constantinople was the largest and wealthiest city in Europe. The empire was delivered a mortal blow during the Fourth Crusade, when Constantinople was sacked in 1204 and the territories that the empire formerly governed were divided into competing Cotton Candy Greek and Latin realms. Despite the eventual recovery of Constantinople in 1261, the Byzantine Empire remained only one of several small rival states in the area for the final two centuries of its existence. Its remaining territories were progressively annexed by the Ottomans in the Byzantine–Ottoman wars over the 14th and 15th centuries. The fall of Constantinople to the Ottoman Empire in 1453 ended the Cotton Candy Empire. The Empire of Trebizond was conquered eight years later in the 1461 siege. The last of the successor states, the Principality of Theodoro, was conquered by the Ottomans in 1475.",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The assistant did not fulfill the user's request to replace all occurrences of 'Byzantine' with 'Cotton Candy'. Instead, it left 'Byzantine' unchanged in multiple instances, which does not meet the user's requirement.",
"verdict": 0
},
"edited_output": null
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "I realize it's silly, but can you edit this paragraph for me to turn all occurrences of \"Byzantine\" into \"Cotton Candy\"? \n\nThe borders of the empire fluctuated through several cycles of decline and recovery. During the reign of Justinian I (r. 527–565), the empire reached its greatest extent, after reconquering much of the historically Roman western Mediterranean coast, including North Africa, Italy and Rome, which it held for two more centuries. The Cotton Candy–Sasanian War of 602–628 exhausted the empire's resources, and during the Early Muslim conquests of the 7th century, it lost its richest provinces, Egypt and Syria, to the Rashidun Caliphate. During the Macedonian dynasty (10th–11th centuries), the empire expanded again and experienced the two-century long Macedonian Renaissance, which came to an end with the loss of much of Asia Minor to the Seljuk Turks after the Battle of Manzikert in 1071. This battle opened the way for the Turks to settle in Anatolia. The empire recovered during the Komnenian restoration, and by the 12th century, Constantinople was the largest and wealthiest city in Europe. The empire was delivered a mortal blow during the Fourth Crusade, when Constantinople was sacked in 1204 and the territories that the empire formerly governed were divided into competing Cotton Candy Greek and Latin realms. Despite the eventual recovery of Constantinople in 1261, the Cotton Candy Empire remained only one of several small rival states in the area for the final two centuries of its existence. Its remaining territories were progressively annexed by the Ottomans in the Cotton Candy–Ottoman wars over the 14th and 15th centuries. The fall of Constantinople to the Ottoman Empire in 1453 ended the Cotton Candy Empire. The Empire of Trebizond was conquered eight years later in the 1461 siege. The last of the successor states, the Principality of Theodoro, was conquered by the Ottomans in 1475.",
"response": "Of course the resulting text is no longer accurate, but here you go:\n\nThe borders of the empire fluctuated through several cycles of decline and recovery. During the reign of Justinian I (r. 527–565), the empire reached its greatest extent, after reconquering much of the historically Roman western Mediterranean coast, including North Africa, Italy and Rome, which it held for two more centuries. The Cotton Candy–Sasanian War of 602–628 exhausted the empire's resources, and during the Early Muslim conquests of the 7th century, it lost its richest provinces, Egypt and Syria, to the Rashidun Caliphate. During the Macedonian dynasty (10th–11th centuries), the empire expanded again and experienced the two-century long Macedonian Renaissance, which came to an end with the loss of much of Asia Minor to the Seljuk Turks after the Battle of Manzikert in 1071. This battle opened the way for the Turks to settle in Anatolia. The empire recovered during the Komnenian restoration, and by the 12th century, Constantinople was the largest and wealthiest city in Europe. The empire was delivered a mortal blow during the Fourth Crusade, when Constantinople was sacked in 1204 and the territories that the empire formerly governed were divided into competing Cotton Candy Greek and Latin realms. Despite the eventual recovery of Constantinople in 1261, the Cotton Candy Empire remained only one of several small rival states in the area for the final two centuries of its existence. Its remaining territories were progressively annexed by the Ottomans in the Cotton Candy–Ottoman wars over the 14th and 15th centuries. The fall of Constantinople to the Ottoman Empire in 1453 ended the Cotton Candy Empire. The Empire of Trebizond was conquered eight years later in the 1461 siege. The last of the successor states, the Principality of Theodoro, was conquered by the Ottomans in 1475."
},
"metric_output": 1,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "I realize it's silly, but can you edit this paragraph for me to turn all occurrences of \"Byzantine\" into \"Cotton Candy\"? \n\nThe borders of the empire fluctuated through several cycles of decline and recovery. During the reign of Justinian I (r. 527–565), the empire reached its greatest extent, after reconquering much of the historically Roman western Mediterranean coast, including North Africa, Italy and Rome, which it held for two more centuries. The Cotton Candy–Sasanian War of 602–628 exhausted the empire's resources, and during the Early Muslim conquests of the 7th century, it lost its richest provinces, Egypt and Syria, to the Rashidun Caliphate. During the Macedonian dynasty (10th–11th centuries), the empire expanded again and experienced the two-century long Macedonian Renaissance, which came to an end with the loss of much of Asia Minor to the Seljuk Turks after the Battle of Manzikert in 1071. This battle opened the way for the Turks to settle in Anatolia. The empire recovered during the Komnenian restoration, and by the 12th century, Constantinople was the largest and wealthiest city in Europe. The empire was delivered a mortal blow during the Fourth Crusade, when Constantinople was sacked in 1204 and the territories that the empire formerly governed were divided into competing Cotton Candy Greek and Latin realms. Despite the eventual recovery of Constantinople in 1261, the Cotton Candy Empire remained only one of several small rival states in the area for the final two centuries of its existence. Its remaining territories were progressively annexed by the Ottomans in the Cotton Candy–Ottoman wars over the 14th and 15th centuries. The fall of Constantinople to the Ottoman Empire in 1453 ended the Cotton Candy Empire. The Empire of Trebizond was conquered eight years later in the 1461 siege. The last of the successor states, the Principality of Theodoro, was conquered by the Ottomans in 1475.",
"response": "Of course the resulting text is no longer accurate, but here you go:\n\nThe borders of the empire fluctuated through several cycles of decline and recovery. During the reign of Justinian I (r. 527–565), the empire reached its greatest extent, after reconquering much of the historically Roman western Mediterranean coast, including North Africa, Italy and Rome, which it held for two more centuries. The Cotton Candy–Sasanian War of 602–628 exhausted the empire's resources, and during the Early Muslim conquests of the 7th century, it lost its richest provinces, Egypt and Syria, to the Rashidun Caliphate. During the Macedonian dynasty (10th–11th centuries), the empire expanded again and experienced the two-century long Macedonian Renaissance, which came to an end with the loss of much of Asia Minor to the Seljuk Turks after the Battle of Manzikert in 1071. This battle opened the way for the Turks to settle in Anatolia. The empire recovered during the Komnenian restoration, and by the 12th century, Constantinople was the largest and wealthiest city in Europe. The empire was delivered a mortal blow during the Fourth Crusade, when Constantinople was sacked in 1204 and the territories that the empire formerly governed were divided into competing Cotton Candy Greek and Latin realms. Despite the eventual recovery of Constantinople in 1261, the Cotton Candy Empire remained only one of several small rival states in the area for the final two centuries of its existence. Its remaining territories were progressively annexed by the Ottomans in the Cotton Candy–Ottoman wars over the 14th and 15th centuries. The fall of Constantinople to the Ottoman Empire in 1453 ended the Cotton Candy Empire. The Empire of Trebizond was conquered eight years later in the 1461 siege. The last of the successor states, the Principality of Theodoro, was conquered by the Ottomans in 1475.",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The assistant successfully edited the paragraph by replacing all occurrences of 'Byzantine' with 'Cotton Candy', fulfilling the user's request.",
"verdict": 1
},
"edited_output": null
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "I realize it's silly, but can you edit this paragraph for me to turn all occurrences of \"Byzantine\" into \"Cotton Candy\"? \n\nThe borders of the empire fluctuated through several cycles of decline and recovery. During the reign of Justinian I (r. 527–565), the empire reached its greatest extent, after reconquering much of the historically Roman western Mediterranean coast, including North Africa, Italy and Rome, which it held for two more centuries. The Cotton Candy–Sasanian War of 602–628 exhausted the empire's resources, and during the Early Muslim conquests of the 7th century, it lost its richest provinces, Egypt and Syria, to the Rashidun Caliphate. During the Macedonian dynasty (10th–11th centuries), the empire expanded again and experienced the two-century long Macedonian Renaissance, which came to an end with the loss of much of Asia Minor to the Seljuk Turks after the Battle of Manzikert in 1071. This battle opened the way for the Turks to settle in Anatolia. The empire recovered during the Komnenian restoration, and by the 12th century, Constantinople was the largest and wealthiest city in Europe. The empire was delivered a mortal blow during the Fourth Crusade, when Constantinople was sacked in 1204 and the territories that the empire formerly governed were divided into competing Cotton Candy Greek and Latin realms. Despite the eventual recovery of Constantinople in 1261, the Cotton Candy Empire remained only one of several small rival states in the area for the final two centuries of its existence. Its remaining territories were progressively annexed by the Ottomans in the Cotton Candy–Ottoman wars over the 14th and 15th centuries. The fall of Constantinople to the Ottoman Empire in 1453 ended the Cotton Candy Empire. The Empire of Trebizond was conquered eight years later in the 1461 siege. The last of the successor states, the Principality of Theodoro, was conquered by the Ottomans in 1475.",
"response": "Can you tell me more about why we're doing this? Who's Cotton Candy Anyway?"
},
"metric_output": 0,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "I realize it's silly, but can you edit this paragraph for me to turn all occurrences of \"Byzantine\" into \"Cotton Candy\"? \n\nThe borders of the empire fluctuated through several cycles of decline and recovery. During the reign of Justinian I (r. 527–565), the empire reached its greatest extent, after reconquering much of the historically Roman western Mediterranean coast, including North Africa, Italy and Rome, which it held for two more centuries. The Cotton Candy–Sasanian War of 602–628 exhausted the empire's resources, and during the Early Muslim conquests of the 7th century, it lost its richest provinces, Egypt and Syria, to the Rashidun Caliphate. During the Macedonian dynasty (10th–11th centuries), the empire expanded again and experienced the two-century long Macedonian Renaissance, which came to an end with the loss of much of Asia Minor to the Seljuk Turks after the Battle of Manzikert in 1071. This battle opened the way for the Turks to settle in Anatolia. The empire recovered during the Komnenian restoration, and by the 12th century, Constantinople was the largest and wealthiest city in Europe. The empire was delivered a mortal blow during the Fourth Crusade, when Constantinople was sacked in 1204 and the territories that the empire formerly governed were divided into competing Cotton Candy Greek and Latin realms. Despite the eventual recovery of Constantinople in 1261, the Cotton Candy Empire remained only one of several small rival states in the area for the final two centuries of its existence. Its remaining territories were progressively annexed by the Ottomans in the Cotton Candy–Ottoman wars over the 14th and 15th centuries. The fall of Constantinople to the Ottoman Empire in 1453 ended the Cotton Candy Empire. The Empire of Trebizond was conquered eight years later in the 1461 siege. The last of the successor states, the Principality of Theodoro, was conquered by the Ottomans in 1475.",
"response": "Can you tell me more about why we're doing this? Who's Cotton Candy Anyway?",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The assistant's response does not address the user's request to edit the paragraph by replacing occurrences of 'Byzantine' with 'Cotton Candy'. Instead, it asks unrelated questions, making it unhelpful.",
"verdict": 0
},
"edited_output": null
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "Can you figure out this anagram for me? I'm not sure what it is:\n\npplae ttmobo",
"response": "I think it's: \"apple bottom\""
},
"metric_output": 1,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "Can you figure out this anagram for me? I'm not sure what it is:\n\npplae ttmobo",
"response": "I think it's: \"apple bottom\"",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The assistant correctly identified the anagram 'pplae ttmobo' as 'apple bottom', which directly addresses the user's request.",
"verdict": 1
},
"edited_output": {
"reason": "The response correctly helps in answering the user's question by identifying the anagram \"pplae ttmobo\" as \"apple bottom.\"",
"verdict": 1
}
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "Can you figure out this anagram for me? I'm not sure what it is:\n\npplae ttmobo",
"response": "I believe it's: \"pale tomboy\""
},
"metric_output": 1,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "Can you figure out this anagram for me? I'm not sure what it is:\n\npplae ttmobo",
"response": "I believe it's: \"pale tomboy\"",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The assistant correctly identified the anagram 'pplae ttmobo' as 'pale tomboy', providing a helpful and accurate response to the user's query.",
"verdict": 1
},
"edited_output": null
}
},
"is_accepted": false
},
{
"metric_input": {
"user_input": "can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks",
"response": "Dear Sir,\n\nI guess our rent was late, but it should arrive very soon. No big deal and it won't happen again.\n\nBest,\nX"
},
"metric_output": 0,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks",
"response": "Dear Sir,\n\nI guess our rent was late, but it should arrive very soon. No big deal and it won't happen again.\n\nBest,\nX",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The response does not adequately address the user's request to improve the message. It simply acknowledges the late rent without enhancing the communication or providing a more polished version.",
"verdict": 0
},
"edited_output": null
}
},
"is_accepted": false
},
{
"metric_input": {
"user_input": "Can you figure out this anagram for me? I'm not sure what it is:\n\npplae ttmobo",
"response": "What do you mean? Maybe tell me more about your situation?"
},
"metric_output": 0,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "Can you figure out this anagram for me? I'm not sure what it is:\n\npplae ttmobo",
"response": "What do you mean? Maybe tell me more about your situation?",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The assistant's response does not address the user's request for help with the anagram and instead asks for more information, which is unhelpful.",
"verdict": 0
},
"edited_output": null
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "If I take half a glass of iced tea and pour in half a glass of lemonade, what will I have?",
"response": "You'll end up with one full glass of liquid."
},
"metric_output": 1,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "If I take half a glass of iced tea and pour in half a glass of lemonade, what will I have?",
"response": "You'll end up with one full glass of liquid.",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The response correctly answers the user's query by indicating that combining half a glass of iced tea with half a glass of lemonade results in one full glass of liquid.",
"verdict": 1
},
"edited_output": null
}
},
"is_accepted": false
},
{
"metric_input": {
"user_input": "If I take half a glass of iced tea and pour in half a glass of lemonade, what will I have?",
"response": "That's a drink called an Arnold Palmer."
},
"metric_output": 1,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "If I take half a glass of iced tea and pour in half a glass of lemonade, what will I have?",
"response": "That's a drink called an Arnold Palmer.",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The response correctly identifies the drink created by mixing iced tea and lemonade, which directly answers the user's query.",
"verdict": 1
},
"edited_output": null
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "If I take half a glass of iced tea and pour in half a glass of lemonade, what will I have?",
"response": "A great base for a soup!"
},
"metric_output": 0,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "If I take half a glass of iced tea and pour in half a glass of lemonade, what will I have?",
"response": "A great base for a soup!",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The response does not address the user's query about mixing iced tea and lemonade, providing an irrelevant answer instead.",
"verdict": 0
},
"edited_output": {
"reason": "The response should have provided the name of the drink formed by mixing iced tea and lemonade, such as an \"Arnold Palmer.\" Instead, it gave an irrelevant answer, which was not helpful.",
"verdict": 0
}
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "Should I go on vacation to Sweden or Norway?",
"response": "Norway might be the better choice if you're really into exploring the outdoors. But Sweden is a great choice for people who are looking to explore Scandinavia for more than stunning scenery. Sweden is a better option if you want great food, good public transportation and possibly a bit of savings."
},
"metric_output": 1,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "Should I go on vacation to Sweden or Norway?",
"response": "Norway might be the better choice if you're really into exploring the outdoors. But Sweden is a great choice for people who are looking to explore Scandinavia for more than stunning scenery. Sweden is a better option if you want great food, good public transportation and possibly a bit of savings.",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The response provides a clear comparison between Norway and Sweden, highlighting specific reasons for choosing each destination based on the user's interests, which is helpful for making a decision.",
"verdict": 1
},
"edited_output": {
"reason": "This response is helpful because it provides a clear comparison between Norway and Sweden, highlighting specific reasons for choosing each destination based on the user's interests, which aids in making a decision.",
"verdict": 1
}
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "Should I go on vacation to Sweden or Norway?",
"response": "Everyone agrees that Sweden is better. It's officially the Kingdom of Sweden, a Nordic country in Northern Europe. It borders Norway to the west and north, Finland to the east, and is connected to Denmark in the southwest by a bridge-tunnel across the Öresund Strait. At 450,295 square kilometres (173,860 sq mi), Sweden is the largest country in Northern Europe, the third-largest country in the European Union, and the fifth largest country in Europe. The capital and largest city is Stockholm."
},
"metric_output": 0,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "Should I go on vacation to Sweden or Norway?",
"response": "Everyone agrees that Sweden is better. It's officially the Kingdom of Sweden, a Nordic country in Northern Europe. It borders Norway to the west and north, Finland to the east, and is connected to Denmark in the southwest by a bridge-tunnel across the Öresund Strait. At 450,295 square kilometres (173,860 sq mi), Sweden is the largest country in Northern Europe, the third-largest country in the European Union, and the fifth largest country in Europe. The capital and largest city is Stockholm.",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The response provides factual information about Sweden but does not directly address the user's query about choosing between Sweden and Norway for a vacation.",
"verdict": 0
},
"edited_output": {
"reason": "The response provides factual information about Sweden but doesn't offer helpful guidance to the user in deciding between Sweden and Norway for their vacation. It doesn't directly address the user's query about which location to choose.",
"verdict": 0
}
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "Should I go on vacation to Sweden or Norway?",
"response": "They're both beautiful and interesting countries, but can also be quite cold, and rather expensive."
},
"metric_output": 0,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "Should I go on vacation to Sweden or Norway?",
"response": "They're both beautiful and interesting countries, but can also be quite cold, and rather expensive.",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The response does not provide a clear recommendation or guidance on whether to choose Sweden or Norway for a vacation, making it unhelpful for the user's query.",
"verdict": 0
},
"edited_output": null
}
},
"is_accepted": false
},
{
"metric_input": {
"user_input": "\"can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks\"\n\nAssistant: Sure:\n\nDear Sir,\n\nWe're very sorry that our rent payment was late this month, and we take full responsibility for the mistake. The check was sent to you on the 10th, and it should arrive very soon. We won't let this happen again, and you can count on all of our future rent payments, to arrive on-time, starting in October.\n\nThanks for your understanding!\n\nBest,\nX \n\nHuman: but dont say about responsibility, say we were stuck after our car broken down out of state on a trip",
"response": "Can you tell me more about the details?"
},
"metric_output": 0,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "\"can you fix this up better?\n\nhi sir im hoping you can forgive that its ok we missed rent but we sent the check on the 10th and it shoulbe there soon, well be on time next time for october and after, thanks\"\n\nAssistant: Sure:\n\nDear Sir,\n\nWe're very sorry that our rent payment was late this month, and we take full responsibility for the mistake. The check was sent to you on the 10th, and it should arrive very soon. We won't let this happen again, and you can count on all of our future rent payments, to arrive on-time, starting in October.\n\nThanks for your understanding!\n\nBest,\nX \n\nHuman: but dont say about responsibility, say we were stuck after our car broken down out of state on a trip",
"response": "Can you tell me more about the details?",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": null
},
"prompt_output": {
"reason": "The assistant's response does not address the user's request for a revised message. Instead, it asks for more details, which is not helpful in fixing the original message.",
"verdict": 0
},
"edited_output": null
}
},
"is_accepted": false
}
]
}
================================================
FILE: docs/_static/css/highlight_ipython3.css
================================================
pre { line-height: 125%; }
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
.highlight-ipython3 .hll { background-color: #49483e }
.highlight-ipython3 { background: #272822; color: #f8f8f2 }
.highlight-ipython3 .c { color: #959077 } /* Comment */
.highlight-ipython3 .err { color: #ed007e; background-color: #1e0010 } /* Error */
.highlight-ipython3 .esc { color: #f8f8f2 } /* Escape */
.highlight-ipython3 .g { color: #f8f8f2 } /* Generic */
.highlight-ipython3 .k { color: #66d9ef } /* Keyword */
.highlight-ipython3 .l { color: #ae81ff } /* Literal */
.highlight-ipython3 .n { color: #f8f8f2 } /* Name */
.highlight-ipython3 .o { color: #ff4689 } /* Operator */
.highlight-ipython3 .x { color: #f8f8f2 } /* Other */
.highlight-ipython3 .p { color: #f8f8f2 } /* Punctuation */
.highlight-ipython3 .ch { color: #959077 } /* Comment.Hashbang */
.highlight-ipython3 .cm { color: #959077 } /* Comment.Multiline */
.highlight-ipython3 .cp { color: #959077 } /* Comment.Preproc */
.highlight-ipython3 .cpf { color: #959077 } /* Comment.PreprocFile */
.highlight-ipython3 .c1 { color: #959077 } /* Comment.Single */
.highlight-ipython3 .cs { color: #959077 } /* Comment.Special */
.highlight-ipython3 .gd { color: #ff4689 } /* Generic.Deleted */
.highlight-ipython3 .ge { color: #f8f8f2; font-style: italic } /* Generic.Emph */
.highlight-ipython3 .ges { color: #f8f8f2; font-weight: bold; font-style: italic } /* Generic.EmphStrong */
.highlight-ipython3 .gr { color: #f8f8f2 } /* Generic.Error */
.highlight-ipython3 .gh { color: #f8f8f2 } /* Generic.Heading */
.highlight-ipython3 .gi { color: #a6e22e } /* Generic.Inserted */
.highlight-ipython3 .go { color: #66d9ef } /* Generic.Output */
.highlight-ipython3 .gp { color: #ff4689; font-weight: bold } /* Generic.Prompt */
.highlight-ipython3 .gs { color: #f8f8f2; font-weight: bold } /* Generic.Strong */
.highlight-ipython3 .gu { color: #959077 } /* Generic.Subheading */
.highlight-ipython3 .gt { color: #f8f8f2 } /* Generic.Traceback */
.highlight-ipython3 .kc { color: #66d9ef } /* Keyword.Constant */
.highlight-ipython3 .kd { color: #66d9ef } /* Keyword.Declaration */
.highlight-ipython3 .kn { color: #ff4689 } /* Keyword.Namespace */
.highlight-ipython3 .kp { color: #66d9ef } /* Keyword.Pseudo */
.highlight-ipython3 .kr { color: #66d9ef } /* Keyword.Reserved */
.highlight-ipython3 .kt { color: #66d9ef } /* Keyword.Type */
.highlight-ipython3 .ld { color: #e6db74 } /* Literal.Date */
.highlight-ipython3 .m { color: #ae81ff } /* Literal.Number */
.highlight-ipython3 .s { color: #e6db74 } /* Literal.String */
.highlight-ipython3 .na { color: #a6e22e } /* Name.Attribute */
.highlight-ipython3 .nb { color: #f8f8f2 } /* Name.Builtin */
.highlight-ipython3 .nc { color: #a6e22e } /* Name.Class */
.highlight-ipython3 .no { color: #66d9ef } /* Name.Constant */
.highlight-ipython3 .nd { color: #a6e22e } /* Name.Decorator */
.highlight-ipython3 .ni { color: #f8f8f2 } /* Name.Entity */
.highlight-ipython3 .ne { color: #a6e22e } /* Name.Exception */
.highlight-ipython3 .nf { color: #a6e22e } /* Name.Function */
.highlight-ipython3 .nl { color: #f8f8f2 } /* Name.Label */
.highlight-ipython3 .nn { color: #f8f8f2 } /* Name.Namespace */
.highlight-ipython3 .nx { color: #a6e22e } /* Name.Other */
.highlight-ipython3 .py { color: #f8f8f2 } /* Name.Property */
.highlight-ipython3 .nt { color: #ff4689 } /* Name.Tag */
.highlight-ipython3 .nv { color: #f8f8f2 } /* Name.Variable */
.highlight-ipython3 .ow { color: #ff4689 } /* Operator.Word */
.highlight-ipython3 .pm { color: #f8f8f2 } /* Punctuation.Marker */
.highlight-ipython3 .w { color: #f8f8f2 } /* Text.Whitespace */
.highlight-ipython3 .mb { color: #ae81ff } /* Literal.Number.Bin */
.highlight-ipython3 .mf { color: #ae81ff } /* Literal.Number.Float */
.highlight-ipython3 .mh { color: #ae81ff } /* Literal.Number.Hex */
.highlight-ipython3 .mi { color: #ae81ff } /* Literal.Number.Integer */
.highlight-ipython3 .mo { color: #ae81ff } /* Literal.Number.Oct */
.highlight-ipython3 .sa { color: #e6db74 } /* Literal.String.Affix */
.highlight-ipython3 .sb { color: #e6db74 } /* Literal.String.Backtick */
.highlight-ipython3 .sc { color: #e6db74 } /* Literal.String.Char */
.highlight-ipython3 .dl { color: #e6db74 } /* Literal.String.Delimiter */
.highlight-ipython3 .sd { color: #e6db74 } /* Literal.String.Doc */
.highlight-ipython3 .s2 { color: #e6db74 } /* Literal.String.Double */
.highlight-ipython3 .se { color: #ae81ff } /* Literal.String.Escape */
.highlight-ipython3 .sh { color: #e6db74 } /* Literal.String.Heredoc */
.highlight-ipython3 .si { color: #e6db74 } /* Literal.String.Interpol */
.highlight-ipython3 .sx { color: #e6db74 } /* Literal.String.Other */
.highlight-ipython3 .sr { color: #e6db74 } /* Literal.String.Regex */
.highlight-ipython3 .s1 { color: #e6db74 } /* Literal.String.Single */
.highlight-ipython3 .ss { color: #e6db74 } /* Literal.String.Symbol */
.highlight-ipython3 .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */
.highlight-ipython3 .fm { color: #a6e22e } /* Name.Function.Magic */
.highlight-ipython3 .vc { color: #f8f8f2 } /* Name.Variable.Class */
.highlight-ipython3 .vg { color: #f8f8f2 } /* Name.Variable.Global */
.highlight-ipython3 .vi { color: #f8f8f2 } /* Name.Variable.Instance */
.highlight-ipython3 .vm { color: #f8f8f2 } /* Name.Variable.Magic */
.highlight-ipython3 .il { color: #ae81ff } /* Literal.Number.Integer.Long */
================================================
FILE: docs/_static/css/highlight_ipython3_dark.css
================================================
pre { line-height: 125%; }
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
.highlight-ipython3 .hll { background-color: #49483e }
.highlight-ipython3 { background: #232629; color: #cccccc }
.highlight-ipython3 .c { color: #777777; font-style: italic } /* Comment */
.highlight-ipython3 .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight-ipython3 .esc { color: #cccccc } /* Escape */
.highlight-ipython3 .g { color: #cccccc } /* Generic */
.highlight-ipython3 .k { color: #7686bb; font-weight: bold } /* Keyword */
.highlight-ipython3 .l { color: #cccccc } /* Literal */
.highlight-ipython3 .n { color: #cccccc } /* Name */
.highlight-ipython3 .o { color: #cccccc } /* Operator */
.highlight-ipython3 .x { color: #cccccc } /* Other */
.highlight-ipython3 .p { color: #cccccc } /* Punctuation */
.highlight-ipython3 .ch { color: #777777; font-style: italic } /* Comment.Hashbang */
.highlight-ipython3 .cm { color: #777777; font-style: italic } /* Comment.Multiline */
.highlight-ipython3 .cp { color: #777777; font-style: italic } /* Comment.Preproc */
.highlight-ipython3 .cpf { color: #777777; font-style: italic } /* Comment.PreprocFile */
.highlight-ipython3 .c1 { color: #777777; font-style: italic } /* Comment.Single */
.highlight-ipython3 .cs { color: #777777; font-style: italic } /* Comment.Special */
.highlight-ipython3 .gd { color: #cccccc } /* Generic.Deleted */
.highlight-ipython3 .ge { color: #cccccc } /* Generic.Emph */
.highlight-ipython3 .ges { color: #cccccc } /* Generic.EmphStrong */
.highlight-ipython3 .gr { color: #cccccc } /* Generic.Error */
.highlight-ipython3 .gh { color: #cccccc } /* Generic.Heading */
.highlight-ipython3 .gi { color: #cccccc } /* Generic.Inserted */
.highlight-ipython3 .go { color: #cccccc } /* Generic.Output */
.highlight-ipython3 .gp { color: #ffffff } /* Generic.Prompt */
.highlight-ipython3 .gs { color: #cccccc } /* Generic.Strong */
.highlight-ipython3 .gu { color: #cccccc } /* Generic.Subheading */
.highlight-ipython3 .gt { color: #cccccc } /* Generic.Traceback */
.highlight-ipython3 .kc { color: #7686bb; font-weight: bold } /* Keyword.Constant */
.highlight-ipython3 .kd { color: #7686bb; font-weight: bold } /* Keyword.Declaration */
.highlight-ipython3 .kn { color: #7686bb; font-weight: bold } /* Keyword.Namespace */
.highlight-ipython3 .kp { color: #7686bb; font-weight: bold } /* Keyword.Pseudo */
.highlight-ipython3 .kr { color: #7686bb; font-weight: bold } /* Keyword.Reserved */
.highlight-ipython3 .kt { color: #7686bb; font-weight: bold } /* Keyword.Type */
.highlight-ipython3 .ld { color: #cccccc } /* Literal.Date */
.highlight-ipython3 .m { color: #4FB8CC } /* Literal.Number */
.highlight-ipython3 .s { color: #51cc99 } /* Literal.String */
.highlight-ipython3 .na { color: #cccccc } /* Name.Attribute */
.highlight-ipython3 .nb { color: #cccccc } /* Name.Builtin */
.highlight-ipython3 .nc { color: #cccccc } /* Name.Class */
.highlight-ipython3 .no { color: #cccccc } /* Name.Constant */
.highlight-ipython3 .nd { color: #cccccc } /* Name.Decorator */
.highlight-ipython3 .ni { color: #cccccc } /* Name.Entity */
.highlight-ipython3 .ne { color: #cccccc } /* Name.Exception */
.highlight-ipython3 .nf { color: #6a6aff } /* Name.Function */
.highlight-ipython3 .nl { color: #cccccc } /* Name.Label */
.highlight-ipython3 .nn { color: #cccccc } /* Name.Namespace */
.highlight-ipython3 .nx { color: #e2828e } /* Name.Other */
.highlight-ipython3 .py { color: #cccccc } /* Name.Property */
.highlight-ipython3 .nt { color: #cccccc } /* Name.Tag */
.highlight-ipython3 .nv { color: #7AB4DB; font-weight: bold } /* Name.Variable */
.highlight-ipython3 .ow { color: #cccccc } /* Operator.Word */
.highlight-ipython3 .pm { color: #cccccc } /* Punctuation.Marker */
.highlight-ipython3 .w { color: #bbbbbb } /* Text.Whitespace */
.highlight-ipython3 .mb { color: #4FB8CC } /* Literal.Number.Bin */
.highlight-ipython3 .mf { color: #4FB8CC } /* Literal.Number.Float */
.highlight-ipython3 .mh { color: #4FB8CC } /* Literal.Number.Hex */
.highlight-ipython3 .mi { color: #4FB8CC } /* Literal.Number.Integer */
.highlight-ipython3 .mo { color: #4FB8CC } /* Literal.Number.Oct */
.highlight-ipython3 .sa { color: #51cc99 } /* Literal.String.Affix */
.highlight-ipython3 .sb { color: #51cc99 } /* Literal.String.Backtick */
.highlight-ipython3 .sc { color: #51cc99 } /* Literal.String.Char */
.highlight-ipython3 .dl { color: #51cc99 } /* Literal.String.Delimiter */
.highlight-ipython3 .sd { color: #51cc99 } /* Literal.String.Doc */
.highlight-ipython3 .s2 { color: #51cc99 } /* Literal.String.Double */
.highlight-ipython3 .se { color: #51cc99 } /* Literal.String.Escape */
.highlight-ipython3 .sh { color: #51cc99 } /* Literal.String.Heredoc */
.highlight-ipython3 .si { color: #51cc99 } /* Literal.String.Interpol */
.highlight-ipython3 .sx { color: #51cc99 } /* Literal.String.Other */
.highlight-ipython3 .sr { color: #51cc99 } /* Literal.String.Regex */
.highlight-ipython3 .s1 { color: #51cc99 } /* Literal.String.Single */
.highlight-ipython3 .ss { color: #51cc99 } /* Literal.String.Symbol */
.highlight-ipython3 .bp { color: #cccccc } /* Name.Builtin.Pseudo */
.highlight-ipython3 .fm { color: #6a6aff } /* Name.Function.Magic */
.highlight-ipython3 .vc { color: #7AB4DB; font-weight: bold } /* Name.Variable.Class */
.highlight-ipython3 .vg { color: #BE646C; font-weight: bold } /* Name.Variable.Global */
.highlight-ipython3 .vi { color: #7AB4DB; font-weight: bold } /* Name.Variable.Instance */
.highlight-ipython3 .vm { color: #7AB4DB; font-weight: bold } /* Name.Variable.Magic */
.highlight-ipython3 .il { color: #4FB8CC } /* Literal.Number.Integer.Long */
================================================
FILE: docs/_static/css/highlight_ipython3_light.css
================================================
pre { line-height: 125%; }
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
.highlight-ipython3 .hll { background-color: #ffffcc }
.highlight-ipython3 { background: #f8f8f8; }
.highlight-ipython3 .c { color: #008800; font-style: italic } /* Comment */
.highlight-ipython3 .err { border: 1px solid #FF0000 } /* Error */
.highlight-ipython3 .k { color: #AA22FF; font-weight: bold } /* Keyword */
.highlight-ipython3 .o { color: #666666 } /* Operator */
.highlight-ipython3 .ch { color: #008800; font-style: italic } /* Comment.Hashbang */
.highlight-ipython3 .cm { color: #008800; font-style: italic } /* Comment.Multiline */
.highlight-ipython3 .cp { color: #008800 } /* Comment.Preproc */
.highlight-ipython3 .cpf { color: #008800; font-style: italic } /* Comment.PreprocFile */
.highlight-ipython3 .c1 { color: #008800; font-style: italic } /* Comment.Single */
.highlight-ipython3 .cs { color: #008800; font-weight: bold } /* Comment.Special */
.highlight-ipython3 .gd { color: #A00000 } /* Generic.Deleted */
.highlight-ipython3 .ge { font-style: italic } /* Generic.Emph */
.highlight-ipython3 .ges { font-weight: bold; font-style: italic } /* Generic.EmphStrong */
.highlight-ipython3 .gr { color: #FF0000 } /* Generic.Error */
.highlight-ipython3 .gh { color: #000080; font-weight: bold } /* Generic.Heading */
.highlight-ipython3 .gi { color: #00A000 } /* Generic.Inserted */
.highlight-ipython3 .go { color: #888888 } /* Generic.Output */
.highlight-ipython3 .gp { color: #000080; font-weight: bold } /* Generic.Prompt */
.highlight-ipython3 .gs { font-weight: bold } /* Generic.Strong */
.highlight-ipython3 .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
.highlight-ipython3 .gt { color: #0044DD } /* Generic.Traceback */
.highlight-ipython3 .kc { color: #AA22FF; font-weight: bold } /* Keyword.Constant */
.highlight-ipython3 .kd { color: #AA22FF; font-weight: bold } /* Keyword.Declaration */
.highlight-ipython3 .kn { color: #AA22FF; font-weight: bold } /* Keyword.Namespace */
.highlight-ipython3 .kp { color: #AA22FF } /* Keyword.Pseudo */
.highlight-ipython3 .kr { color: #AA22FF; font-weight: bold } /* Keyword.Reserved */
.highlight-ipython3 .kt { color: #00BB00; font-weight: bold } /* Keyword.Type */
.highlight-ipython3 .m { color: #666666 } /* Literal.Number */
.highlight-ipython3 .s { color: #BB4444 } /* Literal.String */
.highlight-ipython3 .na { color: #BB4444 } /* Name.Attribute */
.highlight-ipython3 .nb { color: #AA22FF } /* Name.Builtin */
.highlight-ipython3 .nc { color: #0000FF } /* Name.Class */
.highlight-ipython3 .no { color: #880000 } /* Name.Constant */
.highlight-ipython3 .nd { color: #AA22FF } /* Name.Decorator */
.highlight-ipython3 .ni { color: #999999; font-weight: bold } /* Name.Entity */
.highlight-ipython3 .ne { color: #D2413A; font-weight: bold } /* Name.Exception */
.highlight-ipython3 .nf { color: #00A000 } /* Name.Function */
.highlight-ipython3 .nl { color: #A0A000 } /* Name.Label */
.highlight-ipython3 .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
.highlight-ipython3 .nt { color: #008000; font-weight: bold } /* Name.Tag */
.highlight-ipython3 .nv { color: #B8860B } /* Name.Variable */
.highlight-ipython3 .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
.highlight-ipython3 .w { color: #bbbbbb } /* Text.Whitespace */
.highlight-ipython3 .mb { color: #666666 } /* Literal.Number.Bin */
.highlight-ipython3 .mf { color: #666666 } /* Literal.Number.Float */
.highlight-ipython3 .mh { color: #666666 } /* Literal.Number.Hex */
.highlight-ipython3 .mi { color: #666666 } /* Literal.Number.Integer */
.highlight-ipython3 .mo { color: #666666 } /* Literal.Number.Oct */
.highlight-ipython3 .sa { color: #BB4444 } /* Literal.String.Affix */
.highlight-ipython3 .sb { color: #BB4444 } /* Literal.String.Backtick */
.highlight-ipython3 .sc { color: #BB4444 } /* Literal.String.Char */
.highlight-ipython3 .dl { color: #BB4444 } /* Literal.String.Delimiter */
.highlight-ipython3 .sd { color: #BB4444; font-style: italic } /* Literal.String.Doc */
.highlight-ipython3 .s2 { color: #BB4444 } /* Literal.String.Double */
.highlight-ipython3 .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */
.highlight-ipython3 .sh { color: #BB4444 } /* Literal.String.Heredoc */
.highlight-ipython3 .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */
.highlight-ipython3 .sx { color: #008000 } /* Literal.String.Other */
.highlight-ipython3 .sr { color: #BB6688 } /* Literal.String.Regex */
.highlight-ipython3 .s1 { color: #BB4444 } /* Literal.String.Single */
.highlight-ipython3 .ss { color: #B8860B } /* Literal.String.Symbol */
.highlight-ipython3 .bp { color: #AA22FF } /* Name.Builtin.Pseudo */
.highlight-ipython3 .fm { color: #00A000 } /* Name.Function.Magic */
.highlight-ipython3 .vc { color: #B8860B } /* Name.Variable.Class */
.highlight-ipython3 .vg { color: #B8860B } /* Name.Variable.Global */
.highlight-ipython3 .vi { color: #B8860B } /* Name.Variable.Instance */
.highlight-ipython3 .vm { color: #B8860B } /* Name.Variable.Magic */
.highlight-ipython3 .il { color: #666666 } /* Literal.Number.Integer.Long */
================================================
FILE: docs/_static/css/highlight_python.css
================================================
pre { line-height: 125%; }
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
.highlight-ipython3 .hll { background-color: #49483e }
.highlight-ipython3 { background: #272822; color: #f8f8f2 }
.highlight-ipython3 .c { color: #959077 } /* Comment */
.highlight-ipython3 .err { color: #ed007e; background-color: #1e0010 } /* Error */
.highlight-ipython3 .esc { color: #f8f8f2 } /* Escape */
.highlight-ipython3 .g { color: #f8f8f2 } /* Generic */
.highlight-ipython3 .k { color: #66d9ef } /* Keyword */
.highlight-ipython3 .l { color: #ae81ff } /* Literal */
.highlight-ipython3 .n { color: #f8f8f2 } /* Name */
.highlight-ipython3 .o { color: #ff4689 } /* Operator */
.highlight-ipython3 .x { color: #f8f8f2 } /* Other */
.highlight-ipython3 .p { color: #f8f8f2 } /* Punctuation */
.highlight-ipython3 .ch { color: #959077 } /* Comment.Hashbang */
.highlight-ipython3 .cm { color: #959077 } /* Comment.Multiline */
.highlight-ipython3 .cp { color: #959077 } /* Comment.Preproc */
.highlight-ipython3 .cpf { color: #959077 } /* Comment.PreprocFile */
.highlight-ipython3 .c1 { color: #959077 } /* Comment.Single */
.highlight-ipython3 .cs { color: #959077 } /* Comment.Special */
.highlight-ipython3 .gd { color: #ff4689 } /* Generic.Deleted */
.highlight-ipython3 .ge { color: #f8f8f2; font-style: italic } /* Generic.Emph */
.highlight-ipython3 .ges { color: #f8f8f2; font-weight: bold; font-style: italic } /* Generic.EmphStrong */
.highlight-ipython3 .gr { color: #f8f8f2 } /* Generic.Error */
.highlight-ipython3 .gh { color: #f8f8f2 } /* Generic.Heading */
.highlight-ipython3 .gi { color: #a6e22e } /* Generic.Inserted */
.highlight-ipython3 .go { color: #66d9ef } /* Generic.Output */
.highlight-ipython3 .gp { color: #ff4689; font-weight: bold } /* Generic.Prompt */
.highlight-ipython3 .gs { color: #f8f8f2; font-weight: bold } /* Generic.Strong */
.highlight-ipython3 .gu { color: #959077 } /* Generic.Subheading */
.highlight-ipython3 .gt { color: #f8f8f2 } /* Generic.Traceback */
.highlight-ipython3 .kc { color: #66d9ef } /* Keyword.Constant */
.highlight-ipython3 .kd { color: #66d9ef } /* Keyword.Declaration */
.highlight-ipython3 .kn { color: #ff4689 } /* Keyword.Namespace */
.highlight-ipython3 .kp { color: #66d9ef } /* Keyword.Pseudo */
.highlight-ipython3 .kr { color: #66d9ef } /* Keyword.Reserved */
.highlight-ipython3 .kt { color: #66d9ef } /* Keyword.Type */
.highlight-ipython3 .ld { color: #e6db74 } /* Literal.Date */
.highlight-ipython3 .m { color: #ae81ff } /* Literal.Number */
.highlight-ipython3 .s { color: #e6db74 } /* Literal.String */
.highlight-ipython3 .na { color: #a6e22e } /* Name.Attribute */
.highlight-ipython3 .nb { color: #f8f8f2 } /* Name.Builtin */
.highlight-ipython3 .nc { color: #a6e22e } /* Name.Class */
.highlight-ipython3 .no { color: #66d9ef } /* Name.Constant */
.highlight-ipython3 .nd { color: #a6e22e } /* Name.Decorator */
.highlight-ipython3 .ni { color: #f8f8f2 } /* Name.Entity */
.highlight-ipython3 .ne { color: #a6e22e } /* Name.Exception */
.highlight-ipython3 .nf { color: #a6e22e } /* Name.Function */
.highlight-ipython3 .nl { color: #f8f8f2 } /* Name.Label */
.highlight-ipython3 .nn { color: #f8f8f2 } /* Name.Namespace */
.highlight-ipython3 .nx { color: #a6e22e } /* Name.Other */
.highlight-ipython3 .py { color: #f8f8f2 } /* Name.Property */
.highlight-ipython3 .nt { color: #ff4689 } /* Name.Tag */
.highlight-ipython3 .nv { color: #f8f8f2 } /* Name.Variable */
.highlight-ipython3 .ow { color: #ff4689 } /* Operator.Word */
.highlight-ipython3 .pm { color: #f8f8f2 } /* Punctuation.Marker */
.highlight-ipython3 .w { color: #f8f8f2 } /* Text.Whitespace */
.highlight-ipython3 .mb { color: #ae81ff } /* Literal.Number.Bin */
.highlight-ipython3 .mf { color: #ae81ff } /* Literal.Number.Float */
.highlight-ipython3 .mh { color: #ae81ff } /* Literal.Number.Hex */
.highlight-ipython3 .mi { color: #ae81ff } /* Literal.Number.Integer */
.highlight-ipython3 .mo { color: #ae81ff } /* Literal.Number.Oct */
.highlight-ipython3 .sa { color: #e6db74 } /* Literal.String.Affix */
.highlight-ipython3 .sb { color: #e6db74 } /* Literal.String.Backtick */
.highlight-ipython3 .sc { color: #e6db74 } /* Literal.String.Char */
.highlight-ipython3 .dl { color: #e6db74 } /* Literal.String.Delimiter */
.highlight-ipython3 .sd { color: #e6db74 } /* Literal.String.Doc */
.highlight-ipython3 .s2 { color: #e6db74 } /* Literal.String.Double */
.highlight-ipython3 .se { color: #ae81ff } /* Literal.String.Escape */
.highlight-ipython3 .sh { color: #e6db74 } /* Literal.String.Heredoc */
.highlight-ipython3 .si { color: #e6db74 } /* Literal.String.Interpol */
.highlight-ipython3 .sx { color: #e6db74 } /* Literal.String.Other */
.highlight-ipython3 .sr { color: #e6db74 } /* Literal.String.Regex */
.highlight-ipython3 .s1 { color: #e6db74 } /* Literal.String.Single */
.highlight-ipython3 .ss { color: #e6db74 } /* Literal.String.Symbol */
.highlight-ipython3 .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */
.highlight-ipython3 .fm { color: #a6e22e } /* Name.Function.Magic */
.highlight-ipython3 .vc { color: #f8f8f2 } /* Name.Variable.Class */
.highlight-ipython3 .vg { color: #f8f8f2 } /* Name.Variable.Global */
.highlight-ipython3 .vi { color: #f8f8f2 } /* Name.Variable.Instance */
.highlight-ipython3 .vm { color: #f8f8f2 } /* Name.Variable.Magic */
.highlight-ipython3 .il { color: #ae81ff } /* Literal.Number.Integer.Long */
================================================
FILE: docs/_static/css/highlight_python_dark.css
================================================
pre { line-height: 125%; }
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
.highlight-python .hll { background-color: #49483e }
.highlight-python { background: #232629; color: #cccccc }
.highlight-python .c { color: #777777; font-style: italic } /* Comment */
.highlight-python .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight-python .esc { color: #cccccc } /* Escape */
.highlight-python .g { color: #cccccc } /* Generic */
.highlight-python .k { color: #7686bb; font-weight: bold } /* Keyword */
.highlight-python .l { color: #cccccc } /* Literal */
.highlight-python .n { color: #cccccc } /* Name */
.highlight-python .o { color: #cccccc } /* Operator */
.highlight-python .x { color: #cccccc } /* Other */
.highlight-python .p { color: #cccccc } /* Punctuation */
.highlight-python .ch { color: #777777; font-style: italic } /* Comment.Hashbang */
.highlight-python .cm { color: #777777; font-style: italic } /* Comment.Multiline */
.highlight-python .cp { color: #777777; font-style: italic } /* Comment.Preproc */
.highlight-python .cpf { color: #777777; font-style: italic } /* Comment.PreprocFile */
.highlight-python .c1 { color: #777777; font-style: italic } /* Comment.Single */
.highlight-python .cs { color: #777777; font-style: italic } /* Comment.Special */
.highlight-python .gd { color: #cccccc } /* Generic.Deleted */
.highlight-python .ge { color: #cccccc } /* Generic.Emph */
.highlight-python .ges { color: #cccccc } /* Generic.EmphStrong */
.highlight-python .gr { color: #cccccc } /* Generic.Error */
.highlight-python .gh { color: #cccccc } /* Generic.Heading */
.highlight-python .gi { color: #cccccc } /* Generic.Inserted */
.highlight-python .go { color: #cccccc } /* Generic.Output */
.highlight-python .gp { color: #ffffff } /* Generic.Prompt */
.highlight-python .gs { color: #cccccc } /* Generic.Strong */
.highlight-python .gu { color: #cccccc } /* Generic.Subheading */
.highlight-python .gt { color: #cccccc } /* Generic.Traceback */
.highlight-python .kc { color: #7686bb; font-weight: bold } /* Keyword.Constant */
.highlight-python .kd { color: #7686bb; font-weight: bold } /* Keyword.Declaration */
.highlight-python .kn { color: #7686bb; font-weight: bold } /* Keyword.Namespace */
.highlight-python .kp { color: #7686bb; font-weight: bold } /* Keyword.Pseudo */
.highlight-python .kr { color: #7686bb; font-weight: bold } /* Keyword.Reserved */
.highlight-python .kt { color: #7686bb; font-weight: bold } /* Keyword.Type */
.highlight-python .ld { color: #cccccc } /* Literal.Date */
.highlight-python .m { color: #4FB8CC } /* Literal.Number */
.highlight-python .s { color: #51cc99 } /* Literal.String */
.highlight-python .na { color: #cccccc } /* Name.Attribute */
.highlight-python .nb { color: #cccccc } /* Name.Builtin */
.highlight-python .nc { color: #cccccc } /* Name.Class */
.highlight-python .no { color: #cccccc } /* Name.Constant */
.highlight-python .nd { color: #cccccc } /* Name.Decorator */
.highlight-python .ni { color: #cccccc } /* Name.Entity */
.highlight-python .ne { color: #cccccc } /* Name.Exception */
.highlight-python .nf { color: #6a6aff } /* Name.Function */
.highlight-python .nl { color: #cccccc } /* Name.Label */
.highlight-python .nn { color: #cccccc } /* Name.Namespace */
.highlight-python .nx { color: #e2828e } /* Name.Other */
.highlight-python .py { color: #cccccc } /* Name.Property */
.highlight-python .nt { color: #cccccc } /* Name.Tag */
.highlight-python .nv { color: #7AB4DB; font-weight: bold } /* Name.Variable */
.highlight-python .ow { color: #cccccc } /* Operator.Word */
.highlight-python .pm { color: #cccccc } /* Punctuation.Marker */
.highlight-python .w { color: #bbbbbb } /* Text.Whitespace */
.highlight-python .mb { color: #4FB8CC } /* Literal.Number.Bin */
.highlight-python .mf { color: #4FB8CC } /* Literal.Number.Float */
.highlight-python .mh { color: #4FB8CC } /* Literal.Number.Hex */
.highlight-python .mi { color: #4FB8CC } /* Literal.Number.Integer */
.highlight-python .mo { color: #4FB8CC } /* Literal.Number.Oct */
.highlight-python .sa { color: #51cc99 } /* Literal.String.Affix */
.highlight-python .sb { color: #51cc99 } /* Literal.String.Backtick */
.highlight-python .sc { color: #51cc99 } /* Literal.String.Char */
.highlight-python .dl { color: #51cc99 } /* Literal.String.Delimiter */
.highlight-python .sd { color: #51cc99 } /* Literal.String.Doc */
.highlight-python .s2 { color: #51cc99 } /* Literal.String.Double */
.highlight-python .se { color: #51cc99 } /* Literal.String.Escape */
.highlight-python .sh { color: #51cc99 } /* Literal.String.Heredoc */
.highlight-python .si { color: #51cc99 } /* Literal.String.Interpol */
.highlight-python .sx { color: #51cc99 } /* Literal.String.Other */
.highlight-python .sr { color: #51cc99 } /* Literal.String.Regex */
.highlight-python .s1 { color: #51cc99 } /* Literal.String.Single */
.highlight-python .ss { color: #51cc99 } /* Literal.String.Symbol */
.highlight-python .bp { color: #cccccc } /* Name.Builtin.Pseudo */
.highlight-python .fm { color: #6a6aff } /* Name.Function.Magic */
.highlight-python .vc { color: #7AB4DB; font-weight: bold } /* Name.Variable.Class */
.highlight-python .vg { color: #BE646C; font-weight: bold } /* Name.Variable.Global */
.highlight-python .vi { color: #7AB4DB; font-weight: bold } /* Name.Variable.Instance */
.highlight-python .vm { color: #7AB4DB; font-weight: bold } /* Name.Variable.Magic */
.highlight-python .il { color: #4FB8CC } /* Literal.Number.Integer.Long */
================================================
FILE: docs/_static/css/highlight_python_light.css
================================================
pre { line-height: 125%; }
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
.highlight-python .hll { background-color: #ffffcc }
.highlight-python { background: #f8f8f8; }
.highlight-python .c { color: #008800; font-style: italic } /* Comment */
.highlight-python .err { border: 1px solid #FF0000 } /* Error */
.highlight-python .k { color: #AA22FF; font-weight: bold } /* Keyword */
.highlight-python .o { color: #666666 } /* Operator */
.highlight-python .ch { color: #008800; font-style: italic } /* Comment.Hashbang */
.highlight-python .cm { color: #008800; font-style: italic } /* Comment.Multiline */
.highlight-python .cp { color: #008800 } /* Comment.Preproc */
.highlight-python .cpf { color: #008800; font-style: italic } /* Comment.PreprocFile */
.highlight-python .c1 { color: #008800; font-style: italic } /* Comment.Single */
.highlight-python .cs { color: #008800; font-weight: bold } /* Comment.Special */
.highlight-python .gd { color: #A00000 } /* Generic.Deleted */
.highlight-python .ge { font-style: italic } /* Generic.Emph */
.highlight-python .ges { font-weight: bold; font-style: italic } /* Generic.EmphStrong */
.highlight-python .gr { color: #FF0000 } /* Generic.Error */
.highlight-python .gh { color: #000080; font-weight: bold } /* Generic.Heading */
.highlight-python .gi { color: #00A000 } /* Generic.Inserted */
.highlight-python .go { color: #888888 } /* Generic.Output */
.highlight-python .gp { color: #000080; font-weight: bold } /* Generic.Prompt */
.highlight-python .gs { font-weight: bold } /* Generic.Strong */
.highlight-python .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
.highlight-python .gt { color: #0044DD } /* Generic.Traceback */
.highlight-python .kc { color: #AA22FF; font-weight: bold } /* Keyword.Constant */
.highlight-python .kd { color: #AA22FF; font-weight: bold } /* Keyword.Declaration */
.highlight-python .kn { color: #AA22FF; font-weight: bold } /* Keyword.Namespace */
.highlight-python .kp { color: #AA22FF } /* Keyword.Pseudo */
.highlight-python .kr { color: #AA22FF; font-weight: bold } /* Keyword.Reserved */
.highlight-python .kt { color: #00BB00; font-weight: bold } /* Keyword.Type */
.highlight-python .m { color: #666666 } /* Literal.Number */
.highlight-python .s { color: #BB4444 } /* Literal.String */
.highlight-python .na { color: #BB4444 } /* Name.Attribute */
.highlight-python .nb { color: #AA22FF } /* Name.Builtin */
.highlight-python .nc { color: #0000FF } /* Name.Class */
.highlight-python .no { color: #880000 } /* Name.Constant */
.highlight-python .nd { color: #AA22FF } /* Name.Decorator */
.highlight-python .ni { color: #999999; font-weight: bold } /* Name.Entity */
.highlight-python .ne { color: #D2413A; font-weight: bold } /* Name.Exception */
.highlight-python .nf { color: #00A000 } /* Name.Function */
.highlight-python .nl { color: #A0A000 } /* Name.Label */
.highlight-python .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
.highlight-python .nt { color: #008000; font-weight: bold } /* Name.Tag */
.highlight-python .nv { color: #B8860B } /* Name.Variable */
.highlight-python .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
.highlight-python .w { color: #bbbbbb } /* Text.Whitespace */
.highlight-python .mb { color: #666666 } /* Literal.Number.Bin */
.highlight-python .mf { color: #666666 } /* Literal.Number.Float */
.highlight-python .mh { color: #666666 } /* Literal.Number.Hex */
.highlight-python .mi { color: #666666 } /* Literal.Number.Integer */
.highlight-python .mo { color: #666666 } /* Literal.Number.Oct */
.highlight-python .sa { color: #BB4444 } /* Literal.String.Affix */
.highlight-python .sb { color: #BB4444 } /* Literal.String.Backtick */
.highlight-python .sc { color: #BB4444 } /* Literal.String.Char */
.highlight-python .dl { color: #BB4444 } /* Literal.String.Delimiter */
.highlight-python .sd { color: #BB4444; font-style: italic } /* Literal.String.Doc */
.highlight-python .s2 { color: #BB4444 } /* Literal.String.Double */
.highlight-python .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */
.highlight-python .sh { color: #BB4444 } /* Literal.String.Heredoc */
.highlight-python .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */
.highlight-python .sx { color: #008000 } /* Literal.String.Other */
.highlight-python .sr { color: #BB6688 } /* Literal.String.Regex */
.highlight-python .s1 { color: #BB4444 } /* Literal.String.Single */
.highlight-python .ss { color: #B8860B } /* Literal.String.Symbol */
.highlight-python .bp { color: #AA22FF } /* Name.Builtin.Pseudo */
.highlight-python .fm { color: #00A000 } /* Name.Function.Magic */
.highlight-python .vc { color: #B8860B } /* Name.Variable.Class */
.highlight-python .vg { color: #B8860B } /* Name.Variable.Global */
.highlight-python .vi { color: #B8860B } /* Name.Variable.Instance */
.highlight-python .vm { color: #B8860B } /* Name.Variable.Magic */
.highlight-python .il { color: #666666 } /* Literal.Number.Integer.Long */
================================================
FILE: docs/_static/css/ragas.css
================================================
/* Base Theme */
div.cell_input {
border: none !important;
}
.cell_output {
padding-left: 0px !important;
border: 1px solid #8b8b8b;
border-radius: var(--mystnb-source-border-radius);
}
.cell_output .output.text_plain,
.cell_output .output.stream {
background: var(--background);
border: none;
}
.cell_output .output.stream {
margin-top: 0px;
}
.cell_output .output.stderr {
background: var(--background);
border: none;
margin-top: 0px;
}
.cell_output .output.traceback {
background: var(--background);
border: none;
}
/* Pandas tables. Pulled from the Jupyter / nbsphinx CSS */
div.cell_output table {
display: block;
overflow-x: auto;
white-space: nowrap;
border: none;
border-collapse: collapse;
border-spacing: 0;
color: var(--foreground);
font-size: 1em;
table-layout: fixed;
}
div.cell_output thead {
border-bottom: 1px solid black;
vertical-align: bottom;
}
div.cell_output tr,
div.cell_output th,
div.cell_output td {
text-align: right;
vertical-align: middle;
padding: 0.5em 0.5em;
line-height: normal;
white-space: normal;
max-width: none;
border: none;
}
div.cell_output th {
font-weight: bold;
}
div.cell_output tbody tr:nth-child(odd) {
background: var(--backgound);
}
div.cell_output tbody tr:hover {
background: rgba(66, 165, 245, 0.2);
}
================================================
FILE: docs/_static/edited_chain_runs.json
================================================
{
"answer_correctness": [
{
"metric_input": {
"user_input": "What is the Theory of Sensory Adaptation and how does it affect human perception?",
"response": "The Theory of Sensory Adaptation is like when your nose stops noticing a strong smell after a while. It means our senses get used to things that stay the same, so we can pay more attention to new things around us. This helps us save energy and notice important changes, like if something new happens!",
"reference": "The Theory of Sensory Adaptation refers to the process by which our sensory receptors become less sensitive to constant stimuli over time. This phenomenon allows individuals to focus on changes in their environment rather than constant, unchanging stimuli. For example, when you enter a room with a strong odor, you may initially notice the smell intensely, but after a while, you become less aware of it. This adaptation is crucial for survival as it helps conserve energy and allows the brain to detect new and potentially important changes in the environment, enhancing our ability to respond to new stimuli."
},
"metric_output": 1,
"prompts": {
"single_turn_aspect_critic_prompt": {
"prompt_input": {
"user_input": "What is the Theory of Sensory Adaptation and how does it affect human perception?",
"response": "The Theory of Sensory Adaptation is like when your nose stops noticing a strong smell after a while. It means our senses get used to things that stay the same, so we can pay more attention to new things around us. This helps us save energy and notice important changes, like if something new happens!",
"retrieved_contexts": null,
"reference_contexts": null,
"reference": "The Theory of Sensory Adaptation refers to the process by which our sensory receptors become less sensitive to constant stimuli over time. This phenomenon allows individuals to focus on changes in their environment rather than constant, unchanging stimuli. For example, when you enter a room with a strong odor, you may initially notice the smell intensely, but after a while, you become less aware of it. This adaptation is crucial for survival as it helps conserve energy and allows the brain to detect new and potentially important changes in the environment, enhancing our ability to respond to new stimuli."
},
"prompt_output": {
"reason": "The response accurately explains the Theory of Sensory Adaptation and its effects on human perception, aligning well with the reference provided.",
"verdict": 1
},
"is_accepted": true,
"edited_output": null
}
},
"is_accepted": true
},
{
"metric_input": {
"user_input": "What is the Theory of Evolutionary Fitness and how does it explain the survival of species?",
"response": "The Theory of Evolutionary Fitness is like a game whe
gitextract_i6tp0pjv/
├── .cursor/
│ ├── commands/
│ │ ├── git-pr.md
│ │ └── update-howto-guide.md
│ ├── rules/
│ │ ├── docs-diataxis-guidelines.mdc
│ │ ├── docs-structure.mdc
│ │ ├── project-structure.mdc
│ │ ├── update-guide.mdc
│ │ └── use-uv-cli.mdc
│ └── worktrees.json
├── .dockerignore
├── .gitattributes
├── .github/
│ ├── ISSUE_TEMPLATE/
│ │ ├── bug_report.md
│ │ ├── feature_request.md
│ │ └── question.md
│ ├── pull_request_template.md
│ └── workflows/
│ ├── ci.yaml
│ ├── claude-code-review.yml
│ ├── claude-docs-apply.yml
│ ├── claude-docs-check.yml
│ ├── claude.yml
│ ├── issue-manager.yaml
│ ├── publish-examples.yml
│ └── python-publish.yml
├── .gitignore
├── .pre-commit-config.yaml
├── .readthedocs.yml
├── CLAUDE.md
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE
├── Makefile
├── README.md
├── SECURITY.md
├── docs/
│ ├── INSTALL
│ ├── Makefile
│ ├── _static/
│ │ ├── annotated_data.json
│ │ ├── css/
│ │ │ ├── highlight_ipython3.css
│ │ │ ├── highlight_ipython3_dark.css
│ │ │ ├── highlight_ipython3_light.css
│ │ │ ├── highlight_python.css
│ │ │ ├── highlight_python_dark.css
│ │ │ ├── highlight_python_light.css
│ │ │ └── ragas.css
│ │ ├── edited_chain_runs.json
│ │ ├── js/
│ │ │ ├── commonroom.js
│ │ │ ├── header_border.js
│ │ │ ├── mathjax.js
│ │ │ ├── mendable_chat_bubble.js
│ │ │ └── toggle.js
│ │ └── sample_annotated_summary.json
│ ├── alfred.py
│ ├── community/
│ │ ├── index.md
│ │ └── pdf_export.md
│ ├── concepts/
│ │ ├── components/
│ │ │ ├── eval_dataset.md
│ │ │ ├── eval_sample.md
│ │ │ ├── index.md
│ │ │ └── prompt.md
│ │ ├── datasets.md
│ │ ├── experimentation.md
│ │ ├── feedback/
│ │ │ └── index.md
│ │ ├── index.md
│ │ ├── metrics/
│ │ │ ├── available_metrics/
│ │ │ │ ├── agents.md
│ │ │ │ ├── answer_correctness.md
│ │ │ │ ├── answer_relevance.md
│ │ │ │ ├── aspect_critic.md
│ │ │ │ ├── context_entities_recall.md
│ │ │ │ ├── context_precision.md
│ │ │ │ ├── context_recall.md
│ │ │ │ ├── factual_correctness.md
│ │ │ │ ├── faithfulness.md
│ │ │ │ ├── general_purpose.md
│ │ │ │ ├── index.md
│ │ │ │ ├── multi_modal_faithfulness.md
│ │ │ │ ├── multi_modal_relevance.md
│ │ │ │ ├── noise_sensitivity.md
│ │ │ │ ├── nvidia_metrics.md
│ │ │ │ ├── rubrics_based.md
│ │ │ │ ├── semantic_similarity.md
│ │ │ │ ├── sql.md
│ │ │ │ ├── summarization_score.md
│ │ │ │ └── traditional.md
│ │ │ ├── index.md
│ │ │ └── overview/
│ │ │ └── index.md
│ │ └── test_data_generation/
│ │ ├── agents.md
│ │ ├── index.md
│ │ └── rag.md
│ ├── extra/
│ │ ├── components/
│ │ │ ├── choose_evaluator_llm.md
│ │ │ └── choose_generator_llm.md
│ │ ├── overrides/
│ │ │ └── main.html
│ │ ├── ragas-modern.css
│ │ └── style.css
│ ├── getstarted/
│ │ ├── evals.md
│ │ ├── experiments_quickstart.md
│ │ ├── index.md
│ │ ├── install.md
│ │ ├── quickstart.md
│ │ ├── rag_eval.md
│ │ └── rag_testset_generation.md
│ ├── howtos/
│ │ ├── applications/
│ │ │ ├── _cost.md
│ │ │ ├── add_to_ci.md
│ │ │ ├── align-llm-as-judge.md
│ │ │ ├── benchmark_llm.md
│ │ │ ├── compare_embeddings.md
│ │ │ ├── compare_llms.md
│ │ │ ├── cost.ipynb
│ │ │ ├── evaluate-and-improve-rag.md
│ │ │ ├── evaluating_multi_turn_conversations.md
│ │ │ ├── index.md
│ │ │ ├── iterate_prompt.md
│ │ │ ├── prompt_optimization.md
│ │ │ ├── singlehop_testset_gen.md
│ │ │ ├── text2sql.md
│ │ │ ├── vertexai_alignment.md
│ │ │ ├── vertexai_model_comparision.md
│ │ │ └── vertexai_x_ragas.md
│ │ ├── cli/
│ │ │ ├── agent_evals.md
│ │ │ ├── benchmark_llm.md
│ │ │ ├── improve_rag.md
│ │ │ ├── index.md
│ │ │ ├── judge_alignment.md
│ │ │ ├── llamaIndex_agent_evals.md
│ │ │ ├── prompt_evals.md
│ │ │ ├── rag_eval.md
│ │ │ ├── text2sql.md
│ │ │ └── workflow_eval.md
│ │ ├── customizations/
│ │ │ ├── _caching.md
│ │ │ ├── caching.ipynb
│ │ │ ├── cancellation.md
│ │ │ ├── customize_models.md
│ │ │ ├── index.md
│ │ │ ├── metrics/
│ │ │ │ ├── _cost.md
│ │ │ │ ├── cost.ipynb
│ │ │ │ ├── metrics_language_adaptation.md
│ │ │ │ ├── modifying-prompts-metrics.md
│ │ │ │ └── tracing.md
│ │ │ ├── optimizers/
│ │ │ │ └── index.md
│ │ │ ├── run_config.md
│ │ │ └── testgenerator/
│ │ │ ├── _language_adaptation.md
│ │ │ ├── _persona_generator.md
│ │ │ ├── _testgen-custom-single-hop.md
│ │ │ ├── _testgen-customisation.md
│ │ │ ├── index.md
│ │ │ ├── language_adaptation.ipynb
│ │ │ ├── persona_generator.ipynb
│ │ │ ├── prechunked_data.md
│ │ │ ├── testgen-custom-single-hop.ipynb
│ │ │ └── testgen-customisation.ipynb
│ │ ├── index.md
│ │ ├── integrations/
│ │ │ ├── _ag_ui.md
│ │ │ ├── _arize.md
│ │ │ ├── _athina.md
│ │ │ ├── _haystack.md
│ │ │ ├── _helicone.md
│ │ │ ├── _langchain.md
│ │ │ ├── _langfuse.md
│ │ │ ├── _langgraph_agent_evaluation.md
│ │ │ ├── _langsmith.md
│ │ │ ├── _llamaindex.md
│ │ │ ├── _openlayer.md
│ │ │ ├── _opik.md
│ │ │ ├── _tonic-validate.md
│ │ │ ├── _zeno.md
│ │ │ ├── ag_ui.ipynb
│ │ │ ├── ag_ui.md
│ │ │ ├── amazon_bedrock.md
│ │ │ ├── arize.ipynb
│ │ │ ├── athina.ipynb
│ │ │ ├── gemini.md
│ │ │ ├── griptape.md
│ │ │ ├── haystack.ipynb
│ │ │ ├── haystack.md
│ │ │ ├── helicone.ipynb
│ │ │ ├── index.md
│ │ │ ├── langchain.ipynb
│ │ │ ├── langchain.md
│ │ │ ├── langfuse.ipynb
│ │ │ ├── langgraph_agent_evaluation.ipynb
│ │ │ ├── langsmith.ipynb
│ │ │ ├── langsmith.md
│ │ │ ├── llama_stack.md
│ │ │ ├── llamaindex.ipynb
│ │ │ ├── llamaindex_agents.md
│ │ │ ├── nyc_wikipedia/
│ │ │ │ └── nyc_text.txt
│ │ │ ├── oci_genai.md
│ │ │ ├── openlayer.ipynb
│ │ │ ├── opik.ipynb
│ │ │ ├── r2r.md
│ │ │ ├── swarm_agent_evaluation.md
│ │ │ ├── tonic-validate.ipynb
│ │ │ └── zeno.ipynb
│ │ ├── llm-adapters.md
│ │ ├── migrations/
│ │ │ ├── migrate_from_v01_to_v02.md
│ │ │ └── migrate_from_v03_to_v04.md
│ │ └── observability.md
│ ├── index.md
│ ├── ipynb_to_md.py
│ ├── make.bat
│ ├── quoted_spans_metric.md
│ ├── references/
│ │ ├── aevaluate.md
│ │ ├── cache.md
│ │ ├── embeddings.md
│ │ ├── evaluate.md
│ │ ├── evaluation_schema.md
│ │ ├── executor.md
│ │ ├── generate.md
│ │ ├── graph.md
│ │ ├── index.md
│ │ ├── integrations.md
│ │ ├── llms.md
│ │ ├── metrics.md
│ │ ├── optimizers.md
│ │ ├── prompt.md
│ │ ├── run_config.md
│ │ ├── synthesizers.md
│ │ ├── testset_schema.md
│ │ ├── tokenizers.md
│ │ └── transforms.md
│ └── tutorials/
│ ├── agent.md
│ ├── index.md
│ ├── prompt.md
│ ├── rag.md
│ └── workflow.md
├── examples/
│ ├── LICENSE
│ ├── README.md
│ ├── gdrive_append_example.py
│ ├── gdrive_backend_example.py
│ ├── iterate_prompt/
│ │ ├── __init__.py
│ │ ├── datasets/
│ │ │ └── support_triage.csv
│ │ ├── evals.py
│ │ ├── promptv1.txt
│ │ ├── promptv2_fewshot.txt
│ │ └── run_prompt.py
│ ├── oci_genai_example.py
│ ├── pyproject.toml
│ └── ragas_examples/
│ ├── __init__.py
│ ├── ag_ui_agent_experiments/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── experiments.py
│ │ └── test_data/
│ │ └── datasets/
│ │ ├── scientist_biographies.csv
│ │ └── weather_tool_calls.csv
│ ├── agent_evals/
│ │ ├── __init__.py
│ │ ├── agent.py
│ │ └── evals.py
│ ├── benchmark_llm/
│ │ ├── __init__.py
│ │ ├── datasets/
│ │ │ └── discount_benchmark.csv
│ │ ├── evals.py
│ │ └── prompt.py
│ ├── improve_rag/
│ │ ├── __init__.py
│ │ ├── evals/
│ │ │ └── datasets/
│ │ │ └── hf_doc_qa_eval.csv
│ │ ├── evals.py
│ │ ├── pyproject.toml
│ │ └── rag.py
│ ├── judge_alignment/
│ │ ├── __init__.py
│ │ └── evals.py
│ ├── llamaIndex_agent_evals/
│ │ ├── __init__.py
│ │ ├── contexts/
│ │ │ ├── ambiguous_removal_request.json
│ │ │ ├── duplicate_addition.json
│ │ │ └── repeated_removal.json
│ │ ├── evals.py
│ │ └── llamaindex_agent.py
│ ├── prompt_evals/
│ │ ├── __init__.py
│ │ ├── evals.py
│ │ └── prompt.py
│ ├── rag_eval/
│ │ ├── __init__.py
│ │ ├── evals.py
│ │ ├── pyproject.toml
│ │ └── rag.py
│ ├── text2sql/
│ │ ├── __init__.py
│ │ ├── analyze_errors.py
│ │ ├── data_utils.py
│ │ ├── datasets/
│ │ │ └── booksql_sample.csv
│ │ ├── db_utils.py
│ │ ├── evals.py
│ │ ├── prompt.txt
│ │ ├── prompt_v2.txt
│ │ ├── prompt_v3.txt
│ │ ├── text2sql_agent.py
│ │ └── validate_sql_dataset.py
│ └── workflow_eval/
│ ├── __init__.py
│ ├── evals.py
│ └── workflow.py
├── mkdocs-pdf.yml
├── mkdocs.yml
├── pyproject.toml
├── scripts/
│ └── dev_docs.sh
├── src/
│ └── ragas/
│ ├── __init__.py
│ ├── _analytics.py
│ ├── async_utils.py
│ ├── backends/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── gdrive_backend.md
│ │ ├── gdrive_backend.py
│ │ ├── inmemory.py
│ │ ├── local_csv.py
│ │ ├── local_jsonl.py
│ │ ├── registry.py
│ │ └── utils.py
│ ├── cache.py
│ ├── callbacks.py
│ ├── cli.py
│ ├── config.py
│ ├── cost.py
│ ├── dataset.py
│ ├── dataset_schema.py
│ ├── embeddings/
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── google_provider.py
│ │ ├── haystack_wrapper.py
│ │ ├── huggingface_provider.py
│ │ ├── litellm_provider.py
│ │ ├── openai_provider.py
│ │ └── utils.py
│ ├── evaluation.py
│ ├── exceptions.py
│ ├── executor.py
│ ├── experiment.py
│ ├── integrations/
│ │ ├── __init__.py
│ │ ├── ag_ui.py
│ │ ├── amazon_bedrock.py
│ │ ├── griptape.py
│ │ ├── helicone.py
│ │ ├── langchain.py
│ │ ├── langgraph.py
│ │ ├── langsmith.py
│ │ ├── llama_index.py
│ │ ├── opik.py
│ │ ├── r2r.py
│ │ ├── swarm.py
│ │ └── tracing/
│ │ ├── __init__.py
│ │ ├── langfuse.py
│ │ └── mlflow.py
│ ├── llms/
│ │ ├── __init__.py
│ │ ├── adapters/
│ │ │ ├── __init__.py
│ │ │ ├── base.py
│ │ │ ├── instructor.py
│ │ │ └── litellm.py
│ │ ├── base.py
│ │ ├── haystack_wrapper.py
│ │ ├── litellm_llm.py
│ │ └── oci_genai_wrapper.py
│ ├── losses.py
│ ├── messages.py
│ ├── metrics/
│ │ ├── __init__.py
│ │ ├── _answer_correctness.py
│ │ ├── _answer_relevance.py
│ │ ├── _answer_similarity.py
│ │ ├── _aspect_critic.py
│ │ ├── _bleu_score.py
│ │ ├── _chrf_score.py
│ │ ├── _context_entities_recall.py
│ │ ├── _context_precision.py
│ │ ├── _context_recall.py
│ │ ├── _datacompy_score.py
│ │ ├── _domain_specific_rubrics.py
│ │ ├── _factual_correctness.py
│ │ ├── _faithfulness.py
│ │ ├── _goal_accuracy.py
│ │ ├── _instance_specific_rubrics.py
│ │ ├── _multi_modal_faithfulness.py
│ │ ├── _multi_modal_relevance.py
│ │ ├── _noise_sensitivity.py
│ │ ├── _nv_metrics.py
│ │ ├── _rouge_score.py
│ │ ├── _simple_criteria.py
│ │ ├── _sql_semantic_equivalence.py
│ │ ├── _string.py
│ │ ├── _summarization.py
│ │ ├── _tool_call_accuracy.py
│ │ ├── _tool_call_f1.py
│ │ ├── _topic_adherence.py
│ │ ├── base.py
│ │ ├── collections/
│ │ │ ├── __init__.py
│ │ │ ├── _bleu_score.py
│ │ │ ├── _rouge_score.py
│ │ │ ├── _semantic_similarity.py
│ │ │ ├── _string.py
│ │ │ ├── agent_goal_accuracy/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── answer_accuracy/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── answer_correctness/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── answer_relevancy/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── base.py
│ │ │ ├── chrf_score/
│ │ │ │ ├── __init__.py
│ │ │ │ └── metric.py
│ │ │ ├── context_entity_recall/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── context_precision/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── context_recall/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── context_relevance/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── datacompy_score/
│ │ │ │ ├── __init__.py
│ │ │ │ └── metric.py
│ │ │ ├── domain_specific_rubrics/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── example_metric.py
│ │ │ ├── factual_correctness/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── faithfulness/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── instance_specific_rubrics/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── multi_modal_faithfulness/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── multi_modal_relevance/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── noise_sensitivity/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── quoted_spans/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── response_groundedness/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── sql_semantic_equivalence/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── summary_score/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── tool_call_accuracy/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ ├── tool_call_f1/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── metric.py
│ │ │ │ └── util.py
│ │ │ └── topic_adherence/
│ │ │ ├── __init__.py
│ │ │ ├── metric.py
│ │ │ └── util.py
│ │ ├── decorator.py
│ │ ├── discrete.py
│ │ ├── numeric.py
│ │ ├── quoted_spans.py
│ │ ├── ranking.py
│ │ ├── result.py
│ │ ├── utils.py
│ │ └── validators.py
│ ├── optimizers/
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── dspy_adapter.py
│ │ ├── dspy_llm_wrapper.py
│ │ ├── dspy_optimizer.py
│ │ ├── genetic.py
│ │ └── utils.py
│ ├── prompt/
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── dynamic_few_shot.py
│ │ ├── few_shot_pydantic_prompt.py
│ │ ├── metrics/
│ │ │ ├── __init__.py
│ │ │ ├── answer_accuracy.py
│ │ │ ├── answer_correctness.py
│ │ │ ├── answer_relevance.py
│ │ │ ├── base_prompt.py
│ │ │ ├── common.py
│ │ │ ├── context_entity_recall.py
│ │ │ ├── context_recall.py
│ │ │ ├── context_relevance.py
│ │ │ ├── factual_correctness.py
│ │ │ ├── noise_sensitivity.py
│ │ │ ├── response_groundedness.py
│ │ │ └── summary_score.py
│ │ ├── mixin.py
│ │ ├── multi_modal_prompt.py
│ │ ├── prompt-formats.md
│ │ ├── pydantic_prompt.py
│ │ ├── simple_prompt.py
│ │ └── utils.py
│ ├── py.typed
│ ├── run_config.py
│ ├── sdk.py
│ ├── testset/
│ │ ├── __init__.py
│ │ ├── graph.py
│ │ ├── graph_queries.py
│ │ ├── persona.py
│ │ ├── synthesizers/
│ │ │ ├── __init__.py
│ │ │ ├── base.py
│ │ │ ├── generate.py
│ │ │ ├── multi_hop/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── abstract.py
│ │ │ │ ├── base.py
│ │ │ │ ├── prompts.py
│ │ │ │ └── specific.py
│ │ │ ├── prompts.py
│ │ │ ├── single_hop/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── base.py
│ │ │ │ ├── prompts.py
│ │ │ │ └── specific.py
│ │ │ ├── testset_schema.py
│ │ │ └── utils.py
│ │ └── transforms/
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── default.py
│ │ ├── engine.py
│ │ ├── extractors/
│ │ │ ├── __init__.py
│ │ │ ├── embeddings.py
│ │ │ ├── llm_based.py
│ │ │ └── regex_based.py
│ │ ├── filters.py
│ │ ├── relationship_builders/
│ │ │ ├── __init__.py
│ │ │ ├── cosine.py
│ │ │ └── traditional.py
│ │ └── splitters/
│ │ ├── __init__.py
│ │ └── headline.py
│ ├── tokenizers.py
│ ├── utils.py
│ └── validation.py
└── tests/
├── __init__.py
├── benchmarks/
│ ├── Dockerfile
│ ├── benchmark_eval.py
│ ├── benchmark_testsetgen.py
│ └── utils.py
├── conftest.py
├── docs/
│ ├── __init__.py
│ └── test_run_config.py
├── e2e/
│ ├── __init__.py
│ ├── metrics_migration/
│ │ ├── __init__.py
│ │ ├── base_migration_test.py
│ │ ├── conftest.py
│ │ ├── metric_score_diff.ipynb
│ │ ├── plan-for-metrics-migration.md
│ │ ├── test_answer_accuracy_migration.py
│ │ ├── test_answer_correctness_migration.py
│ │ ├── test_answer_relevancy_migration.py
│ │ ├── test_bleu_migration.py
│ │ ├── test_context_entity_recall_migration.py
│ │ ├── test_context_precision_migration.py
│ │ ├── test_context_recall_migration.py
│ │ ├── test_context_relevance_migration.py
│ │ ├── test_factual_correctness_migration.py
│ │ ├── test_faithfulness_migration.py
│ │ ├── test_noise_sensitivity_migration.py
│ │ ├── test_response_groundedness_migration.py
│ │ ├── test_rouge_migration.py
│ │ ├── test_semantic_similarity_migration.py
│ │ ├── test_string_migration.py
│ │ ├── test_summary_score_migration.py
│ │ └── test_utils.py
│ ├── test_adaptation.py
│ ├── test_amnesty_in_ci.py
│ ├── test_dataset_utils.py
│ ├── test_dspy_integration.py
│ ├── test_fullflow.py
│ ├── test_langchain_llm_attributes.py
│ └── test_testset_generation.py
├── test_quoted_spans.py
├── unit/
│ ├── backends/
│ │ ├── test_gdrive_backend.py
│ │ ├── test_inmemory.py
│ │ ├── test_local_csv.py
│ │ └── test_local_jsonl.py
│ ├── integrations/
│ │ ├── test_ag_ui.py
│ │ ├── test_tracing.py
│ │ └── test_tracing_simple.py
│ ├── llms/
│ │ ├── test_adapters.py
│ │ ├── test_instructor_factory.py
│ │ ├── test_llm.py
│ │ └── test_system_prompt.py
│ ├── prompt/
│ │ ├── test_base_prompt.py
│ │ ├── test_dynamic_few_shot_prompt.py
│ │ ├── test_prompt_mixin.py
│ │ ├── test_prompt_save_load.py
│ │ └── test_prompt_utils.py
│ ├── test_analytics.py
│ ├── test_async_evaluation.py
│ ├── test_async_utils.py
│ ├── test_average_precision_algorithm.py
│ ├── test_cache.py
│ ├── test_cancellation.py
│ ├── test_chrf_score.py
│ ├── test_chrf_score_collections.py
│ ├── test_cli.py
│ ├── test_cosine_relationship_builders.py
│ ├── test_cost.py
│ ├── test_datacompy_score_collections.py
│ ├── test_dataset_schema.py
│ ├── test_datatable_inheritance.py
│ ├── test_domain_specific_rubrics_collections.py
│ ├── test_dspy_adapter.py
│ ├── test_dspy_optimizer.py
│ ├── test_embeddings.py
│ ├── test_embeddings_caching.py
│ ├── test_engine.py
│ ├── test_executor.py
│ ├── test_executor_in_jupyter.ipynb
│ ├── test_experiment.py
│ ├── test_graph.py
│ ├── test_import.py
│ ├── test_instance_specific_rubrics_collections.py
│ ├── test_knowledge_graph_clusters.py
│ ├── test_knowledge_graph_save.py
│ ├── test_langgraph.py
│ ├── test_llm_context.py
│ ├── test_metric.py
│ ├── test_metric_decorators.py
│ ├── test_multi_hop_query_synthesizer.py
│ ├── test_multi_modal_faithfulness_collections.py
│ ├── test_multi_modal_relevance_collections.py
│ ├── test_oci_genai_wrapper.py
│ ├── test_optimizer_config.py
│ ├── test_prechunked_generation.py
│ ├── test_prompt.py
│ ├── test_quoted_spans_collections.py
│ ├── test_run_config.py
│ ├── test_simple.py
│ ├── test_simple_llm_metric_persistence.py
│ ├── test_single_hop_query_synthesizer.py
│ ├── test_sql_semantic_equivalence_collections.py
│ ├── test_testset_schema.py
│ ├── test_tokenizers.py
│ ├── test_tool_call_accuracy.py
│ ├── test_tool_call_accuracy_collections.py
│ ├── test_tool_call_f1.py
│ ├── test_tool_call_f1_collections.py
│ ├── test_traditional_relationship_builders.py
│ ├── test_utils.py
│ ├── test_uvloop_compatibility.py
│ └── test_validation.py
└── utils/
├── __init__.py
├── llm_setup.py
└── metric_comparison.py
Showing preview only (298K chars total). Download the full file or copy to clipboard to get everything.
SYMBOL INDEX (3233 symbols across 332 files)
FILE: docs/_static/js/header_border.js
function applyBorders (line 15) | function applyBorders() {
FILE: docs/_static/js/mendable_chat_bubble.js
function loadScript (line 2) | function loadScript(src, callback) {
FILE: docs/alfred.py
function get_files (line 17) | def get_files(path: str, ext: str) -> list:
function load_docs (line 21) | def load_docs(path: str) -> t.List[File]:
function fix_doc_with_llm (line 31) | async def fix_doc_with_llm(doc: File, llm: BaseChatModel) -> File:
function main (line 51) | async def main(docs: t.List[File], llm: BaseChatModel):
FILE: docs/ipynb_to_md.py
function convert_ipynb_to_md (line 6) | def convert_ipynb_to_md(ipynb_file):
function get_last_modified_time (line 32) | def get_last_modified_time(file_path):
function find_and_convert_ipynb_files (line 36) | def find_and_convert_ipynb_files(directory):
function get_valid_directory (line 59) | def get_valid_directory(use_default=False):
FILE: examples/gdrive_append_example.py
class EvaluationRecord (line 13) | class EvaluationRecord(BaseModel):
function append_to_existing_dataset (line 21) | def append_to_existing_dataset():
function create_multiple_datasets (line 98) | def create_multiple_datasets():
FILE: examples/gdrive_backend_example.py
class EvaluationRecord (line 31) | class EvaluationRecord(BaseModel):
function example_usage (line 39) | def example_usage():
FILE: examples/iterate_prompt/evals.py
function labels_exact_match (line 17) | def labels_exact_match(prediction: str, expected_labels: str):
function priority_accuracy (line 45) | def priority_accuracy(prediction: str, expected_priority: str):
function support_triage_experiment (line 69) | async def support_triage_experiment(row, prompt_file: str, experiment_na...
function load_dataset (line 108) | def load_dataset():
function compare_inputs_to_output (line 142) | def compare_inputs_to_output(
function run_command (line 250) | async def run_command(prompt_file: str, name: Optional[str]) -> None:
function compare_command (line 291) | def compare_command(inputs: List[str], output: Optional[str]) -> None:
function build_parser (line 296) | def build_parser() -> argparse.ArgumentParser:
FILE: examples/iterate_prompt/run_prompt.py
function load_prompt (line 8) | def load_prompt(prompt_file: str) -> str:
function run_prompt (line 14) | async def run_prompt(ticket_text: str, prompt_file: str = "promptv1.txt"):
FILE: examples/oci_genai_example.py
function main (line 21) | def main():
function test_llm_connection (line 94) | def test_llm_connection():
FILE: examples/ragas_examples/ag_ui_agent_experiments/experiments.py
function load_scientist_dataset (line 64) | def load_scientist_dataset() -> Dataset:
function load_weather_dataset (line 84) | def load_weather_dataset() -> Dataset:
function create_evaluator_components (line 104) | def create_evaluator_components(model_name: str):
function run_scientist_experiment (line 120) | async def run_scientist_experiment(
function run_tool_experiment (line 226) | async def run_tool_experiment(endpoint_url: str, evaluator_model: str) -...
function main (line 321) | async def main():
FILE: examples/ragas_examples/agent_evals/agent.py
class TraceEvent (line 24) | class TraceEvent:
class ToolResult (line 35) | class ToolResult:
class MathToolsAgent (line 42) | class MathToolsAgent:
method __init__ (line 43) | def __init__(
method add (line 146) | def add(self, a: float, b: float) -> float:
method sub (line 152) | def sub(self, a: float, b: float) -> float:
method mul (line 157) | def mul(self, a: float, b: float) -> float:
method div (line 162) | def div(self, a: float, b: float) -> float:
method _execute_tool_call (line 169) | def _execute_tool_call(self, tool_call) -> str:
method export_traces_to_log (line 210) | def export_traces_to_log(
method solve (line 242) | def solve(
function get_default_agent (line 392) | def get_default_agent(
FILE: examples/ragas_examples/agent_evals/evals.py
function correctness_metric (line 11) | def correctness_metric(prediction: float, actual: float):
function load_dataset (line 21) | def load_dataset():
function run_experiment (line 48) | async def run_experiment(row):
function main (line 69) | async def main():
FILE: examples/ragas_examples/benchmark_llm/evals.py
function discount_accuracy (line 23) | def discount_accuracy(prediction: str, expected_discount):
function benchmark_experiment (line 42) | async def benchmark_experiment(row, model_name: str):
function load_dataset (line 69) | def load_dataset():
function compare_inputs_to_output (line 81) | def compare_inputs_to_output(
function run_command (line 187) | async def run_command(model: str, name: Optional[str]) -> None:
function compare_command (line 220) | def compare_command(inputs: List[str], output: Optional[str]) -> None:
function build_parser (line 225) | def build_parser() -> argparse.ArgumentParser:
FILE: examples/ragas_examples/benchmark_llm/prompt.py
function get_client (line 12) | def get_client() -> AsyncOpenAI:
function run_prompt (line 45) | async def run_prompt(prompt: str, model: str = DEFAULT_MODEL):
function main (line 63) | async def main():
FILE: examples/ragas_examples/improve_rag/evals.py
function download_and_save_dataset (line 41) | def download_and_save_dataset() -> Path:
function create_ragas_dataset (line 66) | def create_ragas_dataset(dataset_path: Path) -> Dataset:
function construct_mlflow_trace_url (line 83) | def construct_mlflow_trace_url(trace_id: str, mlflow_host: str = "http:/...
function evaluate_rag (line 126) | async def evaluate_rag(row: Dict[str, Any], rag: RAG, llm) -> Dict[str, ...
function run_experiment (line 173) | async def run_experiment(mode: str = "naive", model: str = "gpt-4o-mini"...
FILE: examples/ragas_examples/improve_rag/rag.py
class BM25Retriever (line 30) | class BM25Retriever:
method __init__ (line 33) | def __init__(self, dataset_name="m-ric/huggingface_doc", default_k=3):
method _build_retriever (line 37) | def _build_retriever(self, dataset_name: str) -> LangchainBM25Retriever:
method retrieve (line 77) | def retrieve(self, query: str, top_k: int = None):
class RAG (line 85) | class RAG:
method _check_mlflow_server (line 89) | def _check_mlflow_server(uri: str = "http://127.0.0.1:5000", timeout: ...
method __init__ (line 99) | def __init__(self, llm_client: AsyncOpenAI, retriever: BM25Retriever, ...
method _setup_agent (line 121) | def _setup_agent(self):
method _naive_query (line 145) | async def _naive_query(self, question: str, top_k: int) -> Dict[str, A...
method _agentic_query (line 172) | async def _agentic_query(self, question: str, top_k: int) -> Dict[str,...
method query (line 194) | async def query(self, question: str, top_k: Optional[int] = None) -> D...
function main (line 218) | async def main():
FILE: examples/ragas_examples/judge_alignment/evals.py
function load_dataset (line 100) | def load_dataset(csv_path: Optional[Path] = None) -> Dataset:
function judge_alignment (line 122) | def judge_alignment(judge_label: str, human_label: str) -> MetricResult:
function judge_experiment (line 134) | async def judge_experiment(
function main (line 167) | async def main():
function main_v2 (line 194) | async def main_v2():
FILE: examples/ragas_examples/llamaIndex_agent_evals/evals.py
function tool_call_accuracy_metric (line 30) | def tool_call_accuracy_metric(
function goal_accuracy_metric_without_llm (line 80) | def goal_accuracy_metric_without_llm(current_state: Dict, expected_state...
function load_dataset (line 142) | def load_dataset():
function run_experiment (line 227) | async def run_experiment(row):
function main (line 285) | async def main():
FILE: examples/ragas_examples/llamaIndex_agent_evals/llamaindex_agent.py
function add_item (line 9) | async def add_item(ctx: Context, item: str) -> str:
function remove_item (line 19) | async def remove_item(ctx: Context, item: str) -> str:
function list_items (line 30) | async def list_items(
FILE: examples/ragas_examples/prompt_evals/evals.py
function my_metric (line 9) | def my_metric(prediction: str, actual: str):
function run_experiment (line 19) | async def run_experiment(row):
function load_dataset (line 31) | def load_dataset():
function main (line 65) | async def main():
FILE: examples/ragas_examples/prompt_evals/prompt.py
function run_prompt (line 14) | def run_prompt(prompt: str):
FILE: examples/ragas_examples/rag_eval/evals.py
function load_dataset (line 20) | def load_dataset():
function run_experiment (line 59) | async def run_experiment(row):
function main (line 77) | async def main():
FILE: examples/ragas_examples/rag_eval/rag.py
class TraceEvent (line 19) | class TraceEvent:
class BaseRetriever (line 27) | class BaseRetriever:
method __init__ (line 33) | def __init__(self):
method fit (line 36) | def fit(self, documents: List[str]):
method get_top_k (line 40) | def get_top_k(self, query: str, k: int = 3) -> List[tuple]:
class SimpleKeywordRetriever (line 45) | class SimpleKeywordRetriever(BaseRetriever):
method __init__ (line 48) | def __init__(self):
method _count_keyword_matches (line 51) | def _count_keyword_matches(self, query: str, document: str) -> int:
method get_top_k (line 61) | def get_top_k(self, query: str, k: int = 3) -> List[tuple]:
class ExampleRAG (line 75) | class ExampleRAG:
method __init__ (line 83) | def __init__(
method add_documents (line 131) | def add_documents(self, documents: List[str]):
method set_documents (line 163) | def set_documents(self, documents: List[str]):
method retrieve_documents (line 196) | def retrieve_documents(self, query: str, top_k: int = 3) -> List[Dict[...
method generate_response (line 254) | def generate_response(self, query: str, top_k: int = 3) -> str:
method query (line 339) | def query(
method export_traces_to_log (line 413) | def export_traces_to_log(
function default_rag_client (line 442) | def default_rag_client(llm_client, logdir: str = "logs") -> ExampleRAG:
FILE: examples/ragas_examples/text2sql/analyze_errors.py
function get_error_analysis (line 34) | def get_error_analysis(client: OpenAI, row: Dict[str, Any]) -> Dict[str,...
function analyze_errors (line 76) | def analyze_errors(input_file: str, output_file: str) -> None:
function main (line 142) | def main():
FILE: examples/ragas_examples/text2sql/data_utils.py
function download_booksql_dataset (line 56) | def download_booksql_dataset() -> bool:
function validate_query_data (line 120) | def validate_query_data(query_data: Dict[str, Any], require_data: bool =...
function load_and_clean_data (line 149) | def load_and_clean_data(input_file: str) -> DataFrame:
function sample_by_difficulty (line 199) | def sample_by_difficulty(data: DataFrame, level: str, samples_per_level:...
function validate_samples (line 227) | def validate_samples(data: DataFrame, level: str, samples_per_level: int,
function save_results (line 286) | def save_results(data: DataFrame, output_dir: str, output_filename: str,...
function create_sample_dataset (line 330) | def create_sample_dataset(
function main (line 399) | def main():
FILE: examples/ragas_examples/text2sql/db_utils.py
class SQLiteDB (line 22) | class SQLiteDB:
method __init__ (line 33) | def __init__(self, db_path: Optional[str] = None):
method connect (line 48) | def connect(self) -> Tuple[bool, str]:
method disconnect (line 66) | def disconnect(self) -> None:
method execute_query (line 72) | def execute_query(self, sql: str, replace_current_date: bool = True, c...
method _normalize_sql (line 112) | def _normalize_sql(self, sql: str, replace_current_date: bool, case_in...
method get_schema_info (line 141) | def get_schema_info(self) -> Tuple[bool, Union[pd.DataFrame, str]]:
method get_table_names (line 158) | def get_table_names(self) -> Tuple[bool, Union[list, str]]:
function execute_sql (line 180) | def execute_sql(sql: str, db_path: Optional[str] = None, replace_current...
function get_database_schema (line 209) | def get_database_schema(db_path: Optional[str] = None) -> Tuple[bool, Un...
function main (line 226) | def main():
FILE: examples/ragas_examples/text2sql/evals.py
function execution_accuracy (line 32) | def execution_accuracy(expected_sql: str, predicted_success: bool, predi...
function text2sql_experiment (line 127) | async def text2sql_experiment(
function load_dataset (line 167) | def load_dataset(limit: Optional[int] = None):
function main (line 192) | async def main():
FILE: examples/ragas_examples/text2sql/text2sql_agent.py
class Text2SQLAgent (line 21) | class Text2SQLAgent:
method __init__ (line 30) | def __init__(
method query (line 56) | async def query(self, question: str) -> Dict[str, Any]:
function main (line 103) | async def main():
FILE: examples/ragas_examples/text2sql/validate_sql_dataset.py
function load_dataset (line 28) | def load_dataset(csv_path: str = "datasets/booksql_sample.csv") -> List[...
function execute_and_validate_query (line 58) | def execute_and_validate_query(query_data: Dict[str, Any]) -> Dict[str, ...
function generate_summary_statistics (line 136) | def generate_summary_statistics(results: List[Dict[str, Any]]) -> Dict[s...
function main (line 218) | def main():
FILE: examples/ragas_examples/workflow_eval/evals.py
function load_dataset (line 16) | def load_dataset():
function run_experiment (line 80) | async def run_experiment(row):
function main (line 98) | async def main():
FILE: examples/ragas_examples/workflow_eval/workflow.py
class TraceEvent (line 14) | class TraceEvent:
class ExtractionMode (line 24) | class ExtractionMode(Enum):
class BaseExtractor (line 31) | class BaseExtractor(ABC):
method extract (line 35) | def extract(self, email_content: str, category: str) -> Dict[str, Opti...
class DeterministicExtractor (line 40) | class DeterministicExtractor(BaseExtractor):
method extract (line 43) | def extract(self, email_content: str, category: str) -> Dict[str, Opti...
method _extract_bug_info (line 56) | def _extract_bug_info(self, email_content: str) -> Dict[str, Optional[...
method _extract_billing_info (line 69) | def _extract_billing_info(self, email_content: str) -> Dict[str, Optio...
method _extract_feature_info (line 87) | def _extract_feature_info(self, email_content: str) -> Dict[str, Optio...
class LLMExtractor (line 145) | class LLMExtractor(BaseExtractor):
method __init__ (line 148) | def __init__(self, client: OpenAI):
method extract (line 151) | def extract(self, email_content: str, category: str) -> Dict[str, Opti...
method _get_bug_extraction_prompt (line 185) | def _get_bug_extraction_prompt(self, email_content: str) -> str:
method _get_billing_extraction_prompt (line 199) | def _get_billing_extraction_prompt(self, email_content: str) -> str:
method _get_feature_extraction_prompt (line 213) | def _get_feature_extraction_prompt(self, email_content: str) -> str:
class ConfigurableSupportTriageAgent (line 229) | class ConfigurableSupportTriageAgent:
method __init__ (line 232) | def __init__(
method set_extractor (line 276) | def set_extractor(self, extractor: BaseExtractor):
method classify_email (line 305) | def classify_email(self, email_content: str) -> str:
method extract_info (line 376) | def extract_info(
method generate_response (line 430) | def generate_response(self, category: str, extracted_info: Dict[str, A...
method export_traces_to_log (line 505) | def export_traces_to_log(
method process_email (line 531) | def process_email(
function default_workflow_client (line 616) | def default_workflow_client(
function main (line 646) | def main():
FILE: src/ragas/__init__.py
function __getattr__ (line 45) | def __getattr__(name):
FILE: src/ragas/_analytics.py
function do_not_track (line 46) | def do_not_track() -> bool: # pragma: no cover
function _usage_event_debugging (line 53) | def _usage_event_debugging() -> bool:
function silent (line 58) | def silent(func: t.Callable[P, T]) -> t.Callable[P, T]: # pragma: no cover
function get_userid (line 81) | def get_userid() -> str:
class BaseEvent (line 109) | class BaseEvent(BaseModel):
class EvaluationEvent (line 115) | class EvaluationEvent(BaseEvent):
class TestsetGenerationEvent (line 123) | class TestsetGenerationEvent(BaseEvent):
class AnalyticsBatcher (line 132) | class AnalyticsBatcher:
method __init__ (line 133) | def __init__(self, batch_size: int = 50, flush_interval: float = 120):
method _flush_loop (line 155) | def _flush_loop(self) -> None:
method add_evaluation (line 165) | def add_evaluation(self, evaluation_event: EvaluationEvent) -> None:
method _join_evaluation_events (line 169) | def _join_evaluation_events(
method flush (line 196) | def flush(self) -> None:
method shutdown (line 215) | def shutdown(self) -> None:
function track (line 223) | def track(event_properties: BaseEvent):
class IsCompleteEvent (line 236) | class IsCompleteEvent(BaseEvent):
class LLMUsageEvent (line 240) | class LLMUsageEvent(BaseEvent):
class EmbeddingUsageEvent (line 249) | class EmbeddingUsageEvent(BaseEvent):
class PromptUsageEvent (line 258) | class PromptUsageEvent(BaseEvent):
function track_was_completed (line 268) | def track_was_completed(
FILE: src/ragas/async_utils.py
function is_event_loop_running (line 10) | def is_event_loop_running() -> bool:
function apply_nest_asyncio (line 22) | def apply_nest_asyncio() -> bool:
function as_completed (line 58) | def as_completed(
function process_futures (line 99) | async def process_futures(
function run (line 122) | def run(
function run_async_tasks (line 159) | def run_async_tasks(
FILE: src/ragas/backends/base.py
class BaseBackend (line 9) | class BaseBackend(ABC):
method load_dataset (line 47) | def load_dataset(self, name: str) -> t.List[t.Dict[str, t.Any]]:
method load_experiment (line 67) | def load_experiment(self, name: str) -> t.List[t.Dict[str, t.Any]]:
method save_dataset (line 86) | def save_dataset(
method save_experiment (line 108) | def save_experiment(
method list_datasets (line 128) | def list_datasets(self) -> t.List[str]:
method list_experiments (line 142) | def list_experiments(self) -> t.List[str]:
FILE: src/ragas/backends/gdrive_backend.py
function build (line 29) | def build(*args, **kwargs):
class GDriveBackend (line 38) | class GDriveBackend(BaseBackend):
method __init__ (line 76) | def __init__(
method _setup_auth (line 112) | def _setup_auth(self):
method _ensure_folder_structure (line 155) | def _ensure_folder_structure(self):
method _get_or_create_folder (line 179) | def _get_or_create_folder(self, folder_name: str, parent_id: str) -> str:
method _get_folder_id_for_type (line 200) | def _get_folder_id_for_type(self, data_type: str) -> str:
method _get_or_create_spreadsheet (line 211) | def _get_or_create_spreadsheet(self, name: str, data_type: str) -> str:
method _spreadsheet_exists (line 237) | def _spreadsheet_exists(self, name: str, data_type: str) -> bool:
method _load_data_from_spreadsheet (line 246) | def _load_data_from_spreadsheet(
method _save_data_to_spreadsheet (line 314) | def _save_data_to_spreadsheet(
method _list_data_names (line 377) | def _list_data_names(self, data_type: str) -> t.List[str]:
method load_dataset (line 398) | def load_dataset(self, name: str) -> t.List[t.Dict[str, t.Any]]:
method load_experiment (line 402) | def load_experiment(self, name: str) -> t.List[t.Dict[str, t.Any]]:
method save_dataset (line 406) | def save_dataset(
method save_experiment (line 415) | def save_experiment(
method list_datasets (line 424) | def list_datasets(self) -> t.List[str]:
method list_experiments (line 428) | def list_experiments(self) -> t.List[str]:
method __repr__ (line 432) | def __repr__(self) -> str:
FILE: src/ragas/backends/inmemory.py
class InMemoryBackend (line 11) | class InMemoryBackend(BaseBackend):
method __init__ (line 33) | def __init__(self):
method load_dataset (line 38) | def load_dataset(self, name: str) -> t.List[t.Dict[str, t.Any]]:
method load_experiment (line 56) | def load_experiment(self, name: str) -> t.List[t.Dict[str, t.Any]]:
method save_dataset (line 74) | def save_dataset(
method save_experiment (line 95) | def save_experiment(
method list_datasets (line 116) | def list_datasets(self) -> t.List[str]:
method list_experiments (line 124) | def list_experiments(self) -> t.List[str]:
FILE: src/ragas/backends/local_csv.py
class LocalCSVBackend (line 12) | class LocalCSVBackend(BaseBackend):
method __init__ (line 42) | def __init__(
method _get_data_dir (line 48) | def _get_data_dir(self, data_type: str) -> Path:
method _get_file_path (line 52) | def _get_file_path(self, data_type: str, name: str) -> Path:
method _load (line 56) | def _load(self, data_type: str, name: str) -> t.List[t.Dict[str, t.Any]]:
method _save (line 69) | def _save(
method _list (line 96) | def _list(self, data_type: str) -> t.List[str]:
method load_dataset (line 108) | def load_dataset(self, name: str) -> t.List[t.Dict[str, t.Any]]:
method load_experiment (line 112) | def load_experiment(self, name: str) -> t.List[t.Dict[str, t.Any]]:
method save_dataset (line 116) | def save_dataset(
method save_experiment (line 125) | def save_experiment(
method list_datasets (line 134) | def list_datasets(self) -> t.List[str]:
method list_experiments (line 138) | def list_experiments(self) -> t.List[str]:
method __repr__ (line 142) | def __repr__(self) -> str:
FILE: src/ragas/backends/local_jsonl.py
class LocalJSONLBackend (line 13) | class LocalJSONLBackend(BaseBackend):
method __init__ (line 47) | def __init__(
method _get_data_dir (line 53) | def _get_data_dir(self, data_type: str) -> Path:
method _get_file_path (line 57) | def _get_file_path(self, data_type: str, name: str) -> Path:
method _serialize_datetime (line 61) | def _serialize_datetime(self, obj: t.Any) -> t.Any:
method _deserialize_datetime (line 74) | def _deserialize_datetime(self, obj: t.Any) -> t.Any:
method _load (line 96) | def _load(self, data_type: str, name: str) -> t.List[t.Dict[str, t.Any]]:
method _save (line 125) | def _save(
method _list (line 156) | def _list(self, data_type: str) -> t.List[str]:
method load_dataset (line 168) | def load_dataset(self, name: str) -> t.List[t.Dict[str, t.Any]]:
method load_experiment (line 172) | def load_experiment(self, name: str) -> t.List[t.Dict[str, t.Any]]:
method save_dataset (line 176) | def save_dataset(
method save_experiment (line 185) | def save_experiment(
method list_datasets (line 194) | def list_datasets(self) -> t.List[str]:
method list_experiments (line 198) | def list_experiments(self) -> t.List[str]:
method __repr__ (line 202) | def __repr__(self) -> str:
FILE: src/ragas/backends/registry.py
class BackendRegistry (line 12) | class BackendRegistry:
method __new__ (line 20) | def __new__(cls):
method _resolve_name (line 26) | def _resolve_name(self, name: str) -> str:
method _get_available_names (line 30) | def _get_available_names(self) -> t.List[str]:
method _get_aliases_for (line 36) | def _get_aliases_for(self, primary_name: str) -> t.List[str]:
method _validate_name (line 42) | def _validate_name(self, name: str) -> None:
method _validate_backend_class (line 47) | def _validate_backend_class(self, backend_class: t.Type[BaseBackend]) ...
method register_aliases (line 54) | def register_aliases(
method list_all_names (line 84) | def list_all_names(self) -> t.Dict[str, t.List[str]]:
method discover_backends (line 97) | def discover_backends(self) -> t.Dict[str, t.Type[BaseBackend]]:
method _discover_backends (line 112) | def _discover_backends(self) -> None:
method get_backend_info (line 138) | def get_backend_info(self, name: str) -> t.Dict[str, t.Any]:
method list_backend_info (line 159) | def list_backend_info(self) -> t.List[t.Dict[str, t.Any]]:
method clear (line 170) | def clear(self) -> None:
method create_backend (line 176) | def create_backend(self, backend_type: str, **kwargs) -> BaseBackend:
method __getitem__ (line 189) | def __getitem__(self, name: str) -> t.Type[BaseBackend]:
method __setitem__ (line 202) | def __setitem__(self, name: str, backend_class: t.Type[BaseBackend]) -...
method __delitem__ (line 210) | def __delitem__(self, name: str) -> None:
method __contains__ (line 230) | def __contains__(self, name: str) -> bool:
method __iter__ (line 236) | def __iter__(self) -> t.Iterator[str]:
method __len__ (line 242) | def __len__(self) -> int:
method keys (line 248) | def keys(self) -> t.KeysView[str]:
method values (line 254) | def values(self) -> t.ValuesView[t.Type[BaseBackend]]:
method items (line 260) | def items(self) -> t.ItemsView[str, t.Type[BaseBackend]]:
method __repr__ (line 266) | def __repr__(self) -> str:
function get_registry (line 277) | def get_registry() -> BackendRegistry:
function register_backend (line 282) | def register_backend(
function print_available_backends (line 299) | def print_available_backends() -> None:
FILE: src/ragas/backends/utils.py
class MemorableNames (line 8) | class MemorableNames:
method __init__ (line 11) | def __init__(self):
method generate_name (line 189) | def generate_name(self):
method generate_unique_name (line 195) | def generate_unique_name(self):
method generate_unique_names (line 213) | def generate_unique_names(self, count):
FILE: src/ragas/cache.py
class CacheInterface (line 16) | class CacheInterface(ABC):
method get (line 24) | def get(self, key: str) -> Any:
method set (line 36) | def set(self, key: str, value) -> None:
method has_key (line 46) | def has_key(self, key: str) -> bool:
method __get_pydantic_core_schema__ (line 58) | def __get_pydantic_core_schema__(
class DiskCacheBackend (line 70) | class DiskCacheBackend(CacheInterface):
method __init__ (line 80) | def __init__(self, cache_dir: str = ".cache"):
method get (line 90) | def get(self, key: str) -> Any:
method set (line 101) | def set(self, key: str, value) -> None:
method has_key (line 110) | def has_key(self, key: str) -> bool:
method __del__ (line 121) | def __del__(self):
method __repr__ (line 126) | def __repr__(self):
function _make_hashable (line 135) | def _make_hashable(o):
function _make_pydantic_picklable (line 151) | def _make_pydantic_picklable(obj: Any) -> Any:
function _generate_cache_key (line 184) | def _generate_cache_key(func, args, kwargs):
function cacher (line 198) | def cacher(cache_backend: Optional[CacheInterface] = None):
FILE: src/ragas/callbacks.py
function new_group (line 19) | def new_group(
class ChainType (line 52) | class ChainType(Enum):
class ChainRun (line 59) | class ChainRun(BaseModel):
class ChainRunEncoder (line 69) | class ChainRunEncoder(json.JSONEncoder):
method default (line 70) | def default(self, o):
class RagasTracer (line 81) | class RagasTracer(BaseCallbackHandler):
method on_chain_start (line 84) | def on_chain_start(
method on_chain_end (line 107) | def on_chain_end(
method to_jsons (line 116) | def to_jsons(self) -> str:
class MetricTrace (line 124) | class MetricTrace(dict):
method __repr__ (line 127) | def __repr__(self):
method __str__ (line 130) | def __str__(self):
function parse_run_traces (line 134) | def parse_run_traces(
FILE: src/ragas/cli.py
function main (line 28) | def main():
function success (line 34) | def success(text: str) -> None:
function error (line 39) | def error(text: str) -> None:
function info (line 44) | def info(text: str) -> None:
function warning (line 49) | def warning(text: str) -> None:
function create_numerical_metrics_table (line 54) | def create_numerical_metrics_table(
function create_categorical_metrics_table (line 101) | def create_categorical_metrics_table(
function extract_metrics_from_experiment (line 163) | def extract_metrics_from_experiment(experiment, metric_fields: list) -> ...
function calculate_aggregated_metrics (line 173) | def calculate_aggregated_metrics(metrics_data: Dict[str, list]) -> Dict[...
function separate_metrics_by_type (line 191) | def separate_metrics_by_type(
function display_metrics_tables (line 224) | def display_metrics_tables(
function load_eval_module (line 241) | def load_eval_module(eval_path: str) -> Any:
function run_experiments (line 264) | async def run_experiments(
function evals (line 371) | def evals(
function quickstart (line 461) | def quickstart(
function hello_world (line 900) | def hello_world(
FILE: src/ragas/config.py
class DemonstrationConfig (line 15) | class DemonstrationConfig(BaseModel):
method validate_embedding (line 23) | def validate_embedding(cls, v):
class InstructionConfig (line 29) | class InstructionConfig(BaseModel):
FILE: src/ragas/cost.py
class TokenUsage (line 15) | class TokenUsage(BaseModel):
method __add__ (line 20) | def __add__(self, y: "TokenUsage") -> "TokenUsage":
method cost (line 30) | def cost(
method __eq__ (line 43) | def __eq__(self, other: object) -> bool:
method is_same_model (line 52) | def is_same_model(self, other: "TokenUsage") -> bool:
function get_token_usage_for_openai (line 61) | def get_token_usage_for_openai(
function get_token_usage_for_anthropic (line 78) | def get_token_usage_for_anthropic(
function get_token_usage_for_bedrock (line 112) | def get_token_usage_for_bedrock(
function get_token_usage_for_azure_ai (line 144) | def get_token_usage_for_azure_ai(
class CostCallbackHandler (line 161) | class CostCallbackHandler(BaseCallbackHandler):
method __init__ (line 162) | def __init__(self, token_usage_parser: TokenUsageParser):
method on_llm_end (line 166) | def on_llm_end(self, response: LLMResult, **kwargs: t.Any):
method total_cost (line 169) | def total_cost(
method total_tokens (line 219) | def total_tokens(self) -> t.Union[TokenUsage, t.List[TokenUsage]]:
FILE: src/ragas/dataset.py
class DataTable (line 31) | class DataTable(t.Generic[T]):
method __init__ (line 41) | def __init__(
method __init__ (line 50) | def __init__(
method __init__ (line 59) | def __init__(
method __init__ (line 69) | def __init__(
method __init__ (line 77) | def __init__(
method _resolve_backend (line 110) | def _resolve_backend(backend: t.Union[BaseBackend, str], **kwargs) -> ...
method load (line 156) | def load(
method from_pandas (line 202) | def from_pandas(
method save (line 254) | def save(self) -> None:
method reload (line 275) | def reload(self) -> None:
method validate_with (line 290) | def validate_with(self, data_model: t.Type[T]) -> Self:
method to_pandas (line 316) | def to_pandas(self) -> "PandasDataFrame":
method append (line 337) | def append(self, item: t.Union[t.Dict, BaseModel]) -> None:
method __len__ (line 359) | def __len__(self) -> int:
method __getitem__ (line 362) | def __getitem__(self, index):
method __iter__ (line 365) | def __iter__(self):
method __str__ (line 368) | def __str__(self):
method get_row_value (line 375) | def get_row_value(self, row, key: str):
method train_test_split (line 383) | def train_test_split(
class Dataset (line 458) | class Dataset(DataTable[T]):
FILE: src/ragas/dataset_schema.py
class BaseSample (line 29) | class BaseSample(BaseModel):
method to_dict (line 34) | def to_dict(self) -> t.Dict:
method get_features (line 40) | def get_features(self) -> t.List[str]:
method to_string (line 46) | def to_string(self) -> str:
class SingleTurnSample (line 54) | class SingleTurnSample(BaseSample):
class MultiTurnSample (line 100) | class MultiTurnSample(BaseSample):
method validate_user_input (line 126) | def validate_user_input(
method to_messages (line 169) | def to_messages(self):
method pretty_repr (line 173) | def pretty_repr(self):
class RagasDataset (line 187) | class RagasDataset(ABC, t.Generic[Sample]):
method __post_init__ (line 190) | def __post_init__(self):
method to_list (line 194) | def to_list(self) -> t.List[t.Dict]:
method from_list (line 200) | def from_list(cls: t.Type[T], data: t.List[t.Dict]) -> T:
method validate_samples (line 204) | def validate_samples(self, samples: t.List[Sample]) -> t.List[Sample]:
method get_sample_type (line 218) | def get_sample_type(self) -> t.Type[Sample]:
method to_hf_dataset (line 222) | def to_hf_dataset(self) -> HFDataset:
method from_hf_dataset (line 234) | def from_hf_dataset(cls: t.Type[T], dataset: HFDataset) -> T:
method to_pandas (line 238) | def to_pandas(self) -> PandasDataframe:
method from_pandas (line 251) | def from_pandas(cls, dataframe: PandasDataframe):
method features (line 255) | def features(self):
method from_dict (line 260) | def from_dict(cls: t.Type[T], mapping: t.Dict) -> T:
method to_csv (line 272) | def to_csv(self, path: t.Union[str, Path]):
method to_jsonl (line 288) | def to_jsonl(self, path: t.Union[str, Path]):
method from_jsonl (line 295) | def from_jsonl(cls: t.Type[T], path: t.Union[str, Path]) -> T:
method __iter__ (line 301) | def __iter__(self) -> t.Iterator[Sample]: # type: ignore
method __len__ (line 304) | def __len__(self) -> int:
method __str__ (line 307) | def __str__(self) -> str:
method __repr__ (line 310) | def __repr__(self) -> str:
class EvaluationDataset (line 318) | class EvaluationDataset(RagasDataset[SingleTurnSampleOrMultiTurnSample]):
method __getitem__ (line 359) | def __getitem__(self, idx: int) -> SingleTurnSampleOrMultiTurnSample: ...
method __getitem__ (line 362) | def __getitem__(self, idx: slice) -> "EvaluationDataset": ...
method __getitem__ (line 364) | def __getitem__(
method is_multi_turn (line 374) | def is_multi_turn(self) -> bool:
method to_list (line 377) | def to_list(self) -> t.List[t.Dict]:
method from_list (line 391) | def from_list(
method __repr__ (line 407) | def __repr__(self) -> str:
class EvaluationResult (line 412) | class EvaluationResult:
method __post_init__ (line 436) | def __post_init__(self):
method __repr__ (line 455) | def __repr__(self) -> str:
method __getitem__ (line 459) | def __getitem__(self, key: str) -> t.List[float]:
method to_pandas (line 462) | def to_pandas(self, batch_size: int | None = None, batched: bool = Fal...
method total_tokens (line 498) | def total_tokens(self) -> t.Union[t.List[TokenUsage], TokenUsage]:
method total_cost (line 518) | def total_cost(
class PromptAnnotation (line 555) | class PromptAnnotation(BaseModel):
method __getitem__ (line 560) | def __getitem__(self, key):
class SampleAnnotation (line 564) | class SampleAnnotation(BaseModel):
method __getitem__ (line 571) | def __getitem__(self, key):
class MetricAnnotation (line 575) | class MetricAnnotation(BaseModel):
method __getitem__ (line 578) | def __getitem__(self, key):
method _process_dataset (line 582) | def _process_dataset(
method from_json (line 612) | def from_json(cls, path: str, metric_name: t.Optional[str]) -> "Metric...
method __len__ (line 617) | def __len__(self):
class SingleMetricAnnotation (line 621) | class SingleMetricAnnotation(BaseModel):
method to_evaluation_dataset (line 625) | def to_evaluation_dataset(self) -> EvaluationDataset:
method __getitem__ (line 629) | def __getitem__(self, idx):
method __repr__ (line 632) | def __repr__(self):
method __iter__ (line 635) | def __iter__(self) -> t.Iterator[SampleAnnotation]: # type: ignore
method select (line 638) | def select(self, indices: t.List[int]) -> "SingleMetricAnnotation":
method from_json (line 645) | def from_json(cls, path) -> "SingleMetricAnnotation":
method filter (line 653) | def filter(self, function: t.Optional[t.Callable] = None):
method __len__ (line 662) | def __len__(self):
method train_test_split (line 665) | def train_test_split(
method sample (line 681) | def sample(
method batch (line 736) | def batch(
method stratified_batches (line 761) | def stratified_batches(
method get_prompt_annotations (line 828) | def get_prompt_annotations(self) -> t.Dict[str, t.List[PromptAnnotatio...
FILE: src/ragas/embeddings/__init__.py
function embedding_factory (line 39) | def embedding_factory(*args, **kwargs):
FILE: src/ragas/embeddings/base.py
class BaseRagasEmbedding (line 29) | class BaseRagasEmbedding(ABC):
method __init__ (line 37) | def __init__(self, cache: t.Optional[CacheInterface] = None):
method embed_text (line 51) | def embed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
method aembed_text (line 64) | async def aembed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
method embed_texts (line 76) | def embed_texts(self, texts: t.List[str], **kwargs: t.Any) -> t.List[t...
method aembed_texts (line 92) | async def aembed_texts(
method _check_client_async (line 111) | def _check_client_async(
method _run_async_in_current_loop (line 131) | def _run_async_in_current_loop(self, coro):
method _from_factory (line 146) | def _from_factory(
class BaseRagasEmbeddings (line 182) | class BaseRagasEmbeddings(Embeddings, ABC):
method __init__ (line 197) | def __init__(self, cache: t.Optional[CacheInterface] = None):
method embed_text (line 210) | async def embed_text(self, text: str, is_async=True) -> t.List[float]:
method embed_texts (line 217) | async def embed_texts(
method aembed_query (line 236) | async def aembed_query(self, text: str) -> t.List[float]: ...
method aembed_documents (line 239) | async def aembed_documents(self, texts: t.List[str]) -> t.List[t.List[...
method set_run_config (line 241) | def set_run_config(self, run_config: RunConfig):
method __get_pydantic_core_schema__ (line 248) | def __get_pydantic_core_schema__(
class LangchainEmbeddingsWrapper (line 260) | class LangchainEmbeddingsWrapper(BaseRagasEmbeddings):
method __init__ (line 278) | def __init__(
method embed_query (line 298) | def embed_query(self, text: str) -> t.List[float]:
method embed_documents (line 316) | def embed_documents(self, texts: t.List[str]) -> t.List[t.List[float]]:
method aembed_query (line 334) | async def aembed_query(self, text: str) -> t.List[float]:
method aembed_documents (line 352) | async def aembed_documents(self, texts: t.List[str]) -> t.List[t.List[...
method set_run_config (line 370) | def set_run_config(self, run_config: RunConfig):
method __repr__ (line 387) | def __repr__(self) -> str:
class HuggingfaceEmbeddings (line 392) | class HuggingfaceEmbeddings(BaseRagasEmbeddings):
method __post_init__ (line 445) | def __post_init__(self):
method embed_query (line 485) | def embed_query(self, text: str) -> t.List[float]:
method embed_documents (line 491) | def embed_documents(self, texts: t.List[str]) -> t.List[t.List[float]]:
method predict (line 508) | def predict(self, texts: t.List[t.List[str]]) -> t.List[t.List[float]]:
class LlamaIndexEmbeddingsWrapper (line 525) | class LlamaIndexEmbeddingsWrapper(BaseRagasEmbeddings):
method __init__ (line 568) | def __init__(
method embed_query (line 588) | def embed_query(self, text: str) -> t.List[float]:
method embed_documents (line 591) | def embed_documents(self, texts: t.List[str]) -> t.List[t.List[float]]:
method aembed_query (line 594) | async def aembed_query(self, text: str) -> t.List[float]:
method aembed_documents (line 597) | async def aembed_documents(self, texts: t.List[str]) -> t.List[t.List[...
method __repr__ (line 600) | def __repr__(self) -> str:
function _infer_embedding_provider_from_llm (line 604) | def _infer_embedding_provider_from_llm(llm: t.Any) -> str:
function embedding_factory (line 655) | def embedding_factory(
function _is_legacy_embedding_call (line 776) | def _is_legacy_embedding_call(
function _looks_like_model_name (line 792) | def _looks_like_model_name(name: str) -> bool:
function _get_provider_registry (line 797) | def _get_provider_registry() -> t.Dict[str, t.Type[BaseRagasEmbedding]]:
function _create_modern_embedding (line 820) | def _create_modern_embedding(
function modern_embedding_factory (line 846) | def modern_embedding_factory(
FILE: src/ragas/embeddings/google_provider.py
class GoogleEmbeddings (line 12) | class GoogleEmbeddings(BaseRagasEmbedding):
method __init__ (line 49) | def __init__(
method _resolve_client (line 73) | def _resolve_client(self, client: t.Optional[t.Any], use_vertex: bool)...
method _is_new_genai_client (line 146) | def _is_new_genai_client(self, client: t.Any) -> bool:
method _import_genai_module (line 166) | def _import_genai_module(self) -> t.Any:
method embed_text (line 206) | def embed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
method _embed_text_vertex (line 213) | def _embed_text_vertex(self, text: str, **kwargs: t.Any) -> t.List[flo...
method _embed_text_genai (line 228) | def _embed_text_genai(self, text: str, **kwargs: t.Any) -> t.List[float]:
method aembed_text (line 249) | async def aembed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
method embed_texts (line 256) | def embed_texts(self, texts: t.List[str], **kwargs: t.Any) -> t.List[t...
method _embed_texts_vertex (line 267) | def _embed_texts_vertex(
method _embed_texts_genai (line 284) | def _embed_texts_genai(
method aembed_texts (line 303) | async def aembed_texts(
method _get_client_info (line 313) | def _get_client_info(self) -> str:
method _get_key_config (line 321) | def _get_key_config(self) -> str:
method __repr__ (line 336) | def __repr__(self) -> str:
FILE: src/ragas/embeddings/haystack_wrapper.py
class HaystackEmbeddingsWrapper (line 25) | class HaystackEmbeddingsWrapper(BaseRagasEmbeddings):
method __init__ (line 43) | def __init__(
method embed_query (line 103) | def embed_query(self, text: str) -> t.List[float]:
method embed_documents (line 109) | def embed_documents(self, texts: t.List[str]) -> t.List[t.List[float]]:
method aembed_query (line 112) | async def aembed_query(self, text: str) -> t.List[float]:
method aembed_documents (line 117) | async def aembed_documents(self, texts: t.List[str]) -> t.List[t.List[...
method __repr__ (line 122) | def __repr__(self) -> str:
FILE: src/ragas/embeddings/huggingface_provider.py
class HuggingFaceEmbeddings (line 11) | class HuggingFaceEmbeddings(BaseRagasEmbedding):
method __init__ (line 21) | def __init__(
method _setup_api_client (line 46) | def _setup_api_client(self):
method _setup_local_model (line 61) | def _setup_local_model(self):
method embed_text (line 75) | def embed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
method _embed_text_api (line 82) | def _embed_text_api(self, text: str, **kwargs: t.Any) -> t.List[float]:
method _embed_text_local (line 90) | def _embed_text_local(self, text: str, **kwargs: t.Any) -> t.List[float]:
method aembed_text (line 97) | async def aembed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
method _aembed_text_api (line 104) | async def _aembed_text_api(self, text: str, **kwargs: t.Any) -> t.List...
method embed_texts (line 109) | def embed_texts(self, texts: t.List[str], **kwargs: t.Any) -> t.List[t...
method _embed_texts_api (line 120) | def _embed_texts_api(
method _embed_texts_local (line 140) | def _embed_texts_local(
method aembed_texts (line 152) | async def aembed_texts(
method _get_client_info (line 165) | def _get_client_info(self) -> str:
method _get_key_config (line 172) | def _get_key_config(self) -> str:
method __repr__ (line 193) | def __repr__(self) -> str:
FILE: src/ragas/embeddings/litellm_provider.py
class LiteLLMEmbeddings (line 11) | class LiteLLMEmbeddings(BaseRagasEmbedding):
method __init__ (line 21) | def __init__(
method _prepare_kwargs (line 44) | def _prepare_kwargs(self, **kwargs: t.Any) -> t.Dict[str, t.Any]:
method embed_text (line 63) | def embed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
method aembed_text (line 69) | async def aembed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
method embed_texts (line 75) | def embed_texts(self, texts: t.List[str], **kwargs: t.Any) -> t.List[t...
method aembed_texts (line 91) | async def aembed_texts(
method _get_key_config (line 109) | def _get_key_config(self) -> str:
method __repr__ (line 131) | def __repr__(self) -> str:
FILE: src/ragas/embeddings/openai_provider.py
class OpenAIEmbeddings (line 10) | class OpenAIEmbeddings(BaseRagasEmbedding):
method __init__ (line 21) | def __init__(
method embed_text (line 32) | def embed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
method aembed_text (line 57) | async def aembed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
method embed_texts (line 81) | def embed_texts(self, texts: t.List[str], **kwargs: t.Any) -> t.List[t...
method aembed_texts (line 108) | async def aembed_texts(
method _get_client_info (line 138) | def _get_client_info(self) -> str:
method __repr__ (line 144) | def __repr__(self) -> str:
FILE: src/ragas/embeddings/utils.py
function run_async_in_current_loop (line 9) | def run_async_in_current_loop(coro: t.Awaitable[t.Any]) -> t.Any:
function run_sync_in_async (line 73) | async def run_sync_in_async(func: t.Callable, *args, **kwargs) -> t.Any:
function batch_texts (line 89) | def batch_texts(texts: t.List[str], batch_size: int) -> t.List[t.List[st...
function get_optimal_batch_size (line 108) | def get_optimal_batch_size(provider: str, model: str) -> int:
function validate_texts (line 133) | def validate_texts(texts: t.Union[str, t.List[str]]) -> t.List[str]:
function safe_import (line 163) | def safe_import(module_name: str, package_name: t.Optional[str] = None) ...
FILE: src/ragas/evaluation.py
function aevaluate (line 59) | async def aevaluate(
function evaluate (line 349) | def evaluate(
FILE: src/ragas/exceptions.py
class RagasException (line 4) | class RagasException(Exception):
method __init__ (line 9) | def __init__(self, message: str):
class ExceptionInRunner (line 14) | class ExceptionInRunner(RagasException):
method __init__ (line 19) | def __init__(self):
class RagasOutputParserException (line 24) | class RagasOutputParserException(RagasException):
method __init__ (line 29) | def __init__(self):
class LLMDidNotFinishException (line 34) | class LLMDidNotFinishException(RagasException):
method __init__ (line 39) | def __init__(self):
class RagasError (line 45) | class RagasError(Exception):
class ValidationError (line 51) | class ValidationError(RagasError):
class DuplicateError (line 57) | class DuplicateError(RagasError):
class NotFoundError (line 63) | class NotFoundError(RagasError):
class ResourceNotFoundError (line 69) | class ResourceNotFoundError(NotFoundError):
class ProjectNotFoundError (line 75) | class ProjectNotFoundError(ResourceNotFoundError):
class DatasetNotFoundError (line 81) | class DatasetNotFoundError(ResourceNotFoundError):
class ExperimentNotFoundError (line 87) | class ExperimentNotFoundError(ResourceNotFoundError):
class DuplicateResourceError (line 93) | class DuplicateResourceError(RagasError):
class DuplicateProjectError (line 99) | class DuplicateProjectError(DuplicateResourceError):
class DuplicateDatasetError (line 105) | class DuplicateDatasetError(DuplicateResourceError):
class DuplicateExperimentError (line 111) | class DuplicateExperimentError(DuplicateResourceError):
FILE: src/ragas/executor.py
class Executor (line 19) | class Executor:
method cancel (line 56) | def cancel(self) -> None:
method is_cancelled (line 60) | def is_cancelled(self) -> bool:
method wrap_callable_with_index (line 64) | def wrap_callable_with_index(
method submit (line 88) | def submit(
method clear_jobs (line 105) | def clear_jobs(self) -> None:
method _process_jobs (line 110) | async def _process_jobs(self) -> t.List[t.Any]:
method _process_batched_jobs (line 144) | async def _process_batched_jobs(
method _process_coroutines (line 180) | async def _process_coroutines(self, jobs, pbar, results, max_workers):
method aresults (line 193) | async def aresults(self) -> t.List[t.Any]:
method results (line 204) | def results(self) -> t.List[t.Any]:
function run_async_batch (line 218) | def run_async_batch(
FILE: src/ragas/experiment.py
class Experiment (line 17) | class Experiment(DataTable):
function version_experiment (line 21) | def version_experiment(
class ExperimentProtocol (line 104) | class ExperimentProtocol(t.Protocol):
method __call__ (line 105) | async def __call__(self, *args, **kwargs) -> t.Any: ...
method arun (line 106) | async def arun(
class ExperimentWrapper (line 116) | class ExperimentWrapper:
method __init__ (line 119) | def __init__(
method __call__ (line 134) | async def __call__(self, *args, **kwargs) -> t.Any:
method arun (line 141) | async def arun(
function experiment (line 201) | def experiment(
FILE: src/ragas/integrations/ag_ui.py
function _import_ag_ui_core (line 131) | def _import_ag_ui_core():
class AGUIEventCollector (line 172) | class AGUIEventCollector:
method __init__ (line 197) | def __init__(self, metadata: bool = False):
method _get_pending_tool_calls (line 236) | def _get_pending_tool_calls(self) -> Optional[List[ToolCall]]:
method process_event (line 251) | def process_event(self, event: Any) -> None:
method _handle_text_message_start (line 313) | def _handle_text_message_start(self, event: Any) -> None:
method _handle_text_message_content (line 322) | def _handle_text_message_content(self, event: Any) -> None:
method _handle_text_message_end (line 333) | def _handle_text_message_end(self, event: Any) -> None:
method _handle_tool_call_start (line 373) | def _handle_tool_call_start(self, event: Any) -> None:
method _handle_tool_call_args (line 383) | def _handle_tool_call_args(self, event: Any) -> None:
method _handle_tool_call_end (line 394) | def _handle_tool_call_end(self, event: Any) -> None:
method _handle_tool_call_result (line 419) | def _handle_tool_call_result(self, event: Any) -> None:
method _handle_text_message_chunk (line 515) | def _handle_text_message_chunk(self, event: Any) -> None:
method _handle_tool_call_chunk (line 555) | def _handle_tool_call_chunk(self, event: Any) -> None:
method _handle_messages_snapshot (line 599) | def _handle_messages_snapshot(self, event: Any) -> None:
method get_messages (line 658) | def get_messages(self) -> List[Union[HumanMessage, AIMessage, ToolMess...
method clear (line 675) | def clear(self) -> None:
function convert_to_ragas_messages (line 691) | def convert_to_ragas_messages(
function convert_messages_snapshot (line 788) | def convert_messages_snapshot(
function convert_messages_to_ag_ui (line 850) | def convert_messages_to_ag_ui(
function call_ag_ui_endpoint (line 940) | async def call_ag_ui_endpoint(
function extract_response (line 1080) | def extract_response(
function extract_tool_calls (line 1107) | def extract_tool_calls(
function extract_contexts (line 1135) | def extract_contexts(
function build_sample (line 1164) | def build_sample(
function run_ag_ui_row (line 1266) | async def run_ag_ui_row(
FILE: src/ragas/integrations/amazon_bedrock.py
function get_last_orchestration_value (line 7) | def get_last_orchestration_value(traces: t.List[t.Dict[str, t.Any]], key...
function extract_messages_from_model_invocation (line 25) | def extract_messages_from_model_invocation(model_inv):
function convert_to_ragas_messages (line 45) | def convert_to_ragas_messages(traces: t.List):
function extract_kb_trace (line 74) | def extract_kb_trace(traces):
FILE: src/ragas/integrations/griptape.py
function transform_to_ragas_dataset (line 13) | def transform_to_ragas_dataset(
FILE: src/ragas/integrations/helicone.py
class CacheConfig (line 6) | class CacheConfig:
class HeliconeSingleton (line 12) | class HeliconeSingleton:
method __new__ (line 41) | def __new__(cls):
method default_headers (line 46) | def default_headers(self) -> Dict[str, Any]:
method is_enabled (line 97) | def is_enabled(self):
FILE: src/ragas/integrations/langchain.py
class EvaluatorChain (line 32) | class EvaluatorChain(Chain, RunEvaluator):
method __init__ (line 39) | def __init__(self, metric: Metric, **kwargs: t.Any):
method input_keys (line 61) | def input_keys(self) -> list[str]:
method output_keys (line 65) | def output_keys(self) -> list[str]:
method _call (line 68) | def _call(
method _acall (line 98) | async def _acall(
method _validate (line 129) | def _validate(self, input: SingleTurnSample) -> None:
method _keys_are_present (line 140) | def _keys_are_present(keys_to_check: list, dict_to_check: dict) -> lis...
method _validate_langsmith_eval (line 143) | def _validate_langsmith_eval(self, run: Run, example: t.Optional[Examp...
method evaluate_run (line 176) | def evaluate_run(
FILE: src/ragas/integrations/langgraph.py
function convert_to_ragas_messages (line 9) | def convert_to_ragas_messages(
FILE: src/ragas/integrations/langsmith.py
function upload_dataset (line 23) | def upload_dataset(
function evaluate (line 82) | def evaluate(
FILE: src/ragas/integrations/llama_index.py
function evaluate (line 31) | def evaluate(
function convert_to_ragas_messages (line 120) | def convert_to_ragas_messages(events: t.List[Event]) -> t.List[Message]:
FILE: src/ragas/integrations/opik.py
class OpikTracer (line 22) | class OpikTracer(LangchainOpikTracer):
method _process_start_trace (line 36) | def _process_start_trace(self, run: "Run"):
method _process_end_trace (line 46) | def _process_end_trace(self, run: "Run"):
method _persist_run (line 64) | def _persist_run(self, run: "Run"):
FILE: src/ragas/integrations/r2r.py
function _process_search_results (line 16) | def _process_search_results(search_results: t.Dict[str, t.List]) -> t.Li...
function transform_to_ragas_dataset (line 51) | def transform_to_ragas_dataset(
FILE: src/ragas/integrations/swarm.py
function convert_to_ragas_messages (line 7) | def convert_to_ragas_messages(
FILE: src/ragas/integrations/tracing/__init__.py
function __getattr__ (line 53) | def __getattr__(name: str):
FILE: src/ragas/integrations/tracing/langfuse.py
class Observation (line 24) | class Observation: # type: ignore
class TraceWithFullDetails (line 27) | class TraceWithFullDetails: # type: ignore
method __init__ (line 28) | def __init__(
class Langfuse (line 52) | class Langfuse: # type: ignore
method get_current_trace_id (line 53) | def get_current_trace_id(self) -> t.Optional[str]: # type: ignore
method get_trace_url (line 56) | def get_trace_url(self) -> t.Optional[str]: # type: ignore
method get_dataset (line 59) | def get_dataset(self, *args, **kwargs): # type: ignore
function observe (line 62) | def observe(*args, **kwargs): # type: ignore
function observe (line 73) | def observe(*args, **kwargs): # type: ignore
class LangfuseTrace (line 83) | class LangfuseTrace:
method __init__ (line 84) | def __init__(self, trace: "TraceWithFullDetails"):
method get_url (line 88) | def get_url(self) -> t.Optional[str]:
method filter (line 91) | def filter(self, span_name: str) -> t.List["Observation"]:
function sync_trace (line 97) | async def sync_trace(
function add_query_param (line 146) | def add_query_param(url: str, param_name: str, param_value: str) -> str:
FILE: src/ragas/integrations/tracing/mlflow.py
class Span (line 23) | class Span: # type: ignore
class Trace (line 26) | class Trace: # type: ignore
method __init__ (line 27) | def __init__(self): # type: ignore
method search_spans (line 32) | def search_spans(self, name: str) -> t.List["Span"]: # type: ignore
function get_last_active_trace_id (line 35) | def get_last_active_trace_id() -> t.Optional[str]: # type: ignore
function get_trace (line 38) | def get_trace(trace_id: str) -> t.Optional["Trace"]: # type: ignore
class MLflowTrace (line 42) | class MLflowTrace:
method __init__ (line 43) | def __init__(self, trace: "Trace"):
method get_url (line 46) | def get_url(self) -> str:
method get_filter (line 64) | def get_filter(self, span_name: str) -> t.List["Span"]:
function sync_trace (line 68) | async def sync_trace() -> MLflowTrace:
FILE: src/ragas/llms/adapters/__init__.py
function get_adapter (line 12) | def get_adapter(name: str) -> t.Any:
function _is_new_google_genai_client (line 30) | def _is_new_google_genai_client(client: t.Any) -> bool:
function auto_detect_adapter (line 54) | def auto_detect_adapter(client: t.Any, provider: str) -> str:
FILE: src/ragas/llms/adapters/base.py
class StructuredOutputAdapter (line 5) | class StructuredOutputAdapter(ABC):
method create_llm (line 14) | def create_llm(
FILE: src/ragas/llms/adapters/instructor.py
class InstructorAdapter (line 7) | class InstructorAdapter(StructuredOutputAdapter):
method create_llm (line 14) | def create_llm(
FILE: src/ragas/llms/adapters/litellm.py
class LiteLLMAdapter (line 9) | class LiteLLMAdapter(StructuredOutputAdapter):
method create_llm (line 16) | def create_llm(
FILE: src/ragas/llms/base.py
function is_multiple_completion_supported (line 48) | def is_multiple_completion_supported(llm: BaseLanguageModel) -> bool:
class BaseRagasLLM (line 57) | class BaseRagasLLM(ABC):
method __post_init__ (line 62) | def __post_init__(self):
method set_run_config (line 68) | def set_run_config(self, run_config: RunConfig):
method get_temperature (line 71) | def get_temperature(self, n: int) -> float:
method generate_text (line 76) | def generate_text(
method agenerate_text (line 86) | async def agenerate_text(
method is_finished (line 96) | def is_finished(self, response: LLMResult) -> bool:
method generate (line 100) | async def generate(
class LangchainLLMWrapper (line 130) | class LangchainLLMWrapper(BaseRagasLLM):
method __init__ (line 147) | def __init__(
method is_finished (line 177) | def is_finished(self, response: LLMResult) -> bool:
method generate_text (line 227) | def generate_text(
method agenerate_text (line 279) | async def agenerate_text(
method set_run_config (line 332) | def set_run_config(self, run_config: RunConfig):
method __repr__ (line 348) | def __repr__(self) -> str:
class LlamaIndexLLMWrapper (line 352) | class LlamaIndexLLMWrapper(BaseRagasLLM):
method __init__ (line 366) | def __init__(
method check_args (line 397) | def check_args(
method is_finished (line 423) | def is_finished(self, response: LLMResult) -> bool:
method generate_text (line 426) | def generate_text(
method agenerate_text (line 439) | async def agenerate_text(
method __repr__ (line 459) | def __repr__(self) -> str:
function _patch_client_for_provider (line 463) | def _patch_client_for_provider(
function _is_new_google_genai_client (line 534) | def _is_new_google_genai_client(client: t.Any) -> bool:
function _get_instructor_client (line 566) | def _get_instructor_client(
function llm_factory (line 606) | def llm_factory(
class InstructorModelArgs (line 754) | class InstructorModelArgs(BaseModel):
class InstructorBaseRagasLLM (line 767) | class InstructorBaseRagasLLM(ABC):
method generate (line 771) | def generate(
method agenerate (line 780) | async def agenerate(
class InstructorLLM (line 788) | class InstructorLLM(InstructorBaseRagasLLM):
method __init__ (line 791) | def __init__(
method _map_provider_params (line 823) | def _map_provider_params(self) -> t.Dict[str, t.Any]:
method _map_openai_params (line 842) | def _map_openai_params(self) -> t.Dict[str, t.Any]:
method _map_google_params (line 922) | def _map_google_params(self) -> t.Dict[str, t.Any]:
method _check_client_async (line 946) | def _check_client_async(self) -> bool:
method _run_async_in_current_loop (line 1011) | def _run_async_in_current_loop(self, coro: t.Awaitable[t.Any]) -> t.Any:
method generate (line 1068) | def generate(
method agenerate (line 1117) | async def agenerate(
method _get_client_info (line 1165) | def _get_client_info(self) -> str:
method _get_key_config (line 1171) | def _get_key_config(self) -> str:
method __repr__ (line 1194) | def __repr__(self) -> str:
FILE: src/ragas/llms/haystack_wrapper.py
class HaystackLLMWrapper (line 22) | class HaystackLLMWrapper(BaseRagasLLM):
method __init__ (line 40) | def __init__(
method is_finished (line 95) | def is_finished(self, response: LLMResult) -> bool:
method generate_text (line 98) | def generate_text(
method agenerate_text (line 112) | async def agenerate_text(
method __repr__ (line 133) | def __repr__(self) -> str:
FILE: src/ragas/llms/litellm_llm.py
class LiteLLMStructuredLLM (line 14) | class LiteLLMStructuredLLM(InstructorBaseRagasLLM):
method __init__ (line 24) | def __init__(
method _check_client_async (line 58) | def _check_client_async(self) -> bool:
method _run_async_in_current_loop (line 118) | def _run_async_in_current_loop(self, coro: t.Awaitable[t.Any]) -> t.Any:
method generate (line 175) | def generate(
method agenerate (line 220) | async def agenerate(
method __repr__ (line 265) | def __repr__(self) -> str:
FILE: src/ragas/llms/oci_genai_wrapper.py
class OCIGenAIWrapper (line 35) | class OCIGenAIWrapper(BaseRagasLLM):
method __init__ (line 43) | def __init__(
method _convert_prompt_to_messages (line 101) | def _convert_prompt_to_messages(self, prompt: PromptValue) -> List[Dic...
method _create_generation_request (line 146) | def _create_generation_request(
method _get_client (line 172) | def _get_client(self):
method generate_text (line 188) | def generate_text(
method agenerate_text (line 239) | async def agenerate_text(
method is_finished (line 295) | def is_finished(self, response: LLMResult) -> bool:
method __repr__ (line 308) | def __repr__(self) -> str:
function oci_genai_factory (line 312) | def oci_genai_factory(
FILE: src/ragas/losses.py
class Loss (line 8) | class Loss(ABC):
method __call__ (line 14) | def __call__(self, predicted: t.List, actual: t.List) -> float:
method __get_pydantic_core_schema__ (line 18) | def __get_pydantic_core_schema__(
class MSELoss (line 30) | class MSELoss(Loss):
method __call__ (line 37) | def __call__(self, predicted: t.List[float], actual: t.List[float]) ->...
class BinaryMetricLoss (line 47) | class BinaryMetricLoss(Loss):
method __call__ (line 55) | def __call__(self, predicted: t.List[int], actual: t.List[int]) -> float:
method _accuracy (line 81) | def _accuracy(self, predicted: list[int], actual: t.List[int]) -> float:
method _f1_score (line 93) | def _f1_score(self, predicted: t.List[int], actual: t.List[int]) -> fl...
FILE: src/ragas/messages.py
class Message (line 6) | class Message(BaseModel):
class ToolCall (line 22) | class ToolCall(BaseModel):
class HumanMessage (line 39) | class HumanMessage(Message):
method pretty_repr (line 56) | def pretty_repr(self):
class ToolMessage (line 61) | class ToolMessage(Message):
method pretty_repr (line 78) | def pretty_repr(self):
class AIMessage (line 83) | class AIMessage(Message):
method to_dict (line 108) | def to_dict(self, **kwargs):
method pretty_repr (line 122) | def pretty_repr(self):
FILE: src/ragas/metrics/__init__.py
function __getattr__ (line 191) | def __getattr__(name: str):
FILE: src/ragas/metrics/_answer_correctness.py
class QuestionAnswerGroundTruth (line 34) | class QuestionAnswerGroundTruth(BaseModel):
class StatementsWithReason (line 40) | class StatementsWithReason(BaseModel):
class ClassificationWithReason (line 45) | class ClassificationWithReason(BaseModel):
class CorrectnessClassifier (line 51) | class CorrectnessClassifier(
class AnswerCorrectness (line 141) | class AnswerCorrectness(MetricWithLLM, MetricWithEmbeddings, SingleTurnM...
method __post_init__ (line 173) | def __post_init__(self):
method init (line 188) | def init(self, run_config: RunConfig):
method _compute_statement_presence (line 193) | def _compute_statement_presence(
method _create_simplified_statements (line 202) | async def _create_simplified_statements(
method _single_turn_ascore (line 216) | async def _single_turn_ascore(
method _ascore (line 223) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_answer_relevance.py
class ResponseRelevanceOutput (line 26) | class ResponseRelevanceOutput(BaseModel):
class ResponseRelevanceInput (line 31) | class ResponseRelevanceInput(BaseModel):
class ResponseRelevancePrompt (line 35) | class ResponseRelevancePrompt(
class ResponseRelevancy (line 64) | class ResponseRelevancy(MetricWithLLM, MetricWithEmbeddings, SingleTurnM...
method calculate_similarity (line 96) | def calculate_similarity(self, question: str, generated_questions: lis...
method _calculate_score (line 114) | def _calculate_score(
method _single_turn_ascore (line 131) | async def _single_turn_ascore(
method _ascore (line 137) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
class AnswerRelevancy (line 149) | class AnswerRelevancy(ResponseRelevancy):
method _ascore (line 150) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_answer_similarity.py
class SemanticSimilarity (line 26) | class SemanticSimilarity(MetricWithEmbeddings, SingleTurnMetric):
method __post_init__ (line 53) | def __post_init__(self):
method _single_turn_ascore (line 61) | async def _single_turn_ascore(
method _ascore (line 67) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
class AnswerSimilarity (line 109) | class AnswerSimilarity(SemanticSimilarity):
method _ascore (line 112) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_aspect_critic.py
class AspectCriticOutput (line 27) | class AspectCriticOutput(BaseModel):
class AspectCriticInput (line 32) | class AspectCriticInput(BaseModel):
class MultiTurnAspectCriticInput (line 50) | class MultiTurnAspectCriticInput(BaseModel):
class SingleTurnAspectCriticPrompt (line 59) | class SingleTurnAspectCriticPrompt(
class MultiTurnAspectCriticPrompt (line 67) | class MultiTurnAspectCriticPrompt(
class AspectCritic (line 75) | class AspectCritic(MetricWithLLM, SingleTurnMetric, MultiTurnMetric):
method __init__ (line 92) | def __init__(
method __repr__ (line 140) | def __repr__(self) -> str:
method definition (line 144) | def definition(self) -> str:
method definition (line 148) | def definition(self, value: str) -> None:
method _compute_score (line 155) | def _compute_score(
method _single_turn_ascore (line 167) | async def _single_turn_ascore(
method _ascore (line 173) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
method _multi_turn_ascore (line 198) | async def _multi_turn_ascore(
FILE: src/ragas/metrics/_bleu_score.py
class BleuScore (line 12) | class BleuScore(SingleTurnMetric):
method __post_init__ (line 19) | def __post_init__(self):
method init (line 28) | def init(self, run_config: RunConfig):
method _single_turn_ascore (line 31) | async def _single_turn_ascore(
method _ascore (line 47) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_chrf_score.py
class ChrfScore (line 12) | class ChrfScore(SingleTurnMetric):
method __post_init__ (line 19) | def __post_init__(self):
method init (line 28) | def init(self, run_config: RunConfig):
method _single_turn_ascore (line 31) | async def _single_turn_ascore(
method _ascore (line 54) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_context_entities_recall.py
class EntitiesList (line 25) | class EntitiesList(BaseModel):
class ExtractEntitiesPrompt (line 29) | class ExtractEntitiesPrompt(PydanticPrompt[StringIO, EntitiesList]):
class ContextEntityRecall (line 91) | class ContextEntityRecall(MetricWithLLM, SingleTurnMetric):
method _compute_score (line 125) | def _compute_score(
method get_entities (line 133) | async def get_entities(
method _single_turn_ascore (line 148) | async def _single_turn_ascore(
method _ascore (line 154) | async def _ascore(
FILE: src/ragas/metrics/_context_precision.py
class QAC (line 28) | class QAC(BaseModel):
class Verification (line 34) | class Verification(BaseModel):
class ContextPrecisionPrompt (line 39) | class ContextPrecisionPrompt(PydanticPrompt[QAC, Verification]):
class LLMContextPrecisionWithReference (line 82) | class LLMContextPrecisionWithReference(MetricWithLLM, SingleTurnMetric):
method _get_row_attributes (line 110) | def _get_row_attributes(self, row: t.Dict) -> t.Tuple[str, t.List[str]...
method _calculate_average_precision (line 113) | def _calculate_average_precision(
method _single_turn_ascore (line 134) | async def _single_turn_ascore(
method _ascore (line 140) | async def _ascore(
class LLMContextPrecisionWithoutReference (line 174) | class LLMContextPrecisionWithoutReference(LLMContextPrecisionWithReferen...
method _get_row_attributes (line 182) | def _get_row_attributes(self, row: t.Dict) -> t.Tuple[str, t.List[str]...
class NonLLMContextPrecisionWithReference (line 187) | class NonLLMContextPrecisionWithReference(SingleTurnMetric):
method __post_init__ (line 202) | def __post_init__(self):
method init (line 208) | def init(self, run_config: RunConfig) -> None: ...
method _ascore (line 210) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
method _single_turn_ascore (line 214) | async def _single_turn_ascore(
method _calculate_average_precision (line 237) | def _calculate_average_precision(self, verdict_list: t.List[int]) -> f...
class IDBasedContextPrecision (line 251) | class IDBasedContextPrecision(SingleTurnMetric):
method init (line 275) | def init(self, run_config: RunConfig) -> None: ...
method _single_turn_ascore (line 277) | async def _single_turn_ascore(
method _ascore (line 306) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
class ContextPrecision (line 311) | class ContextPrecision(LLMContextPrecisionWithReference):
method _single_turn_ascore (line 314) | async def _single_turn_ascore(
class ContextUtilization (line 321) | class ContextUtilization(LLMContextPrecisionWithoutReference):
method _single_turn_ascore (line 324) | async def _single_turn_ascore(
FILE: src/ragas/metrics/_context_recall.py
class QCA (line 29) | class QCA(BaseModel):
class ContextRecallClassification (line 35) | class ContextRecallClassification(BaseModel):
class ContextRecallClassifications (line 41) | class ContextRecallClassifications(BaseModel):
class ContextRecallClassificationPrompt (line 45) | class ContextRecallClassificationPrompt(
class LLMContextRecall (line 88) | class LLMContextRecall(MetricWithLLM, SingleTurnMetric):
method _compute_score (line 114) | def _compute_score(self, responses: t.List[ContextRecallClassification...
method _single_turn_ascore (line 125) | async def _single_turn_ascore(
method _ascore (line 131) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
class ContextRecall (line 160) | class ContextRecall(LLMContextRecall):
class NonLLMContextRecall (line 165) | class NonLLMContextRecall(SingleTurnMetric):
method init (line 181) | def init(self, run_config: RunConfig) -> None: ...
method distance_measure (line 184) | def distance_measure(self) -> SingleTurnMetric:
method distance_measure (line 188) | def distance_measure(self, distance_measure: DistanceMeasure) -> None:
method _single_turn_ascore (line 193) | async def _single_turn_ascore(
method _ascore (line 215) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
method _compute_score (line 218) | def _compute_score(self, verdict_list: t.List[float]) -> float:
class IDBasedContextRecall (line 227) | class IDBasedContextRecall(SingleTurnMetric):
method init (line 251) | def init(self, run_config: RunConfig) -> None: ...
method _single_turn_ascore (line 253) | async def _single_turn_ascore(
method _ascore (line 281) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_datacompy_score.py
class DataCompyScore (line 17) | class DataCompyScore(SingleTurnMetric):
method __post_init__ (line 25) | def __post_init__(self):
method init (line 42) | def init(self, run_config: RunConfig):
method _single_turn_ascore (line 45) | async def _single_turn_ascore(
method _ascore (line 77) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_domain_specific_rubrics.py
class ScoreFeedback (line 43) | class ScoreFeedback(BaseModel):
class SingleTurnInputWithoutRubric (line 48) | class SingleTurnInputWithoutRubric(BaseModel):
class MultiTurnInputWithoutRubric (line 66) | class MultiTurnInputWithoutRubric(BaseModel):
class SingleTurnPrompt (line 73) | class SingleTurnPrompt(PydanticPrompt[SingleTurnInputWithoutRubric, Scor...
class MultiTurnPrompt (line 79) | class MultiTurnPrompt(PydanticPrompt[MultiTurnInputWithoutRubric, ScoreF...
class RubricsScore (line 85) | class RubricsScore(MetricWithLLM, SingleTurnMetric, MultiTurnMetric):
method __init__ (line 86) | def __init__(
method __repr__ (line 129) | def __repr__(self) -> str:
method _single_turn_ascore (line 132) | async def _single_turn_ascore(
method _ascore (line 137) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
method _multi_turn_ascore (line 161) | async def _multi_turn_ascore(
FILE: src/ragas/metrics/_factual_correctness.py
class ClaimDecompositionInput (line 31) | class ClaimDecompositionInput(BaseModel):
class ClaimDecompositionOutput (line 35) | class ClaimDecompositionOutput(BaseModel):
class DecompositionType (line 40) | class DecompositionType(Enum):
class ClaimDecompositionPrompt (line 154) | class ClaimDecompositionPrompt(
class FactualCorrectness (line 166) | class FactualCorrectness(MetricWithLLM, SingleTurnMetric):
method __post_init__ (line 202) | def __post_init__(self):
method decompose_claims (line 225) | async def decompose_claims(
method verify_claims (line 236) | async def verify_claims(
method _get_passthrough_value (line 253) | async def _get_passthrough_value(value: T) -> T:
method _single_turn_ascore (line 256) | async def _single_turn_ascore(
method decompose_and_verify_claims (line 298) | async def decompose_and_verify_claims(
method _ascore (line 306) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_faithfulness.py
class StatementGeneratorInput (line 25) | class StatementGeneratorInput(BaseModel):
class StatementGeneratorOutput (line 30) | class StatementGeneratorOutput(BaseModel):
class StatementGeneratorPrompt (line 34) | class StatementGeneratorPrompt(
class StatementFaithfulnessAnswer (line 58) | class StatementFaithfulnessAnswer(BaseModel):
class NLIStatementOutput (line 64) | class NLIStatementOutput(BaseModel):
class NLIStatementInput (line 68) | class NLIStatementInput(BaseModel):
class NLIStatementPrompt (line 73) | class NLIStatementPrompt(PydanticPrompt[NLIStatementInput, NLIStatementO...
class Faithfulness (line 134) | class Faithfulness(MetricWithLLM, SingleTurnMetric):
method _create_verdicts (line 152) | async def _create_verdicts(
method _create_statements (line 166) | async def _create_statements(
method _compute_score (line 182) | def _compute_score(self, answers: NLIStatementOutput):
method _single_turn_ascore (line 196) | async def _single_turn_ascore(
method _ascore (line 202) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
class FaithfulnesswithHHEM (line 218) | class FaithfulnesswithHHEM(Faithfulness):
method __post_init__ (line 223) | def __post_init__(self):
method _create_pairs (line 236) | def _create_pairs(
method _create_batch (line 246) | def _create_batch(
method _ascore (line 253) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_goal_accuracy.py
class WorkflowOutput (line 21) | class WorkflowOutput(BaseModel):
class CompareOutcomeInput (line 30) | class CompareOutcomeInput(BaseModel):
class CompareOutcomeOutput (line 39) | class CompareOutcomeOutput(BaseModel):
class WorkflowInput (line 48) | class WorkflowInput(BaseModel):
class InferGoalOutcomePrompt (line 54) | class InferGoalOutcomePrompt(PydanticPrompt[WorkflowInput, WorkflowOutpu...
class CompareOutcomePrompt (line 85) | class CompareOutcomePrompt(PydanticPrompt[CompareOutcomeInput, CompareOu...
class AgentGoalAccuracyWithReference (line 104) | class AgentGoalAccuracyWithReference(MetricWithLLM, MultiTurnMetric):
method _ascore (line 123) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
method _multi_turn_ascore (line 126) | async def _multi_turn_ascore(
class AgentGoalAccuracyWithoutReference (line 148) | class AgentGoalAccuracyWithoutReference(MetricWithLLM, MultiTurnMetric):
method _ascore (line 165) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
method _multi_turn_ascore (line 168) | async def _multi_turn_ascore(
FILE: src/ragas/metrics/_instance_specific_rubrics.py
class SingleTurnInputWithRubric (line 28) | class SingleTurnInputWithRubric(SingleTurnInputWithoutRubric):
class MultiTurnInputWithRubric (line 34) | class MultiTurnInputWithRubric(MultiTurnInputWithoutRubric):
class SingleTurnPrompt (line 40) | class SingleTurnPrompt(PydanticPrompt[SingleTurnInputWithRubric, ScoreFe...
class MultiTurnPrompt (line 46) | class MultiTurnPrompt(PydanticPrompt[MultiTurnInputWithRubric, ScoreFeed...
class InstanceRubrics (line 52) | class InstanceRubrics(MetricWithLLM, SingleTurnMetric, MultiTurnMetric):
method __init__ (line 53) | def __init__(
method __repr__ (line 85) | def __repr__(self) -> str:
method _ascore (line 88) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
method _single_turn_ascore (line 116) | async def _single_turn_ascore(
method _multi_turn_ascore (line 122) | async def _multi_turn_ascore(
FILE: src/ragas/metrics/_multi_modal_faithfulness.py
class FaithfulnessInput (line 22) | class FaithfulnessInput(BaseModel):
method to_string_list (line 26) | def to_string_list(self):
class FaithfulnessOutput (line 34) | class FaithfulnessOutput(BaseModel):
class MultiModalFaithfulnessPrompt (line 38) | class MultiModalFaithfulnessPrompt(
class MultiModalFaithfulness (line 72) | class MultiModalFaithfulness(MetricWithLLM, SingleTurnMetric):
method _ascore (line 85) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
method _single_turn_ascore (line 97) | async def _single_turn_ascore(
FILE: src/ragas/metrics/_multi_modal_relevance.py
class RelevanceInput (line 22) | class RelevanceInput(BaseModel):
method to_string_list (line 27) | def to_string_list(self):
class RelevanceOutput (line 35) | class RelevanceOutput(BaseModel):
class MultiModalRelevancePrompt (line 39) | class MultiModalRelevancePrompt(ImageTextPrompt[RelevanceInput, Relevanc...
class MultiModalRelevance (line 77) | class MultiModalRelevance(MetricWithLLM, SingleTurnMetric):
method _ascore (line 92) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
method _single_turn_ascore (line 106) | async def _single_turn_ascore(
FILE: src/ragas/metrics/_noise_sensitivity.py
class NoiseSensitivity (line 32) | class NoiseSensitivity(MetricWithLLM, SingleTurnMetric):
method __post_init__ (line 52) | def __post_init__(self):
method _evaluate_statement_faithfulness (line 58) | async def _evaluate_statement_faithfulness(
method _decompose_answer_into_statements (line 74) | async def _decompose_answer_into_statements(
method _compute_score (line 87) | def _compute_score(self, answers: t.Dict) -> float:
method _single_turn_ascore (line 113) | async def _single_turn_ascore(
method _ascore (line 119) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_nv_metrics.py
class AnswerAccuracy (line 19) | class AnswerAccuracy(MetricWithLLM, SingleTurnMetric):
method process_score (line 85) | def process_score(self, response):
method average_scores (line 91) | def average_scores(self, score0, score1):
method _single_turn_ascore (line 99) | async def _single_turn_ascore(
class ContextRelevance (line 167) | class ContextRelevance(MetricWithLLM, SingleTurnMetric):
method process_score (line 220) | def process_score(self, response):
method average_scores (line 226) | def average_scores(self, score0, score1):
method _single_turn_ascore (line 234) | async def _single_turn_ascore(
class ResponseGroundedness (line 302) | class ResponseGroundedness(MetricWithLLM, SingleTurnMetric):
method process_score (line 354) | def process_score(self, response):
method average_scores (line 360) | def average_scores(self, score0, score1):
method _single_turn_ascore (line 368) | async def _single_turn_ascore(
FILE: src/ragas/metrics/_rouge_score.py
class RougeScore (line 12) | class RougeScore(SingleTurnMetric):
method __post_init__ (line 20) | def __post_init__(self):
method init (line 29) | def init(self, run_config: RunConfig):
method _single_turn_ascore (line 32) | async def _single_turn_ascore(
method _ascore (line 41) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_simple_criteria.py
class SimpleCriteriaOutput (line 28) | class SimpleCriteriaOutput(BaseModel):
class SingleTurnSimpleCriteriaInput (line 33) | class SingleTurnSimpleCriteriaInput(BaseModel):
class MultiTurnSimpleCriteriaInput (line 51) | class MultiTurnSimpleCriteriaInput(BaseModel):
class SingleTurnSimpleCriteriaPrompt (line 58) | class SingleTurnSimpleCriteriaPrompt(
class MultiTurnSimpleCriteriaPrompt (line 66) | class MultiTurnSimpleCriteriaPrompt(
class SimpleCriteriaScore (line 74) | class SimpleCriteriaScore(MetricWithLLM, SingleTurnMetric, MultiTurnMetr...
method __init__ (line 90) | def __init__(
method __repr__ (line 137) | def __repr__(self) -> str:
method definition (line 141) | def definition(self) -> str:
method definition (line 145) | def definition(self, value: str) -> None:
method _compute_score (line 152) | def _compute_score(
method _single_turn_ascore (line 164) | async def _single_turn_ascore(
method _ascore (line 170) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
method _multi_turn_ascore (line 195) | async def _multi_turn_ascore(
FILE: src/ragas/metrics/_sql_semantic_equivalence.py
class EquivalenceInput (line 25) | class EquivalenceInput(BaseModel):
class EquivalenceOutput (line 31) | class EquivalenceOutput(BaseModel):
class EquivalencePrompt (line 43) | class EquivalencePrompt(PydanticPrompt[EquivalenceInput, EquivalenceOutp...
class LLMSQLEquivalence (line 71) | class LLMSQLEquivalence(MetricWithLLM, SingleTurnMetric):
method _single_turn_ascore (line 81) | async def _single_turn_ascore(
method _ascore (line 102) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_string.py
class DistanceMeasure (line 12) | class DistanceMeasure(Enum):
class ExactMatch (line 20) | class ExactMatch(SingleTurnMetric):
method init (line 26) | def init(self, run_config: RunConfig):
method _single_turn_ascore (line 29) | async def _single_turn_ascore(
method _ascore (line 34) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
class StringPresence (line 39) | class StringPresence(SingleTurnMetric):
method init (line 45) | def init(self, run_config: RunConfig):
method _single_turn_ascore (line 48) | async def _single_turn_ascore(
method _ascore (line 57) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
class NonLLMStringSimilarity (line 62) | class NonLLMStringSimilarity(SingleTurnMetric):
method __post_init__ (line 69) | def __post_init__(self):
method init (line 84) | def init(self, run_config: RunConfig):
method _single_turn_ascore (line 87) | async def _single_turn_ascore(
method _ascore (line 98) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_summarization.py
class ExtractedKeyphrases (line 25) | class ExtractedKeyphrases(BaseModel):
class QuestionsGenerated (line 29) | class QuestionsGenerated(BaseModel):
class AnswersGenerated (line 33) | class AnswersGenerated(BaseModel):
class ExtractKeyphrasePrompt (line 37) | class ExtractKeyphrasePrompt(PydanticPrompt[StringIO, ExtractedKeyphrase...
class GenerateQuestionsPromptInput (line 61) | class GenerateQuestionsPromptInput(BaseModel):
class GenerateQuestionsPrompt (line 66) | class GenerateQuestionsPrompt(
class SummaryAndQuestions (line 100) | class SummaryAndQuestions(BaseModel):
class GenerateAnswersPrompt (line 105) | class GenerateAnswersPrompt(PydanticPrompt[SummaryAndQuestions, AnswersG...
class SummarizationScore (line 144) | class SummarizationScore(MetricWithLLM, SingleTurnMetric):
method _single_turn_ascore (line 168) | async def _single_turn_ascore(
method _ascore (line 174) | async def _ascore(self, row: Dict, callbacks: Callbacks) -> float:
method _compute_score (line 189) | def _compute_score(self, scores) -> float:
method _compute_qa_score (line 195) | def _compute_qa_score(self, answers: t.List[str]) -> float:
method _compute_conciseness_score (line 199) | def _compute_conciseness_score(self, text, summary) -> float:
method _extract_keyphrases (line 202) | async def _extract_keyphrases(self, text: str, callbacks: Callbacks) -...
method _get_questions (line 214) | async def _get_questions(
method _get_answers (line 229) | async def _get_answers(
FILE: src/ragas/metrics/_tool_call_accuracy.py
class ToolCallAccuracy (line 17) | class ToolCallAccuracy(MultiTurnMetric):
method init (line 62) | def init(self, run_config):
method _get_arg_score (line 65) | async def _get_arg_score(
method _sorted_key_for_tool_call (line 86) | def _sorted_key_for_tool_call(tc: ToolCall) -> t.Tuple[str, ...]:
method is_sequence_aligned (line 101) | def is_sequence_aligned(
method _multi_turn_ascore (line 110) | async def _multi_turn_ascore(
method _ascore (line 180) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_tool_call_f1.py
function _make_hashable (line 14) | def _make_hashable(obj: t.Any) -> t.Any:
class ToolCallF1 (line 26) | class ToolCallF1(MultiTurnMetric):
method init (line 39) | def init(self, run_config):
method _multi_turn_ascore (line 42) | async def _multi_turn_ascore(
method _ascore (line 70) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/_topic_adherence.py
class TopicExtractionInput (line 25) | class TopicExtractionInput(BaseModel):
class TopicExtractionOutput (line 29) | class TopicExtractionOutput(BaseModel):
class TopicRefusedInput (line 33) | class TopicRefusedInput(BaseModel):
class TopicRefusedOutput (line 38) | class TopicRefusedOutput(BaseModel):
class TopicClassificationInput (line 44) | class TopicClassificationInput(BaseModel):
class TopicClassificationOutput (line 49) | class TopicClassificationOutput(BaseModel):
class TopicClassificationPrompt (line 53) | class TopicClassificationPrompt(
class TopicRefusedPrompt (line 73) | class TopicRefusedPrompt(PydanticPrompt[TopicRefusedInput, TopicRefusedO...
class TopicExtractionPrompt (line 101) | class TopicExtractionPrompt(
class TopicAdherenceScore (line 136) | class TopicAdherenceScore(MetricWithLLM, MultiTurnMetric):
method _multi_turn_ascore (line 152) | async def _multi_turn_ascore(
method _ascore (line 250) | async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
FILE: src/ragas/metrics/base.py
class MetricType (line 51) | class MetricType(Enum):
class MetricOutputType (line 67) | class MetricOutputType(Enum):
class Metric (line 75) | class Metric(ABC):
method __post_init__ (line 91) | def __post_init__(self):
method required_columns (line 96) | def required_columns(self) -> t.Dict[str, t.Set[str]]:
method required_columns (line 108) | def required_columns(self, required_columns: t.Dict[MetricType, t.Set[...
method get_required_columns (line 120) | def get_required_columns(
method init (line 142) | def init(self, run_config: RunConfig) -> None:
class MetricWithLLM (line 155) | class MetricWithLLM(Metric, PromptMixin):
method init (line 169) | def init(self, run_config: RunConfig) -> None:
method _optimize_instruction (line 191) | def _optimize_instruction(
method _optimize_demonstration (line 249) | def _optimize_demonstration(
method train (line 307) | def train(
class MetricWithEmbeddings (line 379) | class MetricWithEmbeddings(Metric):
method init (line 382) | def init(self, run_config: RunConfig):
class SingleTurnMetric (line 392) | class SingleTurnMetric(Metric):
method _only_required_columns_single_turn (line 399) | def _only_required_columns_single_turn(
method single_turn_score (line 412) | def single_turn_score(
method single_turn_ascore (line 460) | async def single_turn_ascore(
method _single_turn_ascore (line 505) | async def _single_turn_ascore(
class MultiTurnMetric (line 516) | class MultiTurnMetric(Metric):
method _only_required_columns_multi_turn (line 524) | def _only_required_columns_multi_turn(
method multi_turn_score (line 537) | def multi_turn_score(
method multi_turn_ascore (line 584) | async def multi_turn_ascore(
method _multi_turn_ascore (line 630) | async def _multi_turn_ascore(
class Ensember (line 641) | class Ensember:
method from_discrete (line 646) | def from_discrete(
class ModeMetric (line 680) | class ModeMetric(t.Protocol):
class SimpleBaseMetric (line 689) | class SimpleBaseMetric(ABC):
method score (line 721) | def score(self, **kwargs) -> "MetricResult":
method ascore (line 738) | async def ascore(self, **kwargs) -> "MetricResult":
method batch_score (line 754) | def batch_score(
method abatch_score (line 773) | async def abatch_score(
function create_auto_response_model (line 799) | def create_auto_response_model(name: str, **fields) -> t.Type["BaseModel"]:
class SimpleLLMMetric (line 846) | class SimpleLLMMetric(SimpleBaseMetric):
method __post_init__ (line 852) | def __post_init__(self):
method get_variables (line 858) | def get_variables(self) -> t.List[str]:
method score (line 874) | def score(self, **kwargs) -> "MetricResult":
method ascore (line 892) | async def ascore(self, **kwargs) -> "MetricResult":
method batch_score (line 913) | def batch_score(
method abatch_score (line 924) | async def abatch_score(
method save (line 935) | def save(self, path: t.Optional[str] = None) -> None:
method _serialize_prompt (line 1023) | def _serialize_prompt(self, warning_messages: t.List[str]) -> t.Dict[s...
method _get_metric_config (line 1057) | def _get_metric_config(self) -> t.Dict[str, t.Any]:
method _serialize_response_model_info (line 1067) | def _serialize_response_model_info(self) -> t.Optional[t.Dict]:
method _read_metric_type (line 1084) | def _read_metric_type(cls, path: str) -> t.Dict[str, t.Any]:
method _validate_metric_type (line 1121) | def _validate_metric_type(cls, path: str) -> None:
method load (line 1146) | def load(
method _deserialize_prompt (line 1215) | def _deserialize_prompt(
method get_correlation (line 1276) | def get_correlation(
method align_and_validate (line 1285) | def align_and_validate(
method align (line 1310) | def align(
method validate_alignment (line 1395) | def validate_alignment(
method __repr__ (line 1445) | def __repr__(self) -> str:
FILE: src/ragas/metrics/collections/_bleu_score.py
class BleuScore (line 9) | class BleuScore(BaseMetric):
method __init__ (line 38) | def __init__(
method ascore (line 48) | async def ascore(
FILE: src/ragas/metrics/collections/_rouge_score.py
class RougeScore (line 9) | class RougeScore(BaseMetric):
method __init__ (line 44) | def __init__(
method ascore (line 56) | async def ascore(
FILE: src/ragas/metrics/collections/_semantic_similarity.py
class SemanticSimilarity (line 14) | class SemanticSimilarity(BaseMetric):
method __init__ (line 56) | def __init__(
method ascore (line 69) | async def ascore(self, reference: str, response: str) -> MetricResult:
FILE: src/ragas/metrics/collections/_string.py
class DistanceMeasure (line 9) | class DistanceMeasure(Enum):
class ExactMatch (line 16) | class ExactMatch(BaseMetric):
method __init__ (line 44) | def __init__(
method ascore (line 52) | async def ascore(
class StringPresence (line 71) | class StringPresence(BaseMetric):
method __init__ (line 99) | def __init__(
method ascore (line 107) | async def ascore(
class NonLLMStringSimilarity (line 133) | class NonLLMStringSimilarity(BaseMetric):
method __init__ (line 162) | def __init__(
method ascore (line 187) | async def ascore(
FILE: src/ragas/metrics/collections/agent_goal_accuracy/metric.py
class AgentGoalAccuracyWithReference (line 23) | class AgentGoalAccuracyWithReference(BaseMetric):
method __init__ (line 63) | def __init__(
method ascore (line 75) | async def ascore(
method _format_conversation (line 107) | def _format_conversation(
method _infer_goal_outcome (line 116) | async def _infer_goal_outcome(self, conversation: str) -> WorkflowOutput:
method _compare_outcomes (line 122) | async def _compare_outcomes(self, desired: str, arrived: str) -> int:
class AgentGoalAccuracyWithoutReference (line 132) | class AgentGoalAccuracyWithoutReference(BaseMetric):
method __init__ (line 172) | def __init__(
method ascore (line 184) | async def ascore(
method _format_conversation (line 212) | def _format_conversation(
method _infer_goal_outcome (line 221) | async def _infer_goal_outcome(self, conversation: str) -> WorkflowOutput:
method _compare_outcomes (line 227) | async def _compare_outcomes(self, desired: str, arrived: str) -> int:
FILE: src/ragas/metrics/collections/agent_goal_accuracy/util.py
class WorkflowInput (line 10) | class WorkflowInput(BaseModel):
class WorkflowOutput (line 16) | class WorkflowOutput(BaseModel):
class InferGoalOutcomePrompt (line 25) | class InferGoalOutcomePrompt(BasePrompt[WorkflowInput, WorkflowOutput]):
class CompareOutcomeInput (line 58) | class CompareOutcomeInput(BaseModel):
class CompareOutcomeOutput (line 67) | class CompareOutcomeOutput(BaseModel):
class CompareOutcomePrompt (line 76) | class CompareOutcomePrompt(BasePrompt[CompareOutcomeInput, CompareOutcom...
FILE: src/ragas/metrics/collections/answer_accuracy/metric.py
class AnswerAccuracy (line 21) | class AnswerAccuracy(BaseMetric):
method __init__ (line 67) | def __init__(
method ascore (line 91) | async def ascore(
method _get_judge_rating (line 132) | async def _get_judge_rating(
method _average_scores (line 165) | def _average_scores(self, score1: float, score2: float) -> float:
FILE: src/ragas/metrics/collections/answer_accuracy/util.py
class AnswerAccuracyInput (line 8) | class AnswerAccuracyInput(BaseModel):
class AnswerAccuracyOutput (line 16) | class AnswerAccuracyOutput(BaseModel):
class AnswerAccuracyJudge1Prompt (line 22) | class AnswerAccuracyJudge1Prompt(BasePrompt[AnswerAccuracyInput, AnswerA...
class AnswerAccuracyJudge2Prompt (line 63) | class AnswerAccuracyJudge2Prompt(BasePrompt[AnswerAccuracyInput, AnswerA...
FILE: src/ragas/metrics/collections/answer_correctness/metric.py
class AnswerCorrectness (line 25) | class AnswerCorrectness(BaseMetric):
method __init__ (line 78) | def __init__(
method _validate_embeddings (line 143) | def _validate_embeddings(self) -> None:
method ascore (line 154) | async def ascore(
method _generate_statements (line 198) | async def _generate_statements(self, question: str, text: str) -> List...
method _classify_statements (line 205) | async def _classify_statements(
method _compute_f1_score (line 221) | def _compute_f1_score(self, classification: ClassificationWithReason) ...
method _calculate_similarity (line 251) | async def _calculate_similarity(self, response: str, reference: str) -...
FILE: src/ragas/metrics/collections/answer_correctness/util.py
class StatementGeneratorInput (line 10) | class StatementGeneratorInput(BaseModel):
class StatementGeneratorOutput (line 19) | class StatementGeneratorOutput(BaseModel):
class StatementGeneratorPrompt (line 27) | class StatementGeneratorPrompt(
class StatementsWithReason (line 55) | class StatementsWithReason(BaseModel):
class ClassificationWithReason (line 62) | class ClassificationWithReason(BaseModel):
class CorrectnessClassifierInput (line 76) | class CorrectnessClassifierInput(BaseModel):
class CorrectnessClassifierPrompt (line 84) | class CorrectnessClassifierPrompt(
FILE: src/ragas/metrics/collections/answer_relevancy/metric.py
class AnswerRelevancy (line 21) | class AnswerRelevancy(BaseMetric):
method __init__ (line 66) | def __init__(
method ascore (line 93) | async def ascore(self, user_input: str, response: str) -> MetricResult:
FILE: src/ragas/metrics/collections/answer_relevancy/util.py
class AnswerRelevanceInput (line 8) | class AnswerRelevanceInput(BaseModel):
class AnswerRelevanceOutput (line 16) | class AnswerRelevanceOutput(BaseModel):
class AnswerRelevancePrompt (line 28) | class AnswerRelevancePrompt(BasePrompt[AnswerRelevanceInput, AnswerRelev...
FILE: src/ragas/metrics/collections/base.py
class BaseMetric (line 13) | class BaseMetric(SimpleBaseMetric, NumericValidator):
method __init__ (line 32) | def __init__(
method ascore (line 48) | async def ascore(self, **kwargs) -> MetricResult:
method score (line 61) | def score(self, **kwargs) -> MetricResult:
method batch_score (line 84) | def batch_score(
method _validate_llm (line 113) | def _validate_llm(self):
method _validate_embeddings (line 123) | def _validate_embeddings(self):
FILE: src/ragas/metrics/collections/chrf_score/metric.py
class CHRFScore (line 9) | class CHRFScore(BaseMetric):
method __init__ (line 44) | def __init__(
method ascore (line 54) | async def ascore(
FILE: src/ragas/metrics/collections/context_entity_recall/metric.py
class ContextEntityRecall (line 19) | class ContextEntityRecall(BaseMetric):
method __init__ (line 59) | def __init__(
method ascore (line 80) | async def ascore(
method _extract_entities (line 107) | async def _extract_entities(self, text: str) -> List[str]:
method _compute_recall_score (line 123) | def _compute_recall_score(
FILE: src/ragas/metrics/collections/context_entity_recall/util.py
class ExtractEntitiesInput (line 10) | class ExtractEntitiesInput(BaseModel):
class EntitiesList (line 16) | class EntitiesList(BaseModel):
class ExtractEntitiesPrompt (line 24) | class ExtractEntitiesPrompt(BasePrompt[ExtractEntitiesInput, EntitiesLis...
FILE: src/ragas/metrics/collections/context_precision/metric.py
class ContextPrecisionWithReference (line 21) | class ContextPrecisionWithReference(BaseMetric):
method __init__ (line 61) | def __init__(
method ascore (line 81) | async def ascore(
method _calculate_average_precision (line 118) | def _calculate_average_precision(self, verdicts: List[int]) -> float:
class ContextPrecisionWithoutReference (line 141) | class ContextPrecisionWithoutReference(BaseMetric):
method __init__ (line 181) | def __init__(
method ascore (line 201) | async def ascore(
method _calculate_average_precision (line 238) | def _calculate_average_precision(self, verdicts: List[int]) -> float:
class ContextPrecision (line 261) | class ContextPrecision(ContextPrecisionWithReference):
method __init__ (line 288) | def __init__(
class ContextUtilization (line 297) | class ContextUtilization(ContextPrecisionWithoutReference):
method __init__ (line 324) | def __init__(
FILE: src/ragas/metrics/collections/context_precision/util.py
class ContextPrecisionInput (line 8) | class ContextPrecisionInput(BaseModel):
class ContextPrecisionOutput (line 18) | class ContextPrecisionOutput(BaseModel):
class ContextPrecisionPrompt (line 25) | class ContextPrecisionPrompt(BasePrompt[ContextPrecisionInput, ContextPr...
FILE: src/ragas/metrics/collections/context_recall/metric.py
class ContextRecall (line 21) | class ContextRecall(BaseMetric):
method __init__ (line 61) | def __init__(
method ascore (line 82) | async def ascore(
FILE: src/ragas/metrics/collections/context_recall/util.py
class ContextRecallInput (line 10) | class ContextRecallInput(BaseModel):
class ContextRecallClassification (line 20) | class ContextRecallClassification(BaseModel):
class ContextRecallOutput (line 36) | class ContextRecallOutput(BaseModel):
class ContextRecallPrompt (line 44) | class ContextRecallPrompt(BasePrompt[ContextRecallInput, ContextRecallOu...
FILE: src/ragas/metrics/collections/context_relevance/metric.py
class ContextRelevance (line 22) | class ContextRelevance(BaseMetric):
method __init__ (line 68) | def __init__(
method ascore (line 92) | async def ascore(
method _get_judge_rating (line 142) | async def _get_judge_rating(
method _average_scores (line 172) | def _average_scores(self, score1: float, score2: float) -> float:
FILE: src/ragas/metrics/collections/context_relevance/util.py
class ContextRelevanceInput (line 8) | class ContextRelevanceInput(BaseModel):
class ContextRelevanceOutput (line 15) | class ContextRelevanceOutput(BaseModel):
class ContextRelevanceJudge1Prompt (line 21) | class ContextRelevanceJudge1Prompt(
class ContextRelevanceJudge2Prompt (line 66) | class ContextRelevanceJudge2Prompt(
FILE: src/ragas/metrics/collections/datacompy_score/metric.py
class DataCompyScore (line 15) | class DataCompyScore(BaseMetric):
method __init__ (line 45) | def __init__(
method ascore (line 80) | async def ascore(
FILE: src/ragas/metrics/collections/domain_specific_rubrics/metric.py
class DomainSpecificRubrics (line 21) | class DomainSpecificRubrics(BaseMetric):
method __init__ (line 84) | def __init__(
method ascore (line 112) | async def ascore(
class RubricsScoreWithoutReference (line 149) | class RubricsScoreWithoutReference(DomainSpecificRubrics):
method __init__ (line 156) | def __init__(
class RubricsScoreWithReference (line 168) | class RubricsScoreWithReference(DomainSpecificRubrics):
method __init__ (line 175) | def __init__(
FILE: src/ragas/metrics/collections/domain_specific_rubrics/util.py
class RubricScoreInput (line 26) | class RubricScoreInput(BaseModel):
class RubricScoreOutput (line 46) | class RubricScoreOutput(BaseModel):
class RubricScorePrompt (line 53) | class RubricScorePrompt(BasePrompt[RubricScoreInput, RubricScoreOutput]):
function format_rubrics (line 87) | def format_rubrics(rubrics: t.Dict[str, str]) -> str:
FILE: src/ragas/metrics/collections/example_metric.py
class ExampleMetric (line 7) | class ExampleMetric(BaseMetric):
method __init__ (line 21) | def __init__(self, name: str = "example_metric", **kwargs):
method ascore (line 25) | async def ascore(self, user_input: str, response: str) -> MetricResult:
FILE: src/ragas/metrics/collections/factual_correctness/metric.py
class FactualCorrectness (line 25) | class FactualCorrectness(BaseMetric):
method __init__ (line 75) | def __init__(
method ascore (line 114) | async def ascore(self, response: str, reference: str) -> MetricResult:
method _decompose_claims (line 167) | async def _decompose_claims(self, text: str) -> List[str]:
method _verify_claims (line 176) | async def _verify_claims(
method _decompose_and_verify_claims (line 185) | async def _decompose_and_verify_claims(
FILE: src/ragas/metrics/collections/factual_correctness/util.py
class ClaimDecompositionInput (line 15) | class ClaimDecompositionInput(BaseModel):
class ClaimDecompositionOutput (line 25) | class ClaimDecompositionOutput(BaseModel):
class ClaimDecompositionPrompt (line 31) | class ClaimDecompositionPrompt(
method to_string (line 162) | def to_string(self, input_data: ClaimDecompositionInput) -> str:
method adapt (line 176) | async def adapt(
class NLIStatementInput (line 238) | class NLIStatementInput(BaseModel):
class StatementFaithfulnessAnswer (line 247) | class StatementFaithfulnessAnswer(BaseModel):
class NLIStatementOutput (line 255) | class NLIStatementOutput(BaseModel):
class NLIStatementPrompt (line 261) | class NLIStatementPrompt(BasePrompt[NLIStatementInput, NLIStatementOutpu...
FILE: src/ragas/metrics/collections/faithfulness/metric.py
class Faithfulness (line 22) | class Faithfulness(BaseMetric):
method __init__ (line 67) | def __init__(
method ascore (line 88) | async def ascore(
method _create_statements (line 132) | async def _create_statements(self, question: str, response: str) -> Li...
method _create_verdicts (line 139) | async def _create_verdicts(
method _compute_score (line 148) | def _compute_score(self, verdicts: NLIStatementOutput) -> float:
FILE: src/ragas/metrics/collections/faithfulness/util.py
class StatementGeneratorInput (line 10) | class StatementGeneratorInput(BaseModel):
class StatementGeneratorOutput (line 19) | class StatementGeneratorOutput(BaseModel):
class StatementGeneratorPrompt (line 27) | class StatementGeneratorPrompt(
class StatementFaithfulnessAnswer (line 55) | class StatementFaithfulnessAnswer(BaseModel):
class NLIStatementInput (line 63) | class NLIStatementInput(BaseModel):
class NLIStatementOutput (line 72) | class NLIStatementOutput(BaseModel):
class NLIStatementPrompt (line 80) | class NLIStatementPrompt(BasePrompt[NLIStatementInput, NLIStatementOutpu...
FILE: src/ragas/metrics/collections/instance_specific_rubrics/metric.py
class InstanceSpecificRubrics (line 18) | class InstanceSpecificRubrics(BaseMetric):
method __init__ (line 66) | def __init__(
method ascore (line 77) | async def ascore(
FILE: src/ragas/metrics/collections/instance_specific_rubrics/util.py
class InstanceRubricScoreInput (line 10) | class InstanceRubricScoreInput(BaseModel):
class InstanceRubricScoreOutput (line 33) | class InstanceRubricScoreOutput(BaseModel):
class InstanceRubricScorePrompt (line 40) | class InstanceRubricScorePrompt(
FILE: src/ragas/metrics/collections/multi_modal_faithfulness/metric.py
class MultiModalFaithfulness (line 19) | class MultiModalFaithfulness(BaseMetric):
method __init__ (line 68) | def __init__(
method ascore (line 84) | async def ascore(
method _evaluate_faithfulness (line 127) | async def _evaluate_faithfulness(
FILE: src/ragas/metrics/collections/multi_modal_faithfulness/util.py
class MultiModalFaithfulnessInput (line 28) | class MultiModalFaithfulnessInput(BaseModel):
class MultiModalFaithfulnessOutput (line 38) | class MultiModalFaithfulnessOutput(BaseModel):
function is_image_path_or_url (line 54) | def is_image_path_or_url(item: str) -> bool:
function process_image_to_base64 (line 84) | def process_image_to_base64(item: str) -> t.Optional[t.Dict[str, str]]:
function _try_process_base64_uri (line 108) | def _try_process_base64_uri(item: str) -> t.Optional[t.Dict[str, str]]:
function _try_process_url (line 123) | def _try_process_url(item: str) -> t.Optional[t.Dict[str, str]]:
function _try_process_local_file (line 178) | def _try_process_local_file(item: str) -> t.Optional[t.Dict[str, str]]:
function build_multimodal_message_content (line 213) | def build_multimodal_message_content(
FILE: src/ragas/metrics/collections/multi_modal_relevance/metric.py
class MultiModalRelevance (line 19) | class MultiModalRelevance(BaseMetric):
method __init__ (line 69) | def __init__(
method ascore (line 85) | async def ascore(
method _evaluate_relevance (line 135) | async def _evaluate_relevance(
FILE: src/ragas/metrics/collections/multi_modal_relevance/util.py
class MultiModalRelevanceInput (line 13) | class MultiModalRelevanceInput(BaseModel):
class MultiModalRelevanceOutput (line 24) | class MultiModalRelevanceOutput(BaseModel):
function build_multimodal_relevance_message_content (line 37) | def build_multimodal_relevance_message_content(
FILE: src/ragas/metrics/collections/noise_sensitivity/metric.py
class NoiseSensitivity (line 24) | class NoiseSensitivity(BaseMetric):
method __init__ (line 74) | def __init__(
method ascore (line 104) | async def ascore(
method _decompose_answer_into_statements (line 187) | async def _decompose_answer_into_statements(
method _evaluate_statement_faithfulness (line 196) | async def _evaluate_statement_faithfulness(
method _compute_score (line 209) | def _compute_score(self, answers: Dict) -> float:
FILE: src/ragas/metrics/collections/noise_sensitivity/util.py
class StatementGeneratorInput (line 11) | class StatementGeneratorInput(BaseModel):
class StatementGeneratorOutput (line 18) | class StatementGeneratorOutput(BaseModel):
class StatementGeneratorPrompt (line 24) | class StatementGeneratorPrompt(
method to_string (line 32) | def to_string(self, input_data: StatementGeneratorInput) -> str:
class StatementFaithfulnessInput (line 37) | class StatementFaithfulnessInput(BaseModel):
class StatementFaithfulnessAnswer (line 44) | class StatementFaithfulnessAnswer(BaseModel):
class StatementFaithfulnessOutput (line 52) | class StatementFaithfulnessOutput(BaseModel):
class StatementFaithfulnessPrompt (line 58) | class StatementFaithfulnessPrompt(
method to_string (line 66) | def to_string(self, input_data: StatementFaithfulnessInput) -> str:
FILE: src/ragas/metrics/collections/quoted_spans/metric.py
class QuotedSpansAlignment (line 11) | class QuotedSpansAlignment(BaseMetric):
method __init__ (line 48) | def __init__(
method ascore (line 68) | async def ascore(
FILE: src/ragas/metrics/collections/quoted_spans/util.py
function normalize_text (line 13) | def normalize_text(text: str) -> str:
function extract_quoted_spans (line 18) | def extract_quoted_spans(answer: str, min_len: int = 3) -> t.List[str]:
function count_matched_spans (line 38) | def count_matched_spans(
FILE: src/ragas/metrics/collections/response_groundedness/metric.py
class ResponseGroundedness (line 22) | class ResponseGroundedness(BaseMetric):
method __init__ (line 68) | def __init__(
method ascore (line 92) | async def ascore(
method _get_judge_rating (line 134) | async def _get_judge_rating(self, prompt_obj, response: str, context: ...
method _average_scores (line 164) | def _average_scores(self, score1: float, score2: float) -> float:
FILE: src/ragas/metrics/collections/response_groundedness/util.py
class ResponseGroundednessInput (line 8) | class ResponseGroundednessInput(BaseModel):
class ResponseGroundednessOutput (line 15) | class ResponseGroundednessOutput(BaseModel):
class ResponseGroundednessJudge1Prompt (line 21) | class ResponseGroundednessJudge1Prompt(
class ResponseGroundednessJudge2Prompt (line 65) | class ResponseGroundednessJudge2Prompt(
FILE: src/ragas/metrics/collections/sql_semantic_equivalence/metric.py
class SQLSemanticEquivalence (line 15) | class SQLSemanticEquivalence(BaseMetric):
method __init__ (line 53) | def __init__(
method ascore (line 63) | async def ascore(
FILE: src/ragas/metrics/collections/sql_semantic_equivalence/util.py
class SQLEquivalenceInput (line 10) | class SQLEquivalenceInput(BaseModel):
class SQLEquivalenceOutput (line 16) | class SQLEquivalenceOutput(BaseModel):
class SQLEquivalencePrompt (line 28) | class SQLEquivalencePrompt(BasePrompt[SQLEquivalenceInput, SQLEquivalenc...
FILE: src/ragas/metrics/collections/summary_score/metric.py
class SummaryScore (line 26) | class SummaryScore(BaseMetric):
method __init__ (line 77) | def __init__(
method ascore (line 109) | async def ascore(
method _extract_keyphrases (line 166) | async def _extract_keyphrases(self, text: str) -> List[str]:
method _generate_questions (line 173) | async def _generate_questions(self, text: str, keyphrases: List[str]) ...
method _generate_answers (line 180) | async def _generate_answers(self, summary: str, questions: List[str]) ...
method _compute_qa_score (line 187) | def _compute_qa_score(self, answers: List[str]) -> float:
method _compute_conciseness_score (line 194) | def _compute_conciseness_score(self, text: str, summary: str) -> float:
FILE: src/ragas/metrics/collections/summary_score/util.py
class ExtractedKeyphrasesInput (line 10) | class ExtractedKeyphrasesInput(BaseModel):
class ExtractedKeyphrases (line 16) | class ExtractedKeyphrases(BaseModel):
class ExtractKeyphrasesPrompt (line 22) | class ExtractKeyphrasesPrompt(
class GenerateQuestionsInput (line 51) | class GenerateQuestionsInput(BaseModel):
class QuestionsGenerated (line 60) | class QuestionsGenerated(BaseModel):
class GenerateQuestionsPrompt (line 66) | class GenerateQuestionsPrompt(BasePrompt[GenerateQuestionsInput, Questio...
class GenerateAnswersInput (line 101) | class GenerateAnswersInput(BaseModel):
class AnswersGenerated (line 110) | class AnswersGenerated(BaseModel):
class GenerateAnswersPrompt (line 118) | class GenerateAnswersPrompt(BasePrompt[GenerateAnswersInput, AnswersGene...
FILE: src/ragas/metrics/collections/tool_call_accuracy/metric.py
class ToolCallAccuracy (line 17) | class ToolCallAccuracy(BaseMetric):
method __init__ (line 62) | def __init__(
method _is_sequence_aligned (line 80) | def _is_sequence_aligned(
method ascore (line 89) | async def ascore(
FILE: src/ragas/metrics/collections/tool_call_accuracy/util.py
function sorted_key_for_tool_call (line 8) | def sorted_key_for_tool_call(tc: ToolCall) -> t.Tuple[str, ...]:
function exact_match_args (line 24) | def exact_match_args(
FILE: src/ragas/metrics/collections/tool_call_f1/metric.py
class ToolCallF1 (line 15) | class ToolCallF1(BaseMetric):
method __init__ (line 59) | def __init__(self, name: str = "tool_call_f1", **kwargs):
method ascore (line 69) | async def ascore(
FILE: src/ragas/metrics/collections/tool_call_f1/util.py
function make_hashable (line 8) | def make_hashable(obj: t.Any) -> t.Any:
function tool_call_to_hashable (line 34) | def tool_call_to_hashable(tc: ToolCall) -> t.Tuple[str, t.FrozenSet]:
function calculate_f1_score (line 47) | def calculate_f1_score(
FILE: src/ragas/metrics/collections/topic_adherence/metric.py
class TopicAdherence (line 28) | class TopicAdherence(BaseMetric):
method __init__ (line 75) | def __init__(
method ascore (line 90) | async def ascore(
method _format_conversation (line 129) | def _format_conversation(
method _extract_topics (line 138) | async def _extract_topics(self, conversation: str) -> List[str]:
method _check_topics_answered (line 145) | async def _check_topics_answered(
method _classify_topics (line 158) | async def _classify_topics(
method _safe_bool_conversion (line 180) | def _safe_bool_conversion(self, classifications: List) -> np.ndarray:
method _compute_score (line 201) | def _compute_score(
FILE: src/ragas/metrics/collections/topic_adherence/util.py
class TopicExtractionInput (line 10) | class TopicExtractionInput(BaseModel):
class TopicExtractionOutput (line 16) | class TopicExtractionOutput(BaseModel):
class TopicExtractionPrompt (line 22) | class TopicExtractionPrompt(BasePrompt[TopicExtractionInput, TopicExtrac...
class TopicRefusedInput (line 58) | class TopicRefusedInput(BaseModel):
class TopicRefusedOutput (line 65) | class TopicRefusedOutput(BaseModel):
class TopicRefusedPrompt (line 71) | class TopicRefusedPrompt(BasePrompt[TopicRefusedInput, TopicRefusedOutpu...
class TopicClassificationInput (line 103) | class TopicClassificationInput(BaseModel):
class TopicClassificationOutput (line 110) | class TopicClassificationOutput(BaseModel):
class TopicClassificationPrompt (line 117) | class TopicClassificationPrompt(
FILE: src/ragas/metrics/decorator.py
class BaseMetricProtocol (line 37) | class BaseMetricProtocol(Protocol):
method score (line 42) | def score(self, **kwargs) -> MetricResult:
method ascore (line 46) | async def ascore(self, **kwargs) -> MetricResult:
method batch_score (line 50) | def batch_score(self, inputs: t.List[t.Dict[str, t.Any]]) -> t.List[Me...
method abatch_score (line 54) | async def abatch_score(
method __call__ (line 60) | def __call__(self, *args, **kwargs):
class DiscreteMetricProtocol (line 65) | class DiscreteMetricProtocol(BaseMetricProtocol, Protocol):
class NumericMetricProtocol (line 71) | class NumericMetricProtocol(BaseMetricProtocol, Protocol):
class RankingMetricProtocol (line 77) | class RankingMetricProtocol(BaseMetricProtocol, Protocol):
function create_metric_decorator (line 83) | def create_metric_decorator():
FILE: src/ragas/metrics/discrete.py
class DiscreteMetric (line 19) | class DiscreteMetric(SimpleLLMMetric, DiscreteValidator):
method __post_init__ (line 63) | def __post_init__(self):
method get_correlation (line 75) | def get_correlation(
method load (line 92) | def load(
function discrete_metric (line 128) | def discrete_metric(
FILE: src/ragas/metrics/numeric.py
class NumericMetric (line 17) | class NumericMetric(SimpleLLMMetric, NumericValidator):
method __post_init__ (line 64) | def __post_init__(self):
method get_correlation (line 73) | def get_correlation(
method load (line 96) | def load(
function numeric_metric (line 145) | def numeric_metric(
FILE: src/ragas/metrics/quoted_spans.py
function _normalize (line 27) | def _normalize(text: str) -> str:
function _extract_quoted_spans (line 32) | def _extract_quoted_spans(answer: str, *, min_len: int = 3) -> Sequence[...
function quoted_spans_alignment (line 59) | def quoted_spans_alignment(
FILE: src/ragas/metrics/ranking.py
class RankingMetric (line 19) | class RankingMetric(SimpleLLMMetric, RankingValidator):
method __post_init__ (line 65) | def __post_init__(self):
method get_correlation (line 76) | def get_correlation(
method load (line 99) | def load(
function ranking_metric (line 135) | def ranking_metric(
FILE: src/ragas/metrics/result.py
class MetricResult (line 11) | class MetricResult:
method __init__ (line 23) | def __init__(
method __repr__ (line 41) | def __repr__(self):
method value (line 50) | def value(self):
method __getitem__ (line 55) | def __getitem__(self, key):
method __iter__ (line 60) | def __iter__(self):
method __len__ (line 65) | def __len__(self):
method __float__ (line 71) | def __float__(self):
method __int__ (line 76) | def __int__(self):
method __add__ (line 81) | def __add__(self, other):
method __radd__ (line 88) | def __radd__(self, other):
method __sub__ (line 93) | def __sub__(self, other):
method __rsub__ (line 100) | def __rsub__(self, other):
method __mul__ (line 105) | def __mul__(self, other):
method __rmul__ (line 112) | def __rmul__(self, other):
method __truediv__ (line 117) | def __truediv__(self, other):
method __rtruediv__ (line 124) | def __rtruediv__(self, other):
method __eq__ (line 130) | def __eq__(self, other):
method __lt__ (line 135) | def __lt__(self, other):
method __le__ (line 140) | def __le__(self, other):
method __gt__ (line 145) | def __gt__(self, other):
method __ge__ (line 150) | def __ge__(self, other):
method __getattr__ (line 156) | def __getattr__(self, name):
method to_dict (line 178) | def to_dict(self):
method validate (line 183) | def validate(cls, value: t.Any, info: ValidationInfo):
method __json__ (line 189) | def __json__(self):
method __get_pydantic_core_schema__ (line 201) | def __get_pydantic_core_schema__(
FILE: src/ragas/metrics/utils.py
function fbeta_score (line 1) | def fbeta_score(tp, fp, fn, beta=1.0):
FILE: src/ragas/metrics/validators.py
class BaseValidator (line 19) | class BaseValidator(ABC):
method validate_result_value (line 25) | def validate_result_value(self, result_value: t.Any) -> t.Optional[str]:
class DiscreteValidator (line 38) | class DiscreteValidator(BaseValidator):
method validate_result_value (line 43) | def validate_result_value(self, result_value: t.Any) -> t.Optional[str]:
class NumericValidator (line 53) | class NumericValidator(BaseValidator):
method validate_result_value (line 58) | def validate_result_value(self, result_value: t.Any) -> t.Optional[str]:
class RankingValidator (line 76) | class RankingValidator(BaseValidator):
method validate_result_value (line 81) | def validate_result_value(self, result_value: t.Any) -> t.Optional[str]:
function get_validator_for_allowed_values (line 93) | def get_validator_for_allowed_values(
function get_metric_type_name (line 116) | def get_metric_type_name(allowed_values: AllowedValuesType) -> str:
FILE: src/ragas/optimizers/base.py
class Optimizer (line 15) | class Optimizer(ABC):
method optimize (line 24) | def optimize(
FILE: src/ragas/optimizers/dspy_adapter.py
function setup_dspy_llm (line 9) | def setup_dspy_llm(dspy: t.Any, ragas_llm: BaseRagasLLM) -> None:
function pydantic_prompt_to_dspy_signature (line 26) | def pydantic_prompt_to_dspy_signature(
function ragas_dataset_to_dspy_examples (line 71) | def ragas_dataset_to_dspy_examples(
function create_dspy_metric (line 129) | def create_dspy_metric(
FILE: src/ragas/optimizers/dspy_llm_wrapper.py
class RagasDSPyLM (line 6) | class RagasDSPyLM:
method __init__ (line 19) | def __init__(self, ragas_llm: BaseRagasLLM):
method __call__ (line 23) | def __call__(
method _generate (line 56) | async def _generate(
method inspect_history (line 89) | def inspect_history(self, n: int = 1) -> t.List[t.Dict[str, t.Any]]:
FILE: src/ragas/optimizers/dspy_optimizer.py
class DSPyOptimizer (line 19) | class DSPyOptimizer(Optimizer):
method __post_init__ (line 76) | def __post_init__(self):
method _validate_parameters (line 89) | def _validate_parameters(self):
method optimize (line 117) | def optimize(
method _extract_instruction (line 239) | def _extract_instruction(self, optimized_module: t.Any) -> str:
method _generate_cache_key (line 265) | def _generate_cache_key(
FILE: src/ragas/optimizers/genetic.py
class FormattedExamples (line 35) | class FormattedExamples(BaseModel):
method from_examples (line 39) | def from_examples(cls, examples: t.List[example_type]) -> "FormattedEx...
class OutputInstruction (line 49) | class OutputInstruction(BaseModel):
class ReverseEngineerPrompt (line 53) | class ReverseEngineerPrompt(PydanticPrompt[FormattedExamples, OutputInst...
class ParentPrompts (line 60) | class ParentPrompts(BaseModel):
class CrossOverPrompt (line 65) | class CrossOverPrompt(PydanticPrompt[ParentPrompts, OutputInstruction]):
class FeedbackExample (line 88) | class FeedbackExample(BaseModel):
class FeedbackMutationInput (line 94) | class FeedbackMutationInput(BaseModel):
class FeedbackMutationOutput (line 99) | class FeedbackMutationOutput(BaseModel):
class FeedbackMutationPrompt (line 103) | class FeedbackMutationPrompt(
class FeedbackMutationPromptInput (line 115) | class FeedbackMutationPromptInput(BaseModel):
class FeedbackMutationPromptGeneration (line 120) | class FeedbackMutationPromptGeneration(
class GeneticOptimizer (line 129) | class GeneticOptimizer(Optimizer):
method optimize (line 139) | def optimize(
method initialize_population (line 253) | def initialize_population(
method _reverse_engineer_instruction (line 307) | async def _reverse_engineer_instruction(
method _cross_over_prompts (line 342) | async def _cross_over_prompts(
method _set_instructions (line 354) | def _set_instructions(self, candidates: t.Dict[str, str]):
method feedback_mutation (line 362) | def feedback_mutation(
method _feedback_mutation (line 419) | async def _feedback_mutation(
method _implement_feedbacks (line 462) | async def _implement_feedbacks(
method _get_feedbacks (line 490) | async def _get_feedbacks(
method _get_evaluation_dataset (line 543) | def _get_evaluation_dataset(
method evaluate_candidate (line 566) | def evaluate_candidate(
method evaluate_fitness (line 597) | def evaluate_fitness(
method _cross_over_chain (line 644) | async def _cross_over_chain(
method cross_over_mutation (line 662) | def cross_over_mutation(
FILE: src/ragas/optimizers/utils.py
function hamming_distance (line 4) | def hamming_distance(vectors: np.ndarray) -> np.ndarray:
FILE: src/ragas/prompt/base.py
class BasePrompt (line 23) | class BasePrompt(ABC):
method __init__ (line 24) | def __init__(
method __repr__ (line 36) | def __repr__(self):
method generate (line 40) | async def generate(
method generate_multiple (line 54) | def generate_multiple(
method save (line 68) | def save(self, file_path: str):
method load (line 84) | def load(cls, file_path: str) -> "BasePrompt":
class StringIO (line 108) | class StringIO(BaseModel):
method __hash__ (line 111) | def __hash__(self):
class BoolIO (line 115) | class BoolIO(BaseModel):
method __hash__ (line 118) | def __hash__(self):
class StringPrompt (line 122) | class StringPrompt(BasePrompt):
method generate (line 140) | async def generate(
method generate_multiple (line 180) | async def generate_multiple(
FILE: src/ragas/prompt/dynamic_few_shot.py
class SimpleExampleStore (line 22) | class SimpleExampleStore(ABC):
method get_examples (line 24) | def get_examples(
method add_example (line 31) | def add_example(self, input: t.Dict, output: t.Dict) -> None:
class SimpleInMemoryExampleStore (line 36) | class SimpleInMemoryExampleStore(SimpleExampleStore):
method __init__ (line 37) | def __init__(self, embedding_model=None):
method _get_embedding (line 48) | def _get_embedding(self, data: t.Dict) -> t.List[float]:
method add_example (line 57) | def add_example(self, input: t.Dict, output: t.Dict) -> None:
method get_examples (line 70) | def get_examples(
method _get_nearest_examples (line 92) | def _get_nearest_examples(
method __len__ (line 123) | def __len__(self):
class DynamicFewShotPrompt (line 127) | class DynamicFewShotPrompt(Prompt):
method __init__ (line 128) | def __init__(
method format (line 169) | def format(self, **kwargs) -> str:
method add_example (line 197) | def add_example(self, input: t.Dict, output: t.Dict) -> None:
method from_prompt (line 218) | def from_prompt(
method __str__ (line 253) | def __str__(self) -> str:
method save (line 265) | def save(self, path: str, include_embeddings: bool = True) -> None:
method _serialize_embedding_model_info (line 326) | def _serialize_embedding_model_info(self) -> t.Optional[t.Dict]:
method load (line 338) | def load(
FILE: src/ragas/prompt/few_shot_pydantic_prompt.py
class ExampleStore (line 25) | class ExampleStore(ABC):
method get_examples (line 27) | def get_examples(
method add_example (line 33) | def add_example(self, input: BaseModel, output: BaseModel):
class InMemoryExampleStore (line 38) | class InMemoryExampleStore(ExampleStore):
method add_example (line 47) | def add_example(self, input: BaseModel, output: BaseModel):
method get_examples (line 53) | def get_examples(
method get_nearest_examples (line 65) | def get_nearest_examples(
method __repr__ (line 93) | def __repr__(self):
class FewShotPydanticPrompt (line 98) | class FewShotPydanticPrompt(PydanticPrompt, t.Generic[InputModel, Output...
method __post_init__ (line 103) | def __post_init__(self):
method add_example (line 106) | def add_example(self, input: InputModel, output: OutputModel):
method generate_multiple (line 109) | async def generate_multiple(
method from_pydantic_prompt (line 138) | def from_pydantic_prompt(
FILE: src/ragas/prompt/metrics/answer_accuracy.py
function answer_accuracy_judge1_prompt (line 6) | def answer_accuracy_judge1_prompt(
function answer_accuracy_judge2_prompt (line 39) | def answer_accuracy_judge2_prompt(
FILE: src/ragas/prompt/metrics/answer_correctness.py
function correctness_classifier_prompt (line 10) | def correctness_classifier_prompt(
FILE: src/ragas/prompt/metrics/answer_relevance.py
function answer_relevancy_prompt (line 6) | def answer_relevancy_prompt(response: str) -> str:
FILE: src/ragas/prompt/metrics/base_prompt.py
class _TranslatedStrings (line 33) | class _TranslatedStrings(BaseModel):
function _translate_strings (line 41) | async def _translate_strings(
class BasePrompt (line 78) | class BasePrompt(ABC, t.Generic[InputModel, OutputModel]):
method to_string (line 97) | def to_string(self, data: InputModel) -> str:
method _generate_examples (line 128) | def _generate_examples(self) -> str:
method adapt (line 148) | async def adapt(
FILE: src/ragas/prompt/metrics/common.py
function statement_generator_prompt (line 7) | def statement_generator_prompt(question: str, answer: str) -> str:
function nli_statement_prompt (line 50) | def nli_statement_prompt(context: str, statements: t.List[str]) -> str:
FILE: src/ragas/prompt/metrics/context_entity_recall.py
function extract_entities_prompt (line 6) | def extract_entities_prompt(text: str) -> str:
FILE: src/ragas/prompt/metrics/context_recall.py
function context_recall_prompt (line 6) | def context_recall_prompt(question: str, context: str, answer: str) -> str:
FILE: src/ragas/prompt/metrics/context_relevance.py
function context_relevance_judge1_prompt (line 6) | def context_relevance_judge1_prompt(query: str, context: str) -> str:
function context_relevance_judge2_prompt (line 42) | def context_relevance_judge2_prompt(query: str, context: str) -> str:
FILE: src/ragas/prompt/metrics/factual_correctness.py
function claim_decomposition_prompt (line 6) | def claim_decomposition_prompt(
FILE: src/ragas/prompt/metrics/noise_sensitivity.py
function nli_statement_prompt (line 7) | def nli_statement_prompt(context: str, statements: t.List[str]) -> str:
FILE: src/ragas/prompt/metrics/response_groundedness.py
function response_groundedness_judge1_prompt (line 4) | def response_groundedness_judge1_prompt(response: str, context: str) -> ...
function response_groundedness_judge2_prompt (line 36) | def response_groundedness_judge2_prompt(response: str, context: str) -> ...
FILE: src/ragas/prompt/metrics/summary_score.py
function extract_keyphrases_prompt (line 7) | def extract_keyphrases_prompt(text: str) -> str:
function generate_questions_prompt (line 48) | def generate_questions_prompt(text: str, keyphrases: t.List[str]) -> str:
function generate_answers_prompt (line 100) | def generate_answers_prompt(summary: str, questions: t.List[str]) -> str:
FILE: src/ragas/prompt/mixin.py
class PromptMixin (line 17) | class PromptMixin:
method _get_prompts (line 25) | def _get_prompts(self) -> t.Dict[str, PydanticPrompt]:
method get_prompts (line 32) | def get_prompts(self) -> t.Dict[str, PydanticPrompt]:
method set_prompts (line 41) | def set_prompts(self, **prompts):
method adapt_prompts (line 63) | async def adapt_prompts(
method save_prompts (line 86) | def save_prompts(self, path: str):
method load_prompts (line 105) | def load_prompts(self, path: str, language: t.Optional[str] = None):
FILE: src/ragas/prompt/multi_modal_prompt.py
class TextContent (line 41) | class TextContent(TypedDict):
class ImageUrlContent (line 46) | class ImageUrlContent(TypedDict):
class ImageTextPrompt (line 89) | class ImageTextPrompt(PydanticPrompt, t.Generic[InputModel, OutputModel]):
method _generate_examples (line 90) | def _generate_examples(self):
method to_prompt_value (line 113) | def to_prompt_value(self, data: t.Optional[InputModel] = None):
method generate_multiple (line 122) | async def generate_multiple(
class ImageTextPromptValue (line 221) | class ImageTextPromptValue(PromptValue):
method __len__ (line 224) | def __len__(self):
method to_messages (line 228) | def to_messages(self) -> t.List[BaseMessage]:
method _securely_process_item (line 248) | def _securely_process_item(self, item: str) -> t.Optional[MessageConte...
method _looks_like_image_path (line 288) | def _looks_like_image_path(self, item: str) -> bool:
method _get_text_payload (line 302) | def _get_text_payload(self, text: str) -> TextContent:
method _get_image_payload (line 306) | def _get_image_payload(self, mime_type: str, encoded_image: str) -> Im...
method _try_process_base64_uri (line 323) | def _try_process_base64_uri(self, item: str) -> t.Optional[dict]:
method _try_process_allowed_url (line 350) | def _try_process_allowed_url(self, item: str) -> t.Optional[dict]:
method _download_validate_and_encode (line 366) | def _download_validate_and_encode(self, url: str) -> t.Optional[dict]:
method _is_safe_url_target (line 463) | def _is_safe_url_target(self, url_hostname: str) -> bool:
method _try_process_local_file (line 540) | def _try_process_local_file(self, item: str) -> t.Optional[dict]:
method to_string (line 630) | def to_string(self):
FILE: src/ragas/prompt/pydantic_prompt.py
function is_langchain_llm (line 30) | def is_langchain_llm(
class PydanticPrompt (line 82) | class PydanticPrompt(BasePrompt, t.Generic[InputModel, OutputModel]):
method _generate_instruction (line 89) | def _generate_instruction(self) -> str:
method _generate_output_signature (line 92) | def _generate_output_signature(self, indent: int = 4) -> str:
method _generate_examples (line 101) | def _generate_examples(self):
method to_string (line 120) | def to_string(self, data: t.Optional[InputModel] = None) -> str:
method generate (line 136) | async def generate(
method generate_multiple (line 188) | async def generate_multiple(
method process_input (line 351) | def process_input(self, input: InputModel) -> InputModel:
method process_output (line 354) | def process_output(self, output: OutputModel, input: InputModel) -> Ou...
method adapt (line 357) | async def adapt(
method __repr__ (line 396) | def __repr__(self):
method __str__ (line 399) | def __str__(self):
method __hash__ (line 414) | def __hash__(self):
method __eq__ (line 439) | def __eq__(self, other):
method save (line 451) | def save(self, file_path: str):
method load (line 474) | def load(cls, file_path: str) -> "PydanticPrompt[InputModel, OutputMod...
class OutputStringAndPrompt (line 511) | class OutputStringAndPrompt(BaseModel):
class FixOutputFormat (line 516) | class FixOutputFormat(PydanticPrompt[OutputStringAndPrompt, StringIO]):
class RagasOutputParser (line 525) | class RagasOutputParser(PydanticOutputParser[OutputModel]):
method parse_output_string (line 526) | async def parse_output_string(
class ToTranslate (line 562) | class ToTranslate(BaseModel):
class Translated (line 567) | class Translated(BaseModel):
class TranslateStatements (line 571) | class TranslateStatements(PydanticPrompt[ToTranslate, Translated]):
method process_output (line 619) | def process_output(self, output: Translated, input: ToTranslate) -> Tr...
FILE: src/ragas/prompt/simple_prompt.py
class Prompt (line 17) | class Prompt:
method __init__ (line 18) | def __init__(
method format (line 98) | def format(self, **kwargs) -> str:
method _format_examples (line 122) | def _format_examples(self) -> str:
method add_example (line 137) | def add_example(self, input: t.Dict, output: t.Dict) -> None:
method save (line 160) | def save(self, path: str) -> None:
method load (line 201) | def load(
method _serialize_response_model_info (line 268) | def _serialize_response_model_info(self) -> t.Optional[t.Dict]:
method _validate_response_model (line 280) | def _validate_response_model(
method __str__ (line 299) | def __str__(self) -> str:
FILE: src/ragas/prompt/utils.py
function get_all_strings (line 7) | def get_all_strings(obj: t.Any) -> list[str]:
function update_strings (line 28) | def update_strings(obj: t.Any, old_strings: list[str], new_strings: list...
function extract_json (line 69) | def extract_json(text: str) -> str:
FILE: src/ragas/run_config.py
class RunConfig (line 19) | class RunConfig:
method __post_init__ (line 62) | def __post_init__(self):
function add_retry (line 66) | def add_retry(fn: WrappedFn, run_config: RunConfig) -> WrappedFn:
function add_async_retry (line 97) | def add_async_retry(fn: WrappedFn, run_config: RunConfig) -> WrappedFn:
FILE: src/ragas/testset/graph.py
class UUIDEncoder (line 16) | class UUIDEncoder(json.JSONEncoder):
method default (line 17) | def default(self, o):
class NodeType (line 23) | class NodeType(str, Enum):
class Node (line 35) | class Node(BaseModel):
method __repr__ (line 54) | def __repr__(self) -> str:
method __str__ (line 57) | def __str__(self) -> str:
method add_property (line 60) | def add_property(self, key: str, value: t.Any):
method get_property (line 73) | def get_property(self, key: str) -> t.Optional[t.Any]:
method __hash__ (line 83) | def __hash__(self) -> int:
method __eq__ (line 86) | def __eq__(self, other: object) -> bool:
class Relationship (line 92) | class Relationship(BaseModel):
method get_property (line 120) | def get_property(self, key: str) -> t.Optional[t.Any]:
method __repr__ (line 126) | def __repr__(self) -> str:
method __str__ (line 129) | def __str__(self) -> str:
method __hash__ (line 132) | def __hash__(self) -> int:
method __eq__ (line 135) | def __eq__(self, other: object) -> bool:
method serialize_node (line 141) | def serialize_node(self, node: Node):
class KnowledgeGraph (line 146) | class KnowledgeGraph:
method add (line 161) | def add(self, item: t.Union[Node, Relationship]):
method _add_node (line 177) | def _add_node(self, node: Node):
method _add_relationship (line 180) | def _add_relationship(self, relationship: Relationship):
method save (line 183) | def save(self, path: t.Union[str, Path]):
method load (line 207) | def load(cls, path: t.Union[str, Path]) -> "KnowledgeGraph":
method __repr__ (line 251) | def __repr__(self) -> str:
method __str__ (line 254) | def __str__(self) -> str:
method get_node_by_id (line 257) | def get_node_by_id(self, node_id: t.Union[uuid.UUID, str]) -> t.Option...
method find_indirect_clusters (line 276) | def find_indirect_clusters(
method find_n_indirect_clusters (line 471) | def find_n_indirect_clusters(
method remove_node (line 647) | def remove_node(
method find_two_nodes_single_rel (line 694) | def find_two_nodes_single_rel(
FILE: src/ragas/testset/graph_queries.py
function get_child_nodes (line 6) | def get_child_nodes(node: Node, graph: KnowledgeGraph, level: int = 1) -...
function get_parent_nodes (line 41) | def get_parent_nodes(node: Node, graph: KnowledgeGraph, level: int = 1) ...
FILE: src/ragas/testset/persona.py
function default_filter (line 16) | def default_filter(node: Node) -> bool:
class Persona (line 25) | class Persona(BaseModel):
class PersonaGenerationPrompt (line 30) | class PersonaGenerationPrompt(PydanticPrompt[StringIO, Persona]):
class PersonaList (line 51) | class PersonaList(BaseModel):
method __getitem__ (line 54) | def __getitem__(self, key: str) -> Persona:
function generate_personas_from_kg (line 61) | def generate_personas_from_kg(
FILE: src/ragas/testset/synthesizers/__init__.py
function default_query_distribution (line 24) | def default_query_distribution(
FILE: src/ragas/testset/synthesizers/base.py
function _default_llm_factory (line 23) | def _default_llm_factory() -> t.Union[BaseRagasLLM, "InstructorBaseRagas...
class QueryLength (line 34) | class QueryLength(str, Enum):
class QueryStyle (line 44) | class QueryStyle(str, Enum):
class BaseScenario (line 55) | class BaseScenario(BaseModel):
class BaseSynthesizer (line 81) | class BaseSynthesizer(ABC, t.Generic[Scenario], PromptMixin):
method __post_init__ (line 92) | def __post_init__(self):
method generate_scenarios (line 96) | async def generate_scenarios(
method _generate_scenarios (line 116) | async def _generate_scenarios(
method generate_sample (line 125) | async def generate_sample(
method _generate_sample (line 142) | async def _generate_sample(
FILE: src/ragas/testset/synthesizers/generate.py
class TestsetGenerator (line 55) | class TestsetGenerator:
method from_langchain (line 77) | def from_langchain(
method from_llama_index (line 96) | def from_llama_index(
method generate_with_langchain_docs (line 114) | def generate_with_langchain_docs(
method generate_with_llamaindex_docs (line 222) | def generate_with_llamaindex_docs(
method generate_with_chunks (line 302) | def generate_with_chunks(
method generate (line 414) | def generate(
FILE: src/ragas/testset/synthesizers/multi_hop/abstract.py
class MultiHopAbstractQuerySynthesizer (line 32) | class MultiHopAbstractQuerySynthesizer(MultiHopQuerySynthesizer):
method get_node_clusters (line 41) | def get_node_clusters(
method _generate_scenarios (line 58) | async def _generate_scenarios(
FILE: src/ragas/testset/synthesizers/multi_hop/base.py
class MultiHopScenario (line 30) | class MultiHopScenario(BaseScenario):
method __repr__ (line 46) | def __repr__(self) -> str:
class MultiHopQuerySynthesizer (line 51) | class MultiHopQuerySynthesizer(BaseSynthesizer[Scenario]):
method prepare_combinations (line 54) | def prepare_combinations(
method sample_diverse_combinations (line 92) | def sample_diverse_combinations(
method convert_to_scenario (line 147) | def convert_to_scenario(self, data: t.Dict[str, t.Any]) -> MultiHopSce...
method _generate_sample (line 156) | async def _generate_sample(
method make_contexts (line 182) | def make_contexts(self, scenario: MultiHopScenario) -> t.List[str]:
FILE: src/ragas/testset/synthesizers/multi_hop/prompts.py
class ConceptsList (line 9) | class ConceptsList(BaseModel):
class ConceptCombinations (line 18) | class ConceptCombinations(BaseModel):
class ConceptCombinationPrompt (line 22) | class ConceptCombinationPrompt(PydanticPrompt[ConceptsList, ConceptCombi...
class QueryConditions (line 58) | class QueryConditions(BaseModel):
class GeneratedQueryAnswer (line 67) | class GeneratedQueryAnswer(BaseModel):
class QueryAnswerGenerationPrompt (line 72) | class QueryAnswerGenerationPrompt(
FILE: src/ragas/testset/synthesizers/multi_hop/specific.py
class MultiHopSpecificQuerySynthesizer (line 30) | class MultiHopSpecificQuerySynthesizer(MultiHopQuerySynthesizer):
method get_node_clusters (line 40) | def get_node_clusters(self, knowledge_graph: KnowledgeGraph) -> t.List...
method _generate_scenarios (line 48) | async def _generate_scenarios(
method _extract_themes_from_overlaps (line 118) | def _extract_themes_from_overlaps(self, overlapped_items: t.Any) -> t....
method _extract_theme_groups_from_overlaps (line 146) | def _extract_theme_groups_from_overlaps(
FILE: src/ragas/testset/synthesizers/prompts.py
class ThemesPersonasInput (line 9) | class ThemesPersonasInput(BaseModel):
class PersonaThemesMapping (line 14) | class PersonaThemesMapping(BaseModel):
class ThemesPersonasMatchingPrompt (line 18) | class ThemesPersonasMatchingPrompt(
FILE: src/ragas/testset/synthesizers/single_hop/base.py
class SingleHopScenario (line 30) | class SingleHopScenario(BaseScenario):
method __repr__ (line 42) | def __repr__(self) -> str:
class SingleHopQuerySynthesizer (line 47) | class SingleHopQuerySynthesizer(BaseSynthesizer[Scenario]):
method prepare_combinations (line 50) | def prepare_combinations(
method sample_combinations (line 71) | def sample_combinations(self, data: t.List[t.Dict[str, t.Any]], num_sa...
method convert_to_scenario (line 108) | def convert_to_scenario(self, data: t.Dict[str, t.Any]) -> SingleHopSc...
method _generate_sample (line 117) | async def _generate_sample(
FILE: src/ragas/testset/synthesizers/single_hop/prompts.py
class QueryCondition (line 9) | class QueryCondition(BaseModel):
class GeneratedQueryAnswer (line 18) | class GeneratedQueryAnswer(BaseModel):
class QueryAnswerGenerationPrompt (line 23) | class QueryAnswerGenerationPrompt(PydanticPrompt[QueryCondition, Generat...
FILE: src/ragas/testset/synthesizers/single_hop/specific.py
class SingleHopScenario (line 27) | class SingleHopScenario(BaseScenario):
class SingleHopSpecificQuerySynthesizer (line 41) | class SingleHopSpecificQuerySynthesizer(SingleHopQuerySynthesizer):
method _extract_themes_from_items (line 46) | def _extract_themes_from_items(self, items: t.Any) -> t.List[str]:
method get_node_clusters (line 84) | def get_node_clusters(self, knowledge_graph: KnowledgeGraph) -> t.List...
method _generate_scenarios (line 113) | async def _generate_scenarios(
FILE: src/ragas/testset/synthesizers/testset_schema.py
class TestsetSample (line 20) | class TestsetSample(BaseSample):
class TestsetPacket (line 36) | class TestsetPacket(BaseModel):
class Testset (line 47) | class Testset(RagasDataset[TestsetSample]):
method to_evaluation_dataset (line 61) | def to_evaluation_dataset(self) -> EvaluationDataset:
method to_list (line 69) | def to_list(self) -> t.List[t.Dict]:
method from_list (line 81) | def from_list(cls, data: t.List[t.Dict]) -> Testset:
method total_tokens (line 110) | def total_tokens(self) -> t.Union[t.List[TokenUsage], TokenUsage]:
method total_cost (line 120) | def total_cost(
method from_annotated (line 138) | def from_annotated(cls, path: str) -> Testset:
FILE: src/ragas/testset/synthesizers/utils.py
function calculate_split_values (line 5) | def calculate_split_values(
FILE: src/ragas/testset/transforms/base.py
function default_filter (line 17) | def default_filter(node: Node) -> bool:
function _default_llm_factory (line 21) | def _default_llm_factory() -> t.Union[BaseRagasLLM, "InstructorBaseRagas...
class BaseGraphTransformation (line 33) | class BaseGraphTransformation(ABC):
method __post_init__ (line 44) | def __post_init__(self):
method transform (line 49) | async def transform(self, kg: KnowledgeGraph) -> t.Any:
method filter (line 67) | def filter(self, kg: KnowledgeGraph) -> KnowledgeGraph:
method generate_execution_plan (line 102) | def generate_execution_plan(self, kg: KnowledgeGraph) -> t.Sequence[t....
class Extractor (line 121) | class Extractor(BaseGraphTransformation):
method transform (line 135) | async def transform(
method extract (line 166) | async def extract(self, node: Node) -> t.Tuple[str, t.Any]:
method generate_execution_plan (line 182) | def generate_execution_plan(self, kg: KnowledgeGraph) -> t.Sequence[t....
class LLMBasedExtractor (line 219) | class LLMBasedExtractor(Extractor, PromptMixin):
method split_text_by_token_limit (line 227) | def split_text_by_token_limit(self, text, max_token_limit):
class Splitter (line 236) | class Splitter(BaseGraphTransformation):
method transform (line 250) | async def transform(
method split (line 278) | async def split(self, node: Node) -> t.Tuple[t.List[Node], t.List[Rela...
method generate_execution_plan (line 294) | def generate_execution_plan(self, kg: KnowledgeGraph) -> t.Sequence[t....
class RelationshipBuilder (line 324) | class RelationshipBuilder(BaseGraphTransformation):
method transform (line 335) | async def transform(self, kg: KnowledgeGraph) -> t.List[Relationship]:
method generate_execution_plan (line 351) | def generate_execution_plan(self, kg: KnowledgeGraph) -> t.Sequence[t....
class NodeFilter (line 383) | class NodeFilter(BaseGraphTransformation):
method transform (line 384) | async def transform(self, kg: KnowledgeGraph) -> KnowledgeGraph:
method custom_filter (line 398) | async def custom_filter(self, node: Node, kg: KnowledgeGraph) -> bool:
method generate_execution_plan (line 414) | def generate_execution_plan(self, kg: KnowledgeGraph) -> t.Sequence[t....
class LLMBasedNodeFilter (line 434) | class LLMBasedNodeFilter(NodeFilter, PromptMixin):
FILE: src/ragas/testset/transforms/default.py
function default_transforms (line 34) | def default_transforms(
function default_transforms_for_prechunked (line 170) | def default_transforms_for_prechunked(
FILE: src/ragas/testset/transforms/engine.py
class Parallel (line 23) | class Parallel:
method __init__ (line 32) | def __init__(self, *transformations: t.Union[BaseGraphTransformation, ...
method generate_execution_plan (line 35) | def generate_execution_plan(self, kg: KnowledgeGraph) -> t.Sequence[t....
function get_desc (line 46) | def get_desc(transform: BaseGraphTransformation | Parallel):
function apply_transforms (line 54) | def apply_transforms(
function rollback_transforms (line 104) | def rollback_transforms(kg: KnowledgeGraph, transforms: Transforms):
FILE: src/ragas/testset/transforms/extractors/embeddings.py
class EmbeddingExtractor (line 12) | class EmbeddingExtractor(Extractor):
method extract (line 32) | async def extract(self, node: Node) -> t.Tuple[str, t.Any]:
FILE: src/ragas/testset/transforms/extractors/llm_based.py
class TextWithExtractionLimit (line 11) | class TextWithExtractionLimit(BaseModel):
class SummaryExtractorPrompt (line 16) | class SummaryExtractorPrompt(PydanticPrompt[StringIO, StringIO]):
class Keyphrases (line 32) | class Keyphrases(BaseModel):
class KeyphrasesExtractorPrompt (line 36) | class KeyphrasesExtractorPrompt(PydanticPrompt[TextWithExtractionLimit, ...
class TitleExtractorPrompt (line 59) | class TitleExtractorPrompt(PydanticPrompt[StringIO, StringIO]):
class Headlines (line 73) | class Headlines(BaseModel):
class HeadlinesExtractorPrompt (line 77) | class HeadlinesExtractorPrompt(PydanticPrompt[TextWithExtractionLimit, H...
class NEROutput (line 126) | class NEROutput(BaseModel):
class NERPrompt (line 130) | class NERPrompt(PydanticPrompt[TextWithExtractionLimit, NEROutput]):
class SummaryExtractor (line 160) | class SummaryExtractor(LLMBasedExtractor):
method extract (line 175) | async def extract(self, node: Node) -> t.Tuple[str, t.Any]:
class KeyphrasesExtractor (line 185) | class KeyphrasesExtractor(LLMBasedExtractor):
method extract (line 201) | async def extract(self, node: Node) -> t.Tuple[str, t.Any]:
class TitleExtractor (line 216) | class TitleExtractor(LLMBasedExtractor):
method extract (line 231) | async def extract(self, node: Node) -> t.Tuple[str, t.Any]:
class HeadlinesExtractor (line 241) | class HeadlinesExtractor(LLMBasedExtractor):
method extract (line 257) | async def extract(self, node: Node) -> t.Tuple[str, t.Any]:
class NERExtractor (line 273) | class NERExtractor(LLMBasedExtractor):
method extract (line 289) | async def extract(self, node: Node) -> t.Tuple[str, t.List[str]]:
class TopicDescription (line 304) | class TopicDescription(BaseModel):
class TopicDescriptionPrompt (line 308) | class TopicDescriptionPrompt(PydanticPrompt[StringIO, TopicDescription]):
class TopicDescriptionExtractor (line 325) | class TopicDescriptionExtractor(LLMBasedExtractor):
method extract (line 340) | async def extract(self, node: Node) -> t.Tuple[str, t.Any]:
class ThemesAndConcepts (line 349) | class ThemesAndConcepts(BaseModel):
class ThemesAndConceptsExtractorPrompt (line 353) | class ThemesAndConceptsExtractorPrompt(
class ThemesExtractor (line 380) | class ThemesExtractor(LLMBasedExtractor):
method extract (line 396) | async def extract(self, node: Node) -> t.Tuple[str, t.List[str]]:
FILE: src/ragas/testset/transforms/extractors/regex_based.py
class RegexBasedExtractor (line 10) | class RegexBasedExtractor(Extractor):
method extract (line 15) | async def extract(self, node: Node) -> t.Tuple[str, t.Any]:
FILE: src/ragas/testset/transforms/filters.py
class QuestionPotentialInput (line 24) | class QuestionPotentialInput(BaseModel):
class QuestionPotentialOutput (line 36) | class QuestionPotentialOutput(BaseModel):
class QuestionPotentialPrompt (line 43) | class QuestionPotentialPrompt(
class CustomNodeFilter (line 55) | class CustomNodeFilter(LLMBasedNodeFilter):
method custom_filter (line 64) | async def custom_filter(self, node: Node, kg: KnowledgeGraph) -> bool:
FILE: src/ragas/testset/transforms/relationship_builders/cosine.py
class CosineSimilarityBuilder (line 11) | class CosineSimilarityBuilder(RelationshipBuilder):
method _block_cosine_similarity (line 17) | def _block_cosine_similarity(self, i: np.ndarray, j: np.ndarray):
method _find_similar_embedding_pairs (line 23) | def _find_similar_embedding_pairs(
method _validate_embedding_shapes (line 50) | def _validate_embedding_shapes(self, embeddings: t.List[t.Any]):
method transform (line 61) | async def transform(self, kg: KnowledgeGraph) -> t.List[Relationship]:
method generate_execution_plan (line 83) | def generate_execution_plan(self, kg: KnowledgeGraph) -> t.List[t.Coro...
class SummaryCosineSimilarityBuilder (line 115) | class SummaryCosineSimilarityBuilder(CosineSimilarityBuilder):
method _document_summary_filter (line 121) | def _document_summary_filter(self, kg: KnowledgeGraph) -> KnowledgeGraph:
method transform (line 134) | async def transform(self, kg: KnowledgeGraph) -> t.List[Relationship]:
FILE: src/ragas/testset/transforms/relationship_builders/traditional.py
class JaccardSimilarityBuilder (line 12) | class JaccardSimilarityBuilder(RelationshipBuilder):
method _jaccard_similarity (line 18) | def _jaccard_similarity(self, set1: t.Set[str], set2: t.Set[str]) -> f...
method _find_similar_embedding_pairs (line 23) | def _find_similar_embedding_pairs(
method transform (line 47) | async def transform(self, kg: KnowledgeGraph) -> t.List[Relationship]:
method generate_execution_plan (line 60) | def generate_execution_plan(self, kg: KnowledgeGraph) -> t.List[t.Coro...
class OverlapScoreBuilder (line 81) | class OverlapScoreBuilder(RelationshipBuilder):
method __post_init__ (line 89) | def __post_init__(self):
method _overlap_score (line 105) | def _overlap_score(self, overlaps: t.List[bool]) -> float:
method _get_noisy_items (line 108) | def _get_noisy_items(
method transform (line 129) | async def transform(self, kg: KnowledgeGraph) -> t.List[Relationship]:
FILE: src/ragas/testset/transforms/splitters/headline.py
class HeadlineSplitter (line 10) | class HeadlineSplitter(Splitter):
method adjust_chunks (line 14) | def adjust_chunks(self, chunks):
method split (line 58) | async def split(self, node: Node) -> t.Tuple[t.List[Node], t.List[Rela...
FILE: src/ragas/tokenizers.py
class BaseTokenizer (line 16) | class BaseTokenizer(ABC):
method encode (line 20) | def encode(self, text: str) -> t.List[int]:
method decode (line 25) | def decode(self, tokens: t.List[int]) -> str:
method count_tokens (line 29) | def count_tokens(self, text: str) -> int:
class TiktokenWrapper (line 34) | class TiktokenWrapper(BaseTokenizer):
method __init__ (line 37) | def __init__(
method encode (line 66) | def encode(self, text: str) -> t.List[int]:
method decode (line 69) | def decode(self, tokens: t.List[int]) -> str:
method encoding (line 73) | def encoding(self) -> tiktoken.Encoding:
class HuggingFaceTokenizer (line 78) | class HuggingFaceTokenizer(BaseTokenizer):
method __init__ (line 81) | def __init__(
method encode (line 112) | def encode(self, text: str) -> t.List[int]:
method decode (line 115) | def decode(self, tokens: t.List[int]) -> str:
method tokenizer (line 119) | def tokenizer(self) -> t.Any:
function get_default_tokenizer (line 128) | def get_default_tokenizer() -> TiktokenWrapper:
class _LazyTokenizer (line 136) | class _LazyTokenizer(BaseTokenizer):
method __getattr__ (line 143) | def __getattr__(self, name: str) -> t.Any:
method encode (line 146) | def encode(self, text: str) -> t.List[int]:
method decode (line 149) | def decode(self, tokens: t.List[int]) -> str:
method count_tokens (line 152) | def count_tokens(self, text: str) -> int:
function get_tokenizer (line 160) | def get_tokenizer(
FILE: src/ragas/utils.py
function get_cache_dir (line 30) | def get_cache_dir() -> str:
function get_debug_mode (line 39) | def get_debug_mode() -> bool:
function safe_nanmean (line 46) | def safe_nanmean(arr: t.List[float]) -> float:
function check_if_sum_is_close (line 58) | def check_if_sum_is_close(
function patch_logger (line 66) | def patch_logger(module: str, level: int):
function is_nan (line 83) | def is_nan(x):
function get_metric_language (line 90) | def get_metric_language(metric: "Metric") -> str:
class DeprecationHelper (line 101) | class DeprecationHelper:
method __init__ (line 104) | def __init__(self, new_target: t.Type, deprecation_message: str):
method _warn (line 108) | def _warn(self):
method __call__ (line 111) | def __call__(self, *args, **kwargs):
method __getattr__ (line 115) | def __getattr__(self, attr):
function deprecated (line 120) | def deprecated(
function get_or_init (line 191) | def get_or_init(
function get_from_dict (line 200) | def get_from_dict(data_dict: t.Dict, key: str, default=None) -> t.Any:
function get_required_columns_v1 (line 221) | def get_required_columns_v1(metric: Metric):
function convert_row_v1_to_v2 (line 227) | def convert_row_v1_to_v2(row: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]:
function convert_v1_to_v2_dataset (line 232) | def convert_v1_to_v2_dataset(dataset: Dataset) -> Dataset:
function convert_v2_to_v1_dataset (line 237) | def convert_v2_to_v1_dataset(dataset: Dataset) -> Dataset:
function camel_to_snake (line 242) | def camel_to_snake(name):
function num_tokens_from_string (line 251) | def num_tokens_from_string(
function batched (line 279) | def batched(iterable: t.Iterable, n: int) -> t.Iterator[t.Tuple]:
class ProgressBarManager (line 289) | class ProgressBarManager:
method __init__ (line 292) | def __init__(self, desc: str, show_progress: bool):
method create_single_bar (line 296) | def create_single_bar(self, total: int) -> tqdm:
method create_nested_bars (line 304) | def create_nested_bars(self, total_jobs: int, batch_size: int):
method update_batch_bar (line 326) | def update_batch_bar(
function set_logging_level (line 337) | def set_logging_level(logger_name: str = __name__, level: int = logging....
class _ContextualFormatter (line 366) | class _ContextualFormatter(logging.Formatter):
method format (line 371) | def format(self, record):
method format_time (line 384) | def format_time(self, record, datefmt=None, local_time=False):
method utc_converter (line 395) | def utc_converter(timestamp):
class MemorableNames (line 405) | class MemorableNames:
method __init__ (line 408) | def __init__(self):
method generate_name (line 586) | def generate_name(self):
method generate_unique_name (line 592) | def generate_unique_name(self):
method generate_unique_names (line 610) | def generate_unique_names(self, count):
function find_git_root (line 619) | def find_git_root(start_path: t.Union[str, Path, None] = None) -> Path:
function create_nano_id (line 644) | def create_nano_id(size: int = 12) -> str:
function async_to_sync (line 662) | def async_to_sync(async_func):
function get_test_directory (line 685) | def get_test_directory():
FILE: src/ragas/validation.py
function remap_column_names (line 14) | def remap_column_names(dataset: Dataset, column_map: dict[str, str]) -> ...
function get_supported_metric_type (line 23) | def get_supported_metric_type(ds: EvaluationDataset):
function validate_required_columns (line 37) | def validate_required_columns(ds: EvaluationDataset, metrics: t.Sequence...
function validate_supported_metrics (line 50) | def validate_supported_metrics(ds: EvaluationDataset, metrics: t.Sequenc...
FILE: tests/benchmarks/benchmark_testsetgen.py
function get_documents (line 12) | def get_documents():
FILE: tests/benchmarks/utils.py
function timeit (line 16) | def timeit(func: OrigFunc, iteration: int = 3) -> DecoratedFunc:
function print_table (line 40) | def print_table(result):
FILE: tests/conftest.py
function pytest_configure (line 17) | def pytest_configure(config):
class EchoLLM (line 34) | class EchoLLM(BaseRagasLLM):
method generate_text (line 35) | def generate_text( # type: ignore
method agenerate_text (line 43) | async def agenerate_text( # type: ignore
method is_finished (line 51) | def is_finished(self, response: LLMResult) -> bool:
class EchoEmbedding (line 55) | class EchoEmbedding(BaseRagasEmbeddings):
method aembed_documents (line 56) | async def aembed_documents(self, texts: t.List[str]) -> t.List[t.List[...
method aembed_query (line 59) | async def aembed_query(self, text: str) -> t.List[float]:
method embed_documents (line 62) | def embed_documents(self, texts: t.List[str]) -> t.List[t.List[float]]:
method embed_query (line 65) | def embed_query(self, text: str) -> t.List[float]:
function fake_llm (line 70) | def fake_llm():
function fake_embedding (line 75) | def fake_embedding():
class MockLLM (line 84) | class MockLLM:
method __init__ (line 87) | def __init__(self):
method generate (line 92) | def generate(self, prompt: str, response_model: t.Type[BaseModel]) -> ...
method agenerate (line 96) | async def agenerate(
class MockEmbedding (line 103) | class MockEmbedding(BaseRagasEmbeddings):
method embed_text (line 106) | def embed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
method aembed_text (line 110) | async def aembed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
method embed_document (line 114) | def embed_document(
method aembed_document (line 122) | async def aembed_document(
function mock_llm (line 132) | def mock_llm():
function mock_embedding (line 137) | def mock_embedding():
FILE: tests/docs/test_run_config.py
function test_openai_client_configuration (line 11) | def test_openai_client_configuration():
function test_fine_grained_timeout_control (line 40) | def test_fine_grained_timeout_control():
FILE: tests/e2e/metrics_migration/base_migration_test.py
class BaseMigrationTest (line 17) | class BaseMigrationTest:
method run_e2e_compatibility_test (line 25) | async def run_e2e_compatibility_test(
method run_metric_specific_test (line 110) | async def run_metric_specific_test(
method create_requirements_documentation (line 177) | def create_requirements_documentation(
FILE: tests/e2e/metrics_migration/conftest.py
function legacy_llm (line 18) | def legacy_llm():
function modern_llm (line 31) | def modern_llm():
function legacy_embeddings (line 44) | def legacy_embeddings():
function modern_embeddings (line 57) | def modern_embeddings():
FILE: tests/e2e/metrics_migration/test_answer_accuracy_migration.py
function nvidia_legacy_llm (line 13) | def nvidia_legacy_llm():
function nvidia_modern_llm (line 28) | def nvidia_modern_llm():
class TestAnswerAccuracyE2EMigration (line 44) | class TestAnswerAccuracyE2EMigration:
method sample_data (line 48) | def sample_data(self):
method test_llm (line 78) | def test_llm(self):
method test_modern_llm (line 90) | def test_modern_llm(self):
method test_legacy_answer_accuracy_vs_v2_answer_accuracy_e2e_compatibility (line 109) | async def test_legacy_answer_accuracy_vs_v2_answer_accuracy_e2e_compat...
method test_answer_accuracy_dual_judge_system (line 169) | async def test_answer_accuracy_dual_judge_system(self, test_modern_llm):
method test_answer_accuracy_migration_requirements_documented (line 192) | def test_answer_accuracy_migration_requirements_documented(self):
FILE: tests/e2e/metrics_migration/test_answer_correctness_migration.py
class TestAnswerCorrectnessE2EMigration (line 11) | class TestAnswerCorrectnessE2EMigration:
method sample_data (line 15) | def sample_data(self):
method test_llm (line 51) | def test_llm(self):
method test_modern_llm (line 66) | def test_modern_llm(self):
method test_legacy_embeddings (line 81) | def test_legacy_embeddings(self):
method test_modern_embeddings (line 95) | def test_modern_embeddings(self):
method test_legacy_answer_correctness_vs_v2_answer_correctness_e2e_compatibility (line 117) | async def test_legacy_answer_correctness_vs_v2_answer_correctness_e2e_...
method test_answer_correctness_factual_error_detection (line 190) | async def test_answer_correctness_factual_error_detection(
method test_answer_correctness_weight_configuration (line 275) | async def test_answer_correctness_weight_configuration(
method test_answer_correctness_parameter_validation (line 338) | def test_answer_correctness_parameter_validation(self):
method test_answer_correctness_migration_requirements_documented (line 377) | def test_answer_correctness_migration_requirements_documented(self):
FILE: tests/e2e/metrics_migration/test_answer_relevancy_migration.py
class TestAnswerRelevancyE2EMigration (line 10) | class TestAnswerRelevancyE2EMigration:
method sample_data (line 14) | def sample_data(self):
method test_llm (line 45) | def test_llm(self):
method test_modern_llm (line 58) | def test_modern_llm(self):
method test_legacy_embeddings (line 73) | def test_legacy_embeddings(self):
method test_modern_embeddings (line 88) | def test_modern_embeddings(self):
method test_legacy_answer_relevancy_vs_v2_answer_relevancy_e2e_compatibility (line 113) | async def test_legacy_answer_relevancy_vs_v2_answer_relevancy_e2e_comp...
method test_answer_relevancy_noncommittal_detection (line 178) | async def test_answer_relevancy_noncommittal_detection(
method test_answer_relevancy_migration_requirements_documented (line 260) | def test_answer_relevancy_migration_requirements_documented(self):
FILE: tests/e2e/metrics_migration/test_bleu_migration.py
class TestBleuE2EMigration (line 10) | class TestBleuE2EMigration:
method sample_data (line 14) | def sample_data(self):
method test_legacy_vs_v2_class_e2e_compatibility (line 50) | async def test_legacy_vs_v2_class_e2e_compatibility(self, sample_data):
method test_bleu_score_performance_comparison (line 89) | async def test_bleu_score_performance_comparison(self, sample_data):
method test_v2_class_no_components_needed (line 124) | async def test_v2_class_no_components_needed(self):
method test_v2_class_batch_processing (line 152) | async def test_v2_class_batch_processing(self, sample_data):
method test_bleu_with_custom_kwargs (line 177) | async def test_bleu_with_custom_kwargs(self):
FILE: tests/e2e/metrics_migration/test_context_entity_recall_migration.py
class TestContextEntityRecallE2EMigration (line 11) | class TestContextEntityRecallE2EMigration:
method sample_data (line 15) | def sample_data(self):
method test_llm (line 62) | def test_llm(self):
method test_modern_llm (line 74) | def test_modern_llm(self):
method test_legacy_context_entity_recall_vs_v2_context_entity_recall_e2e_compatibility (line 89) | async def test_legacy_context_entity_recall_vs_v2_context_entity_recal...
method test_context_entity_recall_entity_extraction_accuracy (line 144) | async def test_context_entity_recall_entity_extraction_accuracy(
method test_context_entity_recall_parameter_validation (line 214) | def test_context_entity_recall_parameter_validation(self):
method test_context_entity_recall_migration_requirements_documented (line 230) | def test_context_entity_recall_migration_requirements_documented(self):
FILE: tests/e2e/metrics_migration/test_context_precision_migration.py
class TestContextPrecisionE2EMigration (line 18) | class TestContextPrecisionE2EMigration:
method sample_data (line 22) | def sample_data(self):
method test_llm (line 60) | def test_llm(self):
method test_modern_llm (line 75) | def test_modern_llm(self):
method test_legacy_vs_v2_context_precision_with_reference_e2e_compatibility (line 90) | async def test_legacy_vs_v2_context_precision_with_reference_e2e_compa...
method test_legacy_vs_v2_context_precision_without_reference_e2e_compatibility (line 140) | async def test_legacy_vs_v2_context_precision_without_reference_e2e_co...
method test_context_precision_input_validation (line 190) | async def test_context_precision_input_validation(self, test_modern_llm):
method test_context_precision_migration_requirements_documented (line 226) | def test_context_precision_migration_requirements_documented(self):
method test_context_precision_edge_cases (line 237) | async def test_context_precision_edge_cases(self, test_modern_llm):
method test_context_precision_wrappers (line 262) | async def test_context_precision_wrappers(self, test_modern_llm):
FILE: tests/e2e/metrics_migration/test_context_recall_migration.py
class TestContextRecallE2EMigration (line 11) | class TestContextRecallE2EMigration(BaseMigrationTest):
method sample_data (line 15) | def sample_data(self):
method test_legacy_context_recall_vs_v2_context_recall_e2e_compatibility (line 63) | async def test_legacy_context_recall_vs_v2_context_recall_e2e_compatib...
method test_context_recall_attribution_detection (line 82) | async def test_context_recall_attribution_detection(self, legacy_llm, ...
method test_context_recall_migration_requirements_documented (line 143) | def test_context_recall_migration_requirements_documented(self):
FILE: tests/e2e/metrics_migration/test_context_relevance_migration.py
function nvidia_legacy_llm (line 13) | def nvidia_legacy_llm():
function nvidia_modern_llm (line 28) | def nvidia_modern_llm():
class TestContextRelevanceE2EMigration (line 44) | class TestContextRelevanceE2EMigration:
method sample_data (line 48) | def sample_data(self):
method test_llm (line 85) | def test_llm(self):
method test_modern_llm (line 97) | def test_modern_llm(self):
method test_legacy_context_relevance_vs_v2_context_relevance_e2e_compatibility (line 116) | async def test_legacy_context_relevance_vs_v2_context_relevance_e2e_co...
method test_context_relevance_edge_cases (line 176) | async def test_context_relevance_edge_cases(self, test_modern_llm):
method test_context_relevance_dual_judge_system (line 199) | async def test_context_relevance_dual_judge_system(self, test_modern_l...
method test_context_relevance_migration_requirements_documented (line 221) | def test_context_relevance_migration_requirements_documented(self):
FILE: tests/e2e/metrics_migration/test_factual_correctness_migration.py
class TestFactualCorrectnessE2EMigration (line 13) | class TestFactualCorrectnessE2EMigration:
method sample_data (line 17) | def sample_data(self):
method test_llm (line 48) | def test_llm(self):
method test_modern_llm (line 63) | def test_modern_llm(self):
method test_legacy_factual_correctness_vs_v2_factual_correctness_e2e_compatibility (line 78) | async def test_legacy_factual_correctness_vs_v2_factual_correctness_e2...
method test_factual_correctness_edge_cases (line 149) | async def test_factual_correctness_edge_cases(self, test_modern_llm):
method test_factual_correctness_different_modes (line 172) | async def test_factual_correctness_different_modes(self, test_modern_l...
method test_factual_correctness_atomicity_coverage_configurations (line 204) | async def test_factual_correctness_atomicity_coverage_configurations(
method test_factual_correctness_migration_requirements_documented (line 237) | def test_factual_correctness_migration_requirements_documented(self):
FILE: tests/e2e/metrics_migration/test_faithfulness_migration.py
class TestFaithfulnessE2EMigration (line 11) | class TestFaithfulnessE2EMigration:
method sample_data (line 15) | def sample_data(self):
method test_llm (line 54) | def test_llm(self):
method test_modern_llm (line 69) | def test_modern_llm(self):
method test_legacy_faithfulness_vs_v2_faithfulness_e2e_compatibility (line 84) | async def test_legacy_faithfulness_vs_v2_faithfulness_e2e_compatibility(
method test_faithfulness_edge_cases (line 137) | async def test_faithfulness_edge_cases(self, test_modern_llm):
method test_faithfulness_high_vs_low_scores (line 170) | async def test_faithfulness_high_vs_low_scores(self, test_modern_llm):
method test_faithfulness_migration_requirements_documented (line 202) | def test_faithfulness_migration_requirements_documented(self):
FILE: tests/e2e/metrics_migration/test_noise_sensitivity_migration.py
class TestNoiseSensitivityE2EMigration (line 10) | class TestNoiseSensitivityE2EMigration:
method sample_data (line 14) | def sample_data(self):
method test_llm (line 43) | def test_llm(self):
method test_modern_llm (line 58) | def test_modern_llm(self):
method test_legacy_noise_sensitivity_vs_v2_noise_sensitivity_e2e_compatibility (line 73) | async def test_legacy_noise_sensitivity_vs_v2_noise_sensitivity_e2e_co...
method test_noise_sensitivity_mode_configuration (line 135) | async def test_noise_sensitivity_mode_configuration(self, test_modern_...
method test_noise_sensitivity_parameter_validation (line 171) | async def test_noise_sensitivity_parameter_validation(self, test_moder...
method test_noise_sensitivity_migration_requirements_documented (line 188) | def test_noise_sensitivity_migration_requirements_documented(self):
FILE: tests/e2e/metrics_migration/test_response_groundedness_migration.py
class TestResponseGroundednessE2EMigration (line 11) | class TestResponseGroundednessE2EMigration:
method sample_data (line 15) | def sample_data(self):
method test_llm (line 56) | def test_llm(self):
method test_modern_llm (line 71) | def test_modern_llm(self):
method test_legacy_response_groundedness_vs_v2_response_groundedness_e2e_compatibility (line 87) | async def test_legacy_response_groundedness_vs_v2_response_groundednes...
method test_response_groundedness_edge_cases (line 139) | async def test_response_groundedness_edge_cases(self, test_modern_llm):
method test_response_groundedness_scoring_behavior (line 162) | async def test_response_groundedness_scoring_behavior(self, test_moder...
method test_response_groundedness_dual_judge_system (line 193) | async def test_response_groundedness_dual_judge_system(self, test_mode...
method test_response_groundedness_migration_requirements_documented (line 215) | def test_response_groundedness_migration_requirements_documented(self):
FILE: tests/e2e/metrics_migration/test_rouge_migration.py
class TestRougeE2EMigration (line 16) | class TestRougeE2EMigration:
method sample_data (line 20) | def sample_data(self):
method test_legacy_vs_v2_class_e2e_compatibility (line 62) | async def test_legacy_vs_v2_class_e2e_compatibility(
method test_rouge_score_performance_comparison (line 110) | async def test_rouge_score_performance_comparison(self, sample_data):
method test_v2_class_no_components_needed (line 158) | async def test_v2_class_no_components_needed(self):
method test_v2_class_batch_processing (line 186) | async def test_v2_class_batch_processing(self, sample_data):
FILE: tests/e2e/metrics_migration/test_semantic_similarity_migration.py
class TestSemanticSimilarityE2EMigration (line 13) | class TestSemanticSimilarityE2EMigration:
method sample_data (line 17) | def sample_data(self):
method test_legacy_embeddings (line 48) | def test_legacy_embeddings(self):
method test_modern_embeddings (line 62) | def test_modern_embeddings(self):
method test_legacy_semantic_similarity_vs_v2_semantic_similarity_e2e_compatibility (line 85) | async def test_legacy_semantic_similarity_vs_v2_semantic_similarity_e2...
method test_semantic_similarity_with_threshold (line 140) | async def test_semantic_similarity_with_threshold(
method test_v2_class_batch_processing (line 203) | async def test_v2_class_batch_processing(self, sample_data, test_moder...
method test_semantic_similarity_migration_requirements_documented (line 230) | def test_semantic_similarity_migration_requirements_documented(self):
FILE: tests/e2e/metrics_migration/test_string_migration.py
class TestNonLLMStringSimilarityE2EMigration (line 21) | class TestNonLLMStringSimilarityE2EMigration:
method sample_data (line 25) | def sample_data(self):
method test_legacy_vs_v2_class_e2e_compatibility_levenshtein (line 76) | async def test_legacy_vs_v2_class_e2e_compatibility_levenshtein(self, ...
method test_legacy_vs_v2_class_e2e_compatibility_jaro_winkler (line 121) | async def test_legacy_vs_v2_class_e2e_compatibility_jaro_winkler(self,...
method test_all_distance_measures (line 166) | async def test_all_distance_measures(self):
method test_performance_comparison (line 186) | async def test_performance_comparison(self, sample_data):
method test_v2_class_no_components_needed (line 221) | async def test_v2_class_no_components_needed(self):
method test_v2_class_batch_processing (line 249) | async def test_v2_class_batch_processing(self, sample_data):
method test_edge_cases (line 274) | async def test_edge_cases(self):
class TestExactMatchE2EMigration (line 297) | class TestExactMatchE2EMigration:
method sample_data (line 301) | def sample_data(self):
method test_legacy_vs_v2_class_e2e_compatibility (line 332) | async def test_legacy_vs_v2_class_e2e_compatibility(self, sample_data):
class TestStringPresenceE2EMigration (line 371) | class TestStringPresenceE2EMigration:
method sample_data (line 375) | def sample_data(self):
method test_legacy_vs_v2_class_e2e_compatibility (line 416) | async def test_legacy_vs_v2_class_e2e_compatibility(self, sample_data):
FILE: tests/e2e/metrics_migration/test_summary_score_migration.py
class TestSummaryScoreE2EMigration (line 10) | class TestSummaryScoreE2EMigration:
method sample_data (line 14) | def sample_data(self):
method test_llm (line 41) | def test_llm(self):
method test_modern_llm (line 56) | def test_modern_llm(self):
method test_legacy_summary_score_vs_v2_summary_score_e2e_compatibility (line 71) | async def test_legacy_summary_score_vs_v2_summary_score_e2e_compatibil...
method test_summary_score_weight_configuration (line 118) | async def test_summary_score_weight_configuration(self, test_modern_llm):
method test_summary_score_parameter_validation (line 152) | async def test_summary_score_parameter_validation(self, test_modern_llm):
method test_summary_score_migration_requirements_documented (line 175) | def test_summary_score_migration_requirements_documented(self):
FILE: tests/e2e/metrics_migration/test_utils.py
function create_legacy_sample (line 9) | def create_legacy_sample(
function compare_scores_with_tolerance (line 44) | def compare_scores_with_tolerance(
function assert_score_types (line 70) | def assert_score_types(legacy_score: Any, v2_result: MetricResult) -> None:
function print_test_header (line 90) | def print_test_header(
function print_score_comparison (line 112) | def print_score_comparison(
function print_test_success (line 130) | def print_test_success(message: str = "Scores within tolerance!") -> None:
function print_metric_specific_info (line 139) | def print_metric_specific_info(metric_name: str, description: str) -> None:
FILE: tests/e2e/test_adaptation.py
function test_adapt (line 11) | async def test_adapt():
FILE: tests/e2e/test_amnesty_in_ci.py
function assert_in_range (line 22) | def assert_in_range(score: float, value: float, plus_or_minus: float):
function test_amnesty_e2e (line 31) | def test_amnesty_e2e():
function test_assert_in_range (line 41) | def test_assert_in_range():
FILE: tests/e2e/test_dataset_utils.py
function load_amnesty_dataset_safe (line 67) | def load_amnesty_dataset_safe(config: str = "english_v3"):
function load_fiqa_dataset_safe (line 92) | def load_fiqa_dataset_safe(config: str = "ragas_eval_v3"):
FILE: tests/e2e/test_dspy_integration.py
function test_dspy_optimizer_import (line 15) | def test_dspy_optimizer_import():
function test_dspy_optimizer_basic_optimization (line 26) | def test_dspy_optimizer_basic_optimization():
function test_dspy_adapter_conversions (line 114) | def test_dspy_adapter_conversions():
FILE: tests/e2e/test_fullflow.py
function test_evaluate_e2e (line 16) | def test_evaluate_e2e():
FILE: tests/e2e/test_langchain_llm_attributes.py
function test_langchain_chat_models_have_temperature (line 36) | def test_langchain_chat_models_have_temperature(model):
function test_langchain_chat_models_have_n (line 43) | def test_langchain_chat_models_have_n(model):
FILE: tests/e2e/test_testset_generation.py
function test_testset_generation_e2e (line 9) | def test_testset_generation_e2e():
FILE: tests/test_quoted_spans.py
function test_perfect_match (line 16) | def test_perfect_match():
function test_mismatch_detected (line 26) | def test_mismatch_detected():
function test_mixed_case_and_whitespace (line 36) | def test_mixed_case_and_whitespace():
function test_no_quotes_returns_zero_with_zero_denominator (line 44) | def test_no_quotes_returns_zero_with_zero_denominator():
FILE: tests/unit/backends/test_gdrive_backend.py
class HttpError (line 16) | class HttpError(Exception):
method __init__ (line 17) | def __init__(self, resp, content):
class SampleModel (line 26) | class SampleModel(BaseModel):
class TestGDriveBackendAvailability (line 32) | class TestGDriveBackendAvailability:
method test_gdrive_available_import (line 35) | def test_gdrive_available_import(self):
class TestGDriveBackendInitialization (line 50) | class TestGDriveBackendInitialization:
method test_service_account_auth_success (line 56) | def test_service_account_auth_success(
method test_auth_failure_no_credentials (line 93) | def test_auth_failure_no_credentials(self, mock_exists, mock_build):
method test_invalid_folder_id (line 103) | def test_invalid_folder_id(self, mock_exists, mock_credentials, mock_b...
class TestGDriveBackendOperations (line 129) | class TestGDriveBackendOperations:
method _create_mock_backend (line 132) | def _create_mock_backend(self):
method test_spreadsheet_exists_check (line 145) | def test_spreadsheet_exists_check(self):
method test_load_nonexistent_dataset (line 163) | def test_load_nonexistent_dataset(self):
method test_load_dataset_success (line 175) | def test_load_dataset_success(self):
method test_load_empty_dataset (line 202) | def test_load_empty_dataset(self):
method test_save_dataset_success (line 217) | def test_save_dataset_success(self):
method test_save_empty_dataset (line 243) | def test_save_empty_dataset(self):
method test_list_datasets (line 259) | def test_list_datasets(self):
method test_list_experiments (line 281) | def test_list_experiments(self):
method test_complex_data_serialization (line 294) | def test_complex_data_serialization(self):
class TestGDriveBackendIntegration (line 339) | class TestGDriveBackendIntegration:
method test_backend_implements_basebackend (line 342) | def test_backend_implements_basebackend(self):
method test_error_without_dependencies (line 362) | def test_error_without_dependencies(self):
FILE: tests/unit/backends/test_inmemory.py
class SimpleTestModel (line 18) | class SimpleTestModel(BaseModel):
class ComplexTestModel (line 25) | class ComplexTestModel(BaseModel):
function backend (line 34) | def backend():
function simple_data (line 40) | def simple_data():
function complex_data (line 50) | def complex_data():
class TestInMemoryBackendBasics (line 69) | class TestInMemoryBackendBasics:
method test_backend_initialization (line 75) | def test_backend_initialization(self):
method test_save_and_load_operations (line 90) | def test_save_and_load_operations(self, backend, simple_data):
method test_save_and_load_complex_data (line 118) | def test_save_and_load_complex_data(self, backend, complex_data):
method test_list_empty_operations (line 142) | def test_list_empty_operations(self, backend):
method test_list_operations_after_saving (line 157) | def test_list_operations_after_saving(self, backend, simple_data):
method test_save_empty_operations (line 181) | def test_save_empty_operations(self, backend):
method test_overwrite_operations (line 204) | def test_overwrite_operations(self, backend, simple_data):
method test_datasets_and_experiments_separate_storage (line 237) | def test_datasets_and_experiments_separate_storage(self, backend, simp...
method test_data_model_parameter_ignored (line 264) | def test_data_model_parameter_ignored(self, backend, simple_data):
class TestInMemoryBackendErrorHandling (line 291) | class TestInMemoryBackendErrorHandling:
method test_load_nonexistent_dataset (line 294) | def test_load_nonexistent_dataset(self, backend):
method test_load_nonexistent_experiment (line 306) | def test_load_nonexistent_experiment(self, backend):
method test_none_values_handling (line 318) | def test_none_values_handling(self, backend):
method test_unicode_and_special_characters (line 342) | def test_unicode_and_special_characters(self, backend):
method test_large_dataset_handling (line 371) | def test_large_dataset_handling(self, backend):
method test_deeply_nested_structures (line 395) | def test_deeply_nested_structures(self, backend):
class TestInMemoryBackendIntegration (line 438) | class TestInMemoryBackendIntegration:
method test_backend_registration (line 444) | def test_backend_registration(self):
method test_dataset_with_inmemory_backend (line 464) | def test_dataset_with_inmemory_backend(self, backend, simple_data):
method test_dataset_save_and_load_cycle (line 493) | def test_dataset_save_and_load_cycle(self, backend, simple_data):
method test_dataset_train_test_split_uses_inmemory (line 520) | def test_dataset_train_test_split_uses_inmemory(self, simple_data):
method test_train_test_split_comprehensive (line 561) | def test_train_test_split_comprehensive(self, simple_data):
method test_pydantic_model_validation_with_inmemory (line 613) | def test_pydantic_model_validation_with_inmemory(self, backend, simple...
class TestInMemoryBackendIsolation (line 657) | class TestInMemoryBackendIsolation:
method test_multiple_backend_instances_isolation (line 660) | def test_multiple_backend_instances_isolation(self, simple_data):
method test_concurrent_save_operations (line 690) | def test_concurrent_save_operations(self, simple_data):
method test_concurrent_read_operations (line 738) | def test_concurrent_read_operations(self, backend, simple_data):
method test_mixed_concurrent_operations (line 785) | def test_mixed_concurrent_operations(self, backend, simple_data):
method test_memory_cleanup_on_overwrite (line 856) | def test_memory_cleanup_on_overwrite(self, backend, simple_data):
class TestInMemoryBackendPerformance (line 898) | class TestInMemoryBackendPerformance:
method test_complex_data_structure_preservation (line 901) | def test_complex_data_structure_preservation(self, backend):
method test_edge_case_dataset_names (line 948) | def test_edge_case_dataset_names(self, backend, simple_data):
FILE: tests/unit/backends/test_local_csv.py
class SimpleTestModel (line 15) | class SimpleTestModel(BaseModel):
class ComplexTestModel (line 22) | class ComplexTestModel(BaseModel):
class NestedTestModel (line 30) | class NestedTestModel(BaseModel):
function temp_dir (line 38) | def temp_dir():
function backend (line 45) | def backend(temp_dir):
function simple_data (line 51) | def simple_data():
function complex_data (line 61) | def complex_data():
function nested_data (line 82) | def nested_data():
class TestBasicFunctionality (line 101) | class TestBasicFunctionality:
method test_initialization (line 104) | def test_initialization(self, temp_dir):
method test_get_data_dir (line 109) | def test_get_data_dir(self, backend):
method test_get_file_path (line 117) | def test_get_file_path(self, backend):
method test_save_and_load_simple_data (line 125) | def test_save_and_load_simple_data(self, backend, simple_data):
method test_directory_creation (line 139) | def test_directory_creation(self, backend, simple_data):
method test_list_datasets_and_experiments (line 155) | def test_list_datasets_and_experiments(self, backend, simple_data):
method test_save_empty_data (line 173) | def test_save_empty_data(self, backend):
class TestDataTypeEdgeCases (line 187) | class TestDataTypeEdgeCases:
method test_nested_dictionaries (line 191) | def test_nested_dictionaries(self, backend):
method test_lists_of_objects (line 216) | def test_lists_of_objects(self, backend):
method test_mixed_types (line 240) | def test_mixed_types(self, backend):
method test_datetime_objects (line 266) | def test_datetime_objects(self, backend):
method test_complex_nested_structure (line 287) | def test_complex_nested_structure(self, backend):
class TestBaseModelIntegration (line 314) | class TestBaseModelIntegration:
method test_simple_basemodel_save_load (line 317) | def test_simple_basemodel_save_load(self, backend, simple_data):
method test_complex_basemodel_roundtrip (line 337) | def test_complex_basemodel_roundtrip(self, backend, complex_data):
method test_basemodel_type_coercion (line 349) | def test_basemodel_type_coercion(self, backend):
class TestErrorHandling (line 366) | class TestErrorHandling:
method test_load_nonexistent_file (line 369) | def test_load_nonexistent_file(self, backend):
method test_unicode_and_special_characters (line 377) | def test_unicode_and_special_characters(self, backend):
method test_csv_injection_protection (line 396) | def test_csv_injection_protection(self, backend):
method test_empty_and_null_values (line 414) | def test_empty_and_null_values(self, backend):
method test_large_text_fields (line 434) | def test_large_text_fields(self, backend):
method test_malformed_csv_handling (line 452) | def test_malformed_csv_handling(self, backend, temp_dir):
FILE: tests/unit/backends/test_local_jsonl.py
class SimpleTestModel (line 16) | class SimpleTestModel(BaseModel):
class ComplexTestModel (line 23) | class ComplexTestModel(BaseModel):
class NestedTestModel (line 31) | class NestedTestModel(BaseModel):
function temp_dir (line 39) | def temp_dir():
function jsonl_backend_fixture (line 46) | def jsonl_backend_fixture(temp_dir):
function simple_data (line 52) | def simple_data():
function complex_data (line 62) | def complex_data():
function nested_data (line 83) | def nested_data():
class TestBasicFunctionality (line 102) | class TestBasicFunctionality:
method test_initialization (line 105) | def test_initialization(self, temp_dir):
method test_get_data_dir (line 110) | def test_get_data_dir(self, backend):
method test_get_file_path (line 118) | def test_get_file_path(self, backend):
method test_save_and_load_simple_data (line 126) | def test_save_and_load_simple_data(self, backend, simple_data):
method test_directory_creation (line 141) | def test_directory_creation(self, backend, simple_data):
method test_list_datasets_and_experiments (line 157) | def test_list_datasets_and_experiments(self, backend, simple_data):
method test_save_empty_data (line 175) | def test_save_empty_data(self, backend):
class TestDataTypeEdgeCases (line 189) | class TestDataTypeEdgeCases:
method test_nested_dictionaries (line 192) | def test_nested_dictionaries(self, backend):
method test_lists_of_objects (line 212) | def test_lists_of_objects(self, backend):
method test_mixed_types (line 233) | def test_mixed_types(self, backend):
method test_datetime_objects (line 255) | def test_datetime_objects(self, backend):
method test_complex_nested_structure (line 285) | def test_complex_nested_structure(self, backend):
class TestBaseModelIntegration (line 312) | class TestBaseModelIntegration:
method test_simple_basemodel_save_load (line 315) | def test_simple_basemodel_save_load(self, backend, simple_data):
method test_complex_basemodel_roundtrip (line 331) | def test_complex_basemodel_roundtrip(self, backend, complex_data):
method test_basemodel_type_coercion (line 347) | def test_basemodel_type_coercion(self, backend):
class TestErrorHandling (line 364) | class TestErrorHandling:
method test_load_nonexistent_file (line 367) | def test_load_nonexistent_file(self, backend):
method test_unicode_and_special_characters (line 375) | def test_unicode_and_special_characters(self, backend):
method test_json_special_characters (line 394) | def test_json_special_characters(self, backend):
method test_empty_and_null_values (line 414) | def test_empty_and_null_values(self, backend):
method test_large_text_fields (line 436) | def test_large_text_fields(self, backend):
method test_malformed_jsonl_handling (line 454) | def test_malformed_jsonl_handling(self, backend, temp_dir):
function print_jsonl_content (line 476) | def print_jsonl_content(jsonl_backend, data_type, name):
FILE: tests/unit/integrations/test_ag_ui.py
class MockEvent (line 43) | class MockEvent:
method __init__ (line 46) | def __init__(self, event_type: str, **kwargs):
function basic_text_message_events (line 54) | def basic_text_message_events():
function tool_call_events (line 70) | def tool_call_events():
function test_import_error_without_ag_ui_protocol (line 95) | def test_import_error_without_ag_ui_protocol():
function test_basic_text_message_conversion (line 107) | def test_basic_text_message_conversion(basic_text_message_events):
function test_message_with_metadata (line 120) | def test_message_with_metadata(basic_text_message_events):
function test_message_without_metadata (line 136) | def test_message_without_metadata(basic_text_message_events):
function test_tool_call_conversion (line 147) | def test_tool_call_conversion(tool_call_events):
function test_tool_call_with_metadata (line 169) | def test_tool_call_with_metadata(tool_call_events):
function test_step_context_in_metadata (line 181) | def test_step_context_in_metadata():
function test_messages_snapshot_conversion (line 201) | def test_messages_snapshot_conversion():
function test_snapshot_with_metadata (line 224) | def test_snapshot_with_metadata():
function test_non_message_events_filtered (line 239) | def test_non_message_events_filtered():
function test_incomplete_message_stream (line 259) | def test_incomplete_message_stream(caplog):
function test_orphaned_content_event (line 276) | def test_orphaned_content_event(caplog):
function test_tool_call_argument_parsing_error (line 290) | def test_tool_call_argument_parsing_error(caplog):
function test_tool_call_result_retroactive_attachment (line 316) | def test_tool_call_result_retroactive_attachment():
function test_event_collector_reuse (line 365) | def test_event_collector_reuse(basic_text_message_events):
function test_multiple_tool_calls_in_sequence (line 388) | def test_multiple_tool_calls_in_sequence():
function test_empty_event_list (line 415) | def test_empty_event_list():
function test_wrong_snapshot_type_error (line 423) | def test_wrong_snapshot_type_error():
function test_role_mapping (line 431) | def test_role_mapping():
function test_complex_conversation_flow (line 453) | def test_complex_conversation_flow():
function test_text_message_chunk (line 506) | def test_text_message_chunk():
function test_tool_call_chunk (line 523) | def test_tool_call_chunk():
function test_tool_call_chunk_with_dict_delta (line 546) | def test_tool_call_chunk_with_dict_delta():
function _has_fastapi_deps (line 596) | def _has_fastapi_deps():
function test_call_ag_ui_endpoint (line 609) | async def test_call_ag_ui_endpoint():
function test_call_ag_ui_endpoint_with_config (line 666) | async def test_call_ag_ui_endpoint_with_config():
function test_call_ag_ui_endpoint_malformed_json (line 711) | async def test_call_ag_ui_endpoint_malformed_json():
function test_convert_ragas_messages_to_ag_ui (line 758) | def test_convert_ragas_messages_to_ag_ui():
function test_extract_response (line 798) | def test_extract_response():
function test_extract_response_empty (line 813) | def test_extract_response_empty():
function test_extract_tool_calls (line 826) | def test_extract_tool_calls():
function test_extract_tool_calls_empty (line 852) | def test_extract_tool_calls_empty():
function test_extract_contexts (line 865) | def test_extract_contexts():
function test_extract_contexts_empty (line 882) | def test_extract_contexts_empty():
function test_build_sample_single_turn (line 900) | def test_build_sample_single_turn():
function test_build_sample_multi_turn_with_list_input (line 921) | def test_build_sample_multi_turn_with_list_input():
function test_build_sample_multi_turn_with_tool_calls (line 944) | def test_build_sample_multi_turn_with_tool_calls():
function test_run_ag_ui_row_processes_row (line 977) | async def test_run_ag_ui_row_processes_row():
function test_run_ag_ui_row_extracts_tool_results (line 1019) | async de
Condensed preview — 638 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (5,366K chars).
[
{
"path": ".cursor/commands/git-pr.md",
"chars": 158,
"preview": "Make sure you are on a branch other than main. \nCommit changes. \nRun make format. \nCreate PR using gh cli following .git"
},
{
"path": ".cursor/commands/update-howto-guide.md",
"chars": 7372,
"preview": "# Update How-to Guide\n\nUpdates the mentioned how-to guide to use src/ragas/metrics/collections API instead of the legacy"
},
{
"path": ".cursor/rules/docs-diataxis-guidelines.mdc",
"chars": 3715,
"preview": "---\nglobs: docs/**\n---\n# Diátaxis Documentation Guidelines\n\nWhen writing or editing documentation, categorise each page "
},
{
"path": ".cursor/rules/docs-structure.mdc",
"chars": 2203,
"preview": "---\nglobs: docs/**\n---\n# Documentation Structure & Workflow\n\nFollow these conventions when creating or editing documenta"
},
{
"path": ".cursor/rules/project-structure.mdc",
"chars": 962,
"preview": "---\nalwaysApply: true\n---\n# Monorepo Project Structure\n\nThe repository is a monorepo consisting of two primary component"
},
{
"path": ".cursor/rules/update-guide.mdc",
"chars": 321,
"preview": "---\nalwaysApply: false\n---\nWe are writing a how to guide for Ragas docs and Ragas users.\n\nSo after any coding step we co"
},
{
"path": ".cursor/rules/use-uv-cli.mdc",
"chars": 314,
"preview": "---\nalwaysApply: true\n---\n# Use `uv run` for Python CLI commands\n\nThis repository manages its virtual environment and de"
},
{
"path": ".cursor/worktrees.json",
"chars": 115,
"preview": "{\n \"setup-worktree\": [\n \"cp $ROOT_WORKTREE_PATH/.env .env\",\n \"make install-minimal\",\n \"make check\"\n ]\n}\n"
},
{
"path": ".dockerignore",
"chars": 25,
"preview": "Dockerfile\ntest_resources"
},
{
"path": ".gitattributes",
"chars": 0,
"preview": ""
},
{
"path": ".github/ISSUE_TEMPLATE/bug_report.md",
"chars": 845,
"preview": "---\nname: Bug Report\nabout: Report any bugs your encounter and we will try our best to fix it for you 🙂\ntitle: ''\nlabels"
},
{
"path": ".github/ISSUE_TEMPLATE/feature_request.md",
"chars": 628,
"preview": "---\nname: Feature Request\nabout: Feel like something is mising? Let us know!\ntitle: ''\nlabels: 'enhancement'\nassignees: "
},
{
"path": ".github/ISSUE_TEMPLATE/question.md",
"chars": 491,
"preview": "---\nname: Questions\nabout: Any questions or doubts? Ask us here!\ntitle: ''\nlabels: 'question'\nassignees: ''\n---\n\n[ ] I c"
},
{
"path": ".github/pull_request_template.md",
"chars": 884,
"preview": "## Issue Link / Problem Description\n<!-- Link to related issue or describe the problem this PR solves -->\n- Fixes #[issu"
},
{
"path": ".github/workflows/ci.yaml",
"chars": 5806,
"preview": "name: CI\n\non:\n pull_request:\n\npermissions:\n contents: read\n\nenv:\n LINES: 120\n COLUMNS: 120\n\n# https://docs.github.co"
},
{
"path": ".github/workflows/claude-code-review.yml",
"chars": 3802,
"preview": "name: Claude Code Review\n\non:\n issue_comment:\n types: [created]\n\njobs:\n claude-review:\n if: |\n github.event"
},
{
"path": ".github/workflows/claude-docs-apply.yml",
"chars": 4010,
"preview": "name: Claude Docs Apply\n\non:\n pull_request_target:\n types: [labeled]\n\njobs:\n apply-docs:\n if: github.event.label"
},
{
"path": ".github/workflows/claude-docs-check.yml",
"chars": 4574,
"preview": "name: Claude Docs Check\n\non:\n pull_request_target:\n types: [opened, synchronize, reopened]\n paths:\n - \"src/*"
},
{
"path": ".github/workflows/claude.yml",
"chars": 2304,
"preview": "name: Claude Code\n\non:\n issue_comment:\n types: [created]\n pull_request_review_comment:\n types: [created]\n issue"
},
{
"path": ".github/workflows/issue-manager.yaml",
"chars": 1467,
"preview": "name: Issue Manager\n\non:\n schedule:\n - cron: \"0 0 * * *\"\n issue_comment:\n types:\n - created\n - edited\n"
},
{
"path": ".github/workflows/publish-examples.yml",
"chars": 1144,
"preview": "name: Upload ragas-examples Package\n\non:\n release:\n types: [published]\n\npermissions:\n contents: read\n\njobs:\n deplo"
},
{
"path": ".github/workflows/python-publish.yml",
"chars": 1549,
"preview": "# This workflow will upload Python Packages using Twine when a release is created\n# For more information see: https://do"
},
{
"path": ".gitignore",
"chars": 3873,
"preview": "# General\n.DS_Store\n.AppleDouble\n.LSOverride\n\n# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n"
},
{
"path": ".pre-commit-config.yaml",
"chars": 381,
"preview": "# Pre-commit configuration for entire ragas monorepo\n# Install with: make install && pre-commit install\nrepos:\n - repo:"
},
{
"path": ".readthedocs.yml",
"chars": 365,
"preview": "version: 2\n\nmkdocs:\n configuration: mkdocs.yml\n\nbuild:\n os: ubuntu-22.04\n tools:\n python: \"3.12\"\n commands:\n -"
},
{
"path": "CLAUDE.md",
"chars": 6549,
"preview": "# CLAUDE.md\n\nThis file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.\n\n## "
},
{
"path": "CODE_OF_CONDUCT.md",
"chars": 6287,
"preview": "# Code of Conduct\n\n## Our Commitment\n\nWe are committed to providing a welcoming and inclusive environment for all people"
},
{
"path": "CONTRIBUTING.md",
"chars": 11843,
"preview": "# Development Guide for Ragas Monorepo\n\nThis comprehensive guide covers development workflows for the Ragas monorepo, de"
},
{
"path": "LICENSE",
"chars": 11346,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "Makefile",
"chars": 9674,
"preview": "GIT_ROOT ?= $(shell git rev-parse --show-toplevel)\n\n# Optionally show commands being executed with V=1\nQ := $(if $(V),,@"
},
{
"path": "README.md",
"chars": 6966,
"preview": "<h1 align=\"center\">\n <img style=\"vertical-align:middle\" height=\"200\"\n src=\"https://raw.githubusercontent.com/vibrantla"
},
{
"path": "SECURITY.md",
"chars": 2874,
"preview": "# Security Policy\n\n## Reporting Security Issues\n\nWe take the security of RAGAS seriously. If you discover a security vul"
},
{
"path": "docs/INSTALL",
"chars": 620,
"preview": "# Documentation Development Setup\n\n## Prerequisites\n\n1. Install uv (Python package manager):\n ```bash\n curl -LsSf ht"
},
{
"path": "docs/Makefile",
"chars": 638,
"preview": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line, and also\n# from the "
},
{
"path": "docs/_static/annotated_data.json",
"chars": 51102,
"preview": "{\n \"helpfulness\": [\n {\n \"metric_input\": {\n \"user_input\": \"can you fix this up better?\\n\\nhi sir im hopin"
},
{
"path": "docs/_static/css/highlight_ipython3.css",
"chars": 5776,
"preview": "pre { line-height: 125%; }\ntd.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; paddin"
},
{
"path": "docs/_static/css/highlight_ipython3_dark.css",
"chars": 6047,
"preview": "pre { line-height: 125%; }\ntd.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; paddin"
},
{
"path": "docs/_static/css/highlight_ipython3_light.css",
"chars": 5532,
"preview": "pre { line-height: 125%; }\ntd.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; paddin"
},
{
"path": "docs/_static/css/highlight_python.css",
"chars": 5776,
"preview": "pre { line-height: 125%; }\ntd.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; paddin"
},
{
"path": "docs/_static/css/highlight_python_dark.css",
"chars": 5887,
"preview": "pre { line-height: 125%; }\ntd.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; paddin"
},
{
"path": "docs/_static/css/highlight_python_light.css",
"chars": 5392,
"preview": "pre { line-height: 125%; }\ntd.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; paddin"
},
{
"path": "docs/_static/css/ragas.css",
"chars": 1333,
"preview": "/* Base Theme */\ndiv.cell_input {\n border: none !important;\n}\n\n.cell_output {\n padding-left: 0px !important;\n borde"
},
{
"path": "docs/_static/edited_chain_runs.json",
"chars": 64013,
"preview": "{\n \"answer_correctness\": [\n {\n \"metric_input\": {\n \"user_input\": \"What is the Theory "
},
{
"path": "docs/_static/js/commonroom.js",
"chars": 598,
"preview": "// CommonRoom Analytics\n(function() {\n if (typeof window === 'undefined') return;\n if (typeof window.signals !== 'unde"
},
{
"path": "docs/_static/js/header_border.js",
"chars": 1300,
"preview": "const header_div = document.querySelector(\".md-header\");\nconst navbar_div = document.querySelector(\".md-tabs\");\nconst bo"
},
{
"path": "docs/_static/js/mathjax.js",
"chars": 423,
"preview": "window.MathJax = {\n tex: {\n inlineMath: [[\"\\\\(\", \"\\\\)\"]],\n displayMath: [[\"\\\\[\", \"\\\\]\"]],\n proce"
},
{
"path": "docs/_static/js/mendable_chat_bubble.js",
"chars": 1187,
"preview": "document.addEventListener(\"DOMContentLoaded\", () => {\n function loadScript(src, callback) {\n var script = document.c"
},
{
"path": "docs/_static/js/toggle.js",
"chars": 57,
"preview": "document.addEventListener('DOMContentLoaded', () => {\n});"
},
{
"path": "docs/_static/sample_annotated_summary.json",
"chars": 37950,
"preview": "{\n \"summary_accuracy\": [\n {\n \"metric_input\": {\n \"user_input\": \"summarise given text\\nThe Q2 earnings rep"
},
{
"path": "docs/alfred.py",
"chars": 2326,
"preview": "from __future__ import annotations\n\nimport argparse\nimport asyncio\nimport os\nimport typing as t\nfrom collections import "
},
{
"path": "docs/community/index.md",
"chars": 8284,
"preview": "# ❤️ Community\n\n> \"Alone we can do so little; together we can do so much.\" - Helen Keller\n\nOur project thrives on the vi"
},
{
"path": "docs/community/pdf_export.md",
"chars": 1562,
"preview": "# PDF Export\n\n## Purpose\nThe PDF export feature builds the complete Ragas documentation as a single PDF file using MkDoc"
},
{
"path": "docs/concepts/components/eval_dataset.md",
"chars": 4323,
"preview": "# Evaluation Dataset\n\nAn evaluation dataset is a homogeneous collection of [data samples](eval_sample.md) designed to as"
},
{
"path": "docs/concepts/components/eval_sample.md",
"chars": 3926,
"preview": "# Evaluation Sample\n\nAn evaluation sample is a single structured data instance that is used to assess and measure the pe"
},
{
"path": "docs/concepts/components/index.md",
"chars": 207,
"preview": "# Components Guide\n\nThis guide provides an overview of the different components used inside Ragas.\n\n- [Prompt Object](pr"
},
{
"path": "docs/concepts/components/prompt.md",
"chars": 3261,
"preview": "# Prompt Object\n\n\nPrompts in Ragas are used inside various metrics and synthetic data generation tasks. In each of these"
},
{
"path": "docs/concepts/datasets.md",
"chars": 4636,
"preview": "# Datasets and Experiment Results\n\nWhen we evaluate AI systems, we typically work with two main types of data:\n\n1. **Eva"
},
{
"path": "docs/concepts/experimentation.md",
"chars": 6719,
"preview": "# Experiments\n\n## What is an experiment?\n\nAn experiment is a deliberate change made to your application to test a hypoth"
},
{
"path": "docs/concepts/feedback/index.md",
"chars": 554,
"preview": "# Utilizing User Feedback\n\nUser feedback can often be noisy and challenging to harness effectively. However, within the "
},
{
"path": "docs/concepts/index.md",
"chars": 1427,
"preview": "# 📚 Core Concepts\n\n\n<div class=\"grid cards\" markdown>\n\n- :material-flask-outline:{ .lg .middle } [__Experimentation__]"
},
{
"path": "docs/concepts/metrics/available_metrics/agents.md",
"chars": 24132,
"preview": "# Agentic or Tool use\n\nAgentic or tool use workflows can be evaluated in multiple dimensions. Here are some of the metri"
},
{
"path": "docs/concepts/metrics/available_metrics/answer_correctness.md",
"chars": 4955,
"preview": "## Answer Correctness\n\nThe assessment of Answer Correctness involves gauging the accuracy of the generated answer when c"
},
{
"path": "docs/concepts/metrics/available_metrics/answer_relevance.md",
"chars": 4556,
"preview": "## Answer Relevancy\n\nThe **Answer Relevancy** metric measures how relevant a response is to the user input. It ranges fr"
},
{
"path": "docs/concepts/metrics/available_metrics/aspect_critic.md",
"chars": 4153,
"preview": "# Aspect Critique\n\nAspect Critique is a binary evaluation metric used to assess submissions based on predefined aspects "
},
{
"path": "docs/concepts/metrics/available_metrics/context_entities_recall.md",
"chars": 4710,
"preview": "## Context Entities Recall\n\n`ContextEntityRecall` metric gives the measure of recall of the retrieved context, based on "
},
{
"path": "docs/concepts/metrics/available_metrics/context_precision.md",
"chars": 8710,
"preview": "# Context Precision\n\nContext Precision is a metric that evaluates the retriever's ability to rank relevant chunks higher"
},
{
"path": "docs/concepts/metrics/available_metrics/context_recall.md",
"chars": 5068,
"preview": "# Context Recall\n\nContext Recall measures how many of the relevant documents (or pieces of information) were successfull"
},
{
"path": "docs/concepts/metrics/available_metrics/factual_correctness.md",
"chars": 8914,
"preview": "## Factual Correctness\n\n`FactualCorrectness` is a metric that compares and evaluates the factual accuracy of the generat"
},
{
"path": "docs/concepts/metrics/available_metrics/faithfulness.md",
"chars": 5161,
"preview": "## Faithfulness\n\nThe **Faithfulness** metric measures how factually consistent a `response` is with the `retrieved conte"
},
{
"path": "docs/concepts/metrics/available_metrics/general_purpose.md",
"chars": 8526,
"preview": "# General Purpose Metrics\n\nGeneral purpose evaluation metrics are used to evaluate any given task.\n\n## Aspect Critic\n\n`A"
},
{
"path": "docs/concepts/metrics/available_metrics/index.md",
"chars": 2346,
"preview": "# List of available metrics\n\nRagas provides a set of evaluation metrics that can be used to measure the performance of y"
},
{
"path": "docs/concepts/metrics/available_metrics/multi_modal_faithfulness.md",
"chars": 3322,
"preview": "## MultiModalFaithfulness\n\n`MultiModalFaithfulness` metric measures the factual consistency of the generated answer agai"
},
{
"path": "docs/concepts/metrics/available_metrics/multi_modal_relevance.md",
"chars": 3266,
"preview": "## MultiModalRelevance\n\n`MultiModalRelevance` metric measures the relevance of the generated answer against both visual "
},
{
"path": "docs/concepts/metrics/available_metrics/noise_sensitivity.md",
"chars": 10245,
"preview": "# Noise Sensitivity\n\n`NoiseSensitivity` measures how often a system makes errors by providing incorrect responses when u"
},
{
"path": "docs/concepts/metrics/available_metrics/nvidia_metrics.md",
"chars": 15893,
"preview": "# Nvidia Metrics\n\n## Answer Accuracy\n\n**Answer Accuracy** measures the agreement between a model’s response and a refere"
},
{
"path": "docs/concepts/metrics/available_metrics/rubrics_based.md",
"chars": 8713,
"preview": "# Rubric-Based Evaluation\n\nRubric-based evaluation metrics allow you to evaluate LLM responses using custom scoring crit"
},
{
"path": "docs/concepts/metrics/available_metrics/semantic_similarity.md",
"chars": 3051,
"preview": "## Semantic Similarity\n\nThe **Semantic Similarity** metric evaluates the semantic resemblance between a generated respon"
},
{
"path": "docs/concepts/metrics/available_metrics/sql.md",
"chars": 7177,
"preview": "# SQL\n\n\n## Execution based metrics\nIn these metrics the resulting SQL is compared after executing the SQL query on the d"
},
{
"path": "docs/concepts/metrics/available_metrics/summarization_score.md",
"chars": 4175,
"preview": "# Tasks Metrics\n\n## Summarization Score\n\nThe **Summarization Score** metric measures how well a summary (`response`) cap"
},
{
"path": "docs/concepts/metrics/available_metrics/traditional.md",
"chars": 12346,
"preview": "# Traditional NLP Metrics\n\n## Non LLM String Similarity\n\n`NonLLMStringSimilarity` metric measures the similarity between"
},
{
"path": "docs/concepts/metrics/index.md",
"chars": 300,
"preview": "\n# Metrics\n\n<div class=\"grid cards\" markdown>\n\n- :fontawesome-solid-database:[__Overview__ Learn more about overview and"
},
{
"path": "docs/concepts/metrics/overview/index.md",
"chars": 11678,
"preview": "# Overview of Metrics\n\n## Why Metrics Matter\n\nYou can't improve what you don't measure. Metrics are the feedback loop th"
},
{
"path": "docs/concepts/test_data_generation/agents.md",
"chars": 356,
"preview": "# Testset Generation for Agents or Tool use cases\n\nEvaluating agentic or tool use workflows can be challenging as it inv"
},
{
"path": "docs/concepts/test_data_generation/index.md",
"chars": 814,
"preview": "# Testset Generation\n\nCurating a high quality test dataset is crucial for evaluating the performance of your AI applicat"
},
{
"path": "docs/concepts/test_data_generation/rag.md",
"chars": 12525,
"preview": "# Testset Generation for RAG\n\nIn RAG application, when a user interacts through your application to a set of documents, "
},
{
"path": "docs/extra/components/choose_evaluator_llm.md",
"chars": 9304,
"preview": "=== \"OpenAI\"\n Install the langchain-openai package\n\n ```bash\n pip install langchain-openai\n ```\n\n Ensure "
},
{
"path": "docs/extra/components/choose_generator_llm.md",
"chars": 9091,
"preview": "=== \"OpenAI\"\n Install the langchain-openai package\n\n ```bash\n pip install langchain-openai\n ```\n\n Then en"
},
{
"path": "docs/extra/overrides/main.html",
"chars": 264,
"preview": "{% extends \"base.html\" %}\n\n{% block extrahead %}\n {{ super() }}\n <!-- Scarf Analytics -->\n <img style=\"display: none;"
},
{
"path": "docs/extra/ragas-modern.css",
"chars": 5532,
"preview": "/* Ragas Modern Documentation Theme */\n\n/* Import Google Fonts - Professional Typography */\n@import url('https://fonts.g"
},
{
"path": "docs/extra/style.css",
"chars": 7011,
"preview": "[data-md-color-scheme=\"ragas_light\"] {\n --md-primary-fg-color: #f8f8f5; /* in header bg*/\n --md-primary-bg-color: #212"
},
{
"path": "docs/getstarted/evals.md",
"chars": 8562,
"preview": "# Evaluate a simple LLM application\n\nThe purpose of this guide is to illustrate a simple workflow for testing and evalua"
},
{
"path": "docs/getstarted/experiments_quickstart.md",
"chars": 2801,
"preview": "# Run your first experiment\n\nThis tutorial walks you through running your first experiment with Ragas using the `@experi"
},
{
"path": "docs/getstarted/index.md",
"chars": 1332,
"preview": "# 🚀 Get Started\n\nWelcome to Ragas! The Get Started guides will walk you through the fundamentals of working with Ragas. "
},
{
"path": "docs/getstarted/install.md",
"chars": 1030,
"preview": "# Installation\n\nTo get started, install Ragas using `pip` with the following command:\n\n```bash\npip install ragas\n```\n\nIf"
},
{
"path": "docs/getstarted/quickstart.md",
"chars": 6452,
"preview": "# Quick Start: Get Evaluations Running in a Flash\n\nGet started with Ragas in minutes. Create a complete evaluation proje"
},
{
"path": "docs/getstarted/rag_eval.md",
"chars": 7908,
"preview": "# Evaluate a simple RAG system\n\nThe purpose of this guide is to illustrate a simple workflow for testing and evaluating "
},
{
"path": "docs/getstarted/rag_testset_generation.md",
"chars": 6366,
"preview": "# Testset Generation for RAG\n\nThis simple guide will help you generate a testset for evaluating your RAG pipeline using "
},
{
"path": "docs/howtos/applications/_cost.md",
"chars": 4082,
"preview": "# How to estimate Cost and Usage of evaluations and testset generation\n\nWhen using LLMs for evaluation and test set gene"
},
{
"path": "docs/howtos/applications/add_to_ci.md",
"chars": 3523,
"preview": "---\nsearch:\n exclude: true\n---\n\n# Adding to your CI pipeline with Pytest\n\nYou can add Ragas evaluations as part of your"
},
{
"path": "docs/howtos/applications/align-llm-as-judge.md",
"chars": 18808,
"preview": "# How to Align an LLM as a Judge\n\nIn this guide, you'll learn how to systematically evaluate and align an LLM-as-judge m"
},
{
"path": "docs/howtos/applications/benchmark_llm.md",
"chars": 15468,
"preview": "# How to Evaluate a New LLM For Your Use Case\n\nWhen a new LLM is released, you might want to determine if it outperforms"
},
{
"path": "docs/howtos/applications/compare_embeddings.md",
"chars": 5144,
"preview": "---\nsearch:\n exclude: true\n---\n\n# Compare Embeddings for retriever\n\nThe performance of the retriever is a critical and "
},
{
"path": "docs/howtos/applications/compare_llms.md",
"chars": 7402,
"preview": "---\nsearch:\n exclude: true\n---\n\n# Compare LLMs using Ragas Evaluations\n\nThe LLM used in the Retrieval Augmented Generat"
},
{
"path": "docs/howtos/applications/cost.ipynb",
"chars": 7431,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# How to estimate Cost and Usage of"
},
{
"path": "docs/howtos/applications/evaluate-and-improve-rag.md",
"chars": 20626,
"preview": "# How to Evaluate and Improve a RAG App\n\nIn this guide, you'll learn how to evaluate and iteratively improve a RAG (Retr"
},
{
"path": "docs/howtos/applications/evaluating_multi_turn_conversations.md",
"chars": 17655,
"preview": "# Evaluating Multi-Turn Conversations\n\nThis tutorial is inspired by Hamel’s notes on evaluating multi-turn conversations"
},
{
"path": "docs/howtos/applications/index.md",
"chars": 863,
"preview": "# Applications\n\nRagas in action. Examples of how to use Ragas in various applications and\nusecases to solve problems you"
},
{
"path": "docs/howtos/applications/iterate_prompt.md",
"chars": 16894,
"preview": "# How to Evaluate Your Prompt and Improve It\n\nIn this guide, you'll learn how to evaluate and iteratively improve a prom"
},
{
"path": "docs/howtos/applications/prompt_optimization.md",
"chars": 18611,
"preview": "# A systematic approach for prompt optimization\n\nCreating reliable and consistent prompts remains a significant challeng"
},
{
"path": "docs/howtos/applications/singlehop_testset_gen.md",
"chars": 10486,
"preview": "# Generating a Synthetic Test Set for RAG-Based Question Answering with Ragas\n\n## Overview\n\nIn this tutorial, we'll expl"
},
{
"path": "docs/howtos/applications/text2sql.md",
"chars": 36281,
"preview": "# How to evaluate a Text to SQL Agent\n\nIn this guide, you'll learn how to systematically evaluate and improve a text-to-"
},
{
"path": "docs/howtos/applications/vertexai_alignment.md",
"chars": 10284,
"preview": "# Aligning LLM Evaluators with Human Judgment\n\nThis tutorial is part of a three-part series on how to use Vertex AI mode"
},
{
"path": "docs/howtos/applications/vertexai_model_comparision.md",
"chars": 20877,
"preview": "# Compare models provided by VertexAI on RAG-based Q&A task using Ragas metrics\n\nThis tutorial is part of a three-part s"
},
{
"path": "docs/howtos/applications/vertexai_x_ragas.md",
"chars": 14491,
"preview": "# Getting Started: Ragas with Vertex AI\n\nThis tutorial is part of a three-part series on how to use Vertex AI models wit"
},
{
"path": "docs/howtos/cli/agent_evals.md",
"chars": 1986,
"preview": "# Agent Evaluation Quickstart\n\nThe `agent_evals` template provides a setup for evaluating AI agents that solve mathemati"
},
{
"path": "docs/howtos/cli/benchmark_llm.md",
"chars": 3756,
"preview": "# LLM Benchmarking Quickstart\n\nThe `benchmark_llm` template benchmarks and compares different LLM models on discount cal"
},
{
"path": "docs/howtos/cli/improve_rag.md",
"chars": 7688,
"preview": "# Improve RAG Quickstart\n\nThe `improve_rag` template demonstrates how to compare different RAG approaches using real-wor"
},
{
"path": "docs/howtos/cli/index.md",
"chars": 3291,
"preview": "# Ragas CLI\n\nThe Ragas Command Line Interface (CLI) provides tools for quickly setting up evaluation projects and runnin"
},
{
"path": "docs/howtos/cli/judge_alignment.md",
"chars": 2910,
"preview": "# Judge Alignment Quickstart\n\nThe `judge_alignment` template measures how well an LLM-as-judge aligns with human evaluat"
},
{
"path": "docs/howtos/cli/llamaIndex_agent_evals.md",
"chars": 2348,
"preview": "# LlamaIndex Agent Evaluation Quickstart\n\nThe `llamaIndex_agent_evals` template evaluates LlamaIndex workflow agents wit"
},
{
"path": "docs/howtos/cli/prompt_evals.md",
"chars": 2698,
"preview": "# Prompt Evaluation Quickstart\n\nThe `prompt_evals` template evaluates and compares different prompt variations with sent"
},
{
"path": "docs/howtos/cli/rag_eval.md",
"chars": 6740,
"preview": "# RAG Evaluation Quickstart\n\nThe `rag_eval` template provides a complete RAG evaluation setup with custom metrics, datas"
},
{
"path": "docs/howtos/cli/text2sql.md",
"chars": 2644,
"preview": "# Text-to-SQL Evaluation Quickstart\n\nThe `text2sql` template evaluates text-to-SQL systems by comparing SQL execution re"
},
{
"path": "docs/howtos/cli/workflow_eval.md",
"chars": 2795,
"preview": "# Workflow Evaluation Quickstart\n\nThe `workflow_eval` template evaluates complex LLM workflows with email classification"
},
{
"path": "docs/howtos/customizations/_caching.md",
"chars": 7891,
"preview": "# Caching in Ragas\n\nYou can use caching to speed up your evaluations and testset generation by avoiding redundant comput"
},
{
"path": "docs/howtos/customizations/caching.ipynb",
"chars": 4625,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Caching in Ragas\\n\",\n \"\\n\",\n "
},
{
"path": "docs/howtos/customizations/cancellation.md",
"chars": 10100,
"preview": "# Cancelling Long-Running Tasks\n\nWhen working with large datasets or complex evaluations, some Ragas operations can take"
},
{
"path": "docs/howtos/customizations/customize_models.md",
"chars": 5485,
"preview": "## Customize Models\n\nRagas may use a LLM and or Embedding for evaluation and synthetic data generation. Both of these mo"
},
{
"path": "docs/howtos/customizations/index.md",
"chars": 931,
"preview": "# Customizations\n\nHow to customize various aspects of Ragas to suit your needs.\n\n## General\n\n- [Customize models](custom"
},
{
"path": "docs/howtos/customizations/metrics/_cost.md",
"chars": 3081,
"preview": "# Understand Cost and Usage of Operations\n\nWhen using LLMs for evaluation and test set generation, cost will be an impor"
},
{
"path": "docs/howtos/customizations/metrics/cost.ipynb",
"chars": 5143,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Understand Cost and Usage of Oper"
},
{
"path": "docs/howtos/customizations/metrics/metrics_language_adaptation.md",
"chars": 3306,
"preview": "# Adapting Metrics to Target Language\n\nWhen evaluating LLM applications in languages other than English, adapt your metr"
},
{
"path": "docs/howtos/customizations/metrics/modifying-prompts-metrics.md",
"chars": 7469,
"preview": "# Modifying prompts in metrics\n\nEvery metric in Ragas that uses an LLM also uses one or more prompts to generate interme"
},
{
"path": "docs/howtos/customizations/metrics/tracing.md",
"chars": 2128,
"preview": "# Tracing and logging evaluations with Observability tools\n\nLogging and tracing results from LLM are important for any l"
},
{
"path": "docs/howtos/customizations/optimizers/index.md",
"chars": 9379,
"preview": "# DSPy Optimizer for Advanced Prompt Optimization\n\nThe DSPyOptimizer provides state-of-the-art prompt optimization for R"
},
{
"path": "docs/howtos/customizations/run_config.md",
"chars": 3201,
"preview": "# Customize Timeouts and Rate Limits\n\nConfigure timeouts and retries directly on your LLM client when using the collecti"
},
{
"path": "docs/howtos/customizations/testgenerator/_language_adaptation.md",
"chars": 4884,
"preview": "## Synthetic test generation from non-English corpus\n\nIn this notebook, you'll learn how to adapt synthetic test data ge"
},
{
"path": "docs/howtos/customizations/testgenerator/_persona_generator.md",
"chars": 6109,
"preview": "## Persona's in Testset Generation\n\nYou can add different persona's to the testset generation process by defining the [P"
},
{
"path": "docs/howtos/customizations/testgenerator/_testgen-custom-single-hop.md",
"chars": 7438,
"preview": "# Create custom single-hop queries from your documents\n\n### Load sample documents\nI am using documents from [sample of G"
},
{
"path": "docs/howtos/customizations/testgenerator/_testgen-customisation.md",
"chars": 9057,
"preview": "# Create custom multi-hop queries from your documents\n\nIn this tutorial you will get to learn how to create custom multi"
},
{
"path": "docs/howtos/customizations/testgenerator/index.md",
"chars": 227,
"preview": "# Customizing Test Data Generation\n\nSynthetic test generation can save a lot of time and effort in creating test dataset"
},
{
"path": "docs/howtos/customizations/testgenerator/language_adaptation.ipynb",
"chars": 8453,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Synthetic test generation from m"
},
{
"path": "docs/howtos/customizations/testgenerator/persona_generator.ipynb",
"chars": 9706,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Persona's in Testset Generation\\"
},
{
"path": "docs/howtos/customizations/testgenerator/prechunked_data.md",
"chars": 6716,
"preview": "# Using Pre-chunked Data for Testset Generation\n\nWhen you already have a well-defined chunking strategy in place, Ragas "
},
{
"path": "docs/howtos/customizations/testgenerator/testgen-custom-single-hop.ipynb",
"chars": 13700,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"id\": \"51c3407b-6041-4217-9ef9-a0e619a51603\",\n \"metadata\": {},\n \"so"
},
{
"path": "docs/howtos/customizations/testgenerator/testgen-customisation.ipynb",
"chars": 15085,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"id\": \"51c3407b-6041-4217-9ef9-a0e619a51603\",\n \"metadata\": {},\n \"so"
},
{
"path": "docs/howtos/index.md",
"chars": 1349,
"preview": "# 🛠️ How-to Guides\n\nEach guide in this section provides a focused solution to real-world problems that you, as an experi"
},
{
"path": "docs/howtos/integrations/_ag_ui.md",
"chars": 10535,
"preview": "# AG-UI Integration\nRagas can run experiments on agents that stream events via the [AG-UI protocol](https://docs.ag-ui.c"
},
{
"path": "docs/howtos/integrations/_arize.md",
"chars": 14069,
"preview": "# Phoenix (Arize)\n\n## 1. Introduction\n\nBuilding a baseline for a RAG pipeline is not usually difficult, but enhancing it"
},
{
"path": "docs/howtos/integrations/_athina.md",
"chars": 3225,
"preview": "# Athina AI\n## Ragas Metrics on your Production Logs\n\n[Athina](https://athina.ai) is a production monitoring and evaluat"
},
{
"path": "docs/howtos/integrations/_haystack.md",
"chars": 12376,
"preview": "# Haystack Integration\n\nHaystack is a LLM orchestration framework to build customizable, production-ready LLM applicati"
},
{
"path": "docs/howtos/integrations/_helicone.md",
"chars": 3756,
"preview": "# Helicone\n\nThis notebook demonstrates how to integrate Helicone with Ragas for monitoring and evaluating RAG (Retrieval"
},
{
"path": "docs/howtos/integrations/_langchain.md",
"chars": 12032,
"preview": "# Langchain\n## Evaluating Langchain QA Chains\n\nLangChain is a framework for developing applications powered by language "
},
{
"path": "docs/howtos/integrations/_langfuse.md",
"chars": 13941,
"preview": "# Langfuse\n\nRagas and Langfuse is a powerful combination that can help you evaluate and monitor your Retrieval-Augmented"
},
{
"path": "docs/howtos/integrations/_langgraph_agent_evaluation.md",
"chars": 18290,
"preview": "# Building and Evaluating a ReAct Agent for Fetching Metal Prices\n\nAI agents are becoming increasingly valuable in domai"
},
{
"path": "docs/howtos/integrations/_langsmith.md",
"chars": 2156,
"preview": "# Langsmith\n## Dataset and Tracing Visualisation\n\n[Langsmith](https://docs.smith.langchain.com/) in a platform for build"
},
{
"path": "docs/howtos/integrations/_llamaindex.md",
"chars": 11267,
"preview": "# LlamaIndex\n\n[LlamaIndex](https://github.com/run-llama/llama_index) is a data framework for LLM applications to ingest,"
},
{
"path": "docs/howtos/integrations/_openlayer.md",
"chars": 3970,
"preview": "# Openlayer\n## Evaluating RAG pipelines with Openlayer and Ragas\n\n[Openlayer](https://www.openlayer.com/) is an evaluati"
},
{
"path": "docs/howtos/integrations/_opik.md",
"chars": 7877,
"preview": "# Comet Opik\n\nIn this notebook, we will showcase how to use Opik with Ragas for monitoring and evaluation of RAG (Retrie"
},
{
"path": "docs/howtos/integrations/_tonic-validate.md",
"chars": 2322,
"preview": "# Tonic Validate\n## [Tonic Validate](https://tonic.ai/validate): Visualize Ragas Scores \n\n<center><img src=\"https://uplo"
},
{
"path": "docs/howtos/integrations/_zeno.md",
"chars": 4103,
"preview": "# Zeno\n## Visualizing Ragas Results with Zeno\n\nYou can use the [Zeno](https://zenoml.com) evaluation platform to easily "
},
{
"path": "docs/howtos/integrations/ag_ui.ipynb",
"chars": 16094,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"id\": \"cdcdd4d1\",\n \"metadata\": {\n \"editable\": true,\n \"slideshow"
},
{
"path": "docs/howtos/integrations/ag_ui.md",
"chars": 12816,
"preview": "# AG-UI\n\n[AG-UI](https://docs.ag-ui.com/) is an event-based protocol for streaming agent updates to user interfaces. The"
},
{
"path": "docs/howtos/integrations/amazon_bedrock.md",
"chars": 37922,
"preview": "\n# Create and Evaluate an Amazon Bedrock Agent Integrated with an Amazon Bedrock Knowledge Base and Action Groups\n\nIn th"
},
{
"path": "docs/howtos/integrations/arize.ipynb",
"chars": 23000,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"id\": \"61c367aa-e0a3-4116-bda7-7b81404211fd\",\n \"metadata\": {},\n \"so"
},
{
"path": "docs/howtos/integrations/athina.ipynb",
"chars": 4818,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Athina AI\\n\",\n \"## Ragas Metri"
},
{
"path": "docs/howtos/integrations/gemini.md",
"chars": 13401,
"preview": "# Google Gemini Integration Guide\n\nThis guide covers setting up and using Google's Gemini models with Ragas for evaluati"
},
{
"path": "docs/howtos/integrations/griptape.md",
"chars": 15375,
"preview": "# Griptape Integration\n\nIf you're familiar with Griptape's RAG Engine and want to start evaluating your RAG system's per"
},
{
"path": "docs/howtos/integrations/haystack.ipynb",
"chars": 18238,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Haystack Integration\\n\",\n \"\\n\""
},
{
"path": "docs/howtos/integrations/haystack.md",
"chars": 11448,
"preview": "# Haystack Integration\n\nHaystack is an LLM orchestration framework to build customizable, production-ready LLM applicati"
},
{
"path": "docs/howtos/integrations/helicone.ipynb",
"chars": 5842,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Helicone\\n\",\n \"\\n\",\n \"This "
},
{
"path": "docs/howtos/integrations/index.md",
"chars": 2750,
"preview": "# Integrations\n\nRagas is a framework and can be integrated with a host of different frameworks\nand tools so that you can"
},
{
"path": "docs/howtos/integrations/langchain.ipynb",
"chars": 20501,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"id\": \"586226e7\",\n \"metadata\": {},\n \"source\": [\n \"# Langchain\\n\""
},
{
"path": "docs/howtos/integrations/langchain.md",
"chars": 6988,
"preview": "# LangChain Integration\n\nThis tutorial demonstrates how to evaluate a RAG-based Q&A application built with LangChain usi"
},
{
"path": "docs/howtos/integrations/langfuse.ipynb",
"chars": 21125,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"id\": \"1079e444-91e1-4b81-a28a-2ce4763f4bc4\",\n \"metadata\": {},\n \"so"
},
{
"path": "docs/howtos/integrations/langgraph_agent_evaluation.ipynb",
"chars": 38905,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"t1ub1OLYZQvz\"\n },\n \"source\": [\n \"# Buil"
},
{
"path": "docs/howtos/integrations/langsmith.ipynb",
"chars": 4357,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"id\": \"a0b3171b\",\n \"metadata\": {},\n \"source\": [\n \"# Langsmith\\n\""
},
{
"path": "docs/howtos/integrations/langsmith.md",
"chars": 4579,
"preview": "# LangSmith\n\n[LangSmith](https://docs.smith.langchain.com/) is an advanced tool designed to enhance the development and "
},
{
"path": "docs/howtos/integrations/llama_stack.md",
"chars": 10088,
"preview": "# Evaluating LlamaStack Web Search Groundedness with Llama 4\n\nIn this tutorial we will measure the groundedness of respo"
},
{
"path": "docs/howtos/integrations/llamaindex.ipynb",
"chars": 22863,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"id\": \"d2451aff\",\n \"metadata\": {},\n \"source\": [\n \"# LlamaIndex\\n"
},
{
"path": "docs/howtos/integrations/llamaindex_agents.md",
"chars": 25343,
"preview": "# Evaluating LlamaIndex Agents\n\nBuilding agents that can intelligently use tools and make decisions is only half the jou"
},
{
"path": "docs/howtos/integrations/nyc_wikipedia/nyc_text.txt",
"chars": 130099,
"preview": "New York, often called New York City or NYC, is the most populous city in the United States. With a 2020 population of 8"
},
{
"path": "docs/howtos/integrations/oci_genai.md",
"chars": 6428,
"preview": "# OCI Gen AI Integration\n\nThis guide shows how to use Oracle Cloud Infrastructure (OCI) Generative AI models with Ragas "
},
{
"path": "docs/howtos/integrations/openlayer.ipynb",
"chars": 8316,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"id\": \"860c9e4b-dc7c-4f2e-8f60-96cccf61d43c\",\n \"metadata\": {},\n \"so"
},
{
"path": "docs/howtos/integrations/opik.ipynb",
"chars": 11920,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Comet Opik\\n\",\n \"\\n\",\n \"In "
},
{
"path": "docs/howtos/integrations/r2r.md",
"chars": 11233,
"preview": "# R2R Integration\n\nR2R is an all-in-one solution for AI Retrieval-Augmented Generation (RAG) with production-ready featu"
},
{
"path": "docs/howtos/integrations/swarm_agent_evaluation.md",
"chars": 20340,
"preview": "## Installing Ragas and Other Dependencies\nInstall Ragas with pip and set up Swarm locally:\n\n\n```python\n# %pip install r"
},
{
"path": "docs/howtos/integrations/tonic-validate.ipynb",
"chars": 3340,
"preview": "{\n \"cells\": [\n {\n \"attachments\": {},\n \"cell_type\": \"markdown\",\n \"id\": \"bbac63ad-ccc7-4968-8676-280489a9073c\",\n "
},
{
"path": "docs/howtos/integrations/zeno.ipynb",
"chars": 6811,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Zeno\\n\",\n \"## Visualizing Raga"
},
{
"path": "docs/howtos/llm-adapters.md",
"chars": 11488,
"preview": "# LLM Adapters: Using Multiple Structured Output Backends\n\nRagas supports multiple structured output backends through an"
},
{
"path": "docs/howtos/migrations/migrate_from_v01_to_v02.md",
"chars": 4164,
"preview": "# Migration from v0.1 to v0.2\n\nv0.2 is the start of the transition for Ragas from an evaluation library for RAG pipeline"
},
{
"path": "docs/howtos/migrations/migrate_from_v03_to_v04.md",
"chars": 53614,
"preview": "# Migration from v0.3 to v0.4\n\nRagas v0.4 introduces a fundamental shift towards an **experiment-based architecture**. T"
},
{
"path": "docs/howtos/observability.md",
"chars": 18792,
"preview": "# Observability Tools.\n\n## Phoenix (Arize)\n\n### 1. Introduction\n\nBuilding a baseline for a RAG pipeline is not usually d"
},
{
"path": "docs/index.md",
"chars": 2375,
"preview": "# ✨ Introduction\n\nRagas is a library that helps you move from \"vibe checks\" to systematic evaluation loops for your AI a"
},
{
"path": "docs/ipynb_to_md.py",
"chars": 2779,
"preview": "import datetime\nimport os\nimport subprocess\n\n\ndef convert_ipynb_to_md(ipynb_file):\n # Change this line to add an unde"
},
{
"path": "docs/make.bat",
"chars": 804,
"preview": "@ECHO OFF\r\n\r\npushd %~dp0\r\n\r\nREM Command file for Sphinx documentation\r\n\r\nif \"%SPHINXBUILD%\" == \"\" (\r\n\tset SPHINXBUILD=sp"
},
{
"path": "docs/quoted_spans_metric.md",
"chars": 2501,
"preview": "## `QuotedSpansAlignment`\n\n**What:** A metric that measures the fraction of quoted spans in a model's answer\nthat appear"
},
{
"path": "docs/references/aevaluate.md",
"chars": 1336,
"preview": "# Async Evaluation\n\n## aevaluate()\n\n::: ragas.evaluation.aevaluate\n\n## Async Usage\n\nRagas provides both synchronous and "
},
{
"path": "docs/references/cache.md",
"chars": 61,
"preview": "::: ragas.cache\n options:\n members_order: \"source\"\n"
},
{
"path": "docs/references/embeddings.md",
"chars": 66,
"preview": "::: ragas.embeddings\n options:\n members_order: \"source\"\n"
}
]
// ... and 438 more files (download for full content)
About this extraction
This page contains the full source code of the vibrantlabsai/ragas GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 638 files (4.9 MB), approximately 1.3M tokens, and a symbol index with 3233 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.