Full Code of EleutherAI/lm-evaluation-harness for AI

gitextract_npmvb7su/

├── .github/
│   └── workflows/
│       ├── new_tasks.yml
│       ├── publish.yml
│       └── unit_tests.yml
├── .gitignore
├── .pre-commit-config.yaml
├── CITATION.bib
├── CODEOWNERS
├── LICENSE.md
├── MANIFEST.in
├── README.md
├── docs/
│   ├── API_guide.md
│   ├── CONTRIBUTING.md
│   ├── README.md
│   ├── chat-template-readme.md
│   ├── config_files.md
│   ├── decontamination.md
│   ├── footguns.md
│   ├── interface.md
│   ├── model_guide.md
│   ├── new_task_guide.md
│   ├── python-api.md
│   └── task_guide.md
├── examples/
│   ├── lm-eval-overview.ipynb
│   ├── transformer-lens.py
│   ├── visualize-wandb.ipynb
│   └── visualize-zeno.ipynb
├── ignore.txt
├── lm_eval/
│   ├── __init__.py
│   ├── __main__.py
│   ├── _cli/
│   │   ├── __init__.py
│   │   ├── harness.py
│   │   ├── ls.py
│   │   ├── run.py
│   │   ├── subcommand.py
│   │   ├── utils.py
│   │   └── validate.py
│   ├── api/
│   │   ├── __init__.py
│   │   ├── filter.py
│   │   ├── group.py
│   │   ├── instance.py
│   │   ├── metrics.py
│   │   ├── model.py
│   │   ├── registry.py
│   │   ├── samplers.py
│   │   ├── task.py
│   │   └── utils.py
│   ├── caching/
│   │   ├── __init__.py
│   │   └── cache.py
│   ├── config/
│   │   ├── __init__.py
│   │   ├── evaluate_config.py
│   │   ├── group.py
│   │   └── task.py
│   ├── decontamination/
│   │   ├── __init__.py
│   │   ├── archiver.py
│   │   ├── decontaminate.py
│   │   └── janitor.py
│   ├── defaults.py
│   ├── evaluator.py
│   ├── evaluator_utils.py
│   ├── filters/
│   │   ├── __init__.py
│   │   ├── custom.py
│   │   ├── decontamination.py
│   │   ├── extraction.py
│   │   ├── selection.py
│   │   └── transformation.py
│   ├── loggers/
│   │   ├── __init__.py
│   │   ├── evaluation_tracker.py
│   │   ├── utils.py
│   │   └── wandb_logger.py
│   ├── models/
│   │   ├── __init__.py
│   │   ├── anthropic_llms.py
│   │   ├── api_models.py
│   │   ├── dummy.py
│   │   ├── gguf.py
│   │   ├── hf_audiolm.py
│   │   ├── hf_steered.py
│   │   ├── hf_vlms.py
│   │   ├── huggingface.py
│   │   ├── ibm_watsonx_ai.py
│   │   ├── mamba_lm.py
│   │   ├── megatron_lm.py
│   │   ├── mistral3.py
│   │   ├── nemo_lm.py
│   │   ├── neuron_optimum.py
│   │   ├── openai_completions.py
│   │   ├── optimum_habana.py
│   │   ├── optimum_ipex.py
│   │   ├── optimum_lm.py
│   │   ├── sglang_causallms.py
│   │   ├── sglang_generate_API.py
│   │   ├── textsynth.py
│   │   ├── utils.py
│   │   ├── utils_hf.py
│   │   ├── vllm_causallms.py
│   │   ├── vllm_vlms.py
│   │   └── winml.py
│   ├── prompts/
│   │   └── __init__.py
│   ├── result_schema.py
│   ├── tasks/
│   │   ├── README.md
│   │   ├── __init__.py
│   │   ├── _factory.py
│   │   ├── _index.py
│   │   ├── _yaml_loader.py
│   │   ├── aclue/
│   │   │   ├── README.md
│   │   │   ├── _aclue.yaml
│   │   │   ├── _default_template_yaml
│   │   │   ├── _generate_configs.py
│   │   │   ├── aclue_ancient_chinese_culture.yaml
│   │   │   ├── aclue_ancient_literature.yaml
│   │   │   ├── aclue_ancient_medical.yaml
│   │   │   ├── aclue_ancient_phonetics.yaml
│   │   │   ├── aclue_basic_ancient_chinese.yaml
│   │   │   ├── aclue_couplet_prediction.yaml
│   │   │   ├── aclue_homographic_character_resolution.yaml
│   │   │   ├── aclue_named_entity_recognition.yaml
│   │   │   ├── aclue_poetry_appreciate.yaml
│   │   │   ├── aclue_poetry_context_prediction.yaml
│   │   │   ├── aclue_poetry_quality_assessment.yaml
│   │   │   ├── aclue_poetry_sentiment_analysis.yaml
│   │   │   ├── aclue_polysemy_resolution.yaml
│   │   │   ├── aclue_reading_comprehension.yaml
│   │   │   └── aclue_sentence_segmentation.yaml
│   │   ├── acpbench/
│   │   │   ├── README.md
│   │   │   ├── boolq_cot_2shot/
│   │   │   │   ├── _boolq_cot_2shot_yaml
│   │   │   │   ├── act_reach.yaml
│   │   │   │   ├── app.yaml
│   │   │   │   ├── just.yaml
│   │   │   │   ├── land.yaml
│   │   │   │   ├── prog.yaml
│   │   │   │   ├── reach.yaml
│   │   │   │   └── val.yaml
│   │   │   ├── gen_2shot/
│   │   │   │   ├── _gen_yaml_2shot
│   │   │   │   ├── acp_grammar.lark
│   │   │   │   ├── acp_utils.py
│   │   │   │   ├── act_reach.yaml
│   │   │   │   ├── app.yaml
│   │   │   │   ├── just.yaml
│   │   │   │   ├── land.yaml
│   │   │   │   ├── next_act.yaml
│   │   │   │   ├── prog.yaml
│   │   │   │   ├── reach.yaml
│   │   │   │   └── val.yaml
│   │   │   ├── gen_2shot_with_pddl/
│   │   │   │   ├── _gen_yaml_2shot
│   │   │   │   ├── acp_grammar.lark
│   │   │   │   ├── acp_utils.py
│   │   │   │   ├── act_reach.yaml
│   │   │   │   ├── app.yaml
│   │   │   │   ├── just.yaml
│   │   │   │   ├── land.yaml
│   │   │   │   ├── next_act.yaml
│   │   │   │   ├── prog.yaml
│   │   │   │   ├── reach.yaml
│   │   │   │   └── val.yaml
│   │   │   └── mcq_cot_2shot/
│   │   │       ├── _mcq_cot_2shot_yaml
│   │   │       ├── act_reach.yaml
│   │   │       ├── app.yaml
│   │   │       ├── just.yaml
│   │   │       ├── land.yaml
│   │   │       ├── prog.yaml
│   │   │       ├── reach.yaml
│   │   │       └── val.yaml
│   │   ├── aexams/
│   │   │   ├── README.md
│   │   │   ├── _aexams.yaml
│   │   │   ├── _default_template_yaml
│   │   │   ├── aexams_Biology.yaml
│   │   │   ├── aexams_IslamicStudies.yaml
│   │   │   ├── aexams_Physics.yaml
│   │   │   ├── aexams_Science.yaml
│   │   │   └── aexams_Social.yaml
│   │   ├── afrimgsm/
│   │   │   ├── README.md
│   │   │   ├── direct/
│   │   │   │   ├── afrimgsm.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── afrimgsm_amh.yaml
│   │   │   │   │   ├── afrimgsm_eng.yaml
│   │   │   │   │   ├── afrimgsm_ewe.yaml
│   │   │   │   │   ├── afrimgsm_fra.yaml
│   │   │   │   │   ├── afrimgsm_hau.yaml
│   │   │   │   │   ├── afrimgsm_ibo.yaml
│   │   │   │   │   ├── afrimgsm_kin.yaml
│   │   │   │   │   ├── afrimgsm_lin.yaml
│   │   │   │   │   ├── afrimgsm_lug.yaml
│   │   │   │   │   ├── afrimgsm_orm.yaml
│   │   │   │   │   ├── afrimgsm_sna.yaml
│   │   │   │   │   ├── afrimgsm_sot.yaml
│   │   │   │   │   ├── afrimgsm_swa.yaml
│   │   │   │   │   ├── afrimgsm_twi.yaml
│   │   │   │   │   ├── afrimgsm_vai.yaml
│   │   │   │   │   ├── afrimgsm_wol.yaml
│   │   │   │   │   ├── afrimgsm_xho.yaml
│   │   │   │   │   ├── afrimgsm_yaml
│   │   │   │   │   ├── afrimgsm_yor.yaml
│   │   │   │   │   └── afrimgsm_zul.yaml
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── afrimgsm_amh.yaml
│   │   │   │   │   ├── afrimgsm_eng.yaml
│   │   │   │   │   ├── afrimgsm_ewe.yaml
│   │   │   │   │   ├── afrimgsm_fra.yaml
│   │   │   │   │   ├── afrimgsm_hau.yaml
│   │   │   │   │   ├── afrimgsm_ibo.yaml
│   │   │   │   │   ├── afrimgsm_kin.yaml
│   │   │   │   │   ├── afrimgsm_lin.yaml
│   │   │   │   │   ├── afrimgsm_lug.yaml
│   │   │   │   │   ├── afrimgsm_orm.yaml
│   │   │   │   │   ├── afrimgsm_sna.yaml
│   │   │   │   │   ├── afrimgsm_sot.yaml
│   │   │   │   │   ├── afrimgsm_swa.yaml
│   │   │   │   │   ├── afrimgsm_twi.yaml
│   │   │   │   │   ├── afrimgsm_vai.yaml
│   │   │   │   │   ├── afrimgsm_wol.yaml
│   │   │   │   │   ├── afrimgsm_xho.yaml
│   │   │   │   │   ├── afrimgsm_yaml
│   │   │   │   │   ├── afrimgsm_yor.yaml
│   │   │   │   │   └── afrimgsm_zul.yaml
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── afrimgsm_amh.yaml
│   │   │   │   │   ├── afrimgsm_eng.yaml
│   │   │   │   │   ├── afrimgsm_ewe.yaml
│   │   │   │   │   ├── afrimgsm_fra.yaml
│   │   │   │   │   ├── afrimgsm_hau.yaml
│   │   │   │   │   ├── afrimgsm_ibo.yaml
│   │   │   │   │   ├── afrimgsm_kin.yaml
│   │   │   │   │   ├── afrimgsm_lin.yaml
│   │   │   │   │   ├── afrimgsm_lug.yaml
│   │   │   │   │   ├── afrimgsm_orm.yaml
│   │   │   │   │   ├── afrimgsm_sna.yaml
│   │   │   │   │   ├── afrimgsm_sot.yaml
│   │   │   │   │   ├── afrimgsm_swa.yaml
│   │   │   │   │   ├── afrimgsm_twi.yaml
│   │   │   │   │   ├── afrimgsm_vai.yaml
│   │   │   │   │   ├── afrimgsm_wol.yaml
│   │   │   │   │   ├── afrimgsm_xho.yaml
│   │   │   │   │   ├── afrimgsm_yaml
│   │   │   │   │   ├── afrimgsm_yor.yaml
│   │   │   │   │   └── afrimgsm_zul.yaml
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── afrimgsm_amh.yaml
│   │   │   │   │   ├── afrimgsm_eng.yaml
│   │   │   │   │   ├── afrimgsm_ewe.yaml
│   │   │   │   │   ├── afrimgsm_fra.yaml
│   │   │   │   │   ├── afrimgsm_hau.yaml
│   │   │   │   │   ├── afrimgsm_ibo.yaml
│   │   │   │   │   ├── afrimgsm_kin.yaml
│   │   │   │   │   ├── afrimgsm_lin.yaml
│   │   │   │   │   ├── afrimgsm_lug.yaml
│   │   │   │   │   ├── afrimgsm_orm.yaml
│   │   │   │   │   ├── afrimgsm_sna.yaml
│   │   │   │   │   ├── afrimgsm_sot.yaml
│   │   │   │   │   ├── afrimgsm_swa.yaml
│   │   │   │   │   ├── afrimgsm_twi.yaml
│   │   │   │   │   ├── afrimgsm_vai.yaml
│   │   │   │   │   ├── afrimgsm_wol.yaml
│   │   │   │   │   ├── afrimgsm_xho.yaml
│   │   │   │   │   ├── afrimgsm_yaml
│   │   │   │   │   ├── afrimgsm_yor.yaml
│   │   │   │   │   └── afrimgsm_zul.yaml
│   │   │   │   └── prompt_5/
│   │   │   │       ├── afrimgsm_amh.yaml
│   │   │   │       ├── afrimgsm_eng.yaml
│   │   │   │       ├── afrimgsm_ewe.yaml
│   │   │   │       ├── afrimgsm_fra.yaml
│   │   │   │       ├── afrimgsm_hau.yaml
│   │   │   │       ├── afrimgsm_ibo.yaml
│   │   │   │       ├── afrimgsm_kin.yaml
│   │   │   │       ├── afrimgsm_lin.yaml
│   │   │   │       ├── afrimgsm_lug.yaml
│   │   │   │       ├── afrimgsm_orm.yaml
│   │   │   │       ├── afrimgsm_sna.yaml
│   │   │   │       ├── afrimgsm_sot.yaml
│   │   │   │       ├── afrimgsm_swa.yaml
│   │   │   │       ├── afrimgsm_twi.yaml
│   │   │   │       ├── afrimgsm_vai.yaml
│   │   │   │       ├── afrimgsm_wol.yaml
│   │   │   │       ├── afrimgsm_xho.yaml
│   │   │   │       ├── afrimgsm_yaml
│   │   │   │       ├── afrimgsm_yor.yaml
│   │   │   │       └── afrimgsm_zul.yaml
│   │   │   ├── direct_cot/
│   │   │   │   ├── afrimgsm_cot.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── afrimgsm_cot_amh.yaml
│   │   │   │   │   ├── afrimgsm_cot_eng.yaml
│   │   │   │   │   ├── afrimgsm_cot_ewe.yaml
│   │   │   │   │   ├── afrimgsm_cot_fra.yaml
│   │   │   │   │   ├── afrimgsm_cot_hau.yaml
│   │   │   │   │   ├── afrimgsm_cot_ibo.yaml
│   │   │   │   │   ├── afrimgsm_cot_kin.yaml
│   │   │   │   │   ├── afrimgsm_cot_lin.yaml
│   │   │   │   │   ├── afrimgsm_cot_lug.yaml
│   │   │   │   │   ├── afrimgsm_cot_orm.yaml
│   │   │   │   │   ├── afrimgsm_cot_sna.yaml
│   │   │   │   │   ├── afrimgsm_cot_sot.yaml
│   │   │   │   │   ├── afrimgsm_cot_swa.yaml
│   │   │   │   │   ├── afrimgsm_cot_twi.yaml
│   │   │   │   │   ├── afrimgsm_cot_vai.yaml
│   │   │   │   │   ├── afrimgsm_cot_wol.yaml
│   │   │   │   │   ├── afrimgsm_cot_xho.yaml
│   │   │   │   │   ├── afrimgsm_cot_yaml
│   │   │   │   │   ├── afrimgsm_cot_yor.yaml
│   │   │   │   │   └── afrimgsm_cot_zul.yaml
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── afrimgsm_cot_amh.yaml
│   │   │   │   │   ├── afrimgsm_cot_eng.yaml
│   │   │   │   │   ├── afrimgsm_cot_ewe.yaml
│   │   │   │   │   ├── afrimgsm_cot_fra.yaml
│   │   │   │   │   ├── afrimgsm_cot_hau.yaml
│   │   │   │   │   ├── afrimgsm_cot_ibo.yaml
│   │   │   │   │   ├── afrimgsm_cot_kin.yaml
│   │   │   │   │   ├── afrimgsm_cot_lin.yaml
│   │   │   │   │   ├── afrimgsm_cot_lug.yaml
│   │   │   │   │   ├── afrimgsm_cot_orm.yaml
│   │   │   │   │   ├── afrimgsm_cot_sna.yaml
│   │   │   │   │   ├── afrimgsm_cot_sot.yaml
│   │   │   │   │   ├── afrimgsm_cot_swa.yaml
│   │   │   │   │   ├── afrimgsm_cot_twi.yaml
│   │   │   │   │   ├── afrimgsm_cot_vai.yaml
│   │   │   │   │   ├── afrimgsm_cot_wol.yaml
│   │   │   │   │   ├── afrimgsm_cot_xho.yaml
│   │   │   │   │   ├── afrimgsm_cot_yaml
│   │   │   │   │   ├── afrimgsm_cot_yor.yaml
│   │   │   │   │   └── afrimgsm_cot_zul.yaml
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── afrimgsm_cot_amh.yaml
│   │   │   │   │   ├── afrimgsm_cot_eng.yaml
│   │   │   │   │   ├── afrimgsm_cot_ewe.yaml
│   │   │   │   │   ├── afrimgsm_cot_fra.yaml
│   │   │   │   │   ├── afrimgsm_cot_hau.yaml
│   │   │   │   │   ├── afrimgsm_cot_ibo.yaml
│   │   │   │   │   ├── afrimgsm_cot_kin.yaml
│   │   │   │   │   ├── afrimgsm_cot_lin.yaml
│   │   │   │   │   ├── afrimgsm_cot_lug.yaml
│   │   │   │   │   ├── afrimgsm_cot_orm.yaml
│   │   │   │   │   ├── afrimgsm_cot_sna.yaml
│   │   │   │   │   ├── afrimgsm_cot_sot.yaml
│   │   │   │   │   ├── afrimgsm_cot_swa.yaml
│   │   │   │   │   ├── afrimgsm_cot_twi.yaml
│   │   │   │   │   ├── afrimgsm_cot_vai.yaml
│   │   │   │   │   ├── afrimgsm_cot_wol.yaml
│   │   │   │   │   ├── afrimgsm_cot_xho.yaml
│   │   │   │   │   ├── afrimgsm_cot_yaml
│   │   │   │   │   ├── afrimgsm_cot_yor.yaml
│   │   │   │   │   └── afrimgsm_cot_zul.yaml
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── afrimgsm_cot_amh.yaml
│   │   │   │   │   ├── afrimgsm_cot_eng.yaml
│   │   │   │   │   ├── afrimgsm_cot_ewe.yaml
│   │   │   │   │   ├── afrimgsm_cot_fra.yaml
│   │   │   │   │   ├── afrimgsm_cot_hau.yaml
│   │   │   │   │   ├── afrimgsm_cot_ibo.yaml
│   │   │   │   │   ├── afrimgsm_cot_kin.yaml
│   │   │   │   │   ├── afrimgsm_cot_lin.yaml
│   │   │   │   │   ├── afrimgsm_cot_lug.yaml
│   │   │   │   │   ├── afrimgsm_cot_orm.yaml
│   │   │   │   │   ├── afrimgsm_cot_sna.yaml
│   │   │   │   │   ├── afrimgsm_cot_sot.yaml
│   │   │   │   │   ├── afrimgsm_cot_swa.yaml
│   │   │   │   │   ├── afrimgsm_cot_twi.yaml
│   │   │   │   │   ├── afrimgsm_cot_vai.yaml
│   │   │   │   │   ├── afrimgsm_cot_wol.yaml
│   │   │   │   │   ├── afrimgsm_cot_xho.yaml
│   │   │   │   │   ├── afrimgsm_cot_yaml
│   │   │   │   │   ├── afrimgsm_cot_yor.yaml
│   │   │   │   │   └── afrimgsm_cot_zul.yaml
│   │   │   │   └── prompt_5/
│   │   │   │       ├── afrimgsm_cot_amh.yaml
│   │   │   │       ├── afrimgsm_cot_eng.yaml
│   │   │   │       ├── afrimgsm_cot_ewe.yaml
│   │   │   │       ├── afrimgsm_cot_fra.yaml
│   │   │   │       ├── afrimgsm_cot_hau.yaml
│   │   │   │       ├── afrimgsm_cot_ibo.yaml
│   │   │   │       ├── afrimgsm_cot_kin.yaml
│   │   │   │       ├── afrimgsm_cot_lin.yaml
│   │   │   │       ├── afrimgsm_cot_lug.yaml
│   │   │   │       ├── afrimgsm_cot_orm.yaml
│   │   │   │       ├── afrimgsm_cot_sna.yaml
│   │   │   │       ├── afrimgsm_cot_sot.yaml
│   │   │   │       ├── afrimgsm_cot_swa.yaml
│   │   │   │       ├── afrimgsm_cot_twi.yaml
│   │   │   │       ├── afrimgsm_cot_vai.yaml
│   │   │   │       ├── afrimgsm_cot_wol.yaml
│   │   │   │       ├── afrimgsm_cot_xho.yaml
│   │   │   │       ├── afrimgsm_cot_yaml
│   │   │   │       ├── afrimgsm_cot_yor.yaml
│   │   │   │       └── afrimgsm_cot_zul.yaml
│   │   │   ├── gen_utils.py
│   │   │   ├── gen_yaml.sh
│   │   │   ├── run.sh
│   │   │   ├── translate/
│   │   │   │   ├── afrimgsm_tt.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── afrimgsm_translate_amh.yaml
│   │   │   │   │   ├── afrimgsm_translate_ewe.yaml
│   │   │   │   │   ├── afrimgsm_translate_fra.yaml
│   │   │   │   │   ├── afrimgsm_translate_hau.yaml
│   │   │   │   │   ├── afrimgsm_translate_ibo.yaml
│   │   │   │   │   ├── afrimgsm_translate_kin.yaml
│   │   │   │   │   ├── afrimgsm_translate_lin.yaml
│   │   │   │   │   ├── afrimgsm_translate_lug.yaml
│   │   │   │   │   ├── afrimgsm_translate_orm.yaml
│   │   │   │   │   ├── afrimgsm_translate_sna.yaml
│   │   │   │   │   ├── afrimgsm_translate_sot.yaml
│   │   │   │   │   ├── afrimgsm_translate_swa.yaml
│   │   │   │   │   ├── afrimgsm_translate_twi.yaml
│   │   │   │   │   ├── afrimgsm_translate_wol.yaml
│   │   │   │   │   ├── afrimgsm_translate_xho.yaml
│   │   │   │   │   ├── afrimgsm_translate_yaml
│   │   │   │   │   ├── afrimgsm_translate_yor.yaml
│   │   │   │   │   └── afrimgsm_translate_zul.yaml
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── afrimgsm_translate_amh.yaml
│   │   │   │   │   ├── afrimgsm_translate_ewe.yaml
│   │   │   │   │   ├── afrimgsm_translate_fra.yaml
│   │   │   │   │   ├── afrimgsm_translate_hau.yaml
│   │   │   │   │   ├── afrimgsm_translate_ibo.yaml
│   │   │   │   │   ├── afrimgsm_translate_kin.yaml
│   │   │   │   │   ├── afrimgsm_translate_lin.yaml
│   │   │   │   │   ├── afrimgsm_translate_lug.yaml
│   │   │   │   │   ├── afrimgsm_translate_orm.yaml
│   │   │   │   │   ├── afrimgsm_translate_sna.yaml
│   │   │   │   │   ├── afrimgsm_translate_sot.yaml
│   │   │   │   │   ├── afrimgsm_translate_swa.yaml
│   │   │   │   │   ├── afrimgsm_translate_twi.yaml
│   │   │   │   │   ├── afrimgsm_translate_wol.yaml
│   │   │   │   │   ├── afrimgsm_translate_xho.yaml
│   │   │   │   │   ├── afrimgsm_translate_yaml
│   │   │   │   │   ├── afrimgsm_translate_yor.yaml
│   │   │   │   │   └── afrimgsm_translate_zul.yaml
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── afrimgsm_translate_amh.yaml
│   │   │   │   │   ├── afrimgsm_translate_ewe.yaml
│   │   │   │   │   ├── afrimgsm_translate_fra.yaml
│   │   │   │   │   ├── afrimgsm_translate_hau.yaml
│   │   │   │   │   ├── afrimgsm_translate_ibo.yaml
│   │   │   │   │   ├── afrimgsm_translate_kin.yaml
│   │   │   │   │   ├── afrimgsm_translate_lin.yaml
│   │   │   │   │   ├── afrimgsm_translate_lug.yaml
│   │   │   │   │   ├── afrimgsm_translate_orm.yaml
│   │   │   │   │   ├── afrimgsm_translate_sna.yaml
│   │   │   │   │   ├── afrimgsm_translate_sot.yaml
│   │   │   │   │   ├── afrimgsm_translate_swa.yaml
│   │   │   │   │   ├── afrimgsm_translate_twi.yaml
│   │   │   │   │   ├── afrimgsm_translate_wol.yaml
│   │   │   │   │   ├── afrimgsm_translate_xho.yaml
│   │   │   │   │   ├── afrimgsm_translate_yaml
│   │   │   │   │   ├── afrimgsm_translate_yor.yaml
│   │   │   │   │   └── afrimgsm_translate_zul.yaml
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── afrimgsm_translate_amh.yaml
│   │   │   │   │   ├── afrimgsm_translate_ewe.yaml
│   │   │   │   │   ├── afrimgsm_translate_fra.yaml
│   │   │   │   │   ├── afrimgsm_translate_hau.yaml
│   │   │   │   │   ├── afrimgsm_translate_ibo.yaml
│   │   │   │   │   ├── afrimgsm_translate_kin.yaml
│   │   │   │   │   ├── afrimgsm_translate_lin.yaml
│   │   │   │   │   ├── afrimgsm_translate_lug.yaml
│   │   │   │   │   ├── afrimgsm_translate_orm.yaml
│   │   │   │   │   ├── afrimgsm_translate_sna.yaml
│   │   │   │   │   ├── afrimgsm_translate_sot.yaml
│   │   │   │   │   ├── afrimgsm_translate_swa.yaml
│   │   │   │   │   ├── afrimgsm_translate_twi.yaml
│   │   │   │   │   ├── afrimgsm_translate_wol.yaml
│   │   │   │   │   ├── afrimgsm_translate_xho.yaml
│   │   │   │   │   ├── afrimgsm_translate_yaml
│   │   │   │   │   ├── afrimgsm_translate_yor.yaml
│   │   │   │   │   └── afrimgsm_translate_zul.yaml
│   │   │   │   └── prompt_5/
│   │   │   │       ├── afrimgsm_translate_amh.yaml
│   │   │   │       ├── afrimgsm_translate_ewe.yaml
│   │   │   │       ├── afrimgsm_translate_fra.yaml
│   │   │   │       ├── afrimgsm_translate_hau.yaml
│   │   │   │       ├── afrimgsm_translate_ibo.yaml
│   │   │   │       ├── afrimgsm_translate_kin.yaml
│   │   │   │       ├── afrimgsm_translate_lin.yaml
│   │   │   │       ├── afrimgsm_translate_lug.yaml
│   │   │   │       ├── afrimgsm_translate_orm.yaml
│   │   │   │       ├── afrimgsm_translate_sna.yaml
│   │   │   │       ├── afrimgsm_translate_sot.yaml
│   │   │   │       ├── afrimgsm_translate_swa.yaml
│   │   │   │       ├── afrimgsm_translate_twi.yaml
│   │   │   │       ├── afrimgsm_translate_wol.yaml
│   │   │   │       ├── afrimgsm_translate_xho.yaml
│   │   │   │       ├── afrimgsm_translate_yaml
│   │   │   │       ├── afrimgsm_translate_yor.yaml
│   │   │   │       └── afrimgsm_translate_zul.yaml
│   │   │   ├── translate_cot/
│   │   │   │   ├── afrimgsm_tt_cot.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── afrimgsm_cot_translate_amh.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_ewe.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_fra.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_hau.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_ibo.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_kin.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_lin.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_lug.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_orm.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_sna.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_sot.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_swa.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_twi.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_vai.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_wol.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_xho.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_yor.yaml
│   │   │   │   │   └── afrimgsm_cot_translate_zul.yaml
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── afrimgsm_cot_translate_amh.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_ewe.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_fra.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_hau.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_ibo.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_kin.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_lin.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_lug.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_orm.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_sna.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_sot.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_swa.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_twi.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_vai.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_wol.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_xho.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_yor.yaml
│   │   │   │   │   └── afrimgsm_cot_translate_zul.yaml
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── afrimgsm_cot_translate_amh.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_ewe.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_fra.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_hau.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_ibo.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_kin.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_lin.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_lug.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_orm.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_sna.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_sot.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_swa.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_twi.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_vai.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_wol.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_xho.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_yor.yaml
│   │   │   │   │   └── afrimgsm_cot_translate_zul.yaml
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── afrimgsm_cot_translate_amh.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_ewe.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_fra.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_hau.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_ibo.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_kin.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_lin.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_lug.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_orm.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_sna.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_sot.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_swa.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_twi.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_vai.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_wol.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_xho.yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_yaml
│   │   │   │   │   ├── afrimgsm_cot_translate_yor.yaml
│   │   │   │   │   └── afrimgsm_cot_translate_zul.yaml
│   │   │   │   └── prompt_5/
│   │   │   │       ├── afrimgsm_cot_translate_amh.yaml
│   │   │   │       ├── afrimgsm_cot_translate_ewe.yaml
│   │   │   │       ├── afrimgsm_cot_translate_fra.yaml
│   │   │   │       ├── afrimgsm_cot_translate_hau.yaml
│   │   │   │       ├── afrimgsm_cot_translate_ibo.yaml
│   │   │   │       ├── afrimgsm_cot_translate_kin.yaml
│   │   │   │       ├── afrimgsm_cot_translate_lin.yaml
│   │   │   │       ├── afrimgsm_cot_translate_lug.yaml
│   │   │   │       ├── afrimgsm_cot_translate_orm.yaml
│   │   │   │       ├── afrimgsm_cot_translate_sna.yaml
│   │   │   │       ├── afrimgsm_cot_translate_sot.yaml
│   │   │   │       ├── afrimgsm_cot_translate_swa.yaml
│   │   │   │       ├── afrimgsm_cot_translate_twi.yaml
│   │   │   │       ├── afrimgsm_cot_translate_vai.yaml
│   │   │   │       ├── afrimgsm_cot_translate_wol.yaml
│   │   │   │       ├── afrimgsm_cot_translate_xho.yaml
│   │   │   │       ├── afrimgsm_cot_translate_yaml
│   │   │   │       ├── afrimgsm_cot_translate_yor.yaml
│   │   │   │       └── afrimgsm_cot_translate_zul.yaml
│   │   │   └── utils.py
│   │   ├── afrimmlu/
│   │   │   ├── README.md
│   │   │   ├── direct/
│   │   │   │   ├── afrimmlu.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── afrimmlu_direct
│   │   │   │   │   ├── afrimmlu_direct_amh.yaml
│   │   │   │   │   ├── afrimmlu_direct_eng.yaml
│   │   │   │   │   ├── afrimmlu_direct_ewe.yaml
│   │   │   │   │   ├── afrimmlu_direct_fra.yaml
│   │   │   │   │   ├── afrimmlu_direct_hau.yaml
│   │   │   │   │   ├── afrimmlu_direct_ibo.yaml
│   │   │   │   │   ├── afrimmlu_direct_kin.yaml
│   │   │   │   │   ├── afrimmlu_direct_lin.yaml
│   │   │   │   │   ├── afrimmlu_direct_lug.yaml
│   │   │   │   │   ├── afrimmlu_direct_orm.yaml
│   │   │   │   │   ├── afrimmlu_direct_sna.yaml
│   │   │   │   │   ├── afrimmlu_direct_sot.yaml
│   │   │   │   │   ├── afrimmlu_direct_swa.yaml
│   │   │   │   │   ├── afrimmlu_direct_twi.yaml
│   │   │   │   │   ├── afrimmlu_direct_wol.yaml
│   │   │   │   │   ├── afrimmlu_direct_xho.yaml
│   │   │   │   │   ├── afrimmlu_direct_yor.yaml
│   │   │   │   │   ├── afrimmlu_direct_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── afrimmlu_direct
│   │   │   │   │   ├── afrimmlu_direct_amh.yaml
│   │   │   │   │   ├── afrimmlu_direct_eng.yaml
│   │   │   │   │   ├── afrimmlu_direct_ewe.yaml
│   │   │   │   │   ├── afrimmlu_direct_fra.yaml
│   │   │   │   │   ├── afrimmlu_direct_hau.yaml
│   │   │   │   │   ├── afrimmlu_direct_ibo.yaml
│   │   │   │   │   ├── afrimmlu_direct_kin.yaml
│   │   │   │   │   ├── afrimmlu_direct_lin.yaml
│   │   │   │   │   ├── afrimmlu_direct_lug.yaml
│   │   │   │   │   ├── afrimmlu_direct_orm.yaml
│   │   │   │   │   ├── afrimmlu_direct_sna.yaml
│   │   │   │   │   ├── afrimmlu_direct_sot.yaml
│   │   │   │   │   ├── afrimmlu_direct_swa.yaml
│   │   │   │   │   ├── afrimmlu_direct_twi.yaml
│   │   │   │   │   ├── afrimmlu_direct_wol.yaml
│   │   │   │   │   ├── afrimmlu_direct_xho.yaml
│   │   │   │   │   ├── afrimmlu_direct_yor.yaml
│   │   │   │   │   ├── afrimmlu_direct_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── afrimmlu_direct
│   │   │   │   │   ├── afrimmlu_direct_amh.yaml
│   │   │   │   │   ├── afrimmlu_direct_eng.yaml
│   │   │   │   │   ├── afrimmlu_direct_ewe.yaml
│   │   │   │   │   ├── afrimmlu_direct_fra.yaml
│   │   │   │   │   ├── afrimmlu_direct_hau.yaml
│   │   │   │   │   ├── afrimmlu_direct_ibo.yaml
│   │   │   │   │   ├── afrimmlu_direct_kin.yaml
│   │   │   │   │   ├── afrimmlu_direct_lin.yaml
│   │   │   │   │   ├── afrimmlu_direct_lug.yaml
│   │   │   │   │   ├── afrimmlu_direct_orm.yaml
│   │   │   │   │   ├── afrimmlu_direct_sna.yaml
│   │   │   │   │   ├── afrimmlu_direct_sot.yaml
│   │   │   │   │   ├── afrimmlu_direct_swa.yaml
│   │   │   │   │   ├── afrimmlu_direct_twi.yaml
│   │   │   │   │   ├── afrimmlu_direct_wol.yaml
│   │   │   │   │   ├── afrimmlu_direct_xho.yaml
│   │   │   │   │   ├── afrimmlu_direct_yor.yaml
│   │   │   │   │   ├── afrimmlu_direct_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── afrimmlu_direct
│   │   │   │   │   ├── afrimmlu_direct_amh.yaml
│   │   │   │   │   ├── afrimmlu_direct_eng.yaml
│   │   │   │   │   ├── afrimmlu_direct_ewe.yaml
│   │   │   │   │   ├── afrimmlu_direct_fra.yaml
│   │   │   │   │   ├── afrimmlu_direct_hau.yaml
│   │   │   │   │   ├── afrimmlu_direct_ibo.yaml
│   │   │   │   │   ├── afrimmlu_direct_kin.yaml
│   │   │   │   │   ├── afrimmlu_direct_lin.yaml
│   │   │   │   │   ├── afrimmlu_direct_lug.yaml
│   │   │   │   │   ├── afrimmlu_direct_orm.yaml
│   │   │   │   │   ├── afrimmlu_direct_sna.yaml
│   │   │   │   │   ├── afrimmlu_direct_sot.yaml
│   │   │   │   │   ├── afrimmlu_direct_swa.yaml
│   │   │   │   │   ├── afrimmlu_direct_twi.yaml
│   │   │   │   │   ├── afrimmlu_direct_wol.yaml
│   │   │   │   │   ├── afrimmlu_direct_xho.yaml
│   │   │   │   │   ├── afrimmlu_direct_yor.yaml
│   │   │   │   │   ├── afrimmlu_direct_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   └── prompt_5/
│   │   │   │       ├── afrimmlu_direct
│   │   │   │       ├── afrimmlu_direct_amh.yaml
│   │   │   │       ├── afrimmlu_direct_eng.yaml
│   │   │   │       ├── afrimmlu_direct_ewe.yaml
│   │   │   │       ├── afrimmlu_direct_fra.yaml
│   │   │   │       ├── afrimmlu_direct_hau.yaml
│   │   │   │       ├── afrimmlu_direct_ibo.yaml
│   │   │   │       ├── afrimmlu_direct_kin.yaml
│   │   │   │       ├── afrimmlu_direct_lin.yaml
│   │   │   │       ├── afrimmlu_direct_lug.yaml
│   │   │   │       ├── afrimmlu_direct_orm.yaml
│   │   │   │       ├── afrimmlu_direct_sna.yaml
│   │   │   │       ├── afrimmlu_direct_sot.yaml
│   │   │   │       ├── afrimmlu_direct_swa.yaml
│   │   │   │       ├── afrimmlu_direct_twi.yaml
│   │   │   │       ├── afrimmlu_direct_wol.yaml
│   │   │   │       ├── afrimmlu_direct_xho.yaml
│   │   │   │       ├── afrimmlu_direct_yor.yaml
│   │   │   │       ├── afrimmlu_direct_zul.yaml
│   │   │   │       └── utils.py
│   │   │   ├── fewshot.sh
│   │   │   ├── gen_utils.py
│   │   │   ├── translate/
│   │   │   │   ├── afrimmlu_tt.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── afrimmlu_translate
│   │   │   │   │   ├── afrimmlu_translate_amh.yaml
│   │   │   │   │   ├── afrimmlu_translate_ewe.yaml
│   │   │   │   │   ├── afrimmlu_translate_fra.yaml
│   │   │   │   │   ├── afrimmlu_translate_hau.yaml
│   │   │   │   │   ├── afrimmlu_translate_ibo.yaml
│   │   │   │   │   ├── afrimmlu_translate_kin.yaml
│   │   │   │   │   ├── afrimmlu_translate_lin.yaml
│   │   │   │   │   ├── afrimmlu_translate_lug.yaml
│   │   │   │   │   ├── afrimmlu_translate_orm.yaml
│   │   │   │   │   ├── afrimmlu_translate_sna.yaml
│   │   │   │   │   ├── afrimmlu_translate_sot.yaml
│   │   │   │   │   ├── afrimmlu_translate_swa.yaml
│   │   │   │   │   ├── afrimmlu_translate_twi.yaml
│   │   │   │   │   ├── afrimmlu_translate_wol.yaml
│   │   │   │   │   ├── afrimmlu_translate_xho.yaml
│   │   │   │   │   ├── afrimmlu_translate_yor.yaml
│   │   │   │   │   ├── afrimmlu_translate_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── afrimmlu_translate
│   │   │   │   │   ├── afrimmlu_translate_amh.yaml
│   │   │   │   │   ├── afrimmlu_translate_ewe.yaml
│   │   │   │   │   ├── afrimmlu_translate_fra.yaml
│   │   │   │   │   ├── afrimmlu_translate_hau.yaml
│   │   │   │   │   ├── afrimmlu_translate_ibo.yaml
│   │   │   │   │   ├── afrimmlu_translate_kin.yaml
│   │   │   │   │   ├── afrimmlu_translate_lin.yaml
│   │   │   │   │   ├── afrimmlu_translate_lug.yaml
│   │   │   │   │   ├── afrimmlu_translate_orm.yaml
│   │   │   │   │   ├── afrimmlu_translate_sna.yaml
│   │   │   │   │   ├── afrimmlu_translate_sot.yaml
│   │   │   │   │   ├── afrimmlu_translate_swa.yaml
│   │   │   │   │   ├── afrimmlu_translate_twi.yaml
│   │   │   │   │   ├── afrimmlu_translate_wol.yaml
│   │   │   │   │   ├── afrimmlu_translate_xho.yaml
│   │   │   │   │   ├── afrimmlu_translate_yor.yaml
│   │   │   │   │   ├── afrimmlu_translate_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── afrimmlu_translate
│   │   │   │   │   ├── afrimmlu_translate_amh.yaml
│   │   │   │   │   ├── afrimmlu_translate_ewe.yaml
│   │   │   │   │   ├── afrimmlu_translate_fra.yaml
│   │   │   │   │   ├── afrimmlu_translate_hau.yaml
│   │   │   │   │   ├── afrimmlu_translate_ibo.yaml
│   │   │   │   │   ├── afrimmlu_translate_kin.yaml
│   │   │   │   │   ├── afrimmlu_translate_lin.yaml
│   │   │   │   │   ├── afrimmlu_translate_lug.yaml
│   │   │   │   │   ├── afrimmlu_translate_orm.yaml
│   │   │   │   │   ├── afrimmlu_translate_sna.yaml
│   │   │   │   │   ├── afrimmlu_translate_sot.yaml
│   │   │   │   │   ├── afrimmlu_translate_swa.yaml
│   │   │   │   │   ├── afrimmlu_translate_twi.yaml
│   │   │   │   │   ├── afrimmlu_translate_wol.yaml
│   │   │   │   │   ├── afrimmlu_translate_xho.yaml
│   │   │   │   │   ├── afrimmlu_translate_yor.yaml
│   │   │   │   │   ├── afrimmlu_translate_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── afrimmlu_translate
│   │   │   │   │   ├── afrimmlu_translate_amh.yaml
│   │   │   │   │   ├── afrimmlu_translate_ewe.yaml
│   │   │   │   │   ├── afrimmlu_translate_fra.yaml
│   │   │   │   │   ├── afrimmlu_translate_hau.yaml
│   │   │   │   │   ├── afrimmlu_translate_ibo.yaml
│   │   │   │   │   ├── afrimmlu_translate_kin.yaml
│   │   │   │   │   ├── afrimmlu_translate_lin.yaml
│   │   │   │   │   ├── afrimmlu_translate_lug.yaml
│   │   │   │   │   ├── afrimmlu_translate_orm.yaml
│   │   │   │   │   ├── afrimmlu_translate_sna.yaml
│   │   │   │   │   ├── afrimmlu_translate_sot.yaml
│   │   │   │   │   ├── afrimmlu_translate_swa.yaml
│   │   │   │   │   ├── afrimmlu_translate_twi.yaml
│   │   │   │   │   ├── afrimmlu_translate_wol.yaml
│   │   │   │   │   ├── afrimmlu_translate_xho.yaml
│   │   │   │   │   ├── afrimmlu_translate_yor.yaml
│   │   │   │   │   ├── afrimmlu_translate_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   └── prompt_5/
│   │   │   │       ├── afrimmlu_translate
│   │   │   │       ├── afrimmlu_translate_amh.yaml
│   │   │   │       ├── afrimmlu_translate_ewe.yaml
│   │   │   │       ├── afrimmlu_translate_fra.yaml
│   │   │   │       ├── afrimmlu_translate_hau.yaml
│   │   │   │       ├── afrimmlu_translate_ibo.yaml
│   │   │   │       ├── afrimmlu_translate_kin.yaml
│   │   │   │       ├── afrimmlu_translate_lin.yaml
│   │   │   │       ├── afrimmlu_translate_lug.yaml
│   │   │   │       ├── afrimmlu_translate_orm.yaml
│   │   │   │       ├── afrimmlu_translate_sna.yaml
│   │   │   │       ├── afrimmlu_translate_sot.yaml
│   │   │   │       ├── afrimmlu_translate_swa.yaml
│   │   │   │       ├── afrimmlu_translate_twi.yaml
│   │   │   │       ├── afrimmlu_translate_wol.yaml
│   │   │   │       ├── afrimmlu_translate_xho.yaml
│   │   │   │       ├── afrimmlu_translate_yor.yaml
│   │   │   │       ├── afrimmlu_translate_zul.yaml
│   │   │   │       └── utils.py
│   │   │   └── utils.py
│   │   ├── afrixnli/
│   │   │   ├── README.md
│   │   │   ├── anli prompt/
│   │   │   │   ├── en-direct/
│   │   │   │   │   ├── afrixnli_en_direct_amh.yaml
│   │   │   │   │   ├── afrixnli_en_direct_eng.yaml
│   │   │   │   │   ├── afrixnli_en_direct_ewe.yaml
│   │   │   │   │   ├── afrixnli_en_direct_fra.yaml
│   │   │   │   │   ├── afrixnli_en_direct_hau.yaml
│   │   │   │   │   ├── afrixnli_en_direct_ibo.yaml
│   │   │   │   │   ├── afrixnli_en_direct_kin.yaml
│   │   │   │   │   ├── afrixnli_en_direct_lin.yaml
│   │   │   │   │   ├── afrixnli_en_direct_lug.yaml
│   │   │   │   │   ├── afrixnli_en_direct_orm.yaml
│   │   │   │   │   ├── afrixnli_en_direct_sna.yaml
│   │   │   │   │   ├── afrixnli_en_direct_sot.yaml
│   │   │   │   │   ├── afrixnli_en_direct_swa.yaml
│   │   │   │   │   ├── afrixnli_en_direct_twi.yaml
│   │   │   │   │   ├── afrixnli_en_direct_wol.yaml
│   │   │   │   │   ├── afrixnli_en_direct_xho.yaml
│   │   │   │   │   ├── afrixnli_en_direct_yaml
│   │   │   │   │   ├── afrixnli_en_direct_yor.yaml
│   │   │   │   │   ├── afrixnli_en_direct_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── native-direct/
│   │   │   │   │   ├── afrixnli_native_direct_amh.yaml
│   │   │   │   │   ├── afrixnli_native_direct_eng.yaml
│   │   │   │   │   ├── afrixnli_native_direct_ewe.yaml
│   │   │   │   │   ├── afrixnli_native_direct_fra.yaml
│   │   │   │   │   ├── afrixnli_native_direct_hau.yaml
│   │   │   │   │   ├── afrixnli_native_direct_ibo.yaml
│   │   │   │   │   ├── afrixnli_native_direct_kin.yaml
│   │   │   │   │   ├── afrixnli_native_direct_lin.yaml
│   │   │   │   │   ├── afrixnli_native_direct_lug.yaml
│   │   │   │   │   ├── afrixnli_native_direct_orm.yaml
│   │   │   │   │   ├── afrixnli_native_direct_sna.yaml
│   │   │   │   │   ├── afrixnli_native_direct_sot.yaml
│   │   │   │   │   ├── afrixnli_native_direct_swa.yaml
│   │   │   │   │   ├── afrixnli_native_direct_twi.yaml
│   │   │   │   │   ├── afrixnli_native_direct_wol.yaml
│   │   │   │   │   ├── afrixnli_native_direct_xho.yaml
│   │   │   │   │   ├── afrixnli_native_direct_yaml
│   │   │   │   │   ├── afrixnli_native_direct_yor.yaml
│   │   │   │   │   ├── afrixnli_native_direct_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   └── translate/
│   │   │   │       ├── afrixnli_translate_amh.yaml
│   │   │   │       ├── afrixnli_translate_ewe.yaml
│   │   │   │       ├── afrixnli_translate_fra.yaml
│   │   │   │       ├── afrixnli_translate_hau.yaml
│   │   │   │       ├── afrixnli_translate_ibo.yaml
│   │   │   │       ├── afrixnli_translate_kin.yaml
│   │   │   │       ├── afrixnli_translate_lin.yaml
│   │   │   │       ├── afrixnli_translate_lug.yaml
│   │   │   │       ├── afrixnli_translate_orm.yaml
│   │   │   │       ├── afrixnli_translate_sna.yaml
│   │   │   │       ├── afrixnli_translate_sot.yaml
│   │   │   │       ├── afrixnli_translate_swa.yaml
│   │   │   │       ├── afrixnli_translate_twi.yaml
│   │   │   │       ├── afrixnli_translate_wol.yaml
│   │   │   │       ├── afrixnli_translate_xho.yaml
│   │   │   │       ├── afrixnli_translate_yaml
│   │   │   │       ├── afrixnli_translate_yor.yaml
│   │   │   │       ├── afrixnli_translate_zul.yaml
│   │   │   │       └── utils.py
│   │   │   ├── direct/
│   │   │   │   ├── afrixnli.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── afrixnli_amh.yaml
│   │   │   │   │   ├── afrixnli_eng.yaml
│   │   │   │   │   ├── afrixnli_ewe.yaml
│   │   │   │   │   ├── afrixnli_fra.yaml
│   │   │   │   │   ├── afrixnli_hau.yaml
│   │   │   │   │   ├── afrixnli_ibo.yaml
│   │   │   │   │   ├── afrixnli_kin.yaml
│   │   │   │   │   ├── afrixnli_lin.yaml
│   │   │   │   │   ├── afrixnli_lug.yaml
│   │   │   │   │   ├── afrixnli_orm.yaml
│   │   │   │   │   ├── afrixnli_sna.yaml
│   │   │   │   │   ├── afrixnli_sot.yaml
│   │   │   │   │   ├── afrixnli_swa.yaml
│   │   │   │   │   ├── afrixnli_twi.yaml
│   │   │   │   │   ├── afrixnli_wol.yaml
│   │   │   │   │   ├── afrixnli_xho.yaml
│   │   │   │   │   ├── afrixnli_yaml
│   │   │   │   │   ├── afrixnli_yor.yaml
│   │   │   │   │   ├── afrixnli_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── afrixnli_amh.yaml
│   │   │   │   │   ├── afrixnli_eng.yaml
│   │   │   │   │   ├── afrixnli_ewe.yaml
│   │   │   │   │   ├── afrixnli_fra.yaml
│   │   │   │   │   ├── afrixnli_hau.yaml
│   │   │   │   │   ├── afrixnli_ibo.yaml
│   │   │   │   │   ├── afrixnli_kin.yaml
│   │   │   │   │   ├── afrixnli_lin.yaml
│   │   │   │   │   ├── afrixnli_lug.yaml
│   │   │   │   │   ├── afrixnli_orm.yaml
│   │   │   │   │   ├── afrixnli_sna.yaml
│   │   │   │   │   ├── afrixnli_sot.yaml
│   │   │   │   │   ├── afrixnli_swa.yaml
│   │   │   │   │   ├── afrixnli_twi.yaml
│   │   │   │   │   ├── afrixnli_wol.yaml
│   │   │   │   │   ├── afrixnli_xho.yaml
│   │   │   │   │   ├── afrixnli_yaml
│   │   │   │   │   ├── afrixnli_yor.yaml
│   │   │   │   │   ├── afrixnli_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── afrixnli_amh.yaml
│   │   │   │   │   ├── afrixnli_eng.yaml
│   │   │   │   │   ├── afrixnli_ewe.yaml
│   │   │   │   │   ├── afrixnli_fra.yaml
│   │   │   │   │   ├── afrixnli_hau.yaml
│   │   │   │   │   ├── afrixnli_ibo.yaml
│   │   │   │   │   ├── afrixnli_kin.yaml
│   │   │   │   │   ├── afrixnli_lin.yaml
│   │   │   │   │   ├── afrixnli_lug.yaml
│   │   │   │   │   ├── afrixnli_orm.yaml
│   │   │   │   │   ├── afrixnli_sna.yaml
│   │   │   │   │   ├── afrixnli_sot.yaml
│   │   │   │   │   ├── afrixnli_swa.yaml
│   │   │   │   │   ├── afrixnli_twi.yaml
│   │   │   │   │   ├── afrixnli_wol.yaml
│   │   │   │   │   ├── afrixnli_xho.yaml
│   │   │   │   │   ├── afrixnli_yaml
│   │   │   │   │   ├── afrixnli_yor.yaml
│   │   │   │   │   ├── afrixnli_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── afrixnli_amh.yaml
│   │   │   │   │   ├── afrixnli_eng.yaml
│   │   │   │   │   ├── afrixnli_ewe.yaml
│   │   │   │   │   ├── afrixnli_fra.yaml
│   │   │   │   │   ├── afrixnli_hau.yaml
│   │   │   │   │   ├── afrixnli_ibo.yaml
│   │   │   │   │   ├── afrixnli_kin.yaml
│   │   │   │   │   ├── afrixnli_lin.yaml
│   │   │   │   │   ├── afrixnli_lug.yaml
│   │   │   │   │   ├── afrixnli_orm.yaml
│   │   │   │   │   ├── afrixnli_sna.yaml
│   │   │   │   │   ├── afrixnli_sot.yaml
│   │   │   │   │   ├── afrixnli_swa.yaml
│   │   │   │   │   ├── afrixnli_twi.yaml
│   │   │   │   │   ├── afrixnli_wol.yaml
│   │   │   │   │   ├── afrixnli_xho.yaml
│   │   │   │   │   ├── afrixnli_yaml
│   │   │   │   │   ├── afrixnli_yor.yaml
│   │   │   │   │   ├── afrixnli_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   └── prompt_5/
│   │   │   │       ├── afrixnli_amh.yaml
│   │   │   │       ├── afrixnli_eng.yaml
│   │   │   │       ├── afrixnli_ewe.yaml
│   │   │   │       ├── afrixnli_fra.yaml
│   │   │   │       ├── afrixnli_hau.yaml
│   │   │   │       ├── afrixnli_ibo.yaml
│   │   │   │       ├── afrixnli_kin.yaml
│   │   │   │       ├── afrixnli_lin.yaml
│   │   │   │       ├── afrixnli_lug.yaml
│   │   │   │       ├── afrixnli_orm.yaml
│   │   │   │       ├── afrixnli_sna.yaml
│   │   │   │       ├── afrixnli_sot.yaml
│   │   │   │       ├── afrixnli_swa.yaml
│   │   │   │       ├── afrixnli_twi.yaml
│   │   │   │       ├── afrixnli_wol.yaml
│   │   │   │       ├── afrixnli_xho.yaml
│   │   │   │       ├── afrixnli_yaml
│   │   │   │       ├── afrixnli_yor.yaml
│   │   │   │       ├── afrixnli_zul.yaml
│   │   │   │       └── utils.py
│   │   │   ├── gen_utils.py
│   │   │   ├── lai prompt/
│   │   │   │   ├── direct/
│   │   │   │   │   ├── afrixnli_manual_direct_amh.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_eng.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_ewe.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_fra.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_hau.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_ibo.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_kin.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_lin.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_lug.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_orm.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_sna.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_sot.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_swa.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_twi.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_wol.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_xho.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_yaml
│   │   │   │   │   ├── afrixnli_manual_direct_yor.yaml
│   │   │   │   │   ├── afrixnli_manual_direct_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   └── translate/
│   │   │   │       ├── afrixnli_manual_translate_amh.yaml
│   │   │   │       ├── afrixnli_manual_translate_ewe.yaml
│   │   │   │       ├── afrixnli_manual_translate_fra.yaml
│   │   │   │       ├── afrixnli_manual_translate_hau.yaml
│   │   │   │       ├── afrixnli_manual_translate_ibo.yaml
│   │   │   │       ├── afrixnli_manual_translate_kin.yaml
│   │   │   │       ├── afrixnli_manual_translate_lin.yaml
│   │   │   │       ├── afrixnli_manual_translate_lug.yaml
│   │   │   │       ├── afrixnli_manual_translate_orm.yaml
│   │   │   │       ├── afrixnli_manual_translate_sna.yaml
│   │   │   │       ├── afrixnli_manual_translate_sot.yaml
│   │   │   │       ├── afrixnli_manual_translate_swa.yaml
│   │   │   │       ├── afrixnli_manual_translate_twi.yaml
│   │   │   │       ├── afrixnli_manual_translate_wol.yaml
│   │   │   │       ├── afrixnli_manual_translate_xho.yaml
│   │   │   │       ├── afrixnli_manual_translate_yaml
│   │   │   │       ├── afrixnli_manual_translate_yor.yaml
│   │   │   │       ├── afrixnli_manual_translate_zul.yaml
│   │   │   │       └── utils.py
│   │   │   ├── translate/
│   │   │   │   ├── afrixnli_tt.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── afrixnli_translate_amh.yaml
│   │   │   │   │   ├── afrixnli_translate_ewe.yaml
│   │   │   │   │   ├── afrixnli_translate_fra.yaml
│   │   │   │   │   ├── afrixnli_translate_hau.yaml
│   │   │   │   │   ├── afrixnli_translate_ibo.yaml
│   │   │   │   │   ├── afrixnli_translate_kin.yaml
│   │   │   │   │   ├── afrixnli_translate_lin.yaml
│   │   │   │   │   ├── afrixnli_translate_lug.yaml
│   │   │   │   │   ├── afrixnli_translate_orm.yaml
│   │   │   │   │   ├── afrixnli_translate_sna.yaml
│   │   │   │   │   ├── afrixnli_translate_sot.yaml
│   │   │   │   │   ├── afrixnli_translate_swa.yaml
│   │   │   │   │   ├── afrixnli_translate_twi.yaml
│   │   │   │   │   ├── afrixnli_translate_wol.yaml
│   │   │   │   │   ├── afrixnli_translate_xho.yaml
│   │   │   │   │   ├── afrixnli_translate_yaml
│   │   │   │   │   ├── afrixnli_translate_yor.yaml
│   │   │   │   │   ├── afrixnli_translate_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── afrixnli_translate_amh.yaml
│   │   │   │   │   ├── afrixnli_translate_ewe.yaml
│   │   │   │   │   ├── afrixnli_translate_fra.yaml
│   │   │   │   │   ├── afrixnli_translate_hau.yaml
│   │   │   │   │   ├── afrixnli_translate_ibo.yaml
│   │   │   │   │   ├── afrixnli_translate_kin.yaml
│   │   │   │   │   ├── afrixnli_translate_lin.yaml
│   │   │   │   │   ├── afrixnli_translate_lug.yaml
│   │   │   │   │   ├── afrixnli_translate_orm.yaml
│   │   │   │   │   ├── afrixnli_translate_sna.yaml
│   │   │   │   │   ├── afrixnli_translate_sot.yaml
│   │   │   │   │   ├── afrixnli_translate_swa.yaml
│   │   │   │   │   ├── afrixnli_translate_twi.yaml
│   │   │   │   │   ├── afrixnli_translate_wol.yaml
│   │   │   │   │   ├── afrixnli_translate_xho.yaml
│   │   │   │   │   ├── afrixnli_translate_yaml
│   │   │   │   │   ├── afrixnli_translate_yor.yaml
│   │   │   │   │   ├── afrixnli_translate_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── afrixnli_translate_amh.yaml
│   │   │   │   │   ├── afrixnli_translate_ewe.yaml
│   │   │   │   │   ├── afrixnli_translate_fra.yaml
│   │   │   │   │   ├── afrixnli_translate_hau.yaml
│   │   │   │   │   ├── afrixnli_translate_ibo.yaml
│   │   │   │   │   ├── afrixnli_translate_kin.yaml
│   │   │   │   │   ├── afrixnli_translate_lin.yaml
│   │   │   │   │   ├── afrixnli_translate_lug.yaml
│   │   │   │   │   ├── afrixnli_translate_orm.yaml
│   │   │   │   │   ├── afrixnli_translate_sna.yaml
│   │   │   │   │   ├── afrixnli_translate_sot.yaml
│   │   │   │   │   ├── afrixnli_translate_swa.yaml
│   │   │   │   │   ├── afrixnli_translate_twi.yaml
│   │   │   │   │   ├── afrixnli_translate_wol.yaml
│   │   │   │   │   ├── afrixnli_translate_xho.yaml
│   │   │   │   │   ├── afrixnli_translate_yaml
│   │   │   │   │   ├── afrixnli_translate_yor.yaml
│   │   │   │   │   ├── afrixnli_translate_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── afrixnli_translate_amh.yaml
│   │   │   │   │   ├── afrixnli_translate_ewe.yaml
│   │   │   │   │   ├── afrixnli_translate_fra.yaml
│   │   │   │   │   ├── afrixnli_translate_hau.yaml
│   │   │   │   │   ├── afrixnli_translate_ibo.yaml
│   │   │   │   │   ├── afrixnli_translate_kin.yaml
│   │   │   │   │   ├── afrixnli_translate_lin.yaml
│   │   │   │   │   ├── afrixnli_translate_lug.yaml
│   │   │   │   │   ├── afrixnli_translate_orm.yaml
│   │   │   │   │   ├── afrixnli_translate_sna.yaml
│   │   │   │   │   ├── afrixnli_translate_sot.yaml
│   │   │   │   │   ├── afrixnli_translate_swa.yaml
│   │   │   │   │   ├── afrixnli_translate_twi.yaml
│   │   │   │   │   ├── afrixnli_translate_wol.yaml
│   │   │   │   │   ├── afrixnli_translate_xho.yaml
│   │   │   │   │   ├── afrixnli_translate_yaml
│   │   │   │   │   ├── afrixnli_translate_yor.yaml
│   │   │   │   │   ├── afrixnli_translate_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   └── prompt_5/
│   │   │   │       ├── afrixnli_translate_amh.yaml
│   │   │   │       ├── afrixnli_translate_ewe.yaml
│   │   │   │       ├── afrixnli_translate_fra.yaml
│   │   │   │       ├── afrixnli_translate_hau.yaml
│   │   │   │       ├── afrixnli_translate_ibo.yaml
│   │   │   │       ├── afrixnli_translate_kin.yaml
│   │   │   │       ├── afrixnli_translate_lin.yaml
│   │   │   │       ├── afrixnli_translate_lug.yaml
│   │   │   │       ├── afrixnli_translate_orm.yaml
│   │   │   │       ├── afrixnli_translate_sna.yaml
│   │   │   │       ├── afrixnli_translate_sot.yaml
│   │   │   │       ├── afrixnli_translate_swa.yaml
│   │   │   │       ├── afrixnli_translate_twi.yaml
│   │   │   │       ├── afrixnli_translate_wol.yaml
│   │   │   │       ├── afrixnli_translate_xho.yaml
│   │   │   │       ├── afrixnli_translate_yaml
│   │   │   │       ├── afrixnli_translate_yor.yaml
│   │   │   │       ├── afrixnli_translate_zul.yaml
│   │   │   │       └── utils.py
│   │   │   └── utils.py
│   │   ├── afrobench/
│   │   │   ├── README.md
│   │   │   ├── adr/
│   │   │   │   ├── README.md
│   │   │   │   ├── afridiacritics.yaml
│   │   │   │   ├── gen_utils.py
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── afridiacritics_bbj.yaml
│   │   │   │   │   ├── afridiacritics_fon.yaml
│   │   │   │   │   ├── afridiacritics_ibo.yaml
│   │   │   │   │   ├── afridiacritics_wol.yaml
│   │   │   │   │   ├── afridiacritics_yaml
│   │   │   │   │   └── afridiacritics_yor.yaml
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── afridiacritics_bbj.yaml
│   │   │   │   │   ├── afridiacritics_fon.yaml
│   │   │   │   │   ├── afridiacritics_ibo.yaml
│   │   │   │   │   ├── afridiacritics_wol.yaml
│   │   │   │   │   ├── afridiacritics_yaml
│   │   │   │   │   └── afridiacritics_yor.yaml
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── afridiacritics_bbj.yaml
│   │   │   │   │   ├── afridiacritics_fon.yaml
│   │   │   │   │   ├── afridiacritics_ibo.yaml
│   │   │   │   │   ├── afridiacritics_wol.yaml
│   │   │   │   │   ├── afridiacritics_yaml
│   │   │   │   │   └── afridiacritics_yor.yaml
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── afridiacritics_bbj.yaml
│   │   │   │   │   ├── afridiacritics_fon.yaml
│   │   │   │   │   ├── afridiacritics_ibo.yaml
│   │   │   │   │   ├── afridiacritics_wol.yaml
│   │   │   │   │   ├── afridiacritics_yaml
│   │   │   │   │   └── afridiacritics_yor.yaml
│   │   │   │   └── prompt_5/
│   │   │   │       ├── afridiacritics_bbj.yaml
│   │   │   │       ├── afridiacritics_fon.yaml
│   │   │   │       ├── afridiacritics_ibo.yaml
│   │   │   │       ├── afridiacritics_wol.yaml
│   │   │   │       ├── afridiacritics_yaml
│   │   │   │       └── afridiacritics_yor.yaml
│   │   │   ├── afriqa/
│   │   │   │   ├── README.md
│   │   │   │   ├── afriqa.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── afriqa
│   │   │   │   │   ├── afriqa_bem.yaml
│   │   │   │   │   ├── afriqa_fon.yaml
│   │   │   │   │   ├── afriqa_hau.yaml
│   │   │   │   │   ├── afriqa_ibo.yaml
│   │   │   │   │   ├── afriqa_kin.yaml
│   │   │   │   │   ├── afriqa_swa.yaml
│   │   │   │   │   ├── afriqa_twi.yaml
│   │   │   │   │   ├── afriqa_yor.yaml
│   │   │   │   │   ├── afriqa_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── afriqa
│   │   │   │   │   ├── afriqa_bem.yaml
│   │   │   │   │   ├── afriqa_fon.yaml
│   │   │   │   │   ├── afriqa_hau.yaml
│   │   │   │   │   ├── afriqa_ibo.yaml
│   │   │   │   │   ├── afriqa_kin.yaml
│   │   │   │   │   ├── afriqa_swa.yaml
│   │   │   │   │   ├── afriqa_twi.yaml
│   │   │   │   │   ├── afriqa_yor.yaml
│   │   │   │   │   ├── afriqa_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── afriqa
│   │   │   │   │   ├── afriqa_bem.yaml
│   │   │   │   │   ├── afriqa_fon.yaml
│   │   │   │   │   ├── afriqa_hau.yaml
│   │   │   │   │   ├── afriqa_ibo.yaml
│   │   │   │   │   ├── afriqa_kin.yaml
│   │   │   │   │   ├── afriqa_swa.yaml
│   │   │   │   │   ├── afriqa_twi.yaml
│   │   │   │   │   ├── afriqa_yor.yaml
│   │   │   │   │   ├── afriqa_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── afriqa
│   │   │   │   │   ├── afriqa_bem.yaml
│   │   │   │   │   ├── afriqa_fon.yaml
│   │   │   │   │   ├── afriqa_hau.yaml
│   │   │   │   │   ├── afriqa_ibo.yaml
│   │   │   │   │   ├── afriqa_kin.yaml
│   │   │   │   │   ├── afriqa_swa.yaml
│   │   │   │   │   ├── afriqa_twi.yaml
│   │   │   │   │   ├── afriqa_yor.yaml
│   │   │   │   │   ├── afriqa_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_5/
│   │   │   │   │   ├── afriqa
│   │   │   │   │   ├── afriqa_bem.yaml
│   │   │   │   │   ├── afriqa_fon.yaml
│   │   │   │   │   ├── afriqa_hau.yaml
│   │   │   │   │   ├── afriqa_ibo.yaml
│   │   │   │   │   ├── afriqa_kin.yaml
│   │   │   │   │   ├── afriqa_swa.yaml
│   │   │   │   │   ├── afriqa_twi.yaml
│   │   │   │   │   ├── afriqa_yor.yaml
│   │   │   │   │   ├── afriqa_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   └── utils.py
│   │   │   ├── afrisenti/
│   │   │   │   ├── README.md
│   │   │   │   ├── afrisenti.yaml
│   │   │   │   ├── fewshot.sh
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── afrisenti
│   │   │   │   │   ├── afrisenti_amh.yaml
│   │   │   │   │   ├── afrisenti_arq.yaml
│   │   │   │   │   ├── afrisenti_ary.yaml
│   │   │   │   │   ├── afrisenti_hau.yaml
│   │   │   │   │   ├── afrisenti_ibo.yaml
│   │   │   │   │   ├── afrisenti_kin.yaml
│   │   │   │   │   ├── afrisenti_orm.yaml
│   │   │   │   │   ├── afrisenti_pcm.yaml
│   │   │   │   │   ├── afrisenti_por.yaml
│   │   │   │   │   ├── afrisenti_swa.yaml
│   │   │   │   │   ├── afrisenti_tir.yaml
│   │   │   │   │   ├── afrisenti_tso.yaml
│   │   │   │   │   ├── afrisenti_twi.yaml
│   │   │   │   │   ├── afrisenti_yor.yaml
│   │   │   │   │   ├── run.sh
│   │   │   │   │   ├── utils.py
│   │   │   │   │   └── xx.py
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── afrisenti
│   │   │   │   │   ├── afrisenti_amh.yaml
│   │   │   │   │   ├── afrisenti_arq.yaml
│   │   │   │   │   ├── afrisenti_ary.yaml
│   │   │   │   │   ├── afrisenti_hau.yaml
│   │   │   │   │   ├── afrisenti_ibo.yaml
│   │   │   │   │   ├── afrisenti_kin.yaml
│   │   │   │   │   ├── afrisenti_orm.yaml
│   │   │   │   │   ├── afrisenti_pcm.yaml
│   │   │   │   │   ├── afrisenti_por.yaml
│   │   │   │   │   ├── afrisenti_swa.yaml
│   │   │   │   │   ├── afrisenti_tir.yaml
│   │   │   │   │   ├── afrisenti_tso.yaml
│   │   │   │   │   ├── afrisenti_twi.yaml
│   │   │   │   │   ├── afrisenti_yor.yaml
│   │   │   │   │   ├── run.sh
│   │   │   │   │   ├── utils.py
│   │   │   │   │   └── xx.py
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── afrisenti
│   │   │   │   │   ├── afrisenti_amh.yaml
│   │   │   │   │   ├── afrisenti_arq.yaml
│   │   │   │   │   ├── afrisenti_ary.yaml
│   │   │   │   │   ├── afrisenti_hau.yaml
│   │   │   │   │   ├── afrisenti_ibo.yaml
│   │   │   │   │   ├── afrisenti_kin.yaml
│   │   │   │   │   ├── afrisenti_orm.yaml
│   │   │   │   │   ├── afrisenti_pcm.yaml
│   │   │   │   │   ├── afrisenti_por.yaml
│   │   │   │   │   ├── afrisenti_swa.yaml
│   │   │   │   │   ├── afrisenti_tir.yaml
│   │   │   │   │   ├── afrisenti_tso.yaml
│   │   │   │   │   ├── afrisenti_twi.yaml
│   │   │   │   │   ├── afrisenti_yor.yaml
│   │   │   │   │   ├── utils.py
│   │   │   │   │   └── xx.py
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── afrisenti
│   │   │   │   │   ├── afrisenti_amh.yaml
│   │   │   │   │   ├── afrisenti_arq.yaml
│   │   │   │   │   ├── afrisenti_ary.yaml
│   │   │   │   │   ├── afrisenti_hau.yaml
│   │   │   │   │   ├── afrisenti_ibo.yaml
│   │   │   │   │   ├── afrisenti_kin.yaml
│   │   │   │   │   ├── afrisenti_orm.yaml
│   │   │   │   │   ├── afrisenti_pcm.yaml
│   │   │   │   │   ├── afrisenti_por.yaml
│   │   │   │   │   ├── afrisenti_swa.yaml
│   │   │   │   │   ├── afrisenti_tir.yaml
│   │   │   │   │   ├── afrisenti_tso.yaml
│   │   │   │   │   ├── afrisenti_twi.yaml
│   │   │   │   │   ├── afrisenti_yor.yaml
│   │   │   │   │   ├── utils.py
│   │   │   │   │   └── xx.py
│   │   │   │   ├── prompt_5/
│   │   │   │   │   ├── afrisenti
│   │   │   │   │   ├── afrisenti_amh.yaml
│   │   │   │   │   ├── afrisenti_arq.yaml
│   │   │   │   │   ├── afrisenti_ary.yaml
│   │   │   │   │   ├── afrisenti_hau.yaml
│   │   │   │   │   ├── afrisenti_ibo.yaml
│   │   │   │   │   ├── afrisenti_kin.yaml
│   │   │   │   │   ├── afrisenti_orm.yaml
│   │   │   │   │   ├── afrisenti_pcm.yaml
│   │   │   │   │   ├── afrisenti_por.yaml
│   │   │   │   │   ├── afrisenti_swa.yaml
│   │   │   │   │   ├── afrisenti_tir.yaml
│   │   │   │   │   ├── afrisenti_tso.yaml
│   │   │   │   │   ├── afrisenti_twi.yaml
│   │   │   │   │   ├── afrisenti_yor.yaml
│   │   │   │   │   ├── utils.py
│   │   │   │   │   └── xx.py
│   │   │   │   └── utils.py
│   │   │   ├── afrobench-lite.yaml
│   │   │   ├── afrobench.yaml
│   │   │   ├── belebele/
│   │   │   │   ├── README.md
│   │   │   │   ├── belebele.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── belebele
│   │   │   │   │   ├── belebele_afr.yaml
│   │   │   │   │   ├── belebele_amh.yaml
│   │   │   │   │   ├── belebele_ary.yaml
│   │   │   │   │   ├── belebele_arz.yaml
│   │   │   │   │   ├── belebele_bam.yaml
│   │   │   │   │   ├── belebele_eng.yaml
│   │   │   │   │   ├── belebele_fra.yaml
│   │   │   │   │   ├── belebele_fuv.yaml
│   │   │   │   │   ├── belebele_gaz.yaml
│   │   │   │   │   ├── belebele_hau.yaml
│   │   │   │   │   ├── belebele_ibo.yaml
│   │   │   │   │   ├── belebele_kea.yaml
│   │   │   │   │   ├── belebele_kin.yaml
│   │   │   │   │   ├── belebele_lin.yaml
│   │   │   │   │   ├── belebele_lug.yaml
│   │   │   │   │   ├── belebele_luo.yaml
│   │   │   │   │   ├── belebele_nya.yaml
│   │   │   │   │   ├── belebele_plt.yaml
│   │   │   │   │   ├── belebele_por.yaml
│   │   │   │   │   ├── belebele_sna.yaml
│   │   │   │   │   ├── belebele_som.yaml
│   │   │   │   │   ├── belebele_sot.yaml
│   │   │   │   │   ├── belebele_ssw.yaml
│   │   │   │   │   ├── belebele_swa.yaml
│   │   │   │   │   ├── belebele_tir.yaml
│   │   │   │   │   ├── belebele_tsn.yaml
│   │   │   │   │   ├── belebele_tso.yaml
│   │   │   │   │   ├── belebele_wol.yaml
│   │   │   │   │   ├── belebele_xho.yaml
│   │   │   │   │   ├── belebele_yor.yaml
│   │   │   │   │   └── belebele_zul.yaml
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── belebele
│   │   │   │   │   ├── belebele_afr.yaml
│   │   │   │   │   ├── belebele_amh.yaml
│   │   │   │   │   ├── belebele_ary.yaml
│   │   │   │   │   ├── belebele_arz.yaml
│   │   │   │   │   ├── belebele_bam.yaml
│   │   │   │   │   ├── belebele_eng.yaml
│   │   │   │   │   ├── belebele_fra.yaml
│   │   │   │   │   ├── belebele_fuv.yaml
│   │   │   │   │   ├── belebele_gaz.yaml
│   │   │   │   │   ├── belebele_hau.yaml
│   │   │   │   │   ├── belebele_ibo.yaml
│   │   │   │   │   ├── belebele_kea.yaml
│   │   │   │   │   ├── belebele_kin.yaml
│   │   │   │   │   ├── belebele_lin.yaml
│   │   │   │   │   ├── belebele_lug.yaml
│   │   │   │   │   ├── belebele_luo.yaml
│   │   │   │   │   ├── belebele_nya.yaml
│   │   │   │   │   ├── belebele_plt.yaml
│   │   │   │   │   ├── belebele_por.yaml
│   │   │   │   │   ├── belebele_sna.yaml
│   │   │   │   │   ├── belebele_som.yaml
│   │   │   │   │   ├── belebele_sot.yaml
│   │   │   │   │   ├── belebele_ssw.yaml
│   │   │   │   │   ├── belebele_swa.yaml
│   │   │   │   │   ├── belebele_tir.yaml
│   │   │   │   │   ├── belebele_tsn.yaml
│   │   │   │   │   ├── belebele_tso.yaml
│   │   │   │   │   ├── belebele_wol.yaml
│   │   │   │   │   ├── belebele_xho.yaml
│   │   │   │   │   ├── belebele_yor.yaml
│   │   │   │   │   └── belebele_zul.yaml
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── belebele
│   │   │   │   │   ├── belebele_afr.yaml
│   │   │   │   │   ├── belebele_amh.yaml
│   │   │   │   │   ├── belebele_ary.yaml
│   │   │   │   │   ├── belebele_arz.yaml
│   │   │   │   │   ├── belebele_bam.yaml
│   │   │   │   │   ├── belebele_eng.yaml
│   │   │   │   │   ├── belebele_fra.yaml
│   │   │   │   │   ├── belebele_fuv.yaml
│   │   │   │   │   ├── belebele_gaz.yaml
│   │   │   │   │   ├── belebele_hau.yaml
│   │   │   │   │   ├── belebele_ibo.yaml
│   │   │   │   │   ├── belebele_kea.yaml
│   │   │   │   │   ├── belebele_kin.yaml
│   │   │   │   │   ├── belebele_lin.yaml
│   │   │   │   │   ├── belebele_lug.yaml
│   │   │   │   │   ├── belebele_luo.yaml
│   │   │   │   │   ├── belebele_nya.yaml
│   │   │   │   │   ├── belebele_plt.yaml
│   │   │   │   │   ├── belebele_por.yaml
│   │   │   │   │   ├── belebele_sna.yaml
│   │   │   │   │   ├── belebele_som.yaml
│   │   │   │   │   ├── belebele_sot.yaml
│   │   │   │   │   ├── belebele_ssw.yaml
│   │   │   │   │   ├── belebele_swa.yaml
│   │   │   │   │   ├── belebele_tir.yaml
│   │   │   │   │   ├── belebele_tsn.yaml
│   │   │   │   │   ├── belebele_tso.yaml
│   │   │   │   │   ├── belebele_wol.yaml
│   │   │   │   │   ├── belebele_xho.yaml
│   │   │   │   │   ├── belebele_yor.yaml
│   │   │   │   │   └── belebele_zul.yaml
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── belebele
│   │   │   │   │   ├── belebele_afr.yaml
│   │   │   │   │   ├── belebele_amh.yaml
│   │   │   │   │   ├── belebele_ary.yaml
│   │   │   │   │   ├── belebele_arz.yaml
│   │   │   │   │   ├── belebele_bam.yaml
│   │   │   │   │   ├── belebele_eng.yaml
│   │   │   │   │   ├── belebele_fra.yaml
│   │   │   │   │   ├── belebele_fuv.yaml
│   │   │   │   │   ├── belebele_gaz.yaml
│   │   │   │   │   ├── belebele_hau.yaml
│   │   │   │   │   ├── belebele_ibo.yaml
│   │   │   │   │   ├── belebele_kea.yaml
│   │   │   │   │   ├── belebele_kin.yaml
│   │   │   │   │   ├── belebele_lin.yaml
│   │   │   │   │   ├── belebele_lug.yaml
│   │   │   │   │   ├── belebele_luo.yaml
│   │   │   │   │   ├── belebele_nya.yaml
│   │   │   │   │   ├── belebele_plt.yaml
│   │   │   │   │   ├── belebele_por.yaml
│   │   │   │   │   ├── belebele_sna.yaml
│   │   │   │   │   ├── belebele_som.yaml
│   │   │   │   │   ├── belebele_sot.yaml
│   │   │   │   │   ├── belebele_ssw.yaml
│   │   │   │   │   ├── belebele_swa.yaml
│   │   │   │   │   ├── belebele_tir.yaml
│   │   │   │   │   ├── belebele_tsn.yaml
│   │   │   │   │   ├── belebele_tso.yaml
│   │   │   │   │   ├── belebele_wol.yaml
│   │   │   │   │   ├── belebele_xho.yaml
│   │   │   │   │   ├── belebele_yor.yaml
│   │   │   │   │   └── belebele_zul.yaml
│   │   │   │   ├── prompt_5/
│   │   │   │   │   ├── belebele
│   │   │   │   │   ├── belebele_afr.yaml
│   │   │   │   │   ├── belebele_amh.yaml
│   │   │   │   │   ├── belebele_ary.yaml
│   │   │   │   │   ├── belebele_arz.yaml
│   │   │   │   │   ├── belebele_bam.yaml
│   │   │   │   │   ├── belebele_eng.yaml
│   │   │   │   │   ├── belebele_fra.yaml
│   │   │   │   │   ├── belebele_fuv.yaml
│   │   │   │   │   ├── belebele_gaz.yaml
│   │   │   │   │   ├── belebele_hau.yaml
│   │   │   │   │   ├── belebele_ibo.yaml
│   │   │   │   │   ├── belebele_kea.yaml
│   │   │   │   │   ├── belebele_kin.yaml
│   │   │   │   │   ├── belebele_lin.yaml
│   │   │   │   │   ├── belebele_lug.yaml
│   │   │   │   │   ├── belebele_luo.yaml
│   │   │   │   │   ├── belebele_nya.yaml
│   │   │   │   │   ├── belebele_plt.yaml
│   │   │   │   │   ├── belebele_por.yaml
│   │   │   │   │   ├── belebele_sna.yaml
│   │   │   │   │   ├── belebele_som.yaml
│   │   │   │   │   ├── belebele_sot.yaml
│   │   │   │   │   ├── belebele_ssw.yaml
│   │   │   │   │   ├── belebele_swa.yaml
│   │   │   │   │   ├── belebele_tir.yaml
│   │   │   │   │   ├── belebele_tsn.yaml
│   │   │   │   │   ├── belebele_tso.yaml
│   │   │   │   │   ├── belebele_wol.yaml
│   │   │   │   │   ├── belebele_xho.yaml
│   │   │   │   │   ├── belebele_yor.yaml
│   │   │   │   │   └── belebele_zul.yaml
│   │   │   │   └── utils.py
│   │   │   ├── flores/
│   │   │   │   ├── README.md
│   │   │   │   ├── flores.yaml
│   │   │   │   ├── gen_utils.py
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── african-english/
│   │   │   │   │   │   ├── flores
│   │   │   │   │   │   ├── flores_ace_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_ace_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_acq_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_aeb_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_afr_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_aka_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_amh_Ethi-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_ary_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_arz_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_bam_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_ban_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_bem_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_cjk_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_dik_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_dyu_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_ewe_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_fon_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_fra_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_fuv_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_gaz_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_hau_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_ibo_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kab_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kam_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kbp_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kea_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kik_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kin_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kmb_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_knc_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_knc_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kon_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_lin_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_lua_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_lug_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_luo_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_mos_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_nso_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_nus_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_nya_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_plt_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_run_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_sag_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_sna_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_som_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_sot_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_ssw_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_sun_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_swh_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_taq_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_taq_Tfng-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_tir_Ethi-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_tsn_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_tso_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_tum_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_twi_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_tzm_Tfng-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_umb_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_wol_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_xho_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_yor_Latn-eng_Latn.yaml
│   │   │   │   │   │   └── flores_zul_Latn-eng_Latn.yaml
│   │   │   │   │   ├── english-african/
│   │   │   │   │   │   ├── flores
│   │   │   │   │   │   ├── flores_eng_Latn-ace_Arab.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-ace_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-acq_Arab.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-aeb_Arab.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-afr_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-aka_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-amh_Ethi.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-ary_Arab.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-arz_Arab.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-bam_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-ban_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-bem_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-cjk_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-dik_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-dyu_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-ewe_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-fon_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-fra_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-fuv_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-gaz_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-hau_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-ibo_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kab_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kam_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kbp_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kea_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kik_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kin_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kmb_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-knc_Arab.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-knc_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kon_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-lin_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-lua_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-lug_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-luo_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-mos_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-nso_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-nus_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-nya_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-plt_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-run_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-sag_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-sna_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-som_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-sot_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-ssw_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-sun_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-swh_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-taq_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-taq_Tfng.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-tir_Ethi.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-tsn_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-tso_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-tum_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-twi_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-tzm_Tfng.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-umb_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-wol_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-xho_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-yor_Latn.yaml
│   │   │   │   │   │   └── flores_eng_Latn-zul_Latn.yaml
│   │   │   │   │   └── flores
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── african-english/
│   │   │   │   │   │   ├── flores
│   │   │   │   │   │   ├── flores_ace_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_ace_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_acq_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_aeb_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_afr_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_aka_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_amh_Ethi-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_ary_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_arz_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_bam_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_ban_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_bem_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_cjk_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_dik_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_dyu_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_ewe_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_fon_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_fra_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_fuv_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_gaz_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_hau_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_ibo_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kab_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kam_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kbp_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kea_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kik_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kin_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kmb_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_knc_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_knc_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_kon_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_lin_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_lua_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_lug_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_luo_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_mos_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_nso_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_nus_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_nya_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_plt_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_run_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_sag_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_sna_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_som_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_sot_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_ssw_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_sun_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_swh_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_taq_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_taq_Tfng-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_tir_Ethi-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_tsn_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_tso_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_tum_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_twi_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_tzm_Tfng-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_umb_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_wol_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_xho_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── flores_yor_Latn-eng_Latn.yaml
│   │   │   │   │   │   └── flores_zul_Latn-eng_Latn.yaml
│   │   │   │   │   ├── english-african/
│   │   │   │   │   │   ├── flores
│   │   │   │   │   │   ├── flores_eng_Latn-ace_Arab.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-ace_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-acq_Arab.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-aeb_Arab.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-afr_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-aka_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-amh_Ethi.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-ary_Arab.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-arz_Arab.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-bam_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-ban_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-bem_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-cjk_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-dik_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-dyu_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-ewe_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-fon_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-fra_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-fuv_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-gaz_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-hau_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-ibo_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kab_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kam_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kbp_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kea_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kik_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kin_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kmb_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-knc_Arab.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-knc_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-kon_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-lin_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-lua_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-lug_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-luo_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-mos_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-nso_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-nus_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-nya_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-plt_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-run_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-sag_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-sna_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-som_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-sot_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-ssw_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-sun_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-swh_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-taq_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-taq_Tfng.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-tir_Ethi.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-tsn_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-tso_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-tum_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-twi_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-tzm_Tfng.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-umb_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-wol_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-xho_Latn.yaml
│   │   │   │   │   │   ├── flores_eng_Latn-yor_Latn.yaml
│   │   │   │   │   │   └── flores_eng_Latn-zul_Latn.yaml
│   │   │   │   │   └── flores
│   │   │   │   └── prompt_3/
│   │   │   │       ├── african-english/
│   │   │   │       │   ├── flores
│   │   │   │       │   ├── flores_ace_Arab-eng_Latn.yaml
│   │   │   │       │   ├── flores_ace_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_acq_Arab-eng_Latn.yaml
│   │   │   │       │   ├── flores_aeb_Arab-eng_Latn.yaml
│   │   │   │       │   ├── flores_afr_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_aka_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_amh_Ethi-eng_Latn.yaml
│   │   │   │       │   ├── flores_ary_Arab-eng_Latn.yaml
│   │   │   │       │   ├── flores_arz_Arab-eng_Latn.yaml
│   │   │   │       │   ├── flores_bam_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_ban_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_bem_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_cjk_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_dik_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_dyu_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_ewe_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_fon_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_fra_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_fuv_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_gaz_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_hau_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_ibo_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_kab_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_kam_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_kbp_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_kea_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_kik_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_kin_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_kmb_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_knc_Arab-eng_Latn.yaml
│   │   │   │       │   ├── flores_knc_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_kon_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_lin_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_lua_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_lug_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_luo_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_mos_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_nso_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_nus_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_nya_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_plt_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_run_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_sag_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_sna_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_som_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_sot_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_ssw_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_sun_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_swh_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_taq_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_taq_Tfng-eng_Latn.yaml
│   │   │   │       │   ├── flores_tir_Ethi-eng_Latn.yaml
│   │   │   │       │   ├── flores_tsn_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_tso_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_tum_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_twi_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_tzm_Tfng-eng_Latn.yaml
│   │   │   │       │   ├── flores_umb_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_wol_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_xho_Latn-eng_Latn.yaml
│   │   │   │       │   ├── flores_yor_Latn-eng_Latn.yaml
│   │   │   │       │   └── flores_zul_Latn-eng_Latn.yaml
│   │   │   │       ├── english-african/
│   │   │   │       │   ├── flores
│   │   │   │       │   ├── flores_eng_Latn-ace_Arab.yaml
│   │   │   │       │   ├── flores_eng_Latn-ace_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-acq_Arab.yaml
│   │   │   │       │   ├── flores_eng_Latn-aeb_Arab.yaml
│   │   │   │       │   ├── flores_eng_Latn-afr_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-aka_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-amh_Ethi.yaml
│   │   │   │       │   ├── flores_eng_Latn-ary_Arab.yaml
│   │   │   │       │   ├── flores_eng_Latn-arz_Arab.yaml
│   │   │   │       │   ├── flores_eng_Latn-bam_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-ban_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-bem_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-cjk_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-dik_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-dyu_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-ewe_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-fon_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-fra_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-fuv_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-gaz_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-hau_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-ibo_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-kab_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-kam_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-kbp_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-kea_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-kik_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-kin_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-kmb_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-knc_Arab.yaml
│   │   │   │       │   ├── flores_eng_Latn-knc_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-kon_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-lin_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-lua_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-lug_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-luo_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-mos_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-nso_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-nus_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-nya_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-plt_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-run_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-sag_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-sna_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-som_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-sot_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-ssw_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-sun_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-swh_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-taq_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-taq_Tfng.yaml
│   │   │   │       │   ├── flores_eng_Latn-tir_Ethi.yaml
│   │   │   │       │   ├── flores_eng_Latn-tsn_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-tso_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-tum_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-twi_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-tzm_Tfng.yaml
│   │   │   │       │   ├── flores_eng_Latn-umb_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-wol_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-xho_Latn.yaml
│   │   │   │       │   ├── flores_eng_Latn-yor_Latn.yaml
│   │   │   │       │   └── flores_eng_Latn-zul_Latn.yaml
│   │   │   │       └── flores
│   │   │   ├── injongointent/
│   │   │   │   ├── README.md
│   │   │   │   ├── gen_utils.py
│   │   │   │   ├── injongointent.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── injongointent
│   │   │   │   │   ├── injongointent_amh.yaml
│   │   │   │   │   ├── injongointent_eng.yaml
│   │   │   │   │   ├── injongointent_ewe.yaml
│   │   │   │   │   ├── injongointent_hau.yaml
│   │   │   │   │   ├── injongointent_ibo.yaml
│   │   │   │   │   ├── injongointent_kin.yaml
│   │   │   │   │   ├── injongointent_lin.yaml
│   │   │   │   │   ├── injongointent_lug.yaml
│   │   │   │   │   ├── injongointent_orm.yaml
│   │   │   │   │   ├── injongointent_sna.yaml
│   │   │   │   │   ├── injongointent_sot.yaml
│   │   │   │   │   ├── injongointent_swa.yaml
│   │   │   │   │   ├── injongointent_twi.yaml
│   │   │   │   │   ├── injongointent_wol.yaml
│   │   │   │   │   ├── injongointent_xho.yaml
│   │   │   │   │   ├── injongointent_yor.yaml
│   │   │   │   │   ├── injongointent_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── injongointent
│   │   │   │   │   ├── injongointent_amh.yaml
│   │   │   │   │   ├── injongointent_eng.yaml
│   │   │   │   │   ├── injongointent_ewe.yaml
│   │   │   │   │   ├── injongointent_hau.yaml
│   │   │   │   │   ├── injongointent_ibo.yaml
│   │   │   │   │   ├── injongointent_kin.yaml
│   │   │   │   │   ├── injongointent_lin.yaml
│   │   │   │   │   ├── injongointent_lug.yaml
│   │   │   │   │   ├── injongointent_orm.yaml
│   │   │   │   │   ├── injongointent_sna.yaml
│   │   │   │   │   ├── injongointent_sot.yaml
│   │   │   │   │   ├── injongointent_swa.yaml
│   │   │   │   │   ├── injongointent_twi.yaml
│   │   │   │   │   ├── injongointent_wol.yaml
│   │   │   │   │   ├── injongointent_xho.yaml
│   │   │   │   │   ├── injongointent_yor.yaml
│   │   │   │   │   ├── injongointent_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── injongointent
│   │   │   │   │   ├── injongointent_amh.yaml
│   │   │   │   │   ├── injongointent_eng.yaml
│   │   │   │   │   ├── injongointent_ewe.yaml
│   │   │   │   │   ├── injongointent_hau.yaml
│   │   │   │   │   ├── injongointent_ibo.yaml
│   │   │   │   │   ├── injongointent_kin.yaml
│   │   │   │   │   ├── injongointent_lin.yaml
│   │   │   │   │   ├── injongointent_lug.yaml
│   │   │   │   │   ├── injongointent_orm.yaml
│   │   │   │   │   ├── injongointent_sna.yaml
│   │   │   │   │   ├── injongointent_sot.yaml
│   │   │   │   │   ├── injongointent_swa.yaml
│   │   │   │   │   ├── injongointent_twi.yaml
│   │   │   │   │   ├── injongointent_wol.yaml
│   │   │   │   │   ├── injongointent_xho.yaml
│   │   │   │   │   ├── injongointent_yor.yaml
│   │   │   │   │   ├── injongointent_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── injongointent
│   │   │   │   │   ├── injongointent_amh.yaml
│   │   │   │   │   ├── injongointent_eng.yaml
│   │   │   │   │   ├── injongointent_ewe.yaml
│   │   │   │   │   ├── injongointent_hau.yaml
│   │   │   │   │   ├── injongointent_ibo.yaml
│   │   │   │   │   ├── injongointent_kin.yaml
│   │   │   │   │   ├── injongointent_lin.yaml
│   │   │   │   │   ├── injongointent_lug.yaml
│   │   │   │   │   ├── injongointent_orm.yaml
│   │   │   │   │   ├── injongointent_sna.yaml
│   │   │   │   │   ├── injongointent_sot.yaml
│   │   │   │   │   ├── injongointent_swa.yaml
│   │   │   │   │   ├── injongointent_twi.yaml
│   │   │   │   │   ├── injongointent_wol.yaml
│   │   │   │   │   ├── injongointent_xho.yaml
│   │   │   │   │   ├── injongointent_yor.yaml
│   │   │   │   │   ├── injongointent_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   └── prompt_5/
│   │   │   │       ├── injongointent
│   │   │   │       ├── injongointent_amh.yaml
│   │   │   │       ├── injongointent_eng.yaml
│   │   │   │       ├── injongointent_ewe.yaml
│   │   │   │       ├── injongointent_hau.yaml
│   │   │   │       ├── injongointent_ibo.yaml
│   │   │   │       ├── injongointent_kin.yaml
│   │   │   │       ├── injongointent_lin.yaml
│   │   │   │       ├── injongointent_lug.yaml
│   │   │   │       ├── injongointent_orm.yaml
│   │   │   │       ├── injongointent_sna.yaml
│   │   │   │       ├── injongointent_sot.yaml
│   │   │   │       ├── injongointent_swa.yaml
│   │   │   │       ├── injongointent_twi.yaml
│   │   │   │       ├── injongointent_wol.yaml
│   │   │   │       ├── injongointent_xho.yaml
│   │   │   │       ├── injongointent_yor.yaml
│   │   │   │       ├── injongointent_zul.yaml
│   │   │   │       └── utils.py
│   │   │   ├── mafand/
│   │   │   │   ├── README.md
│   │   │   │   ├── gen_utils.py
│   │   │   │   ├── mafand.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── african-english/
│   │   │   │   │   │   ├── mafand
│   │   │   │   │   │   ├── mafand_amh-en.yaml
│   │   │   │   │   │   ├── mafand_bam-fr.yaml
│   │   │   │   │   │   ├── mafand_bbj-fr.yaml
│   │   │   │   │   │   ├── mafand_ewe-fr.yaml
│   │   │   │   │   │   ├── mafand_fon-fr.yaml
│   │   │   │   │   │   ├── mafand_hau-en.yaml
│   │   │   │   │   │   ├── mafand_ibo-en.yaml
│   │   │   │   │   │   ├── mafand_kin-en.yaml
│   │   │   │   │   │   ├── mafand_lug-en.yaml
│   │   │   │   │   │   ├── mafand_luo-en.yaml
│   │   │   │   │   │   ├── mafand_mos-fr.yaml
│   │   │   │   │   │   ├── mafand_nya-en.yaml
│   │   │   │   │   │   ├── mafand_pcm-en.yaml
│   │   │   │   │   │   ├── mafand_sna-en.yaml
│   │   │   │   │   │   ├── mafand_swa-en.yaml
│   │   │   │   │   │   ├── mafand_tsn-en.yaml
│   │   │   │   │   │   ├── mafand_twi-en.yaml
│   │   │   │   │   │   ├── mafand_wol-fr.yaml
│   │   │   │   │   │   ├── mafand_xho-en.yaml
│   │   │   │   │   │   ├── mafand_yor-en.yaml
│   │   │   │   │   │   ├── mafand_zul-en.yaml
│   │   │   │   │   │   └── utils.py
│   │   │   │   │   └── english-african/
│   │   │   │   │       ├── mafand
│   │   │   │   │       ├── mafand_en-amh.yaml
│   │   │   │   │       ├── mafand_en-hau.yaml
│   │   │   │   │       ├── mafand_en-ibo.yaml
│   │   │   │   │       ├── mafand_en-kin.yaml
│   │   │   │   │       ├── mafand_en-lug.yaml
│   │   │   │   │       ├── mafand_en-luo.yaml
│   │   │   │   │       ├── mafand_en-nya.yaml
│   │   │   │   │       ├── mafand_en-pcm.yaml
│   │   │   │   │       ├── mafand_en-sna.yaml
│   │   │   │   │       ├── mafand_en-swa.yaml
│   │   │   │   │       ├── mafand_en-tsn.yaml
│   │   │   │   │       ├── mafand_en-twi.yaml
│   │   │   │   │       ├── mafand_en-xho.yaml
│   │   │   │   │       ├── mafand_en-yor.yaml
│   │   │   │   │       ├── mafand_en-zul.yaml
│   │   │   │   │       ├── mafand_fr-bam.yaml
│   │   │   │   │       ├── mafand_fr-bbj.yaml
│   │   │   │   │       ├── mafand_fr-ewe.yaml
│   │   │   │   │       ├── mafand_fr-fon.yaml
│   │   │   │   │       ├── mafand_fr-mos.yaml
│   │   │   │   │       ├── mafand_fr-wol.yaml
│   │   │   │   │       └── utils.py
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── african-english/
│   │   │   │   │   │   ├── mafand
│   │   │   │   │   │   ├── mafand_amh-en.yaml
│   │   │   │   │   │   ├── mafand_bam-fr.yaml
│   │   │   │   │   │   ├── mafand_bbj-fr.yaml
│   │   │   │   │   │   ├── mafand_ewe-fr.yaml
│   │   │   │   │   │   ├── mafand_fon-fr.yaml
│   │   │   │   │   │   ├── mafand_hau-en.yaml
│   │   │   │   │   │   ├── mafand_ibo-en.yaml
│   │   │   │   │   │   ├── mafand_kin-en.yaml
│   │   │   │   │   │   ├── mafand_lug-en.yaml
│   │   │   │   │   │   ├── mafand_luo-en.yaml
│   │   │   │   │   │   ├── mafand_mos-fr.yaml
│   │   │   │   │   │   ├── mafand_nya-en.yaml
│   │   │   │   │   │   ├── mafand_pcm-en.yaml
│   │   │   │   │   │   ├── mafand_sna-en.yaml
│   │   │   │   │   │   ├── mafand_swa-en.yaml
│   │   │   │   │   │   ├── mafand_tsn-en.yaml
│   │   │   │   │   │   ├── mafand_twi-en.yaml
│   │   │   │   │   │   ├── mafand_wol-fr.yaml
│   │   │   │   │   │   ├── mafand_xho-en.yaml
│   │   │   │   │   │   ├── mafand_yor-en.yaml
│   │   │   │   │   │   ├── mafand_zul-en.yaml
│   │   │   │   │   │   └── utils.py
│   │   │   │   │   └── english-african/
│   │   │   │   │       ├── mafand
│   │   │   │   │       ├── mafand_en-amh.yaml
│   │   │   │   │       ├── mafand_en-hau.yaml
│   │   │   │   │       ├── mafand_en-ibo.yaml
│   │   │   │   │       ├── mafand_en-kin.yaml
│   │   │   │   │       ├── mafand_en-lug.yaml
│   │   │   │   │       ├── mafand_en-luo.yaml
│   │   │   │   │       ├── mafand_en-nya.yaml
│   │   │   │   │       ├── mafand_en-pcm.yaml
│   │   │   │   │       ├── mafand_en-sna.yaml
│   │   │   │   │       ├── mafand_en-swa.yaml
│   │   │   │   │       ├── mafand_en-tsn.yaml
│   │   │   │   │       ├── mafand_en-twi.yaml
│   │   │   │   │       ├── mafand_en-xho.yaml
│   │   │   │   │       ├── mafand_en-yor.yaml
│   │   │   │   │       ├── mafand_en-zul.yaml
│   │   │   │   │       ├── mafand_fr-bam.yaml
│   │   │   │   │       ├── mafand_fr-bbj.yaml
│   │   │   │   │       ├── mafand_fr-ewe.yaml
│   │   │   │   │       ├── mafand_fr-fon.yaml
│   │   │   │   │       ├── mafand_fr-mos.yaml
│   │   │   │   │       ├── mafand_fr-wol.yaml
│   │   │   │   │       └── utils.py
│   │   │   │   └── prompt_3/
│   │   │   │       ├── african-english/
│   │   │   │       │   ├── mafand
│   │   │   │       │   ├── mafand_amh-en.yaml
│   │   │   │       │   ├── mafand_bam-fr.yaml
│   │   │   │       │   ├── mafand_bbj-fr.yaml
│   │   │   │       │   ├── mafand_ewe-fr.yaml
│   │   │   │       │   ├── mafand_fon-fr.yaml
│   │   │   │       │   ├── mafand_hau-en.yaml
│   │   │   │       │   ├── mafand_ibo-en.yaml
│   │   │   │       │   ├── mafand_kin-en.yaml
│   │   │   │       │   ├── mafand_lug-en.yaml
│   │   │   │       │   ├── mafand_luo-en.yaml
│   │   │   │       │   ├── mafand_mos-fr.yaml
│   │   │   │       │   ├── mafand_nya-en.yaml
│   │   │   │       │   ├── mafand_pcm-en.yaml
│   │   │   │       │   ├── mafand_sna-en.yaml
│   │   │   │       │   ├── mafand_swa-en.yaml
│   │   │   │       │   ├── mafand_tsn-en.yaml
│   │   │   │       │   ├── mafand_twi-en.yaml
│   │   │   │       │   ├── mafand_wol-fr.yaml
│   │   │   │       │   ├── mafand_xho-en.yaml
│   │   │   │       │   ├── mafand_yor-en.yaml
│   │   │   │       │   ├── mafand_zul-en.yaml
│   │   │   │       │   └── utils.py
│   │   │   │       └── english-african/
│   │   │   │           ├── mafand
│   │   │   │           ├── mafand_en-amh.yaml
│   │   │   │           ├── mafand_en-hau.yaml
│   │   │   │           ├── mafand_en-ibo.yaml
│   │   │   │           ├── mafand_en-kin.yaml
│   │   │   │           ├── mafand_en-lug.yaml
│   │   │   │           ├── mafand_en-luo.yaml
│   │   │   │           ├── mafand_en-nya.yaml
│   │   │   │           ├── mafand_en-pcm.yaml
│   │   │   │           ├── mafand_en-sna.yaml
│   │   │   │           ├── mafand_en-swa.yaml
│   │   │   │           ├── mafand_en-tsn.yaml
│   │   │   │           ├── mafand_en-twi.yaml
│   │   │   │           ├── mafand_en-xho.yaml
│   │   │   │           ├── mafand_en-yor.yaml
│   │   │   │           ├── mafand_en-zul.yaml
│   │   │   │           ├── mafand_fr-bam.yaml
│   │   │   │           ├── mafand_fr-bbj.yaml
│   │   │   │           ├── mafand_fr-ewe.yaml
│   │   │   │           ├── mafand_fr-fon.yaml
│   │   │   │           ├── mafand_fr-mos.yaml
│   │   │   │           ├── mafand_fr-wol.yaml
│   │   │   │           └── utils.py
│   │   │   ├── masakhaner/
│   │   │   │   ├── README.md
│   │   │   │   ├── gen_utils.py
│   │   │   │   ├── masakhaner.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── masakhaner
│   │   │   │   │   ├── masakhaner_am.yaml
│   │   │   │   │   ├── masakhaner_bbj.yaml
│   │   │   │   │   ├── masakhaner_bm.yaml
│   │   │   │   │   ├── masakhaner_ee.yaml
│   │   │   │   │   ├── masakhaner_ha.yaml
│   │   │   │   │   ├── masakhaner_ig.yaml
│   │   │   │   │   ├── masakhaner_lg.yaml
│   │   │   │   │   ├── masakhaner_luo.yaml
│   │   │   │   │   ├── masakhaner_mos.yaml
│   │   │   │   │   ├── masakhaner_ny.yaml
│   │   │   │   │   ├── masakhaner_pcm.yaml
│   │   │   │   │   ├── masakhaner_rw.yaml
│   │   │   │   │   ├── masakhaner_sn.yaml
│   │   │   │   │   ├── masakhaner_sw.yaml
│   │   │   │   │   ├── masakhaner_tn.yaml
│   │   │   │   │   ├── masakhaner_tw.yaml
│   │   │   │   │   ├── masakhaner_wo.yaml
│   │   │   │   │   ├── masakhaner_xh.yaml
│   │   │   │   │   ├── masakhaner_yo.yaml
│   │   │   │   │   ├── masakhaner_zu.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── masakhaner
│   │   │   │   │   ├── masakhaner_am.yaml
│   │   │   │   │   ├── masakhaner_bbj.yaml
│   │   │   │   │   ├── masakhaner_bm.yaml
│   │   │   │   │   ├── masakhaner_ee.yaml
│   │   │   │   │   ├── masakhaner_ha.yaml
│   │   │   │   │   ├── masakhaner_ig.yaml
│   │   │   │   │   ├── masakhaner_lg.yaml
│   │   │   │   │   ├── masakhaner_luo.yaml
│   │   │   │   │   ├── masakhaner_mos.yaml
│   │   │   │   │   ├── masakhaner_ny.yaml
│   │   │   │   │   ├── masakhaner_pcm.yaml
│   │   │   │   │   ├── masakhaner_rw.yaml
│   │   │   │   │   ├── masakhaner_sn.yaml
│   │   │   │   │   ├── masakhaner_sw.yaml
│   │   │   │   │   ├── masakhaner_tn.yaml
│   │   │   │   │   ├── masakhaner_tw.yaml
│   │   │   │   │   ├── masakhaner_wo.yaml
│   │   │   │   │   ├── masakhaner_xh.yaml
│   │   │   │   │   ├── masakhaner_yo.yaml
│   │   │   │   │   ├── masakhaner_zu.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── masakhaner
│   │   │   │   │   ├── masakhaner_am.yaml
│   │   │   │   │   ├── masakhaner_bbj.yaml
│   │   │   │   │   ├── masakhaner_bm.yaml
│   │   │   │   │   ├── masakhaner_ee.yaml
│   │   │   │   │   ├── masakhaner_ha.yaml
│   │   │   │   │   ├── masakhaner_ig.yaml
│   │   │   │   │   ├── masakhaner_lg.yaml
│   │   │   │   │   ├── masakhaner_luo.yaml
│   │   │   │   │   ├── masakhaner_mos.yaml
│   │   │   │   │   ├── masakhaner_ny.yaml
│   │   │   │   │   ├── masakhaner_pcm.yaml
│   │   │   │   │   ├── masakhaner_rw.yaml
│   │   │   │   │   ├── masakhaner_sn.yaml
│   │   │   │   │   ├── masakhaner_sw.yaml
│   │   │   │   │   ├── masakhaner_tn.yaml
│   │   │   │   │   ├── masakhaner_tw.yaml
│   │   │   │   │   ├── masakhaner_wo.yaml
│   │   │   │   │   ├── masakhaner_xh.yaml
│   │   │   │   │   ├── masakhaner_yo.yaml
│   │   │   │   │   ├── masakhaner_zu.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── masakhaner
│   │   │   │   │   ├── masakhaner_am.yaml
│   │   │   │   │   ├── masakhaner_bbj.yaml
│   │   │   │   │   ├── masakhaner_bm.yaml
│   │   │   │   │   ├── masakhaner_ee.yaml
│   │   │   │   │   ├── masakhaner_ha.yaml
│   │   │   │   │   ├── masakhaner_ig.yaml
│   │   │   │   │   ├── masakhaner_lg.yaml
│   │   │   │   │   ├── masakhaner_luo.yaml
│   │   │   │   │   ├── masakhaner_mos.yaml
│   │   │   │   │   ├── masakhaner_ny.yaml
│   │   │   │   │   ├── masakhaner_pcm.yaml
│   │   │   │   │   ├── masakhaner_rw.yaml
│   │   │   │   │   ├── masakhaner_sn.yaml
│   │   │   │   │   ├── masakhaner_sw.yaml
│   │   │   │   │   ├── masakhaner_tn.yaml
│   │   │   │   │   ├── masakhaner_tw.yaml
│   │   │   │   │   ├── masakhaner_wo.yaml
│   │   │   │   │   ├── masakhaner_xh.yaml
│   │   │   │   │   ├── masakhaner_yo.yaml
│   │   │   │   │   ├── masakhaner_zu.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   └── prompt_5/
│   │   │   │       ├── masakhaner
│   │   │   │       ├── masakhaner_am.yaml
│   │   │   │       ├── masakhaner_bbj.yaml
│   │   │   │       ├── masakhaner_bm.yaml
│   │   │   │       ├── masakhaner_ee.yaml
│   │   │   │       ├── masakhaner_ha.yaml
│   │   │   │       ├── masakhaner_ig.yaml
│   │   │   │       ├── masakhaner_lg.yaml
│   │   │   │       ├── masakhaner_luo.yaml
│   │   │   │       ├── masakhaner_mos.yaml
│   │   │   │       ├── masakhaner_ny.yaml
│   │   │   │       ├── masakhaner_pcm.yaml
│   │   │   │       ├── masakhaner_rw.yaml
│   │   │   │       ├── masakhaner_sn.yaml
│   │   │   │       ├── masakhaner_sw.yaml
│   │   │   │       ├── masakhaner_tn.yaml
│   │   │   │       ├── masakhaner_tw.yaml
│   │   │   │       ├── masakhaner_wo.yaml
│   │   │   │       ├── masakhaner_xh.yaml
│   │   │   │       ├── masakhaner_yo.yaml
│   │   │   │       ├── masakhaner_zu.yaml
│   │   │   │       └── utils.py
│   │   │   ├── masakhanews/
│   │   │   │   ├── README.md
│   │   │   │   ├── masakhanews.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── masakhanews
│   │   │   │   │   ├── masakhanews_amh.yaml
│   │   │   │   │   ├── masakhanews_eng.yaml
│   │   │   │   │   ├── masakhanews_fra.yaml
│   │   │   │   │   ├── masakhanews_hau.yaml
│   │   │   │   │   ├── masakhanews_ibo.yaml
│   │   │   │   │   ├── masakhanews_lin.yaml
│   │   │   │   │   ├── masakhanews_lug.yaml
│   │   │   │   │   ├── masakhanews_orm.yaml
│   │   │   │   │   ├── masakhanews_pcm.yaml
│   │   │   │   │   ├── masakhanews_run.yaml
│   │   │   │   │   ├── masakhanews_sna.yaml
│   │   │   │   │   ├── masakhanews_som.yaml
│   │   │   │   │   ├── masakhanews_swa.yaml
│   │   │   │   │   ├── masakhanews_tir.yaml
│   │   │   │   │   ├── masakhanews_xho.yaml
│   │   │   │   │   ├── masakhanews_yor.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── masakhanews
│   │   │   │   │   ├── masakhanews_amh.yaml
│   │   │   │   │   ├── masakhanews_eng.yaml
│   │   │   │   │   ├── masakhanews_fra.yaml
│   │   │   │   │   ├── masakhanews_hau.yaml
│   │   │   │   │   ├── masakhanews_ibo.yaml
│   │   │   │   │   ├── masakhanews_lin.yaml
│   │   │   │   │   ├── masakhanews_lug.yaml
│   │   │   │   │   ├── masakhanews_orm.yaml
│   │   │   │   │   ├── masakhanews_pcm.yaml
│   │   │   │   │   ├── masakhanews_run.yaml
│   │   │   │   │   ├── masakhanews_sna.yaml
│   │   │   │   │   ├── masakhanews_som.yaml
│   │   │   │   │   ├── masakhanews_swa.yaml
│   │   │   │   │   ├── masakhanews_tir.yaml
│   │   │   │   │   ├── masakhanews_xho.yaml
│   │   │   │   │   ├── masakhanews_yor.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── masakhanews
│   │   │   │   │   ├── masakhanews_amh.yaml
│   │   │   │   │   ├── masakhanews_eng.yaml
│   │   │   │   │   ├── masakhanews_fra.yaml
│   │   │   │   │   ├── masakhanews_hau.yaml
│   │   │   │   │   ├── masakhanews_ibo.yaml
│   │   │   │   │   ├── masakhanews_lin.yaml
│   │   │   │   │   ├── masakhanews_lug.yaml
│   │   │   │   │   ├── masakhanews_orm.yaml
│   │   │   │   │   ├── masakhanews_pcm.yaml
│   │   │   │   │   ├── masakhanews_run.yaml
│   │   │   │   │   ├── masakhanews_sna.yaml
│   │   │   │   │   ├── masakhanews_som.yaml
│   │   │   │   │   ├── masakhanews_swa.yaml
│   │   │   │   │   ├── masakhanews_tir.yaml
│   │   │   │   │   ├── masakhanews_xho.yaml
│   │   │   │   │   ├── masakhanews_yor.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── masakhanews
│   │   │   │   │   ├── masakhanews_amh.yaml
│   │   │   │   │   ├── masakhanews_eng.yaml
│   │   │   │   │   ├── masakhanews_fra.yaml
│   │   │   │   │   ├── masakhanews_hau.yaml
│   │   │   │   │   ├── masakhanews_ibo.yaml
│   │   │   │   │   ├── masakhanews_lin.yaml
│   │   │   │   │   ├── masakhanews_lug.yaml
│   │   │   │   │   ├── masakhanews_orm.yaml
│   │   │   │   │   ├── masakhanews_pcm.yaml
│   │   │   │   │   ├── masakhanews_run.yaml
│   │   │   │   │   ├── masakhanews_sna.yaml
│   │   │   │   │   ├── masakhanews_som.yaml
│   │   │   │   │   ├── masakhanews_swa.yaml
│   │   │   │   │   ├── masakhanews_tir.yaml
│   │   │   │   │   ├── masakhanews_xho.yaml
│   │   │   │   │   ├── masakhanews_yor.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_5/
│   │   │   │   │   ├── masakhanews
│   │   │   │   │   ├── masakhanews_amh.yaml
│   │   │   │   │   ├── masakhanews_eng.yaml
│   │   │   │   │   ├── masakhanews_fra.yaml
│   │   │   │   │   ├── masakhanews_hau.yaml
│   │   │   │   │   ├── masakhanews_ibo.yaml
│   │   │   │   │   ├── masakhanews_lin.yaml
│   │   │   │   │   ├── masakhanews_lug.yaml
│   │   │   │   │   ├── masakhanews_orm.yaml
│   │   │   │   │   ├── masakhanews_pcm.yaml
│   │   │   │   │   ├── masakhanews_run.yaml
│   │   │   │   │   ├── masakhanews_sna.yaml
│   │   │   │   │   ├── masakhanews_som.yaml
│   │   │   │   │   ├── masakhanews_swa.yaml
│   │   │   │   │   ├── masakhanews_tir.yaml
│   │   │   │   │   ├── masakhanews_xho.yaml
│   │   │   │   │   ├── masakhanews_yor.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   └── utils.py
│   │   │   ├── masakhapos/
│   │   │   │   ├── README.md
│   │   │   │   ├── gen_utils.py
│   │   │   │   ├── masakhapos.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── masakhapos_bam.yaml
│   │   │   │   │   ├── masakhapos_bbj.yaml
│   │   │   │   │   ├── masakhapos_ewe.yaml
│   │   │   │   │   ├── masakhapos_fon.yaml
│   │   │   │   │   ├── masakhapos_hau.yaml
│   │   │   │   │   ├── masakhapos_ibo.yaml
│   │   │   │   │   ├── masakhapos_kin.yaml
│   │   │   │   │   ├── masakhapos_lug.yaml
│   │   │   │   │   ├── masakhapos_luo.yaml
│   │   │   │   │   ├── masakhapos_mos.yaml
│   │   │   │   │   ├── masakhapos_nya.yaml
│   │   │   │   │   ├── masakhapos_pcm.yaml
│   │   │   │   │   ├── masakhapos_sna.yaml
│   │   │   │   │   ├── masakhapos_swa.yaml
│   │   │   │   │   ├── masakhapos_tsn.yaml
│   │   │   │   │   ├── masakhapos_twi.yaml
│   │   │   │   │   ├── masakhapos_wol.yaml
│   │   │   │   │   ├── masakhapos_xho.yaml
│   │   │   │   │   ├── masakhapos_yaml
│   │   │   │   │   ├── masakhapos_yor.yaml
│   │   │   │   │   ├── masakhapos_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── masakhapos_bam.yaml
│   │   │   │   │   ├── masakhapos_bbj.yaml
│   │   │   │   │   ├── masakhapos_ewe.yaml
│   │   │   │   │   ├── masakhapos_fon.yaml
│   │   │   │   │   ├── masakhapos_hau.yaml
│   │   │   │   │   ├── masakhapos_ibo.yaml
│   │   │   │   │   ├── masakhapos_kin.yaml
│   │   │   │   │   ├── masakhapos_lug.yaml
│   │   │   │   │   ├── masakhapos_luo.yaml
│   │   │   │   │   ├── masakhapos_mos.yaml
│   │   │   │   │   ├── masakhapos_nya.yaml
│   │   │   │   │   ├── masakhapos_pcm.yaml
│   │   │   │   │   ├── masakhapos_sna.yaml
│   │   │   │   │   ├── masakhapos_swa.yaml
│   │   │   │   │   ├── masakhapos_tsn.yaml
│   │   │   │   │   ├── masakhapos_twi.yaml
│   │   │   │   │   ├── masakhapos_wol.yaml
│   │   │   │   │   ├── masakhapos_xho.yaml
│   │   │   │   │   ├── masakhapos_yaml
│   │   │   │   │   ├── masakhapos_yor.yaml
│   │   │   │   │   ├── masakhapos_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── masakhapos_bam.yaml
│   │   │   │   │   ├── masakhapos_bbj.yaml
│   │   │   │   │   ├── masakhapos_ewe.yaml
│   │   │   │   │   ├── masakhapos_fon.yaml
│   │   │   │   │   ├── masakhapos_hau.yaml
│   │   │   │   │   ├── masakhapos_ibo.yaml
│   │   │   │   │   ├── masakhapos_kin.yaml
│   │   │   │   │   ├── masakhapos_lug.yaml
│   │   │   │   │   ├── masakhapos_luo.yaml
│   │   │   │   │   ├── masakhapos_mos.yaml
│   │   │   │   │   ├── masakhapos_nya.yaml
│   │   │   │   │   ├── masakhapos_pcm.yaml
│   │   │   │   │   ├── masakhapos_sna.yaml
│   │   │   │   │   ├── masakhapos_swa.yaml
│   │   │   │   │   ├── masakhapos_tsn.yaml
│   │   │   │   │   ├── masakhapos_twi.yaml
│   │   │   │   │   ├── masakhapos_wol.yaml
│   │   │   │   │   ├── masakhapos_xho.yaml
│   │   │   │   │   ├── masakhapos_yaml
│   │   │   │   │   ├── masakhapos_yor.yaml
│   │   │   │   │   ├── masakhapos_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── masakhapos_bam.yaml
│   │   │   │   │   ├── masakhapos_bbj.yaml
│   │   │   │   │   ├── masakhapos_ewe.yaml
│   │   │   │   │   ├── masakhapos_fon.yaml
│   │   │   │   │   ├── masakhapos_hau.yaml
│   │   │   │   │   ├── masakhapos_ibo.yaml
│   │   │   │   │   ├── masakhapos_kin.yaml
│   │   │   │   │   ├── masakhapos_lug.yaml
│   │   │   │   │   ├── masakhapos_luo.yaml
│   │   │   │   │   ├── masakhapos_mos.yaml
│   │   │   │   │   ├── masakhapos_nya.yaml
│   │   │   │   │   ├── masakhapos_pcm.yaml
│   │   │   │   │   ├── masakhapos_sna.yaml
│   │   │   │   │   ├── masakhapos_swa.yaml
│   │   │   │   │   ├── masakhapos_tsn.yaml
│   │   │   │   │   ├── masakhapos_twi.yaml
│   │   │   │   │   ├── masakhapos_wol.yaml
│   │   │   │   │   ├── masakhapos_xho.yaml
│   │   │   │   │   ├── masakhapos_yaml
│   │   │   │   │   ├── masakhapos_yor.yaml
│   │   │   │   │   ├── masakhapos_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_5/
│   │   │   │   │   ├── masakhapos_bam.yaml
│   │   │   │   │   ├── masakhapos_bbj.yaml
│   │   │   │   │   ├── masakhapos_ewe.yaml
│   │   │   │   │   ├── masakhapos_fon.yaml
│   │   │   │   │   ├── masakhapos_hau.yaml
│   │   │   │   │   ├── masakhapos_ibo.yaml
│   │   │   │   │   ├── masakhapos_kin.yaml
│   │   │   │   │   ├── masakhapos_lug.yaml
│   │   │   │   │   ├── masakhapos_luo.yaml
│   │   │   │   │   ├── masakhapos_mos.yaml
│   │   │   │   │   ├── masakhapos_nya.yaml
│   │   │   │   │   ├── masakhapos_pcm.yaml
│   │   │   │   │   ├── masakhapos_sna.yaml
│   │   │   │   │   ├── masakhapos_swa.yaml
│   │   │   │   │   ├── masakhapos_tsn.yaml
│   │   │   │   │   ├── masakhapos_twi.yaml
│   │   │   │   │   ├── masakhapos_wol.yaml
│   │   │   │   │   ├── masakhapos_xho.yaml
│   │   │   │   │   ├── masakhapos_yaml
│   │   │   │   │   ├── masakhapos_yor.yaml
│   │   │   │   │   ├── masakhapos_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   └── utils.py
│   │   │   ├── naijarc/
│   │   │   │   ├── README.md
│   │   │   │   ├── naijarc.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── naijarc
│   │   │   │   │   ├── naijarc_hau.yaml
│   │   │   │   │   ├── naijarc_ibo.yaml
│   │   │   │   │   └── naijarc_yor.yaml
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── naijarc
│   │   │   │   │   ├── naijarc_hau.yaml
│   │   │   │   │   ├── naijarc_ibo.yaml
│   │   │   │   │   └── naijarc_yor.yaml
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── naijarc
│   │   │   │   │   ├── naijarc_hau.yaml
│   │   │   │   │   ├── naijarc_ibo.yaml
│   │   │   │   │   └── naijarc_yor.yaml
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── naijarc
│   │   │   │   │   ├── naijarc_hau.yaml
│   │   │   │   │   ├── naijarc_ibo.yaml
│   │   │   │   │   └── naijarc_yor.yaml
│   │   │   │   ├── prompt_5/
│   │   │   │   │   ├── naijarc
│   │   │   │   │   ├── naijarc_hau.yaml
│   │   │   │   │   ├── naijarc_ibo.yaml
│   │   │   │   │   └── naijarc_yor.yaml
│   │   │   │   └── utils.py
│   │   │   ├── nollysenti/
│   │   │   │   ├── README.md
│   │   │   │   ├── nollysenti.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── nollysenti
│   │   │   │   │   ├── nollysenti_eng.yaml
│   │   │   │   │   ├── nollysenti_hau.yaml
│   │   │   │   │   ├── nollysenti_ibo.yaml
│   │   │   │   │   ├── nollysenti_pcm.yaml
│   │   │   │   │   ├── nollysenti_yor.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── nollysenti
│   │   │   │   │   ├── nollysenti_eng.yaml
│   │   │   │   │   ├── nollysenti_hau.yaml
│   │   │   │   │   ├── nollysenti_ibo.yaml
│   │   │   │   │   ├── nollysenti_pcm.yaml
│   │   │   │   │   ├── nollysenti_yor.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── nollysenti
│   │   │   │   │   ├── nollysenti_eng.yaml
│   │   │   │   │   ├── nollysenti_hau.yaml
│   │   │   │   │   ├── nollysenti_ibo.yaml
│   │   │   │   │   ├── nollysenti_pcm.yaml
│   │   │   │   │   ├── nollysenti_yor.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── nollysenti
│   │   │   │   │   ├── nollysenti_eng.yaml
│   │   │   │   │   ├── nollysenti_hau.yaml
│   │   │   │   │   ├── nollysenti_ibo.yaml
│   │   │   │   │   ├── nollysenti_pcm.yaml
│   │   │   │   │   ├── nollysenti_yor.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   └── prompt_5/
│   │   │   │       ├── nollysenti
│   │   │   │       ├── nollysenti_eng.yaml
│   │   │   │       ├── nollysenti_hau.yaml
│   │   │   │       ├── nollysenti_ibo.yaml
│   │   │   │       ├── nollysenti_pcm.yaml
│   │   │   │       ├── nollysenti_yor.yaml
│   │   │   │       └── utils.py
│   │   │   ├── ntrex/
│   │   │   │   ├── README.md
│   │   │   │   ├── gen_utils.py
│   │   │   │   ├── ntrex.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── african-english/
│   │   │   │   │   │   ├── ntrex
│   │   │   │   │   │   ├── ntrex_afr_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_amh_Ethi-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_arb_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_bem_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_ewe_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_fra_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_hau_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_ibo_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_kin_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_mey_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_mlg_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_msa_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_nde_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_nso_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_nya_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_orm_Ethi-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_shi_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_sna_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_som_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_ssw_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_swa_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_tam_Taml-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_tel_Telu-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_tir_Ethi-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_ton_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_tsn_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_urd_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_ven_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_wol_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_xho_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_yor_Latn-eng_Latn.yaml
│   │   │   │   │   │   └── ntrex_zul_Latn-eng_Latn.yaml
│   │   │   │   │   └── english-african/
│   │   │   │   │       ├── ntrex
│   │   │   │   │       ├── ntrex_eng_Latn-afr_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-amh_Ethi.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-arb_Arab.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-bem_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-ewe_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-fra_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-hau_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-ibo_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-kin_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-mey_Arab.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-mlg_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-msa_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-nde_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-nso_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-nya_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-orm_Ethi.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-shi_Arab.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-sna_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-som_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-ssw_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-swa_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-tam_Taml.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-tel_Telu.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-tir_Ethi.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-ton_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-tsn_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-urd_Arab.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-ven_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-wol_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-xho_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-yor_Latn.yaml
│   │   │   │   │       └── ntrex_eng_Latn-zul_Latn.yaml
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── african-english/
│   │   │   │   │   │   ├── ntrex
│   │   │   │   │   │   ├── ntrex_afr_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_amh_Ethi-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_arb_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_bem_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_ewe_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_fra_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_hau_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_ibo_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_kin_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_mey_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_mlg_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_msa_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_nde_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_nso_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_nya_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_orm_Ethi-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_shi_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_sna_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_som_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_ssw_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_swa_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_tam_Taml-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_tel_Telu-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_tir_Ethi-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_ton_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_tsn_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_urd_Arab-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_ven_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_wol_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_xho_Latn-eng_Latn.yaml
│   │   │   │   │   │   ├── ntrex_yor_Latn-eng_Latn.yaml
│   │   │   │   │   │   └── ntrex_zul_Latn-eng_Latn.yaml
│   │   │   │   │   └── english-african/
│   │   │   │   │       ├── ntrex
│   │   │   │   │       ├── ntrex_eng_Latn-afr_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-amh_Ethi.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-arb_Arab.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-bem_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-ewe_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-fra_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-hau_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-ibo_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-kin_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-mey_Arab.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-mlg_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-msa_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-nde_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-nso_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-nya_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-orm_Ethi.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-shi_Arab.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-sna_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-som_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-ssw_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-swa_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-tam_Taml.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-tel_Telu.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-tir_Ethi.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-ton_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-tsn_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-urd_Arab.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-ven_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-wol_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-xho_Latn.yaml
│   │   │   │   │       ├── ntrex_eng_Latn-yor_Latn.yaml
│   │   │   │   │       └── ntrex_eng_Latn-zul_Latn.yaml
│   │   │   │   └── prompt_3/
│   │   │   │       ├── african-english/
│   │   │   │       │   ├── ntrex
│   │   │   │       │   ├── ntrex_afr_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_amh_Ethi-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_arb_Arab-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_bem_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_ewe_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_fra_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_hau_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_ibo_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_kin_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_mey_Arab-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_mlg_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_msa_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_nde_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_nso_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_nya_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_orm_Ethi-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_shi_Arab-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_sna_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_som_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_ssw_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_swa_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_tam_Taml-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_tel_Telu-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_tir_Ethi-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_ton_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_tsn_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_urd_Arab-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_ven_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_wol_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_xho_Latn-eng_Latn.yaml
│   │   │   │       │   ├── ntrex_yor_Latn-eng_Latn.yaml
│   │   │   │       │   └── ntrex_zul_Latn-eng_Latn.yaml
│   │   │   │       └── english-african/
│   │   │   │           ├── ntrex
│   │   │   │           ├── ntrex_eng_Latn-afr_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-amh_Ethi.yaml
│   │   │   │           ├── ntrex_eng_Latn-arb_Arab.yaml
│   │   │   │           ├── ntrex_eng_Latn-bem_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-ewe_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-fra_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-hau_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-ibo_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-kin_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-mey_Arab.yaml
│   │   │   │           ├── ntrex_eng_Latn-mlg_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-msa_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-nde_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-nso_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-nya_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-orm_Ethi.yaml
│   │   │   │           ├── ntrex_eng_Latn-shi_Arab.yaml
│   │   │   │           ├── ntrex_eng_Latn-sna_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-som_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-ssw_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-swa_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-tam_Taml.yaml
│   │   │   │           ├── ntrex_eng_Latn-tel_Telu.yaml
│   │   │   │           ├── ntrex_eng_Latn-tir_Ethi.yaml
│   │   │   │           ├── ntrex_eng_Latn-ton_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-tsn_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-urd_Arab.yaml
│   │   │   │           ├── ntrex_eng_Latn-ven_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-wol_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-xho_Latn.yaml
│   │   │   │           ├── ntrex_eng_Latn-yor_Latn.yaml
│   │   │   │           └── ntrex_eng_Latn-zul_Latn.yaml
│   │   │   ├── openai_mmlu/
│   │   │   │   ├── README.md
│   │   │   │   ├── openai_mmlu.yaml
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── openai_mmlu
│   │   │   │   │   ├── openai_mmlu_ara.yaml
│   │   │   │   │   ├── openai_mmlu_swa.yaml
│   │   │   │   │   └── openai_mmlu_yor.yaml
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── openai_mmlu
│   │   │   │   │   ├── openai_mmlu_ara.yaml
│   │   │   │   │   ├── openai_mmlu_swa.yaml
│   │   │   │   │   └── openai_mmlu_yor.yaml
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── openai_mmlu
│   │   │   │   │   ├── openai_mmlu_ara.yaml
│   │   │   │   │   ├── openai_mmlu_swa.yaml
│   │   │   │   │   └── openai_mmlu_yor.yaml
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── openai_mmlu
│   │   │   │   │   ├── openai_mmlu_ara.yaml
│   │   │   │   │   ├── openai_mmlu_swa.yaml
│   │   │   │   │   └── openai_mmlu_yor.yaml
│   │   │   │   ├── prompt_5/
│   │   │   │   │   ├── openai_mmlu
│   │   │   │   │   ├── openai_mmlu_ara.yaml
│   │   │   │   │   ├── openai_mmlu_swa.yaml
│   │   │   │   │   └── openai_mmlu_yor.yaml
│   │   │   │   └── utils.py
│   │   │   ├── salt/
│   │   │   │   ├── README.md
│   │   │   │   ├── gen_utils.py
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── salt
│   │   │   │   │   ├── salt_ach-eng.yaml
│   │   │   │   │   ├── salt_eng-ach.yaml
│   │   │   │   │   ├── salt_eng-ibo.yaml
│   │   │   │   │   ├── salt_eng-lgg.yaml
│   │   │   │   │   ├── salt_eng-lug.yaml
│   │   │   │   │   ├── salt_eng-nyn.yaml
│   │   │   │   │   ├── salt_eng-swa.yaml
│   │   │   │   │   ├── salt_eng-teo.yaml
│   │   │   │   │   ├── salt_ibo-eng.yaml
│   │   │   │   │   ├── salt_lgg-eng.yaml
│   │   │   │   │   ├── salt_lug-eng.yaml
│   │   │   │   │   ├── salt_nyn-eng.yaml
│   │   │   │   │   ├── salt_swa-eng.yaml
│   │   │   │   │   └── salt_teo-eng.yaml
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── salt
│   │   │   │   │   ├── salt_ach-eng.yaml
│   │   │   │   │   ├── salt_eng-ach.yaml
│   │   │   │   │   ├── salt_eng-ibo.yaml
│   │   │   │   │   ├── salt_eng-lgg.yaml
│   │   │   │   │   ├── salt_eng-lug.yaml
│   │   │   │   │   ├── salt_eng-nyn.yaml
│   │   │   │   │   ├── salt_eng-swa.yaml
│   │   │   │   │   ├── salt_eng-teo.yaml
│   │   │   │   │   ├── salt_ibo-eng.yaml
│   │   │   │   │   ├── salt_lgg-eng.yaml
│   │   │   │   │   ├── salt_lug-eng.yaml
│   │   │   │   │   ├── salt_nyn-eng.yaml
│   │   │   │   │   ├── salt_swa-eng.yaml
│   │   │   │   │   └── salt_teo-eng.yaml
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── salt
│   │   │   │   │   ├── salt_ach-eng.yaml
│   │   │   │   │   ├── salt_eng-ach.yaml
│   │   │   │   │   ├── salt_eng-ibo.yaml
│   │   │   │   │   ├── salt_eng-lgg.yaml
│   │   │   │   │   ├── salt_eng-lug.yaml
│   │   │   │   │   ├── salt_eng-nyn.yaml
│   │   │   │   │   ├── salt_eng-swa.yaml
│   │   │   │   │   ├── salt_eng-teo.yaml
│   │   │   │   │   ├── salt_ibo-eng.yaml
│   │   │   │   │   ├── salt_lgg-eng.yaml
│   │   │   │   │   ├── salt_lug-eng.yaml
│   │   │   │   │   ├── salt_nyn-eng.yaml
│   │   │   │   │   ├── salt_swa-eng.yaml
│   │   │   │   │   └── salt_teo-eng.yaml
│   │   │   │   └── salt.yaml
│   │   │   ├── sample_run_scripts/
│   │   │   │   ├── run_afrobench.sh
│   │   │   │   └── run_afrobench_lite.sh
│   │   │   ├── sib/
│   │   │   │   ├── README.md
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── sib
│   │   │   │   │   ├── sib_aeb.yaml
│   │   │   │   │   ├── sib_afr.yaml
│   │   │   │   │   ├── sib_aka.yaml
│   │   │   │   │   ├── sib_amh.yaml
│   │   │   │   │   ├── sib_ary.yaml
│   │   │   │   │   ├── sib_arz.yaml
│   │   │   │   │   ├── sib_bam.yaml
│   │   │   │   │   ├── sib_bem.yaml
│   │   │   │   │   ├── sib_cjk.yaml
│   │   │   │   │   ├── sib_dik.yaml
│   │   │   │   │   ├── sib_dyu.yaml
│   │   │   │   │   ├── sib_eng.yaml
│   │   │   │   │   ├── sib_ewe.yaml
│   │   │   │   │   ├── sib_fon.yaml
│   │   │   │   │   ├── sib_fra.yaml
│   │   │   │   │   ├── sib_fuv.yaml
│   │   │   │   │   ├── sib_gaz.yaml
│   │   │   │   │   ├── sib_hau.yaml
│   │   │   │   │   ├── sib_ibo.yaml
│   │   │   │   │   ├── sib_kab.yaml
│   │   │   │   │   ├── sib_kam.yaml
│   │   │   │   │   ├── sib_kbp.yaml
│   │   │   │   │   ├── sib_kea.yaml
│   │   │   │   │   ├── sib_kik.yaml
│   │   │   │   │   ├── sib_kin.yaml
│   │   │   │   │   ├── sib_kmb.yaml
│   │   │   │   │   ├── sib_knc.yaml
│   │   │   │   │   ├── sib_kon.yaml
│   │   │   │   │   ├── sib_lin.yaml
│   │   │   │   │   ├── sib_lua.yaml
│   │   │   │   │   ├── sib_lug.yaml
│   │   │   │   │   ├── sib_luo.yaml
│   │   │   │   │   ├── sib_mos.yaml
│   │   │   │   │   ├── sib_nso.yaml
│   │   │   │   │   ├── sib_nus.yaml
│   │   │   │   │   ├── sib_nya.yaml
│   │   │   │   │   ├── sib_plt.yaml
│   │   │   │   │   ├── sib_por.yaml
│   │   │   │   │   ├── sib_run.yaml
│   │   │   │   │   ├── sib_sag.yaml
│   │   │   │   │   ├── sib_sna.yaml
│   │   │   │   │   ├── sib_som.yaml
│   │   │   │   │   ├── sib_sot.yaml
│   │   │   │   │   ├── sib_ssw.yaml
│   │   │   │   │   ├── sib_swa.yaml
│   │   │   │   │   ├── sib_taq.yaml
│   │   │   │   │   ├── sib_tir.yaml
│   │   │   │   │   ├── sib_tso.yaml
│   │   │   │   │   ├── sib_tum.yaml
│   │   │   │   │   ├── sib_twi.yaml
│   │   │   │   │   ├── sib_tzm.yaml
│   │   │   │   │   ├── sib_umb.yaml
│   │   │   │   │   ├── sib_wol.yaml
│   │   │   │   │   ├── sib_xho.yaml
│   │   │   │   │   ├── sib_yor.yaml
│   │   │   │   │   ├── sib_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── sib
│   │   │   │   │   ├── sib_aeb.yaml
│   │   │   │   │   ├── sib_afr.yaml
│   │   │   │   │   ├── sib_aka.yaml
│   │   │   │   │   ├── sib_amh.yaml
│   │   │   │   │   ├── sib_ary.yaml
│   │   │   │   │   ├── sib_arz.yaml
│   │   │   │   │   ├── sib_bam.yaml
│   │   │   │   │   ├── sib_bem.yaml
│   │   │   │   │   ├── sib_cjk.yaml
│   │   │   │   │   ├── sib_dik.yaml
│   │   │   │   │   ├── sib_dyu.yaml
│   │   │   │   │   ├── sib_eng.yaml
│   │   │   │   │   ├── sib_ewe.yaml
│   │   │   │   │   ├── sib_fon.yaml
│   │   │   │   │   ├── sib_fra.yaml
│   │   │   │   │   ├── sib_fuv.yaml
│   │   │   │   │   ├── sib_gaz.yaml
│   │   │   │   │   ├── sib_hau.yaml
│   │   │   │   │   ├── sib_ibo.yaml
│   │   │   │   │   ├── sib_kab.yaml
│   │   │   │   │   ├── sib_kam.yaml
│   │   │   │   │   ├── sib_kbp.yaml
│   │   │   │   │   ├── sib_kea.yaml
│   │   │   │   │   ├── sib_kik.yaml
│   │   │   │   │   ├── sib_kin.yaml
│   │   │   │   │   ├── sib_kmb.yaml
│   │   │   │   │   ├── sib_knc.yaml
│   │   │   │   │   ├── sib_kon.yaml
│   │   │   │   │   ├── sib_lin.yaml
│   │   │   │   │   ├── sib_lua.yaml
│   │   │   │   │   ├── sib_lug.yaml
│   │   │   │   │   ├── sib_luo.yaml
│   │   │   │   │   ├── sib_mos.yaml
│   │   │   │   │   ├── sib_nso.yaml
│   │   │   │   │   ├── sib_nus.yaml
│   │   │   │   │   ├── sib_nya.yaml
│   │   │   │   │   ├── sib_plt.yaml
│   │   │   │   │   ├── sib_por.yaml
│   │   │   │   │   ├── sib_run.yaml
│   │   │   │   │   ├── sib_sag.yaml
│   │   │   │   │   ├── sib_sna.yaml
│   │   │   │   │   ├── sib_som.yaml
│   │   │   │   │   ├── sib_sot.yaml
│   │   │   │   │   ├── sib_ssw.yaml
│   │   │   │   │   ├── sib_swa.yaml
│   │   │   │   │   ├── sib_taq.yaml
│   │   │   │   │   ├── sib_tir.yaml
│   │   │   │   │   ├── sib_tso.yaml
│   │   │   │   │   ├── sib_tum.yaml
│   │   │   │   │   ├── sib_twi.yaml
│   │   │   │   │   ├── sib_tzm.yaml
│   │   │   │   │   ├── sib_umb.yaml
│   │   │   │   │   ├── sib_wol.yaml
│   │   │   │   │   ├── sib_xho.yaml
│   │   │   │   │   ├── sib_yor.yaml
│   │   │   │   │   ├── sib_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── sib
│   │   │   │   │   ├── sib_aeb.yaml
│   │   │   │   │   ├── sib_afr.yaml
│   │   │   │   │   ├── sib_aka.yaml
│   │   │   │   │   ├── sib_amh.yaml
│   │   │   │   │   ├── sib_ary.yaml
│   │   │   │   │   ├── sib_arz.yaml
│   │   │   │   │   ├── sib_bam.yaml
│   │   │   │   │   ├── sib_bem.yaml
│   │   │   │   │   ├── sib_cjk.yaml
│   │   │   │   │   ├── sib_dik.yaml
│   │   │   │   │   ├── sib_dyu.yaml
│   │   │   │   │   ├── sib_eng.yaml
│   │   │   │   │   ├── sib_ewe.yaml
│   │   │   │   │   ├── sib_fon.yaml
│   │   │   │   │   ├── sib_fra.yaml
│   │   │   │   │   ├── sib_fuv.yaml
│   │   │   │   │   ├── sib_gaz.yaml
│   │   │   │   │   ├── sib_hau.yaml
│   │   │   │   │   ├── sib_ibo.yaml
│   │   │   │   │   ├── sib_kab.yaml
│   │   │   │   │   ├── sib_kam.yaml
│   │   │   │   │   ├── sib_kbp.yaml
│   │   │   │   │   ├── sib_kea.yaml
│   │   │   │   │   ├── sib_kik.yaml
│   │   │   │   │   ├── sib_kin.yaml
│   │   │   │   │   ├── sib_kmb.yaml
│   │   │   │   │   ├── sib_knc.yaml
│   │   │   │   │   ├── sib_kon.yaml
│   │   │   │   │   ├── sib_lin.yaml
│   │   │   │   │   ├── sib_lua.yaml
│   │   │   │   │   ├── sib_lug.yaml
│   │   │   │   │   ├── sib_luo.yaml
│   │   │   │   │   ├── sib_mos.yaml
│   │   │   │   │   ├── sib_nso.yaml
│   │   │   │   │   ├── sib_nus.yaml
│   │   │   │   │   ├── sib_nya.yaml
│   │   │   │   │   ├── sib_plt.yaml
│   │   │   │   │   ├── sib_por.yaml
│   │   │   │   │   ├── sib_run.yaml
│   │   │   │   │   ├── sib_sag.yaml
│   │   │   │   │   ├── sib_sna.yaml
│   │   │   │   │   ├── sib_som.yaml
│   │   │   │   │   ├── sib_sot.yaml
│   │   │   │   │   ├── sib_ssw.yaml
│   │   │   │   │   ├── sib_swa.yaml
│   │   │   │   │   ├── sib_taq.yaml
│   │   │   │   │   ├── sib_tir.yaml
│   │   │   │   │   ├── sib_tso.yaml
│   │   │   │   │   ├── sib_tum.yaml
│   │   │   │   │   ├── sib_twi.yaml
│   │   │   │   │   ├── sib_tzm.yaml
│   │   │   │   │   ├── sib_umb.yaml
│   │   │   │   │   ├── sib_wol.yaml
│   │   │   │   │   ├── sib_xho.yaml
│   │   │   │   │   ├── sib_yor.yaml
│   │   │   │   │   ├── sib_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── sib
│   │   │   │   │   ├── sib_aeb.yaml
│   │   │   │   │   ├── sib_afr.yaml
│   │   │   │   │   ├── sib_aka.yaml
│   │   │   │   │   ├── sib_amh.yaml
│   │   │   │   │   ├── sib_ary.yaml
│   │   │   │   │   ├── sib_arz.yaml
│   │   │   │   │   ├── sib_bam.yaml
│   │   │   │   │   ├── sib_bem.yaml
│   │   │   │   │   ├── sib_cjk.yaml
│   │   │   │   │   ├── sib_dik.yaml
│   │   │   │   │   ├── sib_dyu.yaml
│   │   │   │   │   ├── sib_eng.yaml
│   │   │   │   │   ├── sib_ewe.yaml
│   │   │   │   │   ├── sib_fon.yaml
│   │   │   │   │   ├── sib_fra.yaml
│   │   │   │   │   ├── sib_fuv.yaml
│   │   │   │   │   ├── sib_gaz.yaml
│   │   │   │   │   ├── sib_hau.yaml
│   │   │   │   │   ├── sib_ibo.yaml
│   │   │   │   │   ├── sib_kab.yaml
│   │   │   │   │   ├── sib_kam.yaml
│   │   │   │   │   ├── sib_kbp.yaml
│   │   │   │   │   ├── sib_kea.yaml
│   │   │   │   │   ├── sib_kik.yaml
│   │   │   │   │   ├── sib_kin.yaml
│   │   │   │   │   ├── sib_kmb.yaml
│   │   │   │   │   ├── sib_knc.yaml
│   │   │   │   │   ├── sib_kon.yaml
│   │   │   │   │   ├── sib_lin.yaml
│   │   │   │   │   ├── sib_lua.yaml
│   │   │   │   │   ├── sib_lug.yaml
│   │   │   │   │   ├── sib_luo.yaml
│   │   │   │   │   ├── sib_mos.yaml
│   │   │   │   │   ├── sib_nso.yaml
│   │   │   │   │   ├── sib_nus.yaml
│   │   │   │   │   ├── sib_nya.yaml
│   │   │   │   │   ├── sib_plt.yaml
│   │   │   │   │   ├── sib_por.yaml
│   │   │   │   │   ├── sib_run.yaml
│   │   │   │   │   ├── sib_sag.yaml
│   │   │   │   │   ├── sib_sna.yaml
│   │   │   │   │   ├── sib_som.yaml
│   │   │   │   │   ├── sib_sot.yaml
│   │   │   │   │   ├── sib_ssw.yaml
│   │   │   │   │   ├── sib_swa.yaml
│   │   │   │   │   ├── sib_taq.yaml
│   │   │   │   │   ├── sib_tir.yaml
│   │   │   │   │   ├── sib_tso.yaml
│   │   │   │   │   ├── sib_tum.yaml
│   │   │   │   │   ├── sib_twi.yaml
│   │   │   │   │   ├── sib_tzm.yaml
│   │   │   │   │   ├── sib_umb.yaml
│   │   │   │   │   ├── sib_wol.yaml
│   │   │   │   │   ├── sib_xho.yaml
│   │   │   │   │   ├── sib_yor.yaml
│   │   │   │   │   ├── sib_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_5/
│   │   │   │   │   ├── sib
│   │   │   │   │   ├── sib_aeb.yaml
│   │   │   │   │   ├── sib_afr.yaml
│   │   │   │   │   ├── sib_aka.yaml
│   │   │   │   │   ├── sib_amh.yaml
│   │   │   │   │   ├── sib_ary.yaml
│   │   │   │   │   ├── sib_arz.yaml
│   │   │   │   │   ├── sib_bam.yaml
│   │   │   │   │   ├── sib_bem.yaml
│   │   │   │   │   ├── sib_cjk.yaml
│   │   │   │   │   ├── sib_dik.yaml
│   │   │   │   │   ├── sib_dyu.yaml
│   │   │   │   │   ├── sib_eng.yaml
│   │   │   │   │   ├── sib_ewe.yaml
│   │   │   │   │   ├── sib_fon.yaml
│   │   │   │   │   ├── sib_fra.yaml
│   │   │   │   │   ├── sib_fuv.yaml
│   │   │   │   │   ├── sib_gaz.yaml
│   │   │   │   │   ├── sib_hau.yaml
│   │   │   │   │   ├── sib_ibo.yaml
│   │   │   │   │   ├── sib_kab.yaml
│   │   │   │   │   ├── sib_kam.yaml
│   │   │   │   │   ├── sib_kbp.yaml
│   │   │   │   │   ├── sib_kea.yaml
│   │   │   │   │   ├── sib_kik.yaml
│   │   │   │   │   ├── sib_kin.yaml
│   │   │   │   │   ├── sib_kmb.yaml
│   │   │   │   │   ├── sib_knc.yaml
│   │   │   │   │   ├── sib_kon.yaml
│   │   │   │   │   ├── sib_lin.yaml
│   │   │   │   │   ├── sib_lua.yaml
│   │   │   │   │   ├── sib_lug.yaml
│   │   │   │   │   ├── sib_luo.yaml
│   │   │   │   │   ├── sib_mos.yaml
│   │   │   │   │   ├── sib_nso.yaml
│   │   │   │   │   ├── sib_nus.yaml
│   │   │   │   │   ├── sib_nya.yaml
│   │   │   │   │   ├── sib_plt.yaml
│   │   │   │   │   ├── sib_por.yaml
│   │   │   │   │   ├── sib_run.yaml
│   │   │   │   │   ├── sib_sag.yaml
│   │   │   │   │   ├── sib_sna.yaml
│   │   │   │   │   ├── sib_som.yaml
│   │   │   │   │   ├── sib_sot.yaml
│   │   │   │   │   ├── sib_ssw.yaml
│   │   │   │   │   ├── sib_swa.yaml
│   │   │   │   │   ├── sib_taq.yaml
│   │   │   │   │   ├── sib_tir.yaml
│   │   │   │   │   ├── sib_tso.yaml
│   │   │   │   │   ├── sib_tum.yaml
│   │   │   │   │   ├── sib_twi.yaml
│   │   │   │   │   ├── sib_tzm.yaml
│   │   │   │   │   ├── sib_umb.yaml
│   │   │   │   │   ├── sib_wol.yaml
│   │   │   │   │   ├── sib_xho.yaml
│   │   │   │   │   ├── sib_yor.yaml
│   │   │   │   │   ├── sib_zul.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── sib.yaml
│   │   │   │   └── utils.py
│   │   │   ├── uhura-arc-easy/
│   │   │   │   ├── README.md
│   │   │   │   ├── prompt_1/
│   │   │   │   │   ├── uhura-arc-easy
│   │   │   │   │   ├── uhura-arc-easy_am.yaml
│   │   │   │   │   ├── uhura-arc-easy_en.yaml
│   │   │   │   │   ├── uhura-arc-easy_ha.yaml
│   │   │   │   │   ├── uhura-arc-easy_nso.yaml
│   │   │   │   │   ├── uhura-arc-easy_sw.yaml
│   │   │   │   │   ├── uhura-arc-easy_yo.yaml
│   │   │   │   │   ├── uhura-arc-easy_zu.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_2/
│   │   │   │   │   ├── uhura-arc-easy
│   │   │   │   │   ├── uhura-arc-easy_am.yaml
│   │   │   │   │   ├── uhura-arc-easy_en.yaml
│   │   │   │   │   ├── uhura-arc-easy_ha.yaml
│   │   │   │   │   ├── uhura-arc-easy_nso.yaml
│   │   │   │   │   ├── uhura-arc-easy_sw.yaml
│   │   │   │   │   ├── uhura-arc-easy_yo.yaml
│   │   │   │   │   ├── uhura-arc-easy_zu.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_3/
│   │   │   │   │   ├── uhura-arc-easy
│   │   │   │   │   ├── uhura-arc-easy_am.yaml
│   │   │   │   │   ├── uhura-arc-easy_en.yaml
│   │   │   │   │   ├── uhura-arc-easy_ha.yaml
│   │   │   │   │   ├── uhura-arc-easy_nso.yaml
│   │   │   │   │   ├── uhura-arc-easy_sw.yaml
│   │   │   │   │   ├── uhura-arc-easy_yo.yaml
│   │   │   │   │   ├── uhura-arc-easy_zu.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_4/
│   │   │   │   │   ├── uhura-arc-easy
│   │   │   │   │   ├── uhura-arc-easy_am.yaml
│   │   │   │   │   ├── uhura-arc-easy_en.yaml
│   │   │   │   │   ├── uhura-arc-easy_ha.yaml
│   │   │   │   │   ├── uhura-arc-easy_nso.yaml
│   │   │   │   │   ├── uhura-arc-easy_sw.yaml
│   │   │   │   │   ├── uhura-arc-easy_yo.yaml
│   │   │   │   │   ├── uhura-arc-easy_zu.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── prompt_5/
│   │   │   │   │   ├── uhura-arc-easy
│   │   │   │   │   ├── uhura-arc-easy_am.yaml
│   │   │   │   │   ├── uhura-arc-easy_en.yaml
│   │   │   │   │   ├── uhura-arc-easy_ha.yaml
│   │   │   │   │   ├── uhura-arc-easy_nso.yaml
│   │   │   │   │   ├── uhura-arc-easy_sw.yaml
│   │   │   │   │   ├── uhura-arc-easy_yo.yaml
│   │   │   │   │   ├── uhura-arc-easy_zu.yaml
│   │   │   │   │   └── utils.py
│   │   │   │   ├── uhura.yaml
│   │   │   │   └── utils.py
│   │   │   └── xlsum/
│   │   │       ├── README.md
│   │   │       ├── prompt_1/
│   │   │       │   ├── utils.py
│   │   │       │   ├── xlsum
│   │   │       │   ├── xlsum_amharic.yaml
│   │   │       │   ├── xlsum_arabic.yaml
│   │   │       │   ├── xlsum_hausa.yaml
│   │   │       │   ├── xlsum_igbo.yaml
│   │   │       │   ├── xlsum_kirundi.yaml
│   │   │       │   ├── xlsum_oromo.yaml
│   │   │       │   ├── xlsum_pidgin.yaml
│   │   │       │   ├── xlsum_somali.yaml
│   │   │       │   ├── xlsum_swahili.yaml
│   │   │       │   ├── xlsum_telugu.yaml
│   │   │       │   ├── xlsum_tigrinya.yaml
│   │   │       │   └── xlsum_yoruba.yaml
│   │   │       ├── prompt_2/
│   │   │       │   ├── utils.py
│   │   │       │   ├── xlsum
│   │   │       │   ├── xlsum_amharic.yaml
│   │   │       │   ├── xlsum_arabic.yaml
│   │   │       │   ├── xlsum_hausa.yaml
│   │   │       │   ├── xlsum_igbo.yaml
│   │   │       │   ├── xlsum_kirundi.yaml
│   │   │       │   ├── xlsum_oromo.yaml
│   │   │       │   ├── xlsum_pidgin.yaml
│   │   │       │   ├── xlsum_somali.yaml
│   │   │       │   ├── xlsum_swahili.yaml
│   │   │       │   ├── xlsum_telugu.yaml
│   │   │       │   ├── xlsum_tigrinya.yaml
│   │   │       │   └── xlsum_yoruba.yaml
│   │   │       ├── prompt_3/
│   │   │       │   ├── utils.py
│   │   │       │   ├── xlsum
│   │   │       │   ├── xlsum_amharic.yaml
│   │   │       │   ├── xlsum_arabic.yaml
│   │   │       │   ├── xlsum_hausa.yaml
│   │   │       │   ├── xlsum_igbo.yaml
│   │   │       │   ├── xlsum_kirundi.yaml
│   │   │       │   ├── xlsum_oromo.yaml
│   │   │       │   ├── xlsum_pidgin.yaml
│   │   │       │   ├── xlsum_somali.yaml
│   │   │       │   ├── xlsum_swahili.yaml
│   │   │       │   ├── xlsum_telugu.yaml
│   │   │       │   ├── xlsum_tigrinya.yaml
│   │   │       │   └── xlsum_yoruba.yaml
│   │   │       ├── utils.py
│   │   │       └── xlsum.yaml
│   │   ├── agieval/
│   │   │   ├── README.md
│   │   │   ├── agieval.yaml
│   │   │   ├── agieval_cn.yaml
│   │   │   ├── agieval_en.yaml
│   │   │   ├── agieval_nous.yaml
│   │   │   ├── aqua-rat.yaml
│   │   │   ├── gaokao-biology.yaml
│   │   │   ├── gaokao-chemistry.yaml
│   │   │   ├── gaokao-chinese.yaml
│   │   │   ├── gaokao-english.yaml
│   │   │   ├── gaokao-geography.yaml
│   │   │   ├── gaokao-history.yaml
│   │   │   ├── gaokao-mathcloze.yaml
│   │   │   ├── gaokao-mathqa.yaml
│   │   │   ├── gaokao-physics.yaml
│   │   │   ├── jec-qa-ca.yaml
│   │   │   ├── jec-qa-kd.yaml
│   │   │   ├── logiqa-en.yaml
│   │   │   ├── logiqa-zh.yaml
│   │   │   ├── lsat-ar.yaml
│   │   │   ├── lsat-lr.yaml
│   │   │   ├── lsat-rc.yaml
│   │   │   ├── math.yaml
│   │   │   ├── sat-en-without-passage.yaml
│   │   │   ├── sat-en.yaml
│   │   │   ├── sat-math.yaml
│   │   │   └── utils.py
│   │   ├── aime/
│   │   │   ├── README.md
│   │   │   ├── aime.yaml
│   │   │   ├── aime24.yaml
│   │   │   ├── aime25.yaml
│   │   │   └── utils.py
│   │   ├── alghafa/
│   │   │   ├── copa_ar/
│   │   │   │   ├── README.md
│   │   │   │   └── copa_ar.yaml
│   │   │   └── piqa_ar/
│   │   │       ├── README.md
│   │   │       └── piqa_ar.yaml
│   │   ├── anli/
│   │   │   ├── README.md
│   │   │   ├── anli_r1.yaml
│   │   │   ├── anli_r2.yaml
│   │   │   └── anli_r3.yaml
│   │   ├── arab_culture/
│   │   │   ├── README.md
│   │   │   ├── _arab_culture.yaml
│   │   │   ├── _arab_culture_gulf.yaml
│   │   │   ├── _arab_culture_levant.yaml
│   │   │   ├── _arab_culture_nile_valley.yaml
│   │   │   ├── _arab_culture_north_africa.yaml
│   │   │   ├── _default_arab_culture_mcq_template_yaml
│   │   │   ├── _generate_configs.py
│   │   │   ├── arab_culture_algeria.yaml
│   │   │   ├── arab_culture_egypt.yaml
│   │   │   ├── arab_culture_jordan.yaml
│   │   │   ├── arab_culture_ksa.yaml
│   │   │   ├── arab_culture_lebanon.yaml
│   │   │   ├── arab_culture_libya.yaml
│   │   │   ├── arab_culture_morocco.yaml
│   │   │   ├── arab_culture_palestine.yaml
│   │   │   ├── arab_culture_sudan.yaml
│   │   │   ├── arab_culture_syria.yaml
│   │   │   ├── arab_culture_tunisia.yaml
│   │   │   ├── arab_culture_uae.yaml
│   │   │   ├── arab_culture_yemen.yaml
│   │   │   ├── prompts.py
│   │   │   └── utils_mcq.py
│   │   ├── arab_culture_completion/
│   │   │   ├── README.md
│   │   │   ├── _arab_culture_completion.yaml
│   │   │   ├── _arab_culture_completion_gulf.yaml
│   │   │   ├── _arab_culture_completion_levant.yaml
│   │   │   ├── _arab_culture_completion_nile_valley.yaml
│   │   │   ├── _arab_culture_completion_north_africa.yaml
│   │   │   ├── _default_arab_culture_completion_template_yaml
│   │   │   ├── _generate_configs.py
│   │   │   ├── arab_culture_completion_algeria.yaml
│   │   │   ├── arab_culture_completion_egypt.yaml
│   │   │   ├── arab_culture_completion_jordan.yaml
│   │   │   ├── arab_culture_completion_ksa.yaml
│   │   │   ├── arab_culture_completion_lebanon.yaml
│   │   │   ├── arab_culture_completion_libya.yaml
│   │   │   ├── arab_culture_completion_morocco.yaml
│   │   │   ├── arab_culture_completion_palestine.yaml
│   │   │   ├── arab_culture_completion_sudan.yaml
│   │   │   ├── arab_culture_completion_syria.yaml
│   │   │   ├── arab_culture_completion_tunisia.yaml
│   │   │   ├── arab_culture_completion_uae.yaml
│   │   │   ├── arab_culture_completion_yemen.yaml
│   │   │   ├── prompts.py
│   │   │   └── utils_completion.py
│   │   ├── arabic_leaderboard_complete/
│   │   │   ├── README.md
│   │   │   ├── arabic_leaderboard_alghafa/
│   │   │   │   ├── arabic_leaderboard_alghafa.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_mcq_exams_test_ar.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_meta_ar_dialects.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_meta_ar_msa.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_multiple_choice_sentiment_task.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_exams/
│   │   │   │   ├── arabic_exams.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_exams.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mmlu/
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_abstract_algebra.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_anatomy.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_astronomy.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_business_ethics.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_clinical_knowledge.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_college_biology.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_college_chemistry.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_college_computer_science.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_college_mathematics.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_college_medicine.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_college_physics.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_computer_security.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_conceptual_physics.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_econometrics.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_electrical_engineering.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_elementary_mathematics.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_formal_logic.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_global_facts.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_biology.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_chemistry.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_computer_science.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_european_history.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_geography.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_government_and_politics.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_macroeconomics.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_mathematics.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_microeconomics.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_physics.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_psychology.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_statistics.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_us_history.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_world_history.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_human_aging.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_human_sexuality.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_international_law.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_jurisprudence.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_logical_fallacies.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_machine_learning.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_management.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_marketing.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_medical_genetics.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_miscellaneous.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_moral_disputes.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_moral_scenarios.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_nutrition.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_philosophy.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_prehistory.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_professional_accounting.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_professional_law.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_professional_medicine.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_professional_psychology.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_public_relations.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_security_studies.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_sociology.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_us_foreign_policy.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_virology.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_world_religions.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_arc_challenge/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_arc_challenge.yaml
│   │   │   │   ├── arabic_mt_arc_challenge.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_arc_easy/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_arc_easy.yaml
│   │   │   │   ├── arabic_mt_arc_easy.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_boolq/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_boolq.yaml
│   │   │   │   ├── arabic_mt_boolq.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_copa/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_copa.yaml
│   │   │   │   ├── arabic_mt_copa.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_hellaswag/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_hellaswag.yaml
│   │   │   │   ├── arabic_mt_hellaswag.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_mmlu/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_mmlu.yaml
│   │   │   │   ├── arabic_mt_mmlu.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_openbook_qa/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_openbook_qa.yaml
│   │   │   │   ├── arabic_mt_openbook_qa.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_piqa/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_piqa.yaml
│   │   │   │   ├── arabic_mt_piqa.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_race/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_race.yaml
│   │   │   │   ├── arabic_mt_race.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_sciq/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_sciq.yaml
│   │   │   │   ├── arabic_mt_sciq.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_toxigen/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_toxigen.yaml
│   │   │   │   ├── arabic_mt_toxigen.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_avca/
│   │   │   │   ├── arabic_leaderboard_acva.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Algeria.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Ancient_Egypt.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arab_Empire.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Architecture.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Art.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Astronomy.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Calligraphy.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Ceremony.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Clothing.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Culture.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Food.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Funeral.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Geography.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_History.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Language_Origin.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Literature.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Math.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Medicine.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Music.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Ornament.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Philosophy.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Physics_and_Chemistry.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Wedding.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Bahrain.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Comoros.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Egypt_modern.yaml
│   │   │   │   ├── arabic_leaderboard_acva_InfluenceFromAncientEgypt.yaml
│   │   │   │   ├── arabic_leaderboard_acva_InfluenceFromByzantium.yaml
│   │   │   │   ├── arabic_leaderboard_acva_InfluenceFromChina.yaml
│   │   │   │   ├── arabic_leaderboard_acva_InfluenceFromGreece.yaml
│   │   │   │   ├── arabic_leaderboard_acva_InfluenceFromIslam.yaml
│   │   │   │   ├── arabic_leaderboard_acva_InfluenceFromPersia.yaml
│   │   │   │   ├── arabic_leaderboard_acva_InfluenceFromRome.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Iraq.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Islam_Education.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Islam_branches_and_schools.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Islamic_law_system.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Jordan.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Kuwait.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Lebanon.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Libya.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Mauritania.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Mesopotamia_civilization.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Morocco.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Oman.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Palestine.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Qatar.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Saudi_Arabia.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Somalia.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Sudan.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Syria.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Tunisia.yaml
│   │   │   │   ├── arabic_leaderboard_acva_United_Arab_Emirates.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Yemen.yaml
│   │   │   │   ├── arabic_leaderboard_acva_communication.yaml
│   │   │   │   ├── arabic_leaderboard_acva_computer_and_phone.yaml
│   │   │   │   ├── arabic_leaderboard_acva_daily_life.yaml
│   │   │   │   ├── arabic_leaderboard_acva_entertainment.yaml
│   │   │   │   └── utils.py
│   │   │   └── arabic_leaderboard_complete.yaml
│   │   ├── arabic_leaderboard_light/
│   │   │   ├── README.md
│   │   │   ├── arabic_leaderboard_alghafa_light/
│   │   │   │   ├── arabic_leaderboard_alghafa_light.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_mcq_exams_test_ar_light.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_meta_ar_dialects_light.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_meta_ar_msa_light.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task_light.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task_light.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task_light.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task_light.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task_light.yaml
│   │   │   │   ├── arabic_leaderboard_alghafa_multiple_choice_sentiment_task_light.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_exams_light/
│   │   │   │   ├── arabic_exams_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_exams_light.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mmlu_light/
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_abstract_algebra_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_anatomy_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_astronomy_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_business_ethics_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_clinical_knowledge_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_college_biology_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_college_chemistry_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_college_computer_science_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_college_mathematics_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_college_medicine_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_college_physics_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_computer_security_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_conceptual_physics_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_econometrics_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_electrical_engineering_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_elementary_mathematics_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_formal_logic_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_global_facts_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_biology_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_chemistry_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_computer_science_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_european_history_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_geography_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_government_and_politics_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_macroeconomics_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_mathematics_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_microeconomics_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_physics_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_psychology_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_statistics_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_us_history_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_high_school_world_history_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_human_aging_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_human_sexuality_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_international_law_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_jurisprudence_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_logical_fallacies_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_machine_learning_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_management_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_marketing_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_medical_genetics_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_miscellaneous_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_moral_disputes_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_moral_scenarios_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_nutrition_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_philosophy_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_prehistory_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_professional_accounting_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_professional_law_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_professional_medicine_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_professional_psychology_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_public_relations_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_security_studies_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_sociology_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_us_foreign_policy_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_virology_light.yaml
│   │   │   │   ├── arabic_leaderboard_arabic_mmlu_world_religions_light.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_arc_challenge_light/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_arc_challenge_light.yaml
│   │   │   │   ├── arabic_mt_arc_challenge_light.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_arc_easy_light/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_arc_easy_light.yaml
│   │   │   │   ├── arabic_mt_arc_easy_light.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_boolq_light/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_boolq_light.yaml
│   │   │   │   ├── arabic_mt_boolq_light.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_copa_light/
│   │   │   │   ├── arabic_mt_copa_light.yaml
│   │   │   │   ├── arbic_leaderboard_arabic_mt_copa_light.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_hellaswag_light/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_hellaswag_light.yaml
│   │   │   │   ├── arabic_mt_hellaswag_light.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_mmlu_light/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_mmlu_light.yaml
│   │   │   │   ├── arabic_mt_mmlu_light.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_openbook_qa_light/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_openbook_qa_light.yaml
│   │   │   │   ├── arabic_mt_openbook_qa_light.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_piqa_light/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_piqa_light.yaml
│   │   │   │   ├── arabic_mt_piqa_light.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_race_light/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_race_light.yaml
│   │   │   │   ├── arabic_mt_race_light.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_sciq_light/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_sciq_light.yaml
│   │   │   │   ├── arabic_mt_sciq_light.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_arabic_mt_toxigen_light/
│   │   │   │   ├── arabic_leaderboard_arabic_mt_toxigen_light.yaml
│   │   │   │   ├── arabic_mt_toxigen_light.yaml
│   │   │   │   └── utils.py
│   │   │   ├── arabic_leaderboard_avca_light/
│   │   │   │   ├── arabic_leaderboard_acva_Algeria_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Ancient_Egypt_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arab_Empire_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Architecture_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Art_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Astronomy_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Calligraphy_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Ceremony_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Clothing_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Culture_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Food_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Funeral_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Geography_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_History_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Language_Origin_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Literature_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Math_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Medicine_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Music_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Ornament_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Philosophy_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Physics_and_Chemistry_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Arabic_Wedding_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Bahrain_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Comoros_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Egypt_modern_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_InfluenceFromAncientEgypt_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_InfluenceFromByzantium_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_InfluenceFromChina_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_InfluenceFromGreece_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_InfluenceFromIslam_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_InfluenceFromPersia_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_InfluenceFromRome_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Iraq_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Islam_Education_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Islam_branches_and_schools_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Islamic_law_system_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Jordan_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Kuwait_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Lebanon_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Libya_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Mauritania_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Mesopotamia_civilization_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Morocco_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Oman_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Palestine_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Qatar_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Saudi_Arabia_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Somalia_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Sudan_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Syria_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Tunisia_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_United_Arab_Emirates_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_Yemen_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_communication_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_computer_and_phone_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_daily_life_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_entertainment_light.yaml
│   │   │   │   ├── arabic_leaderboard_acva_light.yaml
│   │   │   │   └── utils.py
│   │   │   └── arabic_leaderboard_light.yaml
│   │   ├── arabicmmlu/
│   │   │   ├── README.md
│   │   │   ├── _arabicmmlu.yaml
│   │   │   ├── _arabicmmlu_humanities.yaml
│   │   │   ├── _arabicmmlu_language.yaml
│   │   │   ├── _arabicmmlu_other.yaml
│   │   │   ├── _arabicmmlu_social_science.yaml
│   │   │   ├── _arabicmmlu_stem.yaml
│   │   │   ├── _default_arabicmmlu_template_yaml
│   │   │   ├── _generate_configs.py
│   │   │   ├── arabicmmlu_accounting_university.yaml
│   │   │   ├── arabicmmlu_arabic_language_general.yaml
│   │   │   ├── arabicmmlu_arabic_language_grammar.yaml
│   │   │   ├── arabicmmlu_arabic_language_high_school.yaml
│   │   │   ├── arabicmmlu_arabic_language_middle_school.yaml
│   │   │   ├── arabicmmlu_arabic_language_primary_school.yaml
│   │   │   ├── arabicmmlu_biology_high_school.yaml
│   │   │   ├── arabicmmlu_civics_high_school.yaml
│   │   │   ├── arabicmmlu_civics_middle_school.yaml
│   │   │   ├── arabicmmlu_computer_science_high_school.yaml
│   │   │   ├── arabicmmlu_computer_science_middle_school.yaml
│   │   │   ├── arabicmmlu_computer_science_primary_school.yaml
│   │   │   ├── arabicmmlu_computer_science_university.yaml
│   │   │   ├── arabicmmlu_driving_test.yaml
│   │   │   ├── arabicmmlu_economics_high_school.yaml
│   │   │   ├── arabicmmlu_economics_middle_school.yaml
│   │   │   ├── arabicmmlu_economics_university.yaml
│   │   │   ├── arabicmmlu_general_knowledge.yaml
│   │   │   ├── arabicmmlu_general_knowledge_middle_school.yaml
│   │   │   ├── arabicmmlu_general_knowledge_primary_school.yaml
│   │   │   ├── arabicmmlu_geography_high_school.yaml
│   │   │   ├── arabicmmlu_geography_middle_school.yaml
│   │   │   ├── arabicmmlu_geography_primary_school.yaml
│   │   │   ├── arabicmmlu_history_high_school.yaml
│   │   │   ├── arabicmmlu_history_middle_school.yaml
│   │   │   ├── arabicmmlu_history_primary_school.yaml
│   │   │   ├── arabicmmlu_islamic_studies.yaml
│   │   │   ├── arabicmmlu_islamic_studies_high_school.yaml
│   │   │   ├── arabicmmlu_islamic_studies_middle_school.yaml
│   │   │   ├── arabicmmlu_islamic_studies_primary_school.yaml
│   │   │   ├── arabicmmlu_law_professional.yaml
│   │   │   ├── arabicmmlu_management_university.yaml
│   │   │   ├── arabicmmlu_math_primary_school.yaml
│   │   │   ├── arabicmmlu_natural_science_middle_school.yaml
│   │   │   ├── arabicmmlu_natural_science_primary_school.yaml
│   │   │   ├── arabicmmlu_philosophy_high_school.yaml
│   │   │   ├── arabicmmlu_physics_high_school.yaml
│   │   │   ├── arabicmmlu_political_science_university.yaml
│   │   │   ├── arabicmmlu_social_science_middle_school.yaml
│   │   │   ├── arabicmmlu_social_science_primary_school.yaml
│   │   │   └── utils.py
│   │   ├── aradice/
│   │   │   ├── ArabicMMLU/
│   │   │   │   ├── EGY/
│   │   │   │   │   ├── AraDiCE_ArabicMMLU.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_high_humanities_history.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_high_humanities_islamic-studies.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_high_humanities_philosophy.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_high_language_arabic-language.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_high_social-science_civics.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_high_social-science_economics.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_high_social-science_geography.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_high_stem_biology.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_high_stem_computer-science.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_high_stem_physics.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_middle_humanities_history.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_middle_humanities_islamic-studies.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_middle_language_arabic-language.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_middle_other_general-knowledge.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_middle_social-science_civics.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_middle_social-science_economics.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_middle_social-science_geography.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_middle_social-science_social-science.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_middle_stem_computer-science.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_middle_stem_natural-science.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_na_humanities_islamic-studies.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_na_language_arabic-language-general.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_na_language_arabic-language-grammar.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_na_other_driving-test.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_na_other_general-knowledge.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_primary_humanities_history.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_primary_humanities_islamic-studies.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_primary_language_arabic-language.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_primary_other_general-knowledge.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_primary_social-science_geography.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_primary_social-science_social-science.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_primary_stem_computer-science.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_primary_stem_math.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_primary_stem_natural-science.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_prof_humanities_law.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_univ_other_management.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_univ_social-science_accounting.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_univ_social-science_economics.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_univ_social-science_political-science.yaml
│   │   │   │   │   ├── AraDiCE_ArabicMMLU_univ_stem_computer-science.yaml
│   │   │   │   │   ├── _default_template_yaml
│   │   │   │   │   ├── metrics.py
│   │   │   │   │   └── utils.py
│   │   │   │   └── LEV/
│   │   │   │       ├── AraDiCE_ArabicMMLU.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_high_humanities_history.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_high_humanities_islamic-studies.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_high_humanities_philosophy.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_high_language_arabic-language.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_high_social-science_civics.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_high_social-science_economics.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_high_social-science_geography.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_high_stem_biology.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_high_stem_computer-science.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_high_stem_physics.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_middle_humanities_history.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_middle_humanities_islamic-studies.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_middle_language_arabic-language.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_middle_other_general-knowledge.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_middle_social-science_civics.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_middle_social-science_economics.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_middle_social-science_geography.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_middle_social-science_social-science.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_middle_stem_computer-science.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_middle_stem_natural-science.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_na_humanities_islamic-studies.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_na_language_arabic-language-general.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_na_language_arabic-language-grammar.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_na_other_driving-test.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_na_other_general-knowledge.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_primary_humanities_history.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_primary_humanities_islamic-studies.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_primary_language_arabic-language.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_primary_other_general-knowledge.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_primary_social-science_geography.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_primary_social-science_social-science.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_primary_stem_computer-science.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_primary_stem_math.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_primary_stem_natural-science.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_prof_humanities_law.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_univ_other_management.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_univ_social-science_accounting.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_univ_social-science_economics.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_univ_social-science_political-science.yaml
│   │   │   │       ├── AraDiCE_ArabicMMLU_univ_stem_computer-science.yaml
│   │   │   │       ├── _default_template_yaml
│   │   │   │       ├── metrics.py
│   │   │   │       └── utils.py
│   │   │   ├── README.md
│   │   │   ├── aradice.yaml
│   │   │   ├── boolq/
│   │   │   │   ├── EGY/
│   │   │   │   │   ├── boolq_egy.yaml
│   │   │   │   │   ├── metrics.py
│   │   │   │   │   └── utils.py
│   │   │   │   ├── ENG/
│   │   │   │   │   ├── boolq_eng.yaml
│   │   │   │   │   ├── metrics.py
│   │   │   │   │   └── utils.py
│   │   │   │   ├── LEV/
│   │   │   │   │   ├── boolq_lev.yaml
│   │   │   │   │   ├── metrics.py
│   │   │   │   │   └── utils.py
│   │   │   │   └── MSA/
│   │   │   │       ├── boolq_msa.yaml
│   │   │   │       ├── metrics.py
│   │   │   │       └── utils.py
│   │   │   ├── cultural-benchmark/
│   │   │   │   ├── egypt.yaml
│   │   │   │   ├── jordan.yaml
│   │   │   │   ├── lebanon.yaml
│   │   │   │   ├── metrics.py
│   │   │   │   ├── palestine.yaml
│   │   │   │   ├── qatar.yaml
│   │   │   │   ├── syria.yaml
│   │   │   │   └── utils.py
│   │   │   ├── openbookqa/
│   │   │   │   ├── metrics.py
│   │   │   │   ├── openbookqa_egy.yaml
│   │   │   │   ├── openbookqa_eng.yaml
│   │   │   │   ├── openbookqa_lev.yaml
│   │   │   │   ├── openbookqa_msa.yaml
│   │   │   │   └── utils.py
│   │   │   ├── piqa/
│   │   │   │   ├── metrics.py
│   │   │   │   ├── piqa_egy.yaml
│   │   │   │   ├── piqa_eng.yaml
│   │   │   │   ├── piqa_lev.yaml
│   │   │   │   └── piqa_msa.yaml
│   │   │   ├── truthfulqa_mcq/
│   │   │   │   ├── metrics.py
│   │   │   │   ├── truthfulqa_mc1_egy.yaml
│   │   │   │   ├── truthfulqa_mc1_eng.yaml
│   │   │   │   ├── truthfulqa_mc1_lev.yaml
│   │   │   │   └── truthfulqa_mc1_msa.yaml
│   │   │   └── winogrande/
│   │   │       ├── metrics.py
│   │   │       ├── utils.py
│   │   │       ├── winogrande_egy.yaml
│   │   │       ├── winogrande_eng.yaml
│   │   │       ├── winogrande_lev.yaml
│   │   │       └── winogrande_msa.yaml
│   │   ├── arc/
│   │   │   ├── README.md
│   │   │   ├── arc_challenge.yaml
│   │   │   ├── arc_challenge_chat.yaml
│   │   │   └── arc_easy.yaml
│   │   ├── arc_mt/
│   │   │   ├── README.md
│   │   │   ├── arc_challenge_mt_da.yaml
│   │   │   ├── arc_challenge_mt_de.yaml
│   │   │   ├── arc_challenge_mt_el.yaml
│   │   │   ├── arc_challenge_mt_es.yaml
│   │   │   ├── arc_challenge_mt_fi.yaml
│   │   │   ├── arc_challenge_mt_hu.yaml
│   │   │   ├── arc_challenge_mt_is.yaml
│   │   │   ├── arc_challenge_mt_it.yaml
│   │   │   ├── arc_challenge_mt_nb.yaml
│   │   │   ├── arc_challenge_mt_pl.yaml
│   │   │   ├── arc_challenge_mt_pt.yaml
│   │   │   └── arc_challenge_mt_sv.yaml
│   │   ├── arithmetic/
│   │   │   ├── README.md
│   │   │   ├── arithmetic_1dc.yaml
│   │   │   ├── arithmetic_2da.yaml
│   │   │   ├── arithmetic_2dm.yaml
│   │   │   ├── arithmetic_2ds.yaml
│   │   │   ├── arithmetic_3da.yaml
│   │   │   ├── arithmetic_3ds.yaml
│   │   │   ├── arithmetic_4da.yaml
│   │   │   ├── arithmetic_4ds.yaml
│   │   │   ├── arithmetic_5da.yaml
│   │   │   └── arithmetic_5ds.yaml
│   │   ├── asdiv/
│   │   │   ├── README.md
│   │   │   ├── asdiv-cot-llama.yaml
│   │   │   └── default.yaml
│   │   ├── babi/
│   │   │   ├── README.md
│   │   │   └── babi.yaml
│   │   ├── babilong/
│   │   │   ├── README.md
│   │   │   ├── _babilong_common_yaml
│   │   │   ├── babilong.yaml
│   │   │   ├── babilong_longctx.yaml
│   │   │   ├── babilong_qa1.yaml
│   │   │   ├── babilong_qa10.yaml
│   │   │   ├── babilong_qa11.yaml
│   │   │   ├── babilong_qa12.yaml
│   │   │   ├── babilong_qa13.yaml
│   │   │   ├── babilong_qa14.yaml
│   │   │   ├── babilong_qa15.yaml
│   │   │   ├── babilong_qa16.yaml
│   │   │   ├── babilong_qa17.yaml
│   │   │   ├── babilong_qa18.yaml
│   │   │   ├── babilong_qa19.yaml
│   │   │   ├── babilong_qa2.yaml
│   │   │   ├── babilong_qa20.yaml
│   │   │   ├── babilong_qa3.yaml
│   │   │   ├── babilong_qa4.yaml
│   │   │   ├── babilong_qa5.yaml
│   │   │   ├── babilong_qa6.yaml
│   │   │   ├── babilong_qa7.yaml
│   │   │   ├── babilong_qa8.yaml
│   │   │   ├── babilong_qa9.yaml
│   │   │   └── common_utils.py
│   │   ├── bangla/
│   │   │   ├── README.md
│   │   │   ├── bangla_boolqa.yaml
│   │   │   ├── bangla_commonsenseqa.yaml
│   │   │   ├── bangla_mmlu.yaml
│   │   │   ├── bangla_openbookqa.yaml
│   │   │   └── bangla_piqa.yaml
│   │   ├── basque_bench/
│   │   │   ├── README.md
│   │   │   ├── arc_eu_challenge.yaml
│   │   │   ├── arc_eu_easy.yaml
│   │   │   ├── basque_bench.yaml
│   │   │   ├── flores_eu/
│   │   │   │   ├── _flores_common_yaml
│   │   │   │   ├── create_yamls_flores_eu.py
│   │   │   │   ├── flores_ca-eu.yaml
│   │   │   │   ├── flores_de-eu.yaml
│   │   │   │   ├── flores_en-eu.yaml
│   │   │   │   ├── flores_es-eu.yaml
│   │   │   │   ├── flores_eu-ca.yaml
│   │   │   │   ├── flores_eu-de.yaml
│   │   │   │   ├── flores_eu-en.yaml
│   │   │   │   ├── flores_eu-es.yaml
│   │   │   │   ├── flores_eu-fr.yaml
│   │   │   │   ├── flores_eu-gl.yaml
│   │   │   │   ├── flores_eu-it.yaml
│   │   │   │   ├── flores_eu-pt.yaml
│   │   │   │   ├── flores_eu.yaml
│   │   │   │   ├── flores_fr-eu.yaml
│   │   │   │   ├── flores_gl-eu.yaml
│   │   │   │   ├── flores_it-eu.yaml
│   │   │   │   └── flores_pt-eu.yaml
│   │   │   ├── mgsm_direct_eu.yaml
│   │   │   ├── mgsm_native_cot_eu.yaml
│   │   │   ├── paws_eu.yaml
│   │   │   ├── piqa_eu.yaml
│   │   │   ├── utils.py
│   │   │   ├── wnli_eu.yaml
│   │   │   └── xcopa_eu.yaml
│   │   ├── basqueglue/
│   │   │   ├── README.md
│   │   │   ├── bec.yaml
│   │   │   ├── bhtc.yaml
│   │   │   ├── coref.yaml
│   │   │   ├── qnli.yaml
│   │   │   ├── utils.py
│   │   │   ├── vaxx.yaml
│   │   │   └── wic.yaml
│   │   ├── bbh/
│   │   │   ├── README.md
│   │   │   ├── _generate_configs.py
│   │   │   ├── cot_fewshot/
│   │   │   │   ├── _bbh.yaml
│   │   │   │   ├── _bbh_cot_fewshot.yaml
│   │   │   │   ├── _cot_fewshot_template_yaml
│   │   │   │   ├── boolean_expressions.yaml
│   │   │   │   ├── causal_judgement.yaml
│   │   │   │   ├── date_understanding.yaml
│   │   │   │   ├── disambiguation_qa.yaml
│   │   │   │   ├── dyck_languages.yaml
│   │   │   │   ├── formal_fallacies.yaml
│   │   │   │   ├── geometric_shapes.yaml
│   │   │   │   ├── hyperbaton.yaml
│   │   │   │   ├── logical_deduction_five_objects.yaml
│   │   │   │   ├── logical_deduction_seven_objects.yaml
│   │   │   │   ├── logical_deduction_three_objects.yaml
│   │   │   │   ├── movie_recommendation.yaml
│   │   │   │   ├── multistep_arithmetic_two.yaml
│   │   │   │   ├── navigate.yaml
│   │   │   │   ├── object_counting.yaml
│   │   │   │   ├── penguins_in_a_table.yaml
│   │   │   │   ├── reasoning_about_colored_objects.yaml
│   │   │   │   ├── ruin_names.yaml
│   │   │   │   ├── salient_translation_error_detection.yaml
│   │   │   │   ├── snarks.yaml
│   │   │   │   ├── sports_understanding.yaml
│   │   │   │   ├── temporal_sequences.yaml
│   │   │   │   ├── tracking_shuffled_objects_five_objects.yaml
│   │   │   │   ├── tracking_shuffled_objects_seven_objects.yaml
│   │   │   │   ├── tracking_shuffled_objects_three_objects.yaml
│   │   │   │   ├── web_of_lies.yaml
│   │   │   │   └── word_sorting.yaml
│   │   │   ├── cot_zeroshot/
│   │   │   │   ├── _bbh_cot_zeroshot.yaml
│   │   │   │   ├── _cot_zeroshot_template_yaml
│   │   │   │   ├── boolean_expressions.yaml
│   │   │   │   ├── causal_judgement.yaml
│   │   │   │   ├── date_understanding.yaml
│   │   │   │   ├── disambiguation_qa.yaml
│   │   │   │   ├── dyck_languages.yaml
│   │   │   │   ├── formal_fallacies.yaml
│   │   │   │   ├── geometric_shapes.yaml
│   │   │   │   ├── hyperbaton.yaml
│   │   │   │   ├── logical_deduction_five_objects.yaml
│   │   │   │   ├── logical_deduction_seven_objects.yaml
│   │   │   │   ├── logical_deduction_three_objects.yaml
│   │   │   │   ├── movie_recommendation.yaml
│   │   │   │   ├── multistep_arithmetic_two.yaml
│   │   │   │   ├── navigate.yaml
│   │   │   │   ├── object_counting.yaml
│   │   │   │   ├── penguins_in_a_table.yaml
│   │   │   │   ├── reasoning_about_colored_objects.yaml
│   │   │   │   ├── ruin_names.yaml
│   │   │   │   ├── salient_translation_error_detection.yaml
│   │   │   │   ├── snarks.yaml
│   │   │   │   ├── sports_understanding.yaml
│   │   │   │   ├── temporal_sequences.yaml
│   │   │   │   ├── tracking_shuffled_objects_five_objects.yaml
│   │   │   │   ├── tracking_shuffled_objects_seven_objects.yaml
│   │   │   │   ├── tracking_shuffled_objects_three_objects.yaml
│   │   │   │   ├── utils.py
│   │   │   │   ├── web_of_lies.yaml
│   │   │   │   └── word_sorting.yaml
│   │   │   ├── fewshot/
│   │   │   │   ├── _bbh_fewshot.yaml
│   │   │   │   ├── _fewshot_template_yaml
│   │   │   │   ├── boolean_expressions.yaml
│   │   │   │   ├── causal_judgement.yaml
│   │   │   │   ├── date_understanding.yaml
│   │   │   │   ├── disambiguation_qa.yaml
│   │   │   │   ├── dyck_languages.yaml
│   │   │   │   ├── formal_fallacies.yaml
│   │   │   │   ├── geometric_shapes.yaml
│   │   │   │   ├── hyperbaton.yaml
│   │   │   │   ├── logical_deduction_five_objects.yaml
│   │   │   │   ├── logical_deduction_seven_objects.yaml
│   │   │   │   ├── logical_deduction_three_objects.yaml
│   │   │   │   ├── movie_recommendation.yaml
│   │   │   │   ├── multistep_arithmetic_two.yaml
│   │   │   │   ├── navigate.yaml
│   │   │   │   ├── object_counting.yaml
│   │   │   │   ├── penguins_in_a_table.yaml
│   │   │   │   ├── reasoning_about_colored_objects.yaml
│   │   │   │   ├── ruin_names.yaml
│   │   │   │   ├── salient_translation_error_detection.yaml
│   │   │   │   ├── snarks.yaml
│   │   │   │   ├── sports_understanding.yaml
│   │   │   │   ├── temporal_sequences.yaml
│   │   │   │   ├── tracking_shuffled_objects_five_objects.yaml
│   │   │   │   ├── tracking_shuffled_objects_seven_objects.yaml
│   │   │   │   ├── tracking_shuffled_objects_three_objects.yaml
│   │   │   │   ├── web_of_lies.yaml
│   │   │   │   └── word_sorting.yaml
│   │   │   └── zeroshot/
│   │   │       ├── _bbh_zeroshot.yaml
│   │   │       ├── _zeroshot_template_yaml
│   │   │       ├── boolean_expressions.yaml
│   │   │       ├── causal_judgement.yaml
│   │   │       ├── date_understanding.yaml
│   │   │       ├── disambiguation_qa.yaml
│   │   │       ├── dyck_languages.yaml
│   │   │       ├── formal_fallacies.yaml
│   │   │       ├── geometric_shapes.yaml
│   │   │       ├── hyperbaton.yaml
│   │   │       ├── logical_deduction_five_objects.yaml
│   │   │       ├── logical_deduction_seven_objects.yaml
│   │   │       ├── logical_deduction_three_objects.yaml
│   │   │       ├── movie_recommendation.yaml
│   │   │       ├── multistep_arithmetic_two.yaml
│   │   │       ├── navigate.yaml
│   │   │       ├── object_counting.yaml
│   │   │       ├── penguins_in_a_table.yaml
│   │   │       ├── reasoning_about_colored_objects.yaml
│   │   │       ├── ruin_names.yaml
│   │   │       ├── salient_translation_error_detection.yaml
│   │   │       ├── snarks.yaml
│   │   │       ├── sports_understanding.yaml
│   │   │       ├── temporal_sequences.yaml
│   │   │       ├── tracking_shuffled_objects_five_objects.yaml
│   │   │       ├── tracking_shuffled_objects_seven_objects.yaml
│   │   │       ├── tracking_shuffled_objects_three_objects.yaml
│   │   │       ├── utils.py
│   │   │       ├── web_of_lies.yaml
│   │   │       └── word_sorting.yaml
│   │   ├── bbq/
│   │   │   ├── README.md
│   │   │   ├── bbq_generate.yaml
│   │   │   ├── bbq_generate_ambig.yaml
│   │   │   ├── bbq_generate_disambig.yaml
│   │   │   ├── bbq_multiple_choice.yaml
│   │   │   ├── bbq_multiple_choice_ambig.yaml
│   │   │   ├── bbq_multiple_choice_disambig.yaml
│   │   │   └── utils.py
│   │   ├── bear/
│   │   │   ├── README.md
│   │   │   ├── bear.yaml
│   │   │   └── bear_big.yaml
│   │   ├── belebele/
│   │   │   ├── README.md
│   │   │   ├── _belebele.yaml
│   │   │   ├── _default_template_yaml
│   │   │   ├── _generate_configs.py
│   │   │   ├── belebele_acm_Arab.yaml
│   │   │   ├── belebele_afr_Latn.yaml
│   │   │   ├── belebele_als_Latn.yaml
│   │   │   ├── belebele_amh_Ethi.yaml
│   │   │   ├── belebele_apc_Arab.yaml
│   │   │   ├── belebele_arb_Arab.yaml
│   │   │   ├── belebele_arb_Latn.yaml
│   │   │   ├── belebele_ars_Arab.yaml
│   │   │   ├── belebele_ary_Arab.yaml
│   │   │   ├── belebele_arz_Arab.yaml
│   │   │   ├── belebele_asm_Beng.yaml
│   │   │   ├── belebele_azj_Latn.yaml
│   │   │   ├── belebele_bam_Latn.yaml
│   │   │   ├── belebele_ben_Beng.yaml
│   │   │   ├── belebele_ben_Latn.yaml
│   │   │   ├── belebele_bod_Tibt.yaml
│   │   │   ├── belebele_bul_Cyrl.yaml
│   │   │   ├── belebele_cat_Latn.yaml
│   │   │   ├── belebele_ceb_Latn.yaml
│   │   │   ├── belebele_ces_Latn.yaml
│   │   │   ├── belebele_ckb_Arab.yaml
│   │   │   ├── belebele_dan_Latn.yaml
│   │   │   ├── belebele_deu_Latn.yaml
│   │   │   ├── belebele_ell_Grek.yaml
│   │   │   ├── belebele_eng_Latn.yaml
│   │   │   ├── belebele_est_Latn.yaml
│   │   │   ├── belebele_eus_Latn.yaml
│   │   │   ├── belebele_fin_Latn.yaml
│   │   │   ├── belebele_fra_Latn.yaml
│   │   │   ├── belebele_fuv_Latn.yaml
│   │   │   ├── belebele_gaz_Latn.yaml
│   │   │   ├── belebele_grn_Latn.yaml
│   │   │   ├── belebele_guj_Gujr.yaml
│   │   │   ├── belebele_hat_Latn.yaml
│   │   │   ├── belebele_hau_Latn.yaml
│   │   │   ├── belebele_heb_Hebr.yaml
│   │   │   ├── belebele_hin_Deva.yaml
│   │   │   ├── belebele_hin_Latn.yaml
│   │   │   ├── belebele_hrv_Latn.yaml
│   │   │   ├── belebele_hun_Latn.yaml
│   │   │   ├── belebele_hye_Armn.yaml
│   │   │   ├── belebele_ibo_Latn.yaml
│   │   │   ├── belebele_ilo_Latn.yaml
│   │   │   ├── belebele_ind_Latn.yaml
│   │   │   ├── belebele_isl_Latn.yaml
│   │   │   ├── belebele_ita_Latn.yaml
│   │   │   ├── belebele_jav_Latn.yaml
│   │   │   ├── belebele_jpn_Jpan.yaml
│   │   │   ├── belebele_kac_Latn.yaml
│   │   │   ├── belebele_kan_Knda.yaml
│   │   │   ├── belebele_kat_Geor.yaml
│   │   │   ├── belebele_kaz_Cyrl.yaml
│   │   │   ├── belebele_kea_Latn.yaml
│   │   │   ├── belebele_khk_Cyrl.yaml
│   │   │   ├── belebele_khm_Khmr.yaml
│   │   │   ├── belebele_kin_Latn.yaml
│   │   │   ├── belebele_kir_Cyrl.yaml
│   │   │   ├── belebele_kor_Hang.yaml
│   │   │   ├── belebele_lao_Laoo.yaml
│   │   │   ├── belebele_lin_Latn.yaml
│   │   │   ├── belebele_lit_Latn.yaml
│   │   │   ├── belebele_lug_Latn.yaml
│   │   │   ├── belebele_luo_Latn.yaml
│   │   │   ├── belebele_lvs_Latn.yaml
│   │   │   ├── belebele_mal_Mlym.yaml
│   │   │   ├── belebele_mar_Deva.yaml
│   │   │   ├── belebele_mkd_Cyrl.yaml
│   │   │   ├── belebele_mlt_Latn.yaml
│   │   │   ├── belebele_mri_Latn.yaml
│   │   │   ├── belebele_mya_Mymr.yaml
│   │   │   ├── belebele_nld_Latn.yaml
│   │   │   ├── belebele_nob_Latn.yaml
│   │   │   ├── belebele_npi_Deva.yaml
│   │   │   ├── belebele_npi_Latn.yaml
│   │   │   ├── belebele_nso_Latn.yaml
│   │   │   ├── belebele_nya_Latn.yaml
│   │   │   ├── belebele_ory_Orya.yaml
│   │   │   ├── belebele_pan_Guru.yaml
│   │   │   ├── belebele_pbt_Arab.yaml
│   │   │   ├── belebele_pes_Arab.yaml
│   │   │   ├── belebele_plt_Latn.yaml
│   │   │   ├── belebele_pol_Latn.yaml
│   │   │   ├── belebele_por_Latn.yaml
│   │   │   ├── belebele_ron_Latn.yaml
│   │   │   ├── belebele_rus_Cyrl.yaml
│   │   │   ├── belebele_shn_Mymr.yaml
│   │   │   ├── belebele_sin_Latn.yaml
│   │   │   ├── belebele_sin_Sinh.yaml
│   │   │   ├── belebele_slk_Latn.yaml
│   │   │   ├── belebele_slv_Latn.yaml
│   │   │   ├── belebele_sna_Latn.yaml
│   │   │   ├── belebele_snd_Arab.yaml
│   │   │   ├── belebele_som_Latn.yaml
│   │   │   ├── belebele_sot_Latn.yaml
│   │   │   ├── belebele_spa_Latn.yaml
│   │   │   ├── belebele_srp_Cyrl.yaml
│   │   │   ├── belebele_ssw_Latn.yaml
│   │   │   ├── belebele_sun_Latn.yaml
│   │   │   ├── belebele_swe_Latn.yaml
│   │   │   ├── belebele_swh_Latn.yaml
│   │   │   ├── belebele_tam_Taml.yaml
│   │   │   ├── belebele_tel_Telu.yaml
│   │   │   ├── belebele_tgk_Cyrl.yaml
│   │   │   ├── belebele_tgl_Latn.yaml
│   │   │   ├── belebele_tha_Thai.yaml
│   │   │   ├── belebele_tir_Ethi.yaml
│   │   │   ├── belebele_tsn_Latn.yaml
│   │   │   ├── belebele_tso_Latn.yaml
│   │   │   ├── belebele_tur_Latn.yaml
│   │   │   ├── belebele_ukr_Cyrl.yaml
│   │   │   ├── belebele_urd_Arab.yaml
│   │   │   ├── belebele_urd_Latn.yaml
│   │   │   ├── belebele_uzn_Latn.yaml
│   │   │   ├── belebele_vie_Latn.yaml
│   │   │   ├── belebele_war_Latn.yaml
│   │   │   ├── belebele_wol_Latn.yaml
│   │   │   ├── belebele_xho_Latn.yaml
│   │   │   ├── belebele_yor_Latn.yaml
│   │   │   ├── belebele_zho_Hans.yaml
│   │   │   ├── belebele_zho_Hant.yaml
│   │   │   ├── belebele_zsm_Latn.yaml
│   │   │   └── belebele_zul_Latn.yaml
│   │   ├── benchmarks/
│   │   │   ├── README.md
│   │   │   ├── flan/
│   │   │   │   ├── _held_in_template_yaml
│   │   │   │   ├── flan_held_in.yaml
│   │   │   │   └── flan_held_out.yaml
│   │   │   ├── minerva_math.yaml
│   │   │   ├── multimedqa/
│   │   │   │   ├── README.md
│   │   │   │   └── multimedqa.yaml
│   │   │   ├── openllm.yaml
│   │   │   ├── pythia.yaml
│   │   │   └── t0_eval.yaml
│   │   ├── bertaqa/
│   │   │   ├── README.md
│   │   │   ├── _bertaqa_template
│   │   │   ├── bertaqa_en.yaml
│   │   │   ├── bertaqa_en_mt_gemma-7b.yaml
│   │   │   ├── bertaqa_en_mt_hitz.yaml
│   │   │   ├── bertaqa_en_mt_itzuli.yaml
│   │   │   ├── bertaqa_en_mt_latxa-13b-v1.1.yaml
│   │   │   ├── bertaqa_en_mt_latxa-13b-v1.yaml
│   │   │   ├── bertaqa_en_mt_latxa-70b-v1.1.yaml
│   │   │   ├── bertaqa_en_mt_latxa-70b-v1.yaml
│   │   │   ├── bertaqa_en_mt_latxa-7b-v1.1.yaml
│   │   │   ├── bertaqa_en_mt_latxa-7b-v1.yaml
│   │   │   ├── bertaqa_en_mt_llama-2-13b.yaml
│   │

SYMBOL INDEX (4474 symbols across 720 files)

FILE: examples/transformer-lens.py
  function evaluate_lm_eval (line 12) | def evaluate_lm_eval(lens_model: HookedTransformer, tasks: list[str], **...

FILE: lm_eval/__init__.py
  function __getattr__ (line 17) | def __getattr__(name):

FILE: lm_eval/__main__.py
  function cli_evaluate (line 5) | def cli_evaluate() -> None:

FILE: lm_eval/_cli/harness.py
  class HarnessCLI (line 10) | class HarnessCLI:
    method __init__ (line 13) | def __init__(self):
    method parse_args (line 46) | def parse_args(self) -> argparse.Namespace:
    method execute (line 58) | def execute(self, args: argparse.Namespace) -> None:

FILE: lm_eval/_cli/ls.py
  class List (line 7) | class List(SubCommand):
    method __init__ (line 10) | def __init__(self, subparsers: argparse._SubParsersAction, *args, **kw...
    method _add_args (line 51) | def _add_args(self) -> None:
    method _execute (line 66) | def _execute(self, args: argparse.Namespace) -> None:

FILE: lm_eval/_cli/run.py
  class Run (line 18) | class Run(SubCommand):
    method __init__ (line 21) | def __init__(self, subparsers: argparse._SubParsersAction, *args, **kw...
    method _add_args (line 49) | def _add_args(self) -> None:
    method _execute (line 338) | def _execute(args: argparse.Namespace) -> None:

FILE: lm_eval/_cli/subcommand.py
  class SubCommand (line 5) | class SubCommand(ABC):
    method __init__ (line 8) | def __init__(self, *args, **kwargs):
    method create (line 12) | def create(cls, subparsers: argparse._SubParsersAction):
    method _add_args (line 17) | def _add_args(self) -> None:

FILE: lm_eval/_cli/utils.py
  function try_parse_json (line 12) | def try_parse_json(value: str | dict[str, Any] | None) -> str | dict[str...
  function _int_or_none_list_arg_type (line 28) | def _int_or_none_list_arg_type(
  function request_caching_arg_to_dict (line 66) | def request_caching_arg_to_dict(cache_requests: str | None) -> dict[str,...
  function check_argument_types (line 81) | def check_argument_types(parser: argparse.ArgumentParser) -> None:
  function handle_cli_value_string (line 95) | def handle_cli_value_string(arg: str) -> bool | int | float | str:
  function key_val_to_dict (line 111) | def key_val_to_dict(args: str) -> dict[str, Any]:
  class MergeDictAction (line 125) | class MergeDictAction(argparse.Action):
    method __call__ (line 128) | def __call__(
  class SplitArgs (line 159) | class SplitArgs(argparse.Action):
    method __call__ (line 160) | def __call__(self, parser, namespace, values, option_string=None):

FILE: lm_eval/_cli/validate.py
  class Validate (line 8) | class Validate(SubCommand):
    method __init__ (line 11) | def __init__(self, subparsers: argparse._SubParsersAction, *args, **kw...
    method _add_args (line 78) | def _add_args(self) -> None:
    method _execute (line 95) | def _execute(self, args: argparse.Namespace) -> None:

FILE: lm_eval/api/filter.py
  class Filter (line 8) | class Filter(ABC):
    method __init__ (line 17) | def __init__(self, **kwargs) -> None:
    method apply (line 23) | def apply(self, resps: Union[List, Iterable], docs: List[dict]) -> Ite...
  class FilterEnsemble (line 34) | class FilterEnsemble:
    method apply (line 45) | def apply(self, instances: List[Instance]) -> None:

FILE: lm_eval/api/group.py
  class Group (line 34) | class Group:
    method add (line 61) | def add(self, item: Task | Group) -> None:
    method pop (line 69) | def pop(self, name: str) -> Group | Task | None:
    method get (line 73) | def get(self, name: str) -> Task | Group | None:
    method __contains__ (line 77) | def __contains__(self, name: str) -> bool:
    method __iter__ (line 81) | def __iter__(self):
    method __len__ (line 85) | def __len__(self) -> int:
    method get_all_tasks (line 91) | def get_all_tasks(self, recursive: bool = True) -> list[Task]:
    method get_all_groups (line 112) | def get_all_groups(self, recursive: bool = True) -> list[Group]:
    method child_names (line 132) | def child_names(self) -> list[str]:
    method version (line 137) | def version(self) -> str:
    method has_aggregation (line 142) | def has_aggregation(self) -> bool:
    method _discover_filters_for_metric (line 149) | def _discover_filters_for_metric(
    method aggregate (line 183) | def aggregate(self, task_metrics: dict[str, _TaskMetrics]) -> _TaskMet...
    method to_dict (line 285) | def to_dict(self) -> dict[str, Any] | None:
    method from_config (line 303) | def from_config(cls, config: GroupConfig | dict[str, Any]) -> Group:
    method __repr__ (line 323) | def __repr__(self):
  class ConfigurableGroup (line 333) | class ConfigurableGroup(Group):
    method __init__ (line 336) | def __init__(self, config: dict | GroupConfig | None = None) -> None:
    method group (line 350) | def group(self):
    method group_alias (line 354) | def group_alias(self):
    method version (line 358) | def version(self) -> str:
    method config (line 364) | def config(self):
    method group_name (line 368) | def group_name(self):
    method from_group (line 372) | def from_group(cls, group: Group) -> ConfigurableGroup:
    method __eq__ (line 385) | def __eq__(self, other):
    method __hash__ (line 390) | def __hash__(self):
    method __repr__ (line 393) | def __repr__(self):

FILE: lm_eval/api/instance.py
  class Instance (line 11) | class Instance:
    method __post_init__ (line 27) | def __post_init__(self) -> None:
    method args (line 32) | def args(self):

FILE: lm_eval/api/metrics.py
  function bypass_agg (line 23) | def bypass_agg(arr):
  function nanmean (line 28) | def nanmean(arr):
  function mean (line 35) | def mean(arr):
  function median (line 40) | def median(arr):
  function perplexity (line 47) | def perplexity(items):
  function weighted_perplexity (line 52) | def weighted_perplexity(items):
  function bits_per_byte (line 57) | def bits_per_byte(items):
  function f1_score (line 62) | def f1_score(items):
  function matthews_corrcoef (line 74) | def matthews_corrcoef(items):
  function bleu (line 84) | def bleu(items):
  function chrf (line 102) | def chrf(items):
  function ter (line 117) | def ter(items):
  function brier_score (line 133) | def brier_score(items):  # This is a passthrough function
  function brier_score_fn (line 148) | def brier_score_fn(items):  # This is a passthrough function
  function acc_fn (line 158) | def acc_fn(items):  # This is a passthrough function
  function acc_norm_fn (line 168) | def acc_norm_fn(items):  # This is a passthrough function
  function acc_mutual_info_fn (line 178) | def acc_mutual_info_fn(items):  # This is a passthrough function
  function acc_bytes_fn (line 188) | def acc_bytes_fn(items):  # This is a passthrough function
  function exact_match_hf_evaluate (line 210) | def exact_match_hf_evaluate(
  function exact_match_fn (line 254) | def exact_match_fn(**kwargs):
  function perplexity_fn (line 264) | def perplexity_fn(items):  # This is a passthrough function
  function likelihood_fn (line 274) | def likelihood_fn(items):  # This is a passthrough function
  function word_perplexity_fn (line 284) | def word_perplexity_fn(items):  # This is a passthrough function
  function byte_perplexity_fn (line 294) | def byte_perplexity_fn(items):  # This is a passthrough function
  function bits_per_byte_fn (line 304) | def bits_per_byte_fn(items):  # This is a passthrough function
  function pop_stddev (line 308) | def pop_stddev(arr):
  function sample_stddev (line 313) | def sample_stddev(arr: Sequence[T]) -> float:
  function mean_stderr (line 318) | def mean_stderr(arr):
  function bypass (line 328) | def bypass(items):
  function mcc_fn (line 338) | def mcc_fn(items):  # This is a passthrough function
  function f1_fn (line 348) | def f1_fn(items):  # This is a passthrough function
  function bleu_fn (line 358) | def bleu_fn(items):  # This is a passthrough function
  function chrf_fn (line 368) | def chrf_fn(items):  # This is a passthrough function
  function ter_fn (line 378) | def ter_fn(items):  # This is a passthrough function
  function acc_all (line 388) | def acc_all(items):
  function acc_all_stderr (line 407) | def acc_all_stderr(items):
  function metric_max_over_ground_truths (line 425) | def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
  function weighted_mean (line 434) | def weighted_mean(items):
  function is_non_str_iterable (line 439) | def is_non_str_iterable(obj):
  function _sacreformat (line 443) | def _sacreformat(refs, preds):
  class _bootstrap_internal (line 474) | class _bootstrap_internal:
    method __init__ (line 480) | def __init__(self, f: Callable[[Sequence[T]], float], n: int) -> None:
    method __call__ (line 484) | def __call__(self, v: tuple[int, Sequence[T]]) -> list[float]:
  function _bootstrap_internal_no_mp (line 494) | def _bootstrap_internal_no_mp(
  function bootstrap_stderr (line 516) | def bootstrap_stderr(
  function stderr_for_metric (line 555) | def stderr_for_metric(
  function pooled_sample_stderr (line 590) | def pooled_sample_stderr(stderrs: List[float], sizes: List[int]):
  function combined_sample_stderr (line 608) | def combined_sample_stderr(stderrs: List[float], sizes: List[int], metri...
  function aggregate_subtask_metrics (line 640) | def aggregate_subtask_metrics(metrics, sizes, weight_by_size=True):

FILE: lm_eval/api/model.py
  class LM (line 25) | class LM(abc.ABC):
    method __init__ (line 32) | def __init__(self) -> None:
    method loglikelihood (line 40) | def loglikelihood(self, requests: list["Instance"]) -> list[tuple[floa...
    method loglikelihood_rolling (line 58) | def loglikelihood_rolling(self, requests: list["Instance"]) -> list[fl...
    method generate_until (line 100) | def generate_until(self, requests: list["Instance"]) -> list[str]:
    method apply_chat_template (line 113) | def apply_chat_template(
    method create_from_arg_string (line 131) | def create_from_arg_string(
    method create_from_arg_obj (line 149) | def create_from_arg_obj(
    method device (line 176) | def device(self):
    method rank (line 180) | def rank(self) -> int:
    method world_size (line 185) | def world_size(self) -> int:
    method all_gather (line 189) | def all_gather(self, tensor):
    method gather_object (line 196) | def gather_object(self, obj, dst=0):
    method barrier (line 203) | def barrier(self) -> None:
    method tokenizer_name (line 208) | def tokenizer_name(self) -> str:
    method chat_template (line 217) | def chat_template(self, chat_template: bool | str = False) -> str | None:
    method set_cache_hook (line 225) | def set_cache_hook(self, cache_hook: "CacheHook") -> None:
  function hash_args (line 230) | def hash_args(attr: str, args: Iterable[Any]) -> str:
  class CacheHook (line 235) | class CacheHook:
    method __init__ (line 236) | def __init__(self, cachinglm: Optional["CachingLM"]) -> None:
    method add_partial (line 243) | def add_partial(self, attr: str, req: Iterable[Any], res: Any) -> None:
  class CachingLM (line 250) | class CachingLM:
    method __init__ (line 251) | def __init__(self, lm: LM, cache_db: str) -> None:
    method __getattr__ (line 269) | def __getattr__(self, attr: str) -> Any:
    method get_cache_hook (line 327) | def get_cache_hook(self) -> "CacheHook":
  class TemplateLM (line 331) | class TemplateLM(LM):
    method eot_token_id (line 343) | def eot_token_id(self) -> int:
    method prefix_token_id (line 347) | def prefix_token_id(self):
    method tok_encode (line 352) | def tok_encode(
    method _loglikelihood_tokens (line 363) | def _loglikelihood_tokens(
    method _encode_pair (line 368) | def _encode_pair(
    method loglikelihood (line 408) | def loglikelihood(
    method loglikelihood_rolling (line 449) | def loglikelihood_rolling(
    method generate_until (line 455) | def generate_until(self, requests, disable_tqdm: bool = False) -> list...
    method chat_template (line 458) | def chat_template(self, chat_template: bool | str = False) -> str | None:

FILE: lm_eval/api/registry.py
  function _materialise_placeholder (line 101) | def _materialise_placeholder(ph: Placeholder) -> Any:
  function _suggest_similar (line 125) | def _suggest_similar(
  function _build_key_error_msg (line 142) | def _build_key_error_msg(name: str, alias: str, keys: Iterable[str]) -> ...
  class Registry (line 156) | class Registry(Generic[T]):
    method __init__ (line 164) | def __init__(
    method register (line 183) | def register(
    method _materialise (line 261) | def _materialise(self, ph: Placeholder) -> T:
    method get (line 273) | def get(self, alias: str) -> T: ...
    method get (line 276) | def get(self, alias: str, default: D) -> T | D: ...
    method get (line 278) | def get(self, alias: str, default: D | Any = _MISSING) -> T | D:
    method __getitem__ (line 329) | def __getitem__(self, alias: str) -> T:
    method __contains__ (line 333) | def __contains__(self, alias: str) -> bool:
    method __iter__ (line 337) | def __iter__(self):
    method __len__ (line 341) | def __len__(self):
    method __repr__ (line 345) | def __repr__(self) -> str:
    method keys (line 352) | def keys(self):
    method values (line 356) | def values(self):
    method items (line 363) | def items(self):
    method origin (line 372) | def origin(self, alias: str) -> str | None:
    method freeze (line 391) | def freeze(self):
    method _clear (line 402) | def _clear(self):  # pragma: no cover
  function freeze_all (line 426) | def freeze_all():
  function register_model (line 465) | def register_model(*names):
  function get_model (line 491) | def get_model(model_name: str):
  function register_filter (line 525) | def register_filter(name: str):
  function get_filter (line 545) | def get_filter(filter_name: str | Callable) -> Callable:
  function register_metric (line 575) | def register_metric(**args):
  function get_metric (line 609) | def get_metric(name: str, hf_evaluate_metric: bool = False) -> Callable ...
  function register_aggregation (line 643) | def register_aggregation(name: str):
  function get_aggregation (line 660) | def get_aggregation(name: str) -> Callable[..., float] | None:
  function get_metric_aggregation (line 680) | def get_metric_aggregation(name: str) -> Callable[..., float] | None:
  function is_higher_better (line 700) | def is_higher_better(metric_name: str) -> bool | None:

FILE: lm_eval/api/samplers.py
  class ContextSampler (line 17) | class ContextSampler:
    method __init__ (line 18) | def __init__(
    method sample (line 31) | def sample(
    method set_rnd (line 69) | def set_rnd(self, rnd: int | None):
    method replace_df (line 73) | def replace_df(self, df: Sequence[dict[str, Any]]):
    method fewshot_docs (line 78) | def fewshot_docs(self):
    method rm_eval_doc (line 88) | def rm_eval_doc(doc: _T, _iter: Iterable[_T], n=None) -> Sequence[_T]:
  class FirstNSampler (line 96) | class FirstNSampler(ContextSampler):
    method sample (line 97) | def sample(self, n: int, eval_doc=None, df=None, **kwargs):
  class BalancedSampler (line 108) | class BalancedSampler(ContextSampler):
    method sample (line 109) | def sample(self, n: int, eval_doc=None, df=None, **kwargs):
  class ManualSampler (line 118) | class ManualSampler(ContextSampler):
    method sample (line 119) | def sample(self, n: int, eval_doc=None, df=None, **kwargs):
  function get_sampler (line 130) | def get_sampler(name: str):

FILE: lm_eval/api/task.py
  class Task (line 64) | class Task(abc.ABC):
    method __init__ (line 85) | def __init__(
    method download (line 125) | def download(
    method config (line 164) | def config(self) -> TaskConfig:
    method has_training_docs (line 169) | def has_training_docs(self):
    method has_validation_docs (line 174) | def has_validation_docs(self):
    method has_test_docs (line 179) | def has_test_docs(self):
    method training_docs (line 183) | def training_docs(self) -> Iterable:
    method validation_docs (line 190) | def validation_docs(self) -> Iterable:
    method test_docs (line 197) | def test_docs(self) -> Iterable:
    method fewshot_docs (line 204) | def fewshot_docs(self) -> Iterable:
    method _process_doc (line 221) | def _process_doc(self, doc: dict) -> dict:
    method instances (line 233) | def instances(self) -> list[Instance]:
    method fewshot_examples (line 239) | def fewshot_examples(self, k, rnd):
    method doc_to_decontamination_query (line 245) | def doc_to_decontamination_query(self, doc):
    method doc_to_text (line 251) | def doc_to_text(self, doc):
    method doc_to_target (line 255) | def doc_to_target(self, doc):
    method doc_to_image (line 259) | def doc_to_image(self, doc):
    method doc_to_audio (line 262) | def doc_to_audio(self, doc):
    method doc_to_prefix (line 265) | def doc_to_prefix(self, doc):
    method build_all_requests (line 268) | def build_all_requests(
    method construct_requests (line 382) | def construct_requests(self, doc, ctx, **kwargs):
    method process_results (line 403) | def process_results(self, doc, results):
    method aggregation (line 416) | def aggregation(self):
    method higher_is_better (line 425) | def higher_is_better(self):
    method get_config (line 433) | def get_config(self, key: str) -> Any:
    method count_bytes (line 437) | def count_bytes(cls, doc):
    method count_words (line 442) | def count_words(cls, doc):
    method fewshot_context (line 447) | def fewshot_context(self, doc, num_fewshot, rnd=None, description=None...
    method apply_filters (line 505) | def apply_filters(self) -> list[Instance] | None:
    method dump_config (line 514) | def dump_config(self) -> dict:
    method set_config (line 520) | def set_config(self, key: str, value: Any, update: bool = False) -> None:
    method override_metric (line 535) | def override_metric(self, metric_name: str) -> None:
    method set_fewshot_seed (line 560) | def set_fewshot_seed(self, seed: int | None = None) -> None:
    method eval_docs (line 566) | def eval_docs(self) -> datasets.Dataset | list[dict]:
    method doc_iterator (line 576) | def doc_iterator(
    method resolve_field (line 609) | def resolve_field(doc: dict[str, Any], field: str | None = None):
    method task_name (line 614) | def task_name(self) -> str:
  class ConfigurableTask (line 618) | class ConfigurableTask(Task):
    method __init__ (line 623) | def __init__(
    method download (line 855) | def download(self, dataset_kwargs: dict[str, Any] | None = None, **kwa...
    method has_training_docs (line 875) | def has_training_docs(self) -> bool:
    method has_validation_docs (line 878) | def has_validation_docs(self) -> bool:
    method has_test_docs (line 881) | def has_test_docs(self) -> bool:
    method training_docs (line 884) | def training_docs(self) -> datasets.Dataset:
    method validation_docs (line 892) | def validation_docs(self) -> datasets.Dataset:
    method test_docs (line 900) | def test_docs(self) -> datasets.Dataset:
    method fewshot_docs (line 906) | def fewshot_docs(self):
    method fewshot_context (line 933) | def fewshot_context(
    method build_qa_turn (line 1044) | def build_qa_turn(
    method multiple_input_context (line 1109) | def multiple_input_context(
    method apply_filters (line 1160) | def apply_filters(self) -> list[Instance] | None:
    method should_decontaminate (line 1169) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 1172) | def doc_to_decontamination_query(self, doc: dict):
    method _process_doc (line 1189) | def _process_doc(self, doc: dict) -> dict:
    method doc_to_text (line 1200) | def doc_to_text(self, doc, doc_to_text=None):
    method doc_to_target (line 1236) | def doc_to_target(self, doc: Mapping, doc_to_target=None) -> int | str...
    method doc_to_choice (line 1282) | def doc_to_choice(self, doc: Any, doc_to_choice=None) -> list[str]:
    method doc_to_image (line 1308) | def doc_to_image(self, doc: Any, doc_to_image=None) -> int | str | lis...
    method doc_to_audio (line 1331) | def doc_to_audio(self, doc: Any, doc_to_audio=None) -> int | str | lis...
    method doc_to_prefix (line 1354) | def doc_to_prefix(self, doc):
    method construct_requests (line 1362) | def construct_requests(
    method process_results (line 1455) | def process_results(self, doc, results):
    method aggregation (line 1666) | def aggregation(self) -> dict:
    method higher_is_better (line 1669) | def higher_is_better(self) -> dict:
    method get_config (line 1672) | def get_config(self, key: str) -> Any:
    method task_name (line 1676) | def task_name(self) -> str:
    method __repr__ (line 1679) | def __repr__(self):
  class MultipleChoiceTask (line 1688) | class MultipleChoiceTask(Task):
    method doc_to_target (line 1691) | def doc_to_target(self, doc: dict) -> str:
    method construct_requests (line 1694) | def construct_requests(self, doc: dict, ctx: str, **kwargs) -> list[In...
    method process_results (line 1707) | def process_results(self, doc: dict, results: Iterable[tuple[float, bo...
    method higher_is_better (line 1722) | def higher_is_better(self) -> dict:
    method aggregation (line 1728) | def aggregation(self) -> dict:
  class PerplexityTask (line 1735) | class PerplexityTask(Task):
    method has_training_docs (line 1738) | def has_training_docs(self) -> bool:
    method fewshot_examples (line 1741) | def fewshot_examples(self, k: int, rnd) -> list:
    method fewshot_context (line 1748) | def fewshot_context(self, doc: dict, num_fewshot: int) -> Literal[""]:
    method higher_is_better (line 1756) | def higher_is_better(self) -> dict:
    method doc_to_decontamination_query (line 1763) | def doc_to_decontamination_query(self, doc):
    method doc_to_text (line 1766) | def doc_to_text(self, doc) -> str:
    method doc_to_target (line 1769) | def doc_to_target(self, doc):
    method construct_requests (line 1772) | def construct_requests(self, doc: dict, ctx: str | None, **kwargs):
    method process_results (line 1784) | def process_results(self, doc: dict, results: tuple[float]) -> dict:
    method aggregation (line 1794) | def aggregation(self) -> dict:
    method count_bytes (line 1802) | def count_bytes(cls, doc) -> int:
    method count_words (line 1806) | def count_words(cls, doc) -> int:

FILE: lm_eval/api/utils.py
  function maybe_delimit (line 7) | def maybe_delimit(prefix: str | None, suffix: str | None, delimiter: str...
  function requires_delimiter (line 20) | def requires_delimiter(prefix: str, suffix: str) -> bool:
  function ends_with_whitespace (line 27) | def ends_with_whitespace(s: str) -> bool:
  class Message (line 33) | class Message:
    method to_dict (line 51) | def to_dict(self) -> dict[str, str]:
    method to_text (line 55) | def to_text(self) -> str:
  function messages_to_text (line 60) | def messages_to_text(messages: list[Message]) -> str:
  function multiturn_to_singleturn (line 65) | def multiturn_to_singleturn(messages: list[Message]) -> list[dict[str, A...
  function format_turn (line 86) | def format_turn(content: str, role: str, type: str | None = None) -> dic...
  function random_task_id (line 95) | def random_task_id():

FILE: lm_eval/caching/cache.py
  function load_from_cache (line 26) | def load_from_cache(file_name: str, cache: bool = False):
  function save_to_cache (line 41) | def save_to_cache(file_name, obj):
  function delete_cache (line 53) | def delete_cache(key: str = ""):

FILE: lm_eval/config/evaluate_config.py
  class EvaluatorConfig (line 29) | class EvaluatorConfig:
    method from_cli (line 196) | def from_cli(cls, namespace: Namespace) -> "EvaluatorConfig":
    method from_config (line 231) | def from_config(cls, config_path: str | Path) -> "EvaluatorConfig":
    method load_yaml_config (line 241) | def load_yaml_config(config_path: str | Path) -> dict[str, Any]:
    method _parse_dict_args (line 261) | def _parse_dict_args(self):
    method _configure (line 268) | def _configure(self):
    method _validate_arguments (line 274) | def _validate_arguments(self):
    method _process_arguments (line 314) | def _process_arguments(self):
    method process_tasks (line 336) | def process_tasks(self, metadata: dict | None = None) -> "TaskManager":
    method _set_trust_remote_code (line 414) | def _set_trust_remote_code(self):

FILE: lm_eval/config/group.py
  class AggMetricConfig (line 7) | class AggMetricConfig:
    method __post_init__ (line 34) | def __post_init__(self):
  class GroupConfig (line 47) | class GroupConfig:
    method __post_init__ (line 93) | def __post_init__(self):
    method to_dict (line 104) | def to_dict(self, keep_callable: bool = False) -> dict[str, str]:
    method serialize_function (line 113) | def serialize_function(

FILE: lm_eval/config/task.py
  class FewshotConfig (line 21) | class FewshotConfig:
    method __post_init__ (line 43) | def __post_init__(self):
    method from_dict (line 50) | def from_dict(
  class TaskConfig (line 82) | class TaskConfig(dict):
    method __post_init__ (line 130) | def __post_init__(self) -> None:
    method __getitem__ (line 170) | def __getitem__(self, item):
    method __setitem__ (line 173) | def __setitem__(self, item, value):
    method to_dict (line 176) | def to_dict(self, keep_callable: bool = False) -> dict:
    method serialize_function (line 204) | def serialize_function(

FILE: lm_eval/decontamination/archiver.py
  function json_serial (line 14) | def json_serial(obj: Any) -> str:
  class Archive (line 23) | class Archive:
    method __init__ (line 24) | def __init__(self, file_path: str, compression_level: int = 3) -> None:
    method add_data (line 33) | def add_data(self, data, meta=None) -> None:
    method commit (line 43) | def commit(self) -> None:
  class Reader (line 50) | class Reader:
    method __init__ (line 51) | def __init__(self) -> None:
    method read (line 54) | def read(
  class TextArchive (line 84) | class TextArchive:
    method __init__ (line 85) | def __init__(self, file_path, mode: str = "rb+") -> None:
    method add_data (line 96) | def add_data(self, data) -> None:
    method commit (line 99) | def commit(self) -> None:
  class TextReader (line 104) | class TextReader:
    method __init__ (line 105) | def __init__(self, file_path) -> None:
    method read_tqdm (line 110) | def read_tqdm(self, update_frequency: int = 10000):
    method read_and_tell (line 134) | def read_and_tell(self):
    method read (line 145) | def read(self):
    method read_slow (line 152) | def read_slow(self):
  class ZStdTextReader (line 164) | class ZStdTextReader:
    method __init__ (line 165) | def __init__(self, file) -> None:
    method read_tqdm (line 168) | def read_tqdm(self):

FILE: lm_eval/decontamination/decontaminate.py
  function get_train_overlap_stub (line 14) | def get_train_overlap_stub(docs: dict, ngrams_path: str, ngrams_n_size: ...
  function get_train_overlap (line 37) | def get_train_overlap(docs_by_task_set: dict, ngrams_path: str, limit: i...

FILE: lm_eval/decontamination/janitor.py
  function form_ngrams (line 25) | def form_ngrams(sequence: Iterator[T], n: int) -> Iterator[Tuple[T, ...]]:
  function word_ngrams (line 42) | def word_ngrams(s: str, n: int) -> Iterator[str]:
  function split_indices (line 74) | def split_indices(s: str) -> Iterator[Tuple[str, Tuple[int, int]]]:
  function word_ngrams_indices (line 81) | def word_ngrams_indices(s: str, n: int) -> Iterator[Tuple[str, Tuple[int...
  class Janitor (line 109) | class Janitor:
    method __init__ (line 111) | def __init__(
    method save_contamination_ngrams (line 140) | def save_contamination_ngrams(self, filename: str) -> None:
    method load_contamination_ngrams (line 144) | def load_contamination_ngrams(self, filename: str) -> None:
    method register_contaminant (line 152) | def register_contaminant(self, dirt_string: str) -> None:
    method clean (line 161) | def clean(self, dirty_string: str) -> List[str]:
    method _split_chunks (line 171) | def _split_chunks(
    method register_contaminant_cpp (line 196) | def register_contaminant_cpp(self, dirt_string) -> None:
    method clean_cpp (line 201) | def clean_cpp(self, dirty_string: str) -> List[str]:
    method normalize_string (line 211) | def normalize_string(self, s: str) -> str:
    method register_contaminant_python (line 214) | def register_contaminant_python(self, dirt_string: str) -> None:
    method clean_python (line 219) | def clean_python(self, dirty_string: str) -> List[str]:

FILE: lm_eval/defaults.py
  function _strtobool (line 13) | def _strtobool(val: str) -> bool:
  function _envbool (line 25) | def _envbool(var: str, default: bool = False) -> bool:
  function default_gen_kwargs (line 38) | def default_gen_kwargs(

FILE: lm_eval/evaluator.py
  function simple_evaluate (line 54) | def simple_evaluate(
  function evaluate (line 414) | def evaluate(

FILE: lm_eval/evaluator_utils.py
  class ResultAcc (line 29) | class ResultAcc(TypedDict):
  function print_writeout (line 37) | def print_writeout(task: Task) -> None:
  function get_sample_size (line 49) | def get_sample_size(task, limit: int | float | None) -> int | None:
  function find_test_root (line 58) | def find_test_root(start_path: pathlib.Path) -> pathlib.Path:
  function run_task_tests (line 76) | def run_task_tests(task_list: list[str]):
  class EvalAcc (line 99) | class EvalAcc:
    method collect (line 120) | def collect(self) -> tuple[dict[str, _TaskMetrics], dict[str, _TaskMet...
    method _to_eval_results (line 134) | def _to_eval_results(
  function _compute_task_aggregations (line 173) | def _compute_task_aggregations(
  function _collect_results (line 222) | def _collect_results(
  function aggregate_groups (line 275) | def aggregate_groups(
  function _get_root_groups (line 302) | def _get_root_groups(groups: dict[str, Group]) -> list[Group]:
  function _collect_groups_bottom_up (line 319) | def _collect_groups_bottom_up(groups: dict[str, Group]) -> list[Group]:
  function _process_results (line 349) | def _process_results(
  function _propagate_num_fewshot (line 395) | def _propagate_num_fewshot(
  function _propagate_higher_is_better (line 404) | def _propagate_higher_is_better(
  function _log_selected_tasks (line 423) | def _log_selected_tasks(
  function _handle_back_comp (line 483) | def _handle_back_comp(

FILE: lm_eval/filters/__init__.py
  function build_filter_ensemble (line 11) | def build_filter_ensemble(

FILE: lm_eval/filters/custom.py
  class CustomFilter (line 6) | class CustomFilter(Filter):
    method __init__ (line 11) | def __init__(self, **kwargs) -> None:
    method apply (line 16) | def apply(self, resps, docs):

FILE: lm_eval/filters/decontamination.py
  class DecontaminationFilter (line 6) | class DecontaminationFilter(Filter):
    method __init__ (line 13) | def __init__(self, path) -> None:
    method apply (line 21) | def apply(self, resps, docs) -> None:

FILE: lm_eval/filters/extraction.py
  class RegexFilter (line 10) | class RegexFilter(Filter):
    method __init__ (line 18) | def __init__(
    method apply (line 33) | def apply(self, resps: list[list[str]], docs: list[dict]) -> list[list...
  class POSFilter (line 63) | class POSFilter(Filter):
    method __init__ (line 66) | def __init__(
    method apply (line 83) | def apply(self, resps, docs):
  class WhitespaceFilter (line 109) | class WhitespaceFilter(Filter):
    method apply (line 112) | def apply(self, resps: list[list[str]], docs: list[dict]) -> list[list...
  class MultiChoiceRegexFilter (line 126) | class MultiChoiceRegexFilter(RegexFilter):
    method __init__ (line 134) | def __init__(
    method apply (line 157) | def apply(self, resps: list[list[str]], docs: list[dict]) -> list[list...

FILE: lm_eval/filters/selection.py
  class TakeFirstFilter (line 13) | class TakeFirstFilter(Filter):
    method __init__ (line 14) | def __init__(self) -> None:
    method apply (line 19) | def apply(self, resps, docs):
  class TakeKFilter (line 27) | class TakeKFilter(Filter):
    method __init__ (line 28) | def __init__(self, **kwargs) -> None:
    method apply (line 33) | def apply(self, resps, docs):
  class MajorityVoteFilter (line 44) | class MajorityVoteFilter(Filter):
    method __init__ (line 45) | def __init__(self) -> None:
    method apply (line 50) | def apply(self, resps, docs):

FILE: lm_eval/filters/transformation.py
  class LowercaseFilter (line 8) | class LowercaseFilter(Filter):
    method __init__ (line 9) | def __init__(self) -> None:
    method apply (line 12) | def apply(self, resps, docs):
  class UppercaseFilter (line 20) | class UppercaseFilter(Filter):
    method __init__ (line 21) | def __init__(self) -> None:
    method apply (line 24) | def apply(self, resps, docs):
  class MapFilter (line 32) | class MapFilter(Filter):
    method __init__ (line 33) | def __init__(self, mapping_dict: dict = None, default_value=None) -> N...
    method apply (line 54) | def apply(self, resps, docs):
  class SPANFilter (line 62) | class SPANFilter(Filter):
    method __init__ (line 63) | def __init__(self) -> None:
    method apply (line 66) | def apply(self, resps, docs):

FILE: lm_eval/loggers/evaluation_tracker.py
  class GeneralConfigTracker (line 38) | class GeneralConfigTracker:
    method __init__ (line 70) | def __init__(self) -> None:
    method _get_model_name (line 75) | def _get_model_name(model_args: str | dict[str, Any] | None) -> str | ...
    method log_experiment_args (line 95) | def log_experiment_args(
    method log_end_time (line 117) | def log_end_time(self) -> None:
  class EvaluationTracker (line 123) | class EvaluationTracker:
    method __init__ (line 130) | def __init__(
    method _api (line 222) | def _api(token: str | None = None) -> "HfApi | None":
    method save_results_aggregated (line 230) | def save_results_aggregated(
    method save_results_samples (line 320) | def save_results_samples(
    method recreate_metadata_card (line 424) | def recreate_metadata_card(self) -> None:

FILE: lm_eval/loggers/utils.py
  function remove_none_pattern (line 15) | def remove_none_pattern(input_string: str) -> tuple[str, bool]:
  function _handle_non_serializable (line 37) | def _handle_non_serializable(o: Any) -> int | str | list:
  function get_commit_from_path (line 56) | def get_commit_from_path(repo_path: Path | str) -> str | None:
  function get_git_commit_hash (line 83) | def get_git_commit_hash():
  function add_env_info (line 97) | def add_env_info(storage: dict[str, Any]):
  function add_tokenizer_info (line 131) | def add_tokenizer_info(storage: dict[str, Any], lm):

FILE: lm_eval/loggers/wandb_logger.py
  function get_wandb_printer (line 16) | def get_wandb_printer() -> Literal["Printer"]:
  class WandbLogger (line 24) | class WandbLogger:
    method __init__ (line 25) | def __init__(self, init_args=None, config_args=None) -> None:
    method post_init (line 66) | def post_init(self, results: Dict[str, Any]) -> None:
    method _get_config (line 71) | def _get_config(self) -> Dict[str, Any]:
    method _sanitize_results_dict (line 82) | def _sanitize_results_dict(self) -> Tuple[Dict[str, str], Dict[str, An...
    method _log_results_as_table (line 118) | def _log_results_as_table(self) -> None:
    method _log_results_as_artifact (line 168) | def _log_results_as_artifact(self) -> None:
    method log_eval_result (line 180) | def log_eval_result(self) -> None:
    method _generate_dataset (line 196) | def _generate_dataset(
    method _log_samples_as_artifact (line 287) | def _log_samples_as_artifact(
    method log_eval_samples (line 307) | def log_eval_samples(self, samples: Dict[str, List[Dict[str, Any]]]) -...

FILE: lm_eval/models/__init__.py
  function _register_all_models (line 60) | def _register_all_models():

FILE: lm_eval/models/anthropic_llms.py
  function anthropic_completion (line 17) | def anthropic_completion(
  function anthropic_chat (line 80) | def anthropic_chat(
  class AnthropicLM (line 145) | class AnthropicLM(LM):
    method __init__ (line 148) | def __init__(
    method eot_token_id (line 186) | def eot_token_id(self):
    method max_length (line 191) | def max_length(self) -> int:
    method max_gen_toks (line 195) | def max_gen_toks(self) -> int:
    method batch_size (line 199) | def batch_size(self):
    method device (line 204) | def device(self):
    method tok_encode (line 208) | def tok_encode(self, string: str) -> List[int]:
    method tok_decode (line 211) | def tok_decode(self, tokens: List[int]) -> str:
    method _loglikelihood_tokens (line 214) | def _loglikelihood_tokens(self, requests, disable_tqdm: bool = False):
    method generate_until (line 217) | def generate_until(self, requests, disable_tqdm: bool = False) -> List...
    method _model_call (line 261) | def _model_call(self, inps):
    method _model_generate (line 265) | def _model_generate(self, context, max_length, eos_token_id):
    method loglikelihood (line 269) | def loglikelihood(self, requests, disable_tqdm: bool = False):
    method loglikelihood_rolling (line 272) | def loglikelihood_rolling(self, requests, disable_tqdm: bool = False):
  class AnthropicChat (line 277) | class AnthropicChat(LocalCompletionsAPI):
    method __init__ (line 278) | def __init__(
    method api_key (line 297) | def api_key(self):
    method header (line 307) | def header(self):
    method _create_payload (line 313) | def _create_payload(
    method parse_generations (line 359) | def parse_generations(
    method tok_encode (line 370) | def tok_encode(
    method loglikelihood (line 379) | def loglikelihood(self, requests, **kwargs):

FILE: lm_eval/models/api_models.py
  class JsonChatStr (line 54) | class JsonChatStr(NamedTuple):
    method encode (line 57) | def encode(self, encoding):
  function create_image_prompt (line 61) | def create_image_prompt(
  class TemplateAPI (line 104) | class TemplateAPI(TemplateLM):
    method __init__ (line 107) | def __init__(
    method _create_payload (line 252) | def _create_payload(
    method create_message (line 265) | def create_message(
    method parse_logprobs (line 297) | def parse_logprobs(
    method parse_generations (line 308) | def parse_generations(outputs: Union[Any, List[Any]], **kwargs) -> Lis...
    method api_key (line 313) | def api_key(self) -> str:
    method header (line 318) | def header(self) -> dict:
    method tokenizer_name (line 323) | def tokenizer_name(self) -> str:
    method apply_chat_template (line 330) | def apply_chat_template(
    method eot_token_id (line 353) | def eot_token_id(self) -> Optional[int]:
    method eos_string (line 365) | def eos_string(self) -> Optional[str]:
    method prefix_token_id (line 382) | def prefix_token_id(self) -> Optional[int]:
    method tok_encode (line 397) | def tok_encode(
    method decode_batch (line 446) | def decode_batch(self, tokens: List[List[int]]) -> List[str]:
    method model_call (line 454) | def model_call(
    method amodel_call (line 490) | async def amodel_call(
    method batch_loglikelihood_requests (line 552) | def batch_loglikelihood_requests(
    method get_batched_requests (line 575) | async def get_batched_requests(
    method _loglikelihood_tokens (line 620) | def _loglikelihood_tokens(self, requests, **kwargs) -> List[Tuple[floa...
    method generate_until (line 683) | def generate_until(
    method loglikelihood_rolling (line 832) | def loglikelihood_rolling(

FILE: lm_eval/models/dummy.py
  class DummyLM (line 11) | class DummyLM(LM):
    method __init__ (line 14) | def __init__(self, *args, write_out: bool = False, **kwargs) -> None:
    method create_from_arg_string (line 19) | def create_from_arg_string(cls, arg_string, additional_config=None):
    method loglikelihood (line 22) | def loglikelihood(self, requests, disable_tqdm: bool = False):
    method generate_until (line 33) | def generate_until(self, requests, disable_tqdm: bool = False):
    method loglikelihood_rolling (line 45) | def loglikelihood_rolling(self, requests, disable_tqdm: bool = False):
    method tokenizer (line 54) | def tokenizer(self):
    method apply_chat_template (line 59) | def apply_chat_template(

FILE: lm_eval/models/gguf.py
  function get_result (line 15) | def get_result(logprobs, context_length):
  class GGUFLM (line 37) | class GGUFLM(LM):
    method __init__ (line 38) | def __init__(self, base_url=None, max_length=2048, **kwargs):
    method gguf_completion (line 46) | def gguf_completion(
    method loglikelihood (line 75) | def loglikelihood(self, requests, disable_tqdm: bool = False):
    method generate_until (line 104) | def generate_until(self, requests, disable_tqdm: bool = False):
    method loglikelihood_rolling (line 129) | def loglikelihood_rolling(self, requests, disable_tqdm: bool = False):

FILE: lm_eval/models/hf_audiolm.py
  class HFAUDIOLMQWEN (line 22) | class HFAUDIOLMQWEN(HFLM):
    method __init__ (line 30) | def __init__(
    method _create_tokenizer (line 42) | def _create_tokenizer(
    method apply_chat_template (line 85) | def apply_chat_template(
    method _model_multimodal_generate (line 98) | def _model_multimodal_generate(self, inputs, max_length, stop, **gener...
    method tok_batch_multimodal_encode (line 124) | def tok_batch_multimodal_encode(
    method generate_until (line 165) | def generate_until(
    method loglikelihood_rolling (line 290) | def loglikelihood_rolling(self, requests: list[Instance]) -> list[float]:
    method loglikelihood (line 296) | def loglikelihood(

FILE: lm_eval/models/hf_steered.py
  function steer (line 23) | def steer(
  class SteeredModel (line 67) | class SteeredModel(HFLM):
    method __init__ (line 70) | def __init__(
    method derive_steer_config (line 147) | def derive_steer_config(cls, steer_path: str):
    method add (line 210) | def add(
    method clamp (line 231) | def clamp(
    method forward (line 270) | def forward(self, *args, **kwargs):
    method _model_call (line 274) | def _model_call(self, *args, **kwargs):
    method _model_generate (line 278) | def _model_generate(self, *args, **kwargs):

FILE: lm_eval/models/hf_vlms.py
  class HFMultimodalLM (line 30) | class HFMultimodalLM(HFLM):
    method __init__ (line 38) | def __init__(
    method _create_tokenizer (line 112) | def _create_tokenizer(
    method tok_multimodal_encode (line 158) | def tok_multimodal_encode(
    method _encode_multimodal_pair (line 188) | def _encode_multimodal_pair(self, context, continuation, images):
    method apply_chat_template (line 218) | def apply_chat_template(
    method chat_template (line 275) | def chat_template(self, chat_template: bool | str = False) -> str | None:
    method tok_batch_multimodal_encode (line 287) | def tok_batch_multimodal_encode(
    method _model_multimodal_call (line 342) | def _model_multimodal_call(self, inps, imgs, attn_mask=None, labels=No...
    method _model_multimodal_generate (line 350) | def _model_multimodal_generate(self, inputs, max_length, stop, **gener...
    method _batch_images (line 376) | def _batch_images(self, image_encs):
    method loglikelihood_rolling (line 394) | def loglikelihood_rolling(self, requests: list[Instance]) -> list[float]:
    method loglikelihood (line 403) | def loglikelihood(
    method _multimodal_loglikelihood_tokens (line 439) | def _multimodal_loglikelihood_tokens(
    method generate_until (line 625) | def generate_until(

FILE: lm_eval/models/huggingface.py
  class HFLM (line 60) | class HFLM(TemplateLM):
    method __init__ (line 70) | def __init__(
    method _get_accelerate_args (line 442) | def _get_accelerate_args(
    method config (line 529) | def config(self):
    method model (line 534) | def model(self):
    method eot_token_id (line 542) | def eot_token_id(self) -> int:
    method prefix_token_id (line 547) | def prefix_token_id(self) -> int:
    method max_length (line 556) | def max_length(self) -> int:
    method max_gen_toks (line 570) | def max_gen_toks(self) -> int:
    method batch_size (line 574) | def batch_size(self):
    method device (line 578) | def device(self):
    method rank (line 582) | def rank(self):
    method world_size (line 586) | def world_size(self):
    method all_gather (line 589) | def all_gather(self, tensor):
    method gather_object (line 594) | def gather_object(self, obj, dst=0):
    method barrier (line 601) | def barrier(self):
    method tokenizer_name (line 606) | def tokenizer_name(self) -> str:
    method _get_backend (line 609) | def _get_backend(
    method _get_config (line 669) | def _get_config(
    method _create_model (line 687) | def _create_model(
    method _create_tokenizer (line 856) | def _create_tokenizer(
    method _detect_batch_size (line 917) | def _detect_batch_size(self, requests: Sequence | None = None, pos: in...
    method tok_encode (line 976) | def tok_encode(
    method tok_batch_encode (line 1001) | def tok_batch_encode(
    method tok_decode (line 1044) | def tok_decode(self, tokens: Iterator[list[str]], skip_special_tokens:...
    method _model_call (line 1047) | def _model_call(
    method _model_generate (line 1089) | def _model_generate(
    method _select_cont_toks (line 1127) | def _select_cont_toks(
    method loglikelihood_rolling (line 1150) | def loglikelihood_rolling(
    method _batch_scheduler (line 1236) | def _batch_scheduler(self, pos, n_reordered_requests):
    method _loglikelihood_tokens (line 1253) | def _loglikelihood_tokens(
    method generate_until (line 1490) | def generate_until(
    method apply_chat_template (line 1634) | def apply_chat_template(
    method get_model_info (line 1661) | def get_model_info(self) -> dict:

FILE: lm_eval/models/ibm_watsonx_ai.py
  class LogLikelihoodResult (line 21) | class LogLikelihoodResult(NamedTuple):
  function _verify_credentials (line 26) | def _verify_credentials(creds: dict) -> None:
  function get_watsonx_credentials (line 73) | def get_watsonx_credentials() -> dict[str, str | None]:
  class WatsonxLLM (line 120) | class WatsonxLLM(LM):
    method create_from_arg_string (line 127) | def create_from_arg_string(
    method __init__ (line 191) | def __init__(
    method _has_stop_token (line 228) | def _has_stop_token(response_tokens: list[str], context_tokens: list[s...
    method _check_model_logprobs_support (line 257) | def _check_model_logprobs_support(self):
    method _get_log_likelihood (line 278) | def _get_log_likelihood(
    method generate_until (line 312) | def generate_until(self, requests: list[Instance]) -> list[str]:
    method loglikelihood (line 349) | def loglikelihood(self, requests: list[Instance]) -> list[tuple[float,...
    method loglikelihood_rolling (line 416) | def loglikelihood_rolling(self, requests) -> list[float]:
    method tokenizer_name (line 470) | def tokenizer_name(self) -> str:
    method apply_chat_template (line 473) | def apply_chat_template(

FILE: lm_eval/models/mamba_lm.py
  class MambaLMWrapper (line 10) | class MambaLMWrapper(HFLM):
    method __init__ (line 11) | def __init__(
    method _get_config (line 66) | def _get_config(
    method _create_model (line 84) | def _create_model(
    method _model_generate (line 114) | def _model_generate(self, context, max_length, stop, **generation_kwar...

FILE: lm_eval/models/megatron_lm.py
  function _add_megatron_to_path (line 74) | def _add_megatron_to_path():
  function _check_dist_ckpt (line 93) | def _check_dist_ckpt(load_path: str) -> bool:
  function _parse_extra_args (line 105) | def _parse_extra_args(extra_args: str | None) -> list[str]:
  class MegatronLMEval (line 130) | class MegatronLMEval(LM):
    method __init__ (line 154) | def __init__(
    method _validate_parallelism_config (line 247) | def _validate_parallelism_config(self, devices: int, tp: int, pp: int,...
    method _initialize_megatron (line 309) | def _initialize_megatron(self, **kwargs):
    method eot_token_id (line 595) | def eot_token_id(self) -> int:
    method prefix_token_id (line 606) | def prefix_token_id(self) -> int:
    method max_length (line 620) | def max_length(self) -> int:
    method max_gen_toks (line 624) | def max_gen_toks(self) -> int:
    method batch_size (line 628) | def batch_size(self) -> int:
    method device (line 632) | def device(self) -> torch.device:
    method rank (line 636) | def rank(self) -> int:
    method world_size (line 640) | def world_size(self) -> int:
    method accelerator (line 644) | def accelerator(self):
    method all_gather (line 648) | def all_gather(self, tensor: torch.Tensor) -> torch.Tensor:
    method gather_object (line 652) | def gather_object(self, obj, dst: int = 0):
    method barrier (line 661) | def barrier(self) -> None:
    class _Accelerator (line 665) | class _Accelerator:
      method __init__ (line 672) | def __init__(self, world_size, device):
      method wait_for_everyone (line 676) | def wait_for_everyone(self):
      method gather (line 681) | def gather(self, local_tensor):
      method gather_object (line 705) | def gather_object(self, local_obj):
    method tok_encode (line 714) | def tok_encode(self, string: str, add_special_tokens: bool = False) ->...
    method tok_decode (line 721) | def tok_decode(self, tokens: list[int]) -> str:
    method _encode_pair (line 728) | def _encode_pair(
    method _model_forward (line 744) | def _model_forward(
    method _distribute_requests (line 823) | def _distribute_requests(self, requests: list) -> tuple[list, list[int]]:
    method _gather_results (line 840) | def _gather_results(self, local_results: list, sizes: list[int]) -> list:
    method loglikelihood (line 860) | def loglikelihood(self, requests: list[Instance]) -> list[tuple[float,...
    method _loglikelihood_tokens (line 891) | def _loglikelihood_tokens(
    method loglikelihood_rolling (line 1010) | def loglikelihood_rolling(
    method generate_until (line 1055) | def generate_until(

FILE: lm_eval/models/mistral3.py
  class Mistral3LM (line 33) | class Mistral3LM(HFLM):
    method __init__ (line 44) | def __init__(self, **kwargs):
    method _get_backend (line 59) | def _get_backend(
    method _model_call (line 74) | def _model_call(
    method max_length (line 99) | def max_length(self) -> int:

FILE: lm_eval/models/nemo_lm.py
  function _patch_pretrained_cfg (line 42) | def _patch_pretrained_cfg(
  function _get_target_from_class (line 72) | def _get_target_from_class(target_class) -> str:
  function load_model (line 76) | def load_model(
  function setup_distributed_environment (line 145) | def setup_distributed_environment(trainer):
  class NeMoLM (line 168) | class NeMoLM(LM):
    method __init__ (line 169) | def __init__(
    method create_from_arg_string (line 275) | def create_from_arg_string(cls, arg_string, additional_config=None):
    method eot_token_id (line 283) | def eot_token_id(self):
    method max_length (line 290) | def max_length(self):
    method max_gen_toks (line 294) | def max_gen_toks(self):
    method batch_size (line 298) | def batch_size(self):
    method device (line 302) | def device(self):
    method rank (line 306) | def rank(self):
    method world_size (line 310) | def world_size(self):
    method all_gather (line 313) | def all_gather(self, tensor):
    method gather_object (line 320) | def gather_object(self, obj, dst=0):
    method barrier (line 327) | def barrier(self):
    method tok_encode (line 331) | def tok_encode(self, string: str):
    method tok_decode (line 334) | def tok_decode(self, tokens):
    method _encode_pair (line 337) | def _encode_pair(self, context, continuation):
    method loglikelihood (line 348) | def loglikelihood(self, requests):
    method loglikelihood_rolling (line 364) | def loglikelihood_rolling(
    method _loglikelihood_tokens (line 398) | def _loglikelihood_tokens(self, requests, disable_tqdm=False):
    method generate_until (line 491) | def generate_until(self, requests):

FILE: lm_eval/models/neuron_optimum.py
  class CustomNeuronModelForCausalLM (line 37) | class CustomNeuronModelForCausalLM(NeuronModelForCausalLM):
    method generate (line 40) | def generate(
  class NEURON_HF (line 126) | class NEURON_HF(TemplateLM):
    method __init__ (line 133) | def __init__(
    method config (line 248) | def config(self):
    method eot_token_id (line 253) | def eot_token_id(self):
    method prefix_token_id (line 258) | def prefix_token_id(self):
    method max_length (line 263) | def max_length(self):
    method max_gen_toks (line 267) | def max_gen_toks(self) -> int:
    method batch_size (line 271) | def batch_size(self):
    method device (line 275) | def device(self):
    method rank (line 280) | def rank(self):
    method world_size (line 284) | def world_size(self):
    method tok_encode (line 287) | def tok_encode(self, string: str, left_truncate_len=None, add_special_...
    method tok_batch_encode (line 300) | def tok_batch_encode(
    method tok_decode (line 329) | def tok_decode(self, tokens):
    method _model_generate (line 332) | def _model_generate(self, context, max_length, stop, **generation_kwar...
    method _select_cont_toks (line 356) | def _select_cont_toks(self, logits, contlen=None, inplen=None):
    method loglikelihood_rolling (line 366) | def loglikelihood_rolling(self, requests, disable_tqdm: bool = False):
    method _loglikelihood_tokens (line 419) | def _loglikelihood_tokens(
    method generate_until (line 568) | def generate_until(self, requests, disable_tqdm: bool = False):

FILE: lm_eval/models/openai_completions.py
  class LocalCompletionsAPI (line 16) | class LocalCompletionsAPI(TemplateAPI):
    method __init__ (line 17) | def __init__(
    method _create_payload (line 61) | def _create_payload(
    method parse_logprobs (line 99) | def parse_logprobs(
    method parse_generations (line 125) | def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> L...
    method api_key (line 137) | def api_key(self):
  class LocalChatCompletion (line 142) | class LocalChatCompletion(LocalCompletionsAPI):
    method __init__ (line 150) | def __init__(
    method _create_payload (line 175) | def _create_payload(
    method parse_generations (line 211) | def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> L...
    method tok_encode (line 229) | def tok_encode(
    method loglikelihood (line 238) | def loglikelihood(self, requests, **kwargs):
  class OpenAICompletionsAPI (line 247) | class OpenAICompletionsAPI(LocalCompletionsAPI):
    method __init__ (line 248) | def __init__(
    method api_key (line 259) | def api_key(self):
    method loglikelihood (line 268) | def loglikelihood(self, requests, **kwargs):
    method chat_template (line 277) | def chat_template(self, chat_template: Union[bool, str] = False) -> Op...
  class OpenAIChatCompletion (line 282) | class OpenAIChatCompletion(LocalChatCompletion):
    method __init__ (line 283) | def __init__(
    method api_key (line 303) | def api_key(self):
    method loglikelihood (line 312) | def loglikelihood(self, requests, **kwargs):
    method _create_payload (line 317) | def _create_payload(
  class AzureOpenaiChatCompletionsLM (line 359) | class AzureOpenaiChatCompletionsLM(OpenAIChatCompletion):
    method __init__ (line 360) | def __init__(
    method api_key (line 384) | def api_key(self):

FILE: lm_eval/models/optimum_habana.py
  class HabanaLM (line 18) | class HabanaLM(HFLM):
    method __init__ (line 30) | def __init__(self, **kwargs) -> None:
    method max_length (line 52) | def max_length(self) -> int:
    method max_length (line 57) | def max_length(self, value: int) -> None:
    method find_bucket (line 60) | def find_bucket(self, length: int, key=lambda b, length: b >= length) ...
    method _model_call (line 75) | def _model_call(self, inps: torch.Tensor) -> torch.Tensor:
    method setup_generation_config_gaudi (line 97) | def setup_generation_config_gaudi(self, **kwargs):
    method _create_model (line 108) | def _create_model(self, *args, **kwargs) -> None:
    method generate_until (line 125) | def generate_until(
    method _model_generate (line 137) | def _model_generate(

FILE: lm_eval/models/optimum_ipex.py
  class IPEXLM (line 13) | class IPEXLM(HFLM):
    method __init__ (line 18) | def __init__(
    method _create_model (line 33) | def _create_model(

FILE: lm_eval/models/optimum_lm.py
  class OptimumLM (line 14) | class OptimumLM(HFLM):
    method __init__ (line 25) | def __init__(
    method _create_model (line 43) | def _create_model(

FILE: lm_eval/models/sglang_causallms.py
  class SGLangLM (line 34) | class SGLangLM(TemplateLM):
    method __init__ (line 37) | def __init__(
    method loglikelihood_rolling (line 124) | def loglikelihood_rolling(
    method generate_until (line 193) | def generate_until(
    method _model_generate (line 288) | def _model_generate(
    method eot_token_id (line 319) | def eot_token_id(self):
    method prefix_token_id (line 324) | def prefix_token_id(self):
    method max_length (line 333) | def max_length(self):
    method max_gen_toks (line 343) | def max_gen_toks(self):
    method tok_encode (line 347) | def tok_encode(
    method tok_decode (line 372) | def tok_decode(self, tokens: List[int]) -> str:
    method tokenizer_name (line 377) | def tokenizer_name(self) -> str:
    method chat_template (line 387) | def chat_template(self, chat_template: Union[bool, str] = False) -> str:
    method apply_chat_template (line 408) | def apply_chat_template(
    method _loglikelihood_tokens (line 423) | def _loglikelihood_tokens(
    method _parse_logprobs (line 483) | def _parse_logprobs(tokens: List, outputs, ctxlen: int) -> Tuple[float...
    method modify_gen_kwargs (line 519) | def modify_gen_kwargs(kwargs: dict) -> dict:

FILE: lm_eval/models/sglang_generate_API.py
  class SGLANGGENERATEAPI (line 9) | class SGLANGGENERATEAPI(LocalCompletionsAPI):
    method __init__ (line 10) | def __init__(
    method _create_payload (line 20) | def _create_payload(
    method parse_logprobs (line 66) | def parse_logprobs(
    method parse_generations (line 90) | def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> L...
    method api_key (line 99) | def api_key(self):

FILE: lm_eval/models/textsynth.py
  function textsynth_completion (line 29) | def textsynth_completion(**kwargs):
  class TextSynthLM (line 51) | class TextSynthLM(LM):
    method __init__ (line 52) | def __init__(self, engine, truncate: bool = False, **kwargs) -> None:
    method eot_token_id (line 68) | def eot_token_id(self):
    method max_length (line 73) | def max_length(self) -> int:
    method max_gen_toks (line 78) | def max_gen_toks(self) -> int:
    method batch_size (line 82) | def batch_size(self):
    method device (line 87) | def device(self):
    method tok_encode (line 91) | def tok_encode(self, string: str):
    method tok_decode (line 95) | def tok_decode(self, tokens):
    method loglikelihood (line 99) | def loglikelihood(self, requests, disable_tqdm: bool = False):
    method loglikelihood_rolling (line 123) | def loglikelihood_rolling(self, requests, disable_tqdm: bool = False):
    method generate_until (line 133) | def generate_until(self, requests, disable_tqdm: bool = False):
    method _model_call (line 166) | def _model_call(self, inps):
    method _model_generate (line 170) | def _model_generate(self, context, max_length, eos_token_id):

FILE: lm_eval/models/utils.py
  class GenKwargs (line 33) | class GenKwargs(TypedDict, total=False):
  function chunks (line 42) | def chunks(iter, n: int = 0, fn=None):
  class MultiChoice (line 80) | class MultiChoice:
    method __init__ (line 81) | def __init__(self, choices) -> None:
    method __contains__ (line 85) | def __contains__(self, values) -> bool:
    method __iter__ (line 94) | def __iter__(self) -> Iterator:
  class Grouper (line 98) | class Grouper:
    method __init__ (line 105) | def __init__(self, arr, fn) -> None:
    method get_grouped (line 123) | def get_grouped(self):
    method get_original (line 134) | def get_original(self, grouped_dict):
  function undistribute (line 156) | def undistribute(iterable):
  function retry_on_specific_exceptions (line 196) | def retry_on_specific_exceptions(
  class Collator (line 236) | class Collator:
    method __init__ (line 249) | def __init__(
    method _group_by_index (line 270) | def _group_by_index(self) -> None:
    method _group_by_context (line 276) | def _group_by_context(self) -> None:
    method get_batched (line 282) | def get_batched(
    method get_cache (line 329) | def get_cache(
    method _reorder (line 390) | def _reorder(self, arr: list | tuple[tuple[int, Any], ...]) -> Iterator:
    method get_original (line 406) | def get_original(self, newarr: list) -> list:
    method __len__ (line 427) | def __len__(self):
    method group (line 431) | def group(
    method get_chunks (line 474) | def get_chunks(
  function configure_pad_token (line 515) | def configure_pad_token(
  function replace_placeholders (line 560) | def replace_placeholders(
  function flatten_image_list (line 594) | def flatten_image_list(images: list[list]):
  function handle_stop_sequences (line 605) | def handle_stop_sequences(until: str | list[str] | None, eos: str | None...
  function normalize_gen_kwargs (line 621) | def normalize_gen_kwargs(
  function resize_image (line 717) | def resize_image(
  function truncate_tokens (line 817) | def truncate_tokens(
  function maybe_truncate (line 836) | def maybe_truncate(
  function postprocess_generated_text (line 910) | def postprocess_generated_text(
  function has_bos_prefix (line 939) | def has_bos_prefix(sequence: str, bos_str: str | Iterable[str] | None = ...
  function _add_special_kwargs (line 948) | def _add_special_kwargs(add_special_tokens: bool | None, add_bos: bool |...

FILE: lm_eval/models/utils_hf.py
  function pad_and_concat (line 8) | def pad_and_concat(
  function clear_torch_cache (line 59) | def clear_torch_cache() -> None:
  function get_dtype (line 64) | def get_dtype(dtype: str | torch.dtype) -> torch.dtype | str:
  class MultiTokenEOSCriteria (line 74) | class MultiTokenEOSCriteria(transformers.StoppingCriteria):
    method __init__ (line 77) | def __init__(
    method __call__ (line 100) | def __call__(self, input_ids, scores, **kwargs) -> bool:
  function stop_sequences_criteria (line 114) | def stop_sequences_criteria(

FILE: lm_eval/models/vllm_causallms.py
  function _vllm_mp_worker (line 68) | def _vllm_mp_worker(
  class VLLM (line 126) | class VLLM(TemplateLM):
    method __init__ (line 130) | def __init__(
    method eot_token_id (line 288) | def eot_token_id(self):
    method prefix_token_id (line 293) | def prefix_token_id(self):
    method max_length (line 302) | def max_length(self) -> int:
    method max_gen_toks (line 319) | def max_gen_toks(self):
    method apply_chat_template (line 322) | def apply_chat_template(
    method tokenizer_name (line 355) | def tokenizer_name(self) -> str:
    method tok_encode (line 359) | def tok_encode(
    method tok_encode (line 363) | def tok_encode(
    method tok_encode (line 367) | def tok_encode(
    method _model_generate (line 428) | def _model_generate(
    method loglikelihood_rolling (line 558) | def loglikelihood_rolling(
    method generate_until (line 627) | def generate_until(
    method _loglikelihood_tokens (line 725) | def _loglikelihood_tokens(
    method _parse_logprobs (line 787) | def _parse_logprobs(tokens: list, outputs, ctxlen: int) -> tuple[float...
    method modify_gen_kwargs (line 850) | def modify_gen_kwargs(

FILE: lm_eval/models/vllm_vlms.py
  class VLLM_VLM (line 33) | class VLLM_VLM(VLLM):
    method __init__ (line 36) | def __init__(
    method tok_batch_multimodal_encode (line 76) | def tok_batch_multimodal_encode(
    method _multimodal_model_generate (line 102) | def _multimodal_model_generate(
    method apply_chat_template (line 157) | def apply_chat_template(
    method generate_until (line 214) | def generate_until(
    method loglikelihood_rolling (line 309) | def loglikelihood_rolling(

FILE: lm_eval/models/winml.py
  class WindowsML (line 32) | class WindowsML(TemplateLM):
    method create_from_arg_obj (line 43) | def create_from_arg_obj(
    method __init__ (line 67) | def __init__(
    method _validate_dependencies (line 120) | def _validate_dependencies(self) -> None:
    method _fix_winrt_runtime (line 150) | def _fix_winrt_runtime(self):
    method _register_winml_providers_to_genai (line 164) | def _register_winml_providers_to_genai(self) -> bool:
    method _setup_winml_devices_and_providers (line 199) | def _setup_winml_devices_and_providers(self) -> None:
    method _load_and_compile_model (line 238) | def _load_and_compile_model(self, model_path: str) -> None:
    method eot_token_id (line 283) | def eot_token_id(self) -> int:
    method prefix_token_id (line 310) | def prefix_token_id(self) -> int | None:
    method max_gen_toks (line 340) | def max_gen_toks(self) -> int:
    method tok_encode (line 349) | def tok_encode(
    method tok_decode (line 375) | def tok_decode(self, tokens: list[int]) -> str:
    method _run_genai_inference_for_full_logits (line 387) | def _run_genai_inference_for_full_logits(self, input_text: str) -> np....
    method _loglikelihood_tokens (line 438) | def _loglikelihood_tokens(
    method loglikelihood (line 461) | def loglikelihood(
    method loglikelihood_rolling (line 574) | def loglikelihood_rolling(
    method generate_until (line 647) | def generate_until(
    method _run_genai_generation (line 694) | def _run_genai_generation(

FILE: lm_eval/prompts/__init__.py
  function get_prompt (line 23) | def get_prompt(prompt_id: str, dataset_name: str = None, subset_name: st...
  function load_prompt_list (line 72) | def load_prompt_list(
  class PromptString (line 115) | class PromptString:
    method __init__ (line 116) | def __init__(self, prompt_string):
    method apply (line 119) | def apply(self, doc):

FILE: lm_eval/result_schema.py
  class _TaskMetrics (line 110) | class _TaskMetrics(TypedDict, Generic[T], extra_items=T):
  class _SampleCount (line 131) | class _SampleCount(TypedDict):
  class _EvalConfig (line 141) | class _EvalConfig(TypedDict, total=False):
  class SampleResult (line 163) | class SampleResult(TypedDict, extra_items=float):

FILE: lm_eval/tasks/__init__.py
  function get_task_name_from_config (line 36) | def get_task_name_from_config(task_config: dict[str, str]) -> str:
  function get_task_name_from_object (line 50) | def get_task_name_from_object(task_object):
  function _check_duplicates (line 63) | def _check_duplicates(task_dict: dict) -> None:
  function _log_task_dict (line 98) | def _log_task_dict(task_dict: dict, task_manager: "TaskManager") -> None:
  function get_task_dict (line 137) | def get_task_dict(

FILE: lm_eval/tasks/_factory.py
  class TaskFactory (line 25) | class TaskFactory:
    method __init__ (line 32) | def __init__(self, *, meta: dict[str, Any] | None = None):
    method build (line 37) | def build(
    method _build_task (line 65) | def _build_task(self, entry: Entry, overrides: dict[str, Any] | None) ...
    method _build_group (line 85) | def _build_group(
    method _build_group_members (line 127) | def _build_group_members(
    method _build_tag (line 234) | def _build_tag(
    method _load_full_config (line 255) | def _load_full_config(
  function _ctor_accepts_config (line 283) | def _ctor_accepts_config(cls) -> bool:

FILE: lm_eval/tasks/_index.py
  class Kind (line 19) | class Kind(Enum):
  class Entry (line 28) | class Entry:
  class TaskIndex (line 36) | class TaskIndex:
    method __init__ (line 41) | def __init__(self, *, meta: dict[str, str] | None = None) -> None:
    method build (line 45) | def build(
    method _iter_yaml_files (line 82) | def _iter_yaml_files(root: Path):
    method process_cfg (line 94) | def process_cfg(
    method _register_tags (line 139) | def _register_tags(
    method _kind_of (line 154) | def _kind_of(cfg: dict) -> Kind:
    method entry_from_path (line 168) | def entry_from_path(path: Path) -> Entry | None:
    method entry_from_config (line 179) | def entry_from_config(cfg: dict[str, Any]) -> Entry | None:
    method _str_to_set (line 192) | def _str_to_set(*args) -> set[str]:

FILE: lm_eval/tasks/_yaml_loader.py
  function _mk_function_ctor (line 17) | def _mk_function_ctor(base_dir: Path, resolve: bool):
  function _make_loader (line 27) | def _make_loader(base_dir: Path, *, resolve_funcs: bool) -> type[yaml.Lo...
  function _load_module_with_cache (line 38) | def _load_module_with_cache(module_path: Path) -> Any:
  function _import_func_in_yml (line 93) | def _import_func_in_yml(qual: str, base_dir: Path):
  function _import_fun_from_str (line 130) | def _import_fun_from_str(path_str: str) -> Any:
  function load_yaml (line 164) | def load_yaml(

FILE: lm_eval/tasks/aclue/_generate_configs.py
  function parse_args (line 35) | def parse_args():

FILE: lm_eval/tasks/acpbench/gen_2shot/acp_utils.py
  class ACPBench_Visitor (line 47) | class ACPBench_Visitor(Visitor):
    method __init__ (line 48) | def __init__(self) -> None:
    method action_list (line 56) | def action_list(self, tree):
    method prog_list (line 59) | def prog_list(self, tree):
    method progression_list (line 64) | def progression_list(self, tree):
    method action_none (line 67) | def action_none(self, tree):
    method action_name (line 70) | def action_name(self, tree):
    method index (line 78) | def index(self, tree):
  class ACPGrammarParser (line 84) | class ACPGrammarParser(object):
    method __init__ (line 85) | def __init__(self, task) -> None:
    method parse (line 91) | def parse(self, input, debug=False):
  function is_on_optimal_plan (line 135) | def is_on_optimal_plan(domain, problem, action, opt):
  function is_plan (line 177) | def is_plan(domain, problem, new_plan):
  function get_action_preconditions (line 196) | def get_action_preconditions(domain, problem, action):
  function generate_optimal_plans_for_problem_state (line 207) | def generate_optimal_plans_for_problem_state(P, state, num_plans, timeout):
  function generate_top_q_plans (line 228) | def generate_top_q_plans(domain, problem, num_plans=10, quality_bound=1....
  function is_unsolvable_new_goal (line 241) | def is_unsolvable_new_goal(domain, problem, new_goal):
  function is_unsolvable (line 247) | def is_unsolvable(domain, problem):
  function extract_goal (line 274) | def extract_goal(prob):
  function entails (line 288) | def entails(state, partialstate):
  function progress (line 292) | def progress(state, act):
  function regress (line 302) | def regress(state, act):
  function get_STRIPS (line 312) | def get_STRIPS(domain, problem):
  function create_tmp_dom_prob_replace_init (line 330) | def create_tmp_dom_prob_replace_init(P, state, result_domain_file, resul...
  function fix_name (line 340) | def fix_name(s):
  function get_atoms_pddl (line 354) | def get_atoms_pddl(d, p, atoms):
  class Action (line 390) | class Action:
    method __init__ (line 391) | def __init__(self, name, pre, add, delete):
    method __str__ (line 397) | def __str__(self):
    method toJSON (line 404) | def toJSON(self):
    method __repr__ (line 416) | def __repr__(self):
    method __eq__ (line 419) | def __eq__(self, action):
    method __hash__ (line 422) | def __hash__(self):
  class STRIPS (line 426) | class STRIPS:
    method __init__ (line 427) | def __init__(self, domain, problem):
    method __str__ (line 453) | def __str__(self):
    method toJSON (line 460) | def toJSON(self):
    method operator_to_action (line 473) | def operator_to_action(self, op, check_fluents=True, check_static=False):
    method fix_pre_name (line 488) | def fix_pre_name(self, precondition):
    method action (line 493) | def action(self, name):
    method get_action_or_none (line 496) | def get_action_or_none(self, name):
    method fluent (line 501) | def fluent(self, name):
    method static_symbols (line 504) | def static_symbols(self):
    method fluent_symbols (line 507) | def fluent_symbols(self):
    method get_grounded_atoms (line 510) | def get_grounded_atoms(self, symbol):
    method get_applicable_actions (line 523) | def get_applicable_actions(self, s):
    method ground_problem (line 526) | def ground_problem(self, problem):
    method get_static (line 551) | def get_static(self):
    method PDDL_replace_init_pddl_parser (line 558) | def PDDL_replace_init_pddl_parser(self, s):
  function parse_ans (line 571) | def parse_ans(response: str, parser: ACPGrammarParser, task: str):
  function remove_garbage (line 582) | def remove_garbage(s):
  function compare_str (line 593) | def compare_str(s1, s2):
  function compare (line 597) | def compare(l1, l2):
  function check_prog_response (line 608) | def check_prog_response(resp):
  function clean_answer (line 618) | def clean_answer(resp, task):
  function get_grammar_task (line 642) | def get_grammar_task(task):
  function fix_action_name (line 666) | def fix_action_name(a):
  function str_remove_before_first_parentheses (line 671) | def str_remove_before_first_parentheses(s):
  function str_remove_after_last_parentheses (line 680) | def str_remove_after_last_parentheses(s):
  function cleanup_answer (line 691) | def cleanup_answer(ans):
  function set_equal (line 710) | def set_equal(ans1, ans2):
  class BaseEvaluator (line 714) | class BaseEvaluator(ABC):
    method __init__ (line 715) | def __init__(self) -> None:
    method get_score (line 719) | def get_score(self, ans, doc):
    method add_scores (line 722) | def add_scores(self, scores):
    method get_avg_score (line 725) | def get_avg_score(self):
  function get_evaluator (line 730) | def get_evaluator(group):
  class ActionReachabilityEvaluator (line 757) | class ActionReachabilityEvaluator(BaseEvaluator):
    method get_score (line 758) | def get_score(self, ans, doc):
  class ApplicabilityEvaluator (line 801) | class ApplicabilityEvaluator(BaseEvaluator):
    method get_score (line 802) | def get_score(self, ans, doc):
  function is_subsequence (line 817) | def is_subsequence(plan, new_plan):
  function is_subsequence_and_plan (line 828) | def is_subsequence_and_plan(domain, problem, plan, new_plan):
  class JustificationEvaluator (line 842) | class JustificationEvaluator(BaseEvaluator):
    method get_score (line 843) | def get_score(self, ans, doc):
  class LandmarksEvaluator (line 883) | class LandmarksEvaluator(BaseEvaluator):
    method get_score (line 884) | def get_score(self, ans, doc):
  class NextActionEvaluator (line 916) | class NextActionEvaluator(BaseEvaluator):
    method get_score (line 917) | def get_score(self, ans, doc):
  class ProgressionEvaluator (line 961) | class ProgressionEvaluator(BaseEvaluator):
    method get_score (line 962) | def get_score(self, ans, doc):
  class ReachabilityEvaluator (line 992) | class ReachabilityEvaluator(BaseEvaluator):
    method get_score (line 993) | def get_score(self, ans, doc):
  class ValidationEvaluator (line 1029) | class ValidationEvaluator(BaseEvaluator):
    method get_score (line 1030) | def get_score(self, ans, doc):
  function dump_item (line 1049) | def dump_item(item, **kwargs):
  function parse_prediction (line 1053) | def parse_prediction(prediction):
  class ACPGrammarFilter (line 1064) | class ACPGrammarFilter(RegexFilter):
    method __init__ (line 1067) | def __init__(self, *args, **kwargs):
    method clean_pos_neg (line 1071) | def clean_pos_neg(self, resp):
    method clean_simplified_plan (line 1082) | def clean_simplified_plan(self, resp):
    method apply (line 1091) | def apply(self, resps, docs):
  function process_acp_results (line 1107) | def process_acp_results(doc, results):
  function get_score (line 1111) | def get_score(references, predictions, **kwargs):

FILE: lm_eval/tasks/acpbench/gen_2shot_with_pddl/acp_utils.py
  class ACPBench_Visitor (line 47) | class ACPBench_Visitor(Visitor):
    method __init__ (line 48) | def __init__(self) -> None:
    method action_list (line 56) | def action_list(self, tree):
    method prog_list (line 59) | def prog_list(self, tree):
    method progression_list (line 64) | def progression_list(self, tree):
    method action_none (line 67) | def action_none(self, tree):
    method action_name (line 70) | def action_name(self, tree):
    method index (line 78) | def index(self, tree):
  class ACPGrammarParser (line 84) | class ACPGrammarParser(object):
    method __init__ (line 85) | def __init__(self, task) -> None:
    method parse (line 91) | def parse(self, input, debug=False):
  function is_on_optimal_plan (line 135) | def is_on_optimal_plan(domain, problem, action, opt):
  function is_plan (line 177) | def is_plan(domain, problem, new_plan):
  function get_action_preconditions (line 196) | def get_action_preconditions(domain, problem, action):
  function generate_optimal_plans_for_problem_state (line 207) | def generate_optimal_plans_for_problem_state(P, state, num_plans, timeout):
  function generate_top_q_plans (line 228) | def generate_top_q_plans(domain, problem, num_plans=10, quality_bound=1....
  function is_unsolvable_new_goal (line 241) | def is_unsolvable_new_goal(domain, problem, new_goal):
  function is_unsolvable (line 247) | def is_unsolvable(domain, problem):
  function extract_goal (line 274) | def extract_goal(prob):
  function entails (line 288) | def entails(state, partialstate):
  function progress (line 292) | def progress(state, act):
  function regress (line 302) | def regress(state, act):
  function get_STRIPS (line 312) | def get_STRIPS(domain, problem):
  function create_tmp_dom_prob_replace_init (line 330) | def create_tmp_dom_prob_replace_init(P, state, result_domain_file, resul...
  function fix_name (line 340) | def fix_name(s):
  function get_atoms_pddl (line 354) | def get_atoms_pddl(d, p, atoms):
  class Action (line 390) | class Action:
    method __init__ (line 391) | def __init__(self, name, pre, add, delete):
    method __str__ (line 397) | def __str__(self):
    method toJSON (line 404) | def toJSON(self):
    method __repr__ (line 416) | def __repr__(self):
    method __eq__ (line 419) | def __eq__(self, action):
    method __hash__ (line 422) | def __hash__(self):
  class STRIPS (line 426) | class STRIPS:
    method __init__ (line 427) | def __init__(self, domain, problem):
    method __str__ (line 453) | def __str__(self):
    method toJSON (line 460) | def toJSON(self):
    method operator_to_action (line 473) | def operator_to_action(self, op, check_fluents=True, check_static=False):
    method fix_pre_name (line 488) | def fix_pre_name(self, precondition):
    method action (line 493) | def action(self, name):
    method get_action_or_none (line 496) | def get_action_or_none(self, name):
    method fluent (line 501) | def fluent(self, name):
    method static_symbols (line 504) | def static_symbols(self):
    method fluent_symbols (line 507) | def fluent_symbols(self):
    method get_grounded_atoms (line 510) | def get_grounded_atoms(self, symbol):
    method get_applicable_actions (line 523) | def get_applicable_actions(self, s):
    method ground_problem (line 526) | def ground_problem(self, problem):
    method get_static (line 551) | def get_static(self):
    method PDDL_replace_init_pddl_parser (line 558) | def PDDL_replace_init_pddl_parser(self, s):
  function parse_ans (line 571) | def parse_ans(response: str, parser: ACPGrammarParser, task: str):
  function remove_garbage (line 582) | def remove_garbage(s):
  function compare_str (line 593) | def compare_str(s1, s2):
  function compare (line 597) | def compare(l1, l2):
  function check_prog_response (line 608) | def check_prog_response(resp):
  function clean_answer (line 618) | def clean_answer(resp, task):
  function get_grammar_task (line 642) | def get_grammar_task(task):
  function fix_action_name (line 666) | def fix_action_name(a):
  function str_remove_before_first_parentheses (line 671) | def str_remove_before_first_parentheses(s):
  function str_remove_after_last_parentheses (line 680) | def str_remove_after_last_parentheses(s):
  function cleanup_answer (line 691) | def cleanup_answer(ans):
  function set_equal (line 710) | def set_equal(ans1, ans2):
  class BaseEvaluator (line 714) | class BaseEvaluator(ABC):
    method __init__ (line 715) | def __init__(self) -> None:
    method get_score (line 719) | def get_score(self, ans, doc):
    method add_scores (line 722) | def add_scores(self, scores):
    method get_avg_score (line 725) | def get_avg_score(self):
  function get_evaluator (line 730) | def get_evaluator(group):
  class ActionReachabilityEvaluator (line 757) | class ActionReachabilityEvaluator(BaseEvaluator):
    method get_score (line 758) | def get_score(self, ans, doc):
  class ApplicabilityEvaluator (line 801) | class ApplicabilityEvaluator(BaseEvaluator):
    method get_score (line 802) | def get_score(self, ans, doc):
  function is_subsequence (line 817) | def is_subsequence(plan, new_plan):
  function is_subsequence_and_plan (line 828) | def is_subsequence_and_plan(domain, problem, plan, new_plan):
  class JustificationEvaluator (line 842) | class JustificationEvaluator(BaseEvaluator):
    method get_score (line 843) | def get_score(self, ans, doc):
  class LandmarksEvaluator (line 883) | class LandmarksEvaluator(BaseEvaluator):
    method get_score (line 884) | def get_score(self, ans, doc):
  class NextActionEvaluator (line 916) | class NextActionEvaluator(BaseEvaluator):
    method get_score (line 917) | def get_score(self, ans, doc):
  class ProgressionEvaluator (line 961) | class ProgressionEvaluator(BaseEvaluator):
    method get_score (line 962) | def get_score(self, ans, doc):
  class ReachabilityEvaluator (line 992) | class ReachabilityEvaluator(BaseEvaluator):
    method get_score (line 993) | def get_score(self, ans, doc):
  class ValidationEvaluator (line 1029) | class ValidationEvaluator(BaseEvaluator):
    method get_score (line 1030) | def get_score(self, ans, doc):
  function dump_item (line 1049) | def dump_item(item, **kwargs):
  function parse_prediction (line 1053) | def parse_prediction(prediction):
  class ACPGrammarFilter (line 1064) | class ACPGrammarFilter(RegexFilter):
    method __init__ (line 1067) | def __init__(self, *args, **kwargs):
    method clean_pos_neg (line 1071) | def clean_pos_neg(self, resp):
    method clean_simplified_plan (line 1082) | def clean_simplified_plan(self, resp):
    method apply (line 1091) | def apply(self, resps, docs):
  function process_acp_results (line 1107) | def process_acp_results(doc, results):
  function get_score (line 1111) | def get_score(references, predictions, **kwargs):

FILE: lm_eval/tasks/afrimgsm/gen_utils.py
  class FunctionTag (line 7) | class FunctionTag:
    method __init__ (line 8) | def __init__(self, value):
  function prompt_func (line 12) | def prompt_func(mode, lang):
  function gen_lang_yamls (line 22) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 96) | def main() -> None:

FILE: lm_eval/tasks/afrimgsm/utils.py
  function add_regex_pattern (line 75) | def add_regex_pattern(regex_pattern):
  function gen_lang_yamls (line 109) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 195) | def main() -> None:

FILE: lm_eval/tasks/afrimmlu/direct/prompt_1/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 9) | def doc_to_text(doc):

FILE: lm_eval/tasks/afrimmlu/direct/prompt_2/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 9) | def doc_to_text(doc):

FILE: lm_eval/tasks/afrimmlu/direct/prompt_3/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 9) | def doc_to_text(doc):

FILE: lm_eval/tasks/afrimmlu/direct/prompt_4/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 9) | def doc_to_text(doc):

FILE: lm_eval/tasks/afrimmlu/direct/prompt_5/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 9) | def doc_to_text(doc):

FILE: lm_eval/tasks/afrimmlu/gen_utils.py
  class FunctionTag (line 7) | class FunctionTag:
    method __init__ (line 8) | def __init__(self, value):
  function gen_lang_yamls (line 12) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 77) | def main() -> None:

FILE: lm_eval/tasks/afrimmlu/translate/prompt_1/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 9) | def doc_to_text(doc):

FILE: lm_eval/tasks/afrimmlu/translate/prompt_2/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 9) | def doc_to_text(doc):

FILE: lm_eval/tasks/afrimmlu/translate/prompt_3/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 9) | def doc_to_text(doc):

FILE: lm_eval/tasks/afrimmlu/translate/prompt_4/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 9) | def doc_to_text(doc):

FILE: lm_eval/tasks/afrimmlu/translate/prompt_5/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 9) | def doc_to_text(doc):

FILE: lm_eval/tasks/afrimmlu/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 9) | def doc_to_text(doc):

FILE: lm_eval/tasks/afrixnli/anli prompt/en-direct/utils.py
  function doc_to_target (line 4) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrixnli/anli prompt/translate/utils.py
  function doc_to_target (line 4) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrixnli/direct/prompt_1/utils.py
  function doc_to_text (line 4) | def doc_to_text(doc):
  function doc_to_target (line 17) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrixnli/direct/prompt_2/utils.py
  function doc_to_target (line 4) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrixnli/direct/prompt_3/utils.py
  function doc_to_target (line 4) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrixnli/direct/prompt_4/utils.py
  function doc_to_text (line 4) | def doc_to_text(doc):
  function doc_to_target (line 17) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrixnli/direct/prompt_5/utils.py
  function doc_to_target (line 4) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrixnli/gen_utils.py
  class FunctionTag (line 7) | class FunctionTag:
    method __init__ (line 8) | def __init__(self, value):
  function prompt_func (line 12) | def prompt_func(mode, lang):
  function gen_lang_yamls (line 30) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 103) | def main() -> None:

FILE: lm_eval/tasks/afrixnli/lai prompt/direct/utils.py
  function doc_to_text (line 4) | def doc_to_text(doc):
  function doc_to_target (line 17) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrixnli/lai prompt/translate/utils.py
  function doc_to_text (line 4) | def doc_to_text(doc):
  function doc_to_target (line 17) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrixnli/translate/prompt_1/utils.py
  function doc_to_text (line 4) | def doc_to_text(doc):
  function doc_to_target (line 17) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrixnli/translate/prompt_2/utils.py
  function doc_to_target (line 4) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrixnli/translate/prompt_3/utils.py
  function doc_to_text (line 4) | def doc_to_text(doc):
  function doc_to_target (line 19) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrixnli/translate/prompt_4/utils.py
  function doc_to_text (line 4) | def doc_to_text(doc):
  function doc_to_target (line 17) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrixnli/translate/prompt_5/utils.py
  function doc_to_target (line 4) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrixnli/utils.py
  class FunctionTag (line 6) | class FunctionTag:
    method __init__ (line 7) | def __init__(self, value):
  function gen_lang_yamls (line 123) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 211) | def main() -> None:

FILE: lm_eval/tasks/afrobench/adr/gen_utils.py
  class FunctionTag (line 7) | class FunctionTag:
    method __init__ (line 8) | def __init__(self, value):
  function prompt_func (line 12) | def prompt_func(mode, lang):
  function gen_lang_yamls (line 30) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 79) | def main() -> None:

FILE: lm_eval/tasks/afrobench/afriqa/prompt_1/utils.py
  function normalize_answer (line 6) | def normalize_answer(s):
  function f1 (line 28) | def f1(items):

FILE: lm_eval/tasks/afrobench/afriqa/prompt_2/utils.py
  function normalize_answer (line 6) | def normalize_answer(s):
  function f1 (line 28) | def f1(items):

FILE: lm_eval/tasks/afrobench/afriqa/prompt_3/utils.py
  function normalize_answer (line 6) | def normalize_answer(s):
  function f1 (line 28) | def f1(items):

FILE: lm_eval/tasks/afrobench/afriqa/prompt_4/utils.py
  function normalize_answer (line 6) | def normalize_answer(s):
  function f1 (line 28) | def f1(items):

FILE: lm_eval/tasks/afrobench/afriqa/prompt_5/utils.py
  function normalize_answer (line 6) | def normalize_answer(s):
  function f1 (line 28) | def f1(items):

FILE: lm_eval/tasks/afrobench/afriqa/utils.py
  class FunctionTag (line 7) | class FunctionTag:
    method __init__ (line 8) | def __init__(self, value):
  function prompt_func (line 12) | def prompt_func(mode, lang):
  function gen_lang_yamls (line 43) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 99) | def main() -> None:

FILE: lm_eval/tasks/afrobench/afrisenti/utils.py
  class FunctionTag (line 6) | class FunctionTag:
    method __init__ (line 7) | def __init__(self, value):
  function prompt_func (line 11) | def prompt_func(mode, lang):
  function gen_lang_yamls (line 35) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 98) | def main() -> None:

FILE: lm_eval/tasks/afrobench/belebele/utils.py
  function prompt_func (line 7) | def prompt_func(mode, lang):
  function gen_lang_yamls (line 18) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 129) | def main() -> None:

FILE: lm_eval/tasks/afrobench/flores/gen_utils.py
  class FunctionTag (line 7) | class FunctionTag:
    method __init__ (line 8) | def __init__(self, value):
  function prompt_func (line 12) | def prompt_func(mode, lang, lang_dict):
  function gen_lang_yamls (line 33) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str, reverse:...
  function main (line 165) | def main() -> None:

FILE: lm_eval/tasks/afrobench/injongointent/gen_utils.py
  class FunctionTag (line 7) | class FunctionTag:
    method __init__ (line 8) | def __init__(self, value):
  function prompt_func (line 12) | def prompt_func(mode, lang, intent):
  function gen_lang_yamls (line 29) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 133) | def main() -> None:

FILE: lm_eval/tasks/afrobench/mafand/gen_utils.py
  class FunctionTag (line 7) | class FunctionTag:
    method __init__ (line 8) | def __init__(self, value):
  function prompt_func (line 12) | def prompt_func(mode, lang, lang_dict):
  function gen_lang_yamls (line 35) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str, reverse:...
  function main (line 110) | def main() -> None:

FILE: lm_eval/tasks/afrobench/mafand/prompt_1/african-english/utils.py
  function get_target (line 26) | def get_target(doc):
  function get_target_reverse (line 35) | def get_target_reverse(doc):
  function create_text_prompt_1 (line 43) | def create_text_prompt_1(doc):
  function create_reverse_prompt_1 (line 57) | def create_reverse_prompt_1(doc):
  function create_text_prompt_2 (line 72) | def create_text_prompt_2(doc):
  function create_reverse_prompt_2 (line 84) | def create_reverse_prompt_2(doc):
  function create_text_prompt_3 (line 97) | def create_text_prompt_3(doc):
  function create_reverse_prompt_3 (line 110) | def create_reverse_prompt_3(doc):

FILE: lm_eval/tasks/afrobench/mafand/prompt_1/english-african/utils.py
  function get_target (line 26) | def get_target(doc):
  function get_target_reverse (line 35) | def get_target_reverse(doc):
  function create_text_prompt_1 (line 43) | def create_text_prompt_1(doc):
  function create_reverse_prompt_1 (line 57) | def create_reverse_prompt_1(doc):
  function create_text_prompt_2 (line 72) | def create_text_prompt_2(doc):
  function create_reverse_prompt_2 (line 84) | def create_reverse_prompt_2(doc):
  function create_text_prompt_3 (line 97) | def create_text_prompt_3(doc):
  function create_reverse_prompt_3 (line 110) | def create_reverse_prompt_3(doc):

FILE: lm_eval/tasks/afrobench/mafand/prompt_2/african-english/utils.py
  function get_target (line 26) | def get_target(doc):
  function get_target_reverse (line 35) | def get_target_reverse(doc):
  function create_text_prompt_1 (line 43) | def create_text_prompt_1(doc):
  function create_reverse_prompt_1 (line 57) | def create_reverse_prompt_1(doc):
  function create_text_prompt_2 (line 72) | def create_text_prompt_2(doc):
  function create_reverse_prompt_2 (line 84) | def create_reverse_prompt_2(doc):
  function create_text_prompt_3 (line 97) | def create_text_prompt_3(doc):
  function create_reverse_prompt_3 (line 110) | def create_reverse_prompt_3(doc):

FILE: lm_eval/tasks/afrobench/mafand/prompt_2/english-african/utils.py
  function get_target (line 26) | def get_target(doc):
  function get_target_reverse (line 35) | def get_target_reverse(doc):
  function create_text_prompt_1 (line 43) | def create_text_prompt_1(doc):
  function create_reverse_prompt_1 (line 57) | def create_reverse_prompt_1(doc):
  function create_text_prompt_2 (line 72) | def create_text_prompt_2(doc):
  function create_reverse_prompt_2 (line 84) | def create_reverse_prompt_2(doc):
  function create_text_prompt_3 (line 97) | def create_text_prompt_3(doc):
  function create_reverse_prompt_3 (line 110) | def create_reverse_prompt_3(doc):

FILE: lm_eval/tasks/afrobench/mafand/prompt_3/african-english/utils.py
  function get_target (line 26) | def get_target(doc):
  function get_target_reverse (line 35) | def get_target_reverse(doc):
  function create_text_prompt_1 (line 43) | def create_text_prompt_1(doc):
  function create_reverse_prompt_1 (line 57) | def create_reverse_prompt_1(doc):
  function create_text_prompt_2 (line 72) | def create_text_prompt_2(doc):
  function create_reverse_prompt_2 (line 84) | def create_reverse_prompt_2(doc):
  function create_text_prompt_3 (line 97) | def create_text_prompt_3(doc):
  function create_reverse_prompt_3 (line 110) | def create_reverse_prompt_3(doc):

FILE: lm_eval/tasks/afrobench/mafand/prompt_3/english-african/utils.py
  function get_target (line 26) | def get_target(doc):
  function get_target_reverse (line 35) | def get_target_reverse(doc):
  function create_text_prompt_1 (line 43) | def create_text_prompt_1(doc):
  function create_reverse_prompt_1 (line 57) | def create_reverse_prompt_1(doc):
  function create_text_prompt_2 (line 72) | def create_text_prompt_2(doc):
  function create_reverse_prompt_2 (line 84) | def create_reverse_prompt_2(doc):
  function create_text_prompt_3 (line 97) | def create_text_prompt_3(doc):
  function create_reverse_prompt_3 (line 110) | def create_reverse_prompt_3(doc):

FILE: lm_eval/tasks/afrobench/masakhaner/gen_utils.py
  class FunctionTag (line 7) | class FunctionTag:
    method __init__ (line 8) | def __init__(self, value):
  function prompt_func (line 12) | def prompt_func(mode, lang):
  function gen_lang_yamls (line 48) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 112) | def main() -> None:

FILE: lm_eval/tasks/afrobench/masakhaner/prompt_1/utils.py
  function doc_to_target (line 7) | def doc_to_target(doc):
  function transform_text (line 11) | def transform_text(text):
  function span_f1_agg (line 42) | def span_f1_agg(items):

FILE: lm_eval/tasks/afrobench/masakhaner/prompt_2/utils.py
  function doc_to_target (line 7) | def doc_to_target(doc):
  function transform_text (line 11) | def transform_text(text):
  function span_f1_agg (line 42) | def span_f1_agg(items):

FILE: lm_eval/tasks/afrobench/masakhaner/prompt_3/utils.py
  function doc_to_target (line 7) | def doc_to_target(doc):
  function transform_text (line 11) | def transform_text(text):
  function span_f1_agg (line 42) | def span_f1_agg(items):

FILE: lm_eval/tasks/afrobench/masakhaner/prompt_4/utils.py
  function doc_to_target (line 7) | def doc_to_target(doc):
  function transform_text (line 11) | def transform_text(text):
  function span_f1_agg (line 42) | def span_f1_agg(items):

FILE: lm_eval/tasks/afrobench/masakhaner/prompt_5/utils.py
  function doc_to_target (line 7) | def doc_to_target(doc):
  function transform_text (line 11) | def transform_text(text):
  function span_f1_agg (line 42) | def span_f1_agg(items):

FILE: lm_eval/tasks/afrobench/masakhanews/utils.py
  function prompt_func (line 7) | def prompt_func(mode, lang):
  function gen_lang_yamls (line 35) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 97) | def main() -> None:

FILE: lm_eval/tasks/afrobench/masakhapos/gen_utils.py
  class FunctionTag (line 7) | class FunctionTag:
    method __init__ (line 8) | def __init__(self, value):
  function prompt_func (line 12) | def prompt_func(mode, lang):
  function gen_lang_yamls (line 61) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 125) | def main() -> None:

FILE: lm_eval/tasks/afrobench/masakhapos/prompt_1/utils.py
  function doc_to_target (line 8) | def doc_to_target(doc):
  function acc_score (line 32) | def acc_score(items):

FILE: lm_eval/tasks/afrobench/masakhapos/prompt_2/utils.py
  function doc_to_target (line 8) | def doc_to_target(doc):
  function acc_score (line 32) | def acc_score(items):

FILE: lm_eval/tasks/afrobench/masakhapos/prompt_3/utils.py
  function doc_to_target (line 8) | def doc_to_target(doc):
  function acc_score (line 32) | def acc_score(items):

FILE: lm_eval/tasks/afrobench/masakhapos/prompt_4/utils.py
  function doc_to_target (line 8) | def doc_to_target(doc):
  function acc_score (line 32) | def acc_score(items):

FILE: lm_eval/tasks/afrobench/masakhapos/prompt_5/utils.py
  function doc_to_target (line 8) | def doc_to_target(doc):
  function acc_score (line 32) | def acc_score(items):

FILE: lm_eval/tasks/afrobench/masakhapos/utils.py
  function doc_to_text (line 4) | def doc_to_text(doc):
  function doc_to_target (line 19) | def doc_to_target(doc):

FILE: lm_eval/tasks/afrobench/naijarc/utils.py
  function prompt_func (line 7) | def prompt_func(mode, lang):
  function gen_lang_yamls (line 18) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 67) | def main() -> None:

FILE: lm_eval/tasks/afrobench/ntrex/gen_utils.py
  class FunctionTag (line 7) | class FunctionTag:
    method __init__ (line 8) | def __init__(self, value):
  function prompt_func (line 12) | def prompt_func(mode, lang, lang_dict):
  function gen_lang_yamls (line 33) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str, reverse:...
  function main (line 134) | def main() -> None:

FILE: lm_eval/tasks/afrobench/openai_mmlu/utils.py
  function prompt_func (line 7) | def prompt_func(mode, lang):
  function gen_lang_yamls (line 18) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 73) | def main() -> None:

FILE: lm_eval/tasks/afrobench/salt/gen_utils.py
  class FunctionTag (line 7) | class FunctionTag:
    method __init__ (line 8) | def __init__(self, value):
  function prompt_func (line 12) | def prompt_func(mode, lang, lang_dict):
  function gen_lang_yamls (line 34) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str, reverse:...
  function main (line 112) | def main() -> None:

FILE: lm_eval/tasks/afrobench/sib/utils.py
  class FunctionTag (line 7) | class FunctionTag:
    method __init__ (line 8) | def __init__(self, value):
  function prompt_func (line 12) | def prompt_func(mode, lang):
  function gen_lang_yamls (line 40) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 201) | def main() -> None:

FILE: lm_eval/tasks/afrobench/uhura-arc-easy/utils.py
  function get_language_from_code (line 8) | def get_language_from_code(code: str) -> str:
  function prompt_func (line 13) | def prompt_func(mode):
  function gen_lang_yamls (line 51) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 99) | def main() -> None:

FILE: lm_eval/tasks/afrobench/xlsum/prompt_1/utils.py
  function rougeL (line 4) | def rougeL(items):
  function rougeL_agg (line 11) | def rougeL_agg(items):

FILE: lm_eval/tasks/afrobench/xlsum/prompt_2/utils.py
  function rougeL (line 4) | def rougeL(items):
  function rougeL_agg (line 11) | def rougeL_agg(items):

FILE: lm_eval/tasks/afrobench/xlsum/prompt_3/utils.py
  function rougeL (line 4) | def rougeL(items):
  function rougeL_agg (line 11) | def rougeL_agg(items):

FILE: lm_eval/tasks/afrobench/xlsum/utils.py
  function prompt_func (line 7) | def prompt_func(mode, lang):
  function gen_lang_yamls (line 29) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 88) | def main() -> None:

FILE: lm_eval/tasks/agieval/utils.py
  function parse_math_answer (line 10) | def parse_math_answer(raw_string):
  function _fix_fracs (line 82) | def _fix_fracs(string):
  function _fix_a_slash_b (line 114) | def _fix_a_slash_b(string):
  function _remove_right_units (line 129) | def _remove_right_units(string):
  function _fix_sqrt (line 139) | def _fix_sqrt(string):
  function _strip_string (line 154) | def _strip_string(string):
  function is_equiv (line 224) | def is_equiv(str1, str2, verbose=False):
  function process_results (line 243) | def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
  function process_results_mcqa (line 262) | def process_results_mcqa(doc, results):

FILE: lm_eval/tasks/aime/utils.py
  function process_results (line 5) | def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
  function is_equiv (line 36) | def is_equiv(str1, str2, verbose=False):
  function remove_boxed (line 53) | def remove_boxed(s):
  function last_boxed_only_string (line 67) | def last_boxed_only_string(string):
  function fix_fracs (line 97) | def fix_fracs(string):
  function fix_a_slash_b (line 129) | def fix_a_slash_b(string):
  function remove_right_units (line 144) | def remove_right_units(string):
  function fix_sqrt (line 154) | def fix_sqrt(string):
  function strip_string (line 169) | def strip_string(string):

FILE: lm_eval/tasks/arab_culture/_generate_configs.py
  function parse_args (line 34) | def parse_args():

FILE: lm_eval/tasks/arab_culture/utils_mcq.py
  function doc_to_text (line 49) | def doc_to_text(doc):
  function doc_to_choice (line 101) | def doc_to_choice(doc):
  function doc_to_target (line 105) | def doc_to_target(doc):

FILE: lm_eval/tasks/arab_culture_completion/_generate_configs.py
  function parse_args (line 34) | def parse_args():

FILE: lm_eval/tasks/arab_culture_completion/utils_completion.py
  function doc_to_text (line 52) | def doc_to_text(doc):
  function doc_to_choice (line 91) | def doc_to_choice(doc):
  function doc_to_target (line 96) | def doc_to_target(doc):

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/utils.py
  function process_docs (line 15) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/utils.py
  function process_docs (line 15) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/utils.py
  function process_docs (line 7) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/utils.py
  function doc_to_text (line 7) | def doc_to_text(doc):
  function process_docs (line 24) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/utils.py
  function process_docs (line 15) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/utils.py
  function process_docs (line 15) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/utils.py
  function process_docs (line 7) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/utils.py
  function doc_to_text (line 7) | def doc_to_text(doc):
  function process_docs (line 24) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/arabicmmlu/_generate_configs.py
  function parse_args (line 60) | def parse_args():

FILE: lm_eval/tasks/arabicmmlu/utils.py
  function doc_to_text (line 14) | def doc_to_text(doc):
  function doc_to_choice (line 43) | def doc_to_choice(doc):

FILE: lm_eval/tasks/aradice/ArabicMMLU/EGY/metrics.py
  function macro_f1_score (line 4) | def macro_f1_score(items):
  function micro_f1_score (line 12) | def micro_f1_score(items):
  function weighted_f1_score (line 20) | def weighted_f1_score(items):

FILE: lm_eval/tasks/aradice/ArabicMMLU/EGY/utils.py
  function process_docs (line 51) | def process_docs(dataset):

FILE: lm_eval/tasks/aradice/ArabicMMLU/LEV/metrics.py
  function macro_f1_score (line 4) | def macro_f1_score(items):
  function micro_f1_score (line 12) | def micro_f1_score(items):
  function weighted_f1_score (line 20) | def weighted_f1_score(items):

FILE: lm_eval/tasks/aradice/ArabicMMLU/LEV/utils.py
  function process_docs (line 50) | def process_docs(dataset):

FILE: lm_eval/tasks/aradice/boolq/EGY/metrics.py
  function macro_f1_score (line 4) | def macro_f1_score(items):
  function micro_f1_score (line 12) | def micro_f1_score(items):
  function weighted_f1_score (line 20) | def weighted_f1_score(items):

FILE: lm_eval/tasks/aradice/boolq/EGY/utils.py
  function process_docs (line 4) | def process_docs(dataset):

FILE: lm_eval/tasks/aradice/boolq/ENG/metrics.py
  function macro_f1_score (line 4) | def macro_f1_score(items):
  function micro_f1_score (line 12) | def micro_f1_score(items):
  function weighted_f1_score (line 20) | def weighted_f1_score(items):

FILE: lm_eval/tasks/aradice/boolq/ENG/utils.py
  function process_docs (line 4) | def process_docs(dataset):

FILE: lm_eval/tasks/aradice/boolq/LEV/metrics.py
  function macro_f1_score (line 4) | def macro_f1_score(items):
  function micro_f1_score (line 12) | def micro_f1_score(items):
  function weighted_f1_score (line 20) | def weighted_f1_score(items):

FILE: lm_eval/tasks/aradice/boolq/LEV/utils.py
  function process_docs (line 4) | def process_docs(dataset):

FILE: lm_eval/tasks/aradice/boolq/MSA/metrics.py
  function macro_f1_score (line 4) | def macro_f1_score(items):
  function micro_f1_score (line 12) | def micro_f1_score(items):
  function weighted_f1_score (line 20) | def weighted_f1_score(items):

FILE: lm_eval/tasks/aradice/boolq/MSA/utils.py
  function process_docs (line 4) | def process_docs(dataset):

FILE: lm_eval/tasks/aradice/cultural-benchmark/metrics.py
  function macro_f1_score (line 4) | def macro_f1_score(items):
  function micro_f1_score (line 12) | def micro_f1_score(items):
  function weighted_f1_score (line 20) | def weighted_f1_score(items):

FILE: lm_eval/tasks/aradice/cultural-benchmark/utils.py
  function process_docs (line 1) | def process_docs(dataset):

FILE: lm_eval/tasks/aradice/openbookqa/metrics.py
  function macro_f1_score (line 4) | def macro_f1_score(items):
  function micro_f1_score (line 12) | def micro_f1_score(items):
  function weighted_f1_score (line 20) | def weighted_f1_score(items):

FILE: lm_eval/tasks/aradice/openbookqa/utils.py
  function doc_to_target (line 1) | def doc_to_target(doc):
  function doc_to_choice (line 12) | def doc_to_choice(doc):
  function doc_to_text (line 17) | def doc_to_text(doc):

FILE: lm_eval/tasks/aradice/piqa/metrics.py
  function macro_f1_score (line 4) | def macro_f1_score(items):
  function micro_f1_score (line 12) | def micro_f1_score(items):
  function weighted_f1_score (line 20) | def weighted_f1_score(items):

FILE: lm_eval/tasks/aradice/truthfulqa_mcq/metrics.py
  function macro_f1_score (line 4) | def macro_f1_score(items):
  function micro_f1_score (line 12) | def micro_f1_score(items):
  function weighted_f1_score (line 20) | def weighted_f1_score(items):

FILE: lm_eval/tasks/aradice/winogrande/metrics.py
  function macro_f1_score (line 4) | def macro_f1_score(items):
  function micro_f1_score (line 12) | def micro_f1_score(items):
  function weighted_f1_score (line 20) | def weighted_f1_score(items):

FILE: lm_eval/tasks/aradice/winogrande/utils.py
  function doc_to_text (line 1) | def doc_to_text(doc):
  function doc_to_target (line 6) | def doc_to_target(doc):
  function doc_to_choice (line 11) | def doc_to_choice(doc):

FILE: lm_eval/tasks/babilong/common_utils.py
  function get_tokenizer (line 18) | def get_tokenizer(
  function postprocess_pred (line 27) | def postprocess_pred(prediction: list[str]) -> list[str]:
  function load_dataset (line 40) | def load_dataset(**kwargs):
  function process_results (line 55) | def process_results(doc: dict, results: list[str]) -> dict[str, float]:

FILE: lm_eval/tasks/basque_bench/flores_eu/create_yamls_flores_eu.py
  function doc_to_text (line 257) | def doc_to_text(src: str, tgt: str) -> str:
  function doc_to_target (line 265) | def doc_to_target(tgt: str) -> str:
  function gen_lang_yamls (line 272) | def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
  function main (line 316) | def main() -> None:

FILE: lm_eval/tasks/basque_bench/utils.py
  function xcopa_doc_to_text (line 6) | def xcopa_doc_to_text(doc):
  function xcopa_doc_to_choice (line 11) | def xcopa_doc_to_choice(doc):
  function paws_process_docs (line 21) | def paws_process_docs(dataset):

FILE: lm_eval/tasks/basqueglue/utils.py
  function general_detokenize (line 7) | def general_detokenize(string):
  function process_doc (line 16) | def process_doc(string):
  function process_wic_docs (line 22) | def process_wic_docs(dataset):
  function coref_doc_to_text (line 36) | def coref_doc_to_text(x):
  function micro_f1_score (line 62) | def micro_f1_score(items):
  function vaxx_f1_score (line 71) | def vaxx_f1_score(items):

FILE: lm_eval/tasks/bbh/_generate_configs.py
  function parse_args (line 15) | def parse_args():

FILE: lm_eval/tasks/bbh/cot_zeroshot/utils.py
  class ExtendedRegexFilter (line 9) | class ExtendedRegexFilter(RegexFilter):
    method __init__ (line 14) | def __init__(
    method filter_ignores (line 28) | def filter_ignores(self, st):
    method find_match (line 41) | def find_match(self, regex, resp, convert_dict={}):
  class MapRegexFilter (line 53) | class MapRegexFilter(ExtendedRegexFilter):
    method __init__ (line 54) | def __init__(
    method apply (line 82) | def apply(self, resps, docs):
  class NumberParseRegexFilter (line 109) | class NumberParseRegexFilter(ExtendedRegexFilter):
    method apply (line 110) | def apply(self, resps, docs):
  class WordSortFilter (line 140) | class WordSortFilter(Filter):
    method apply (line 143) | def apply(self, resps, docs):
  class MultiChoiceRegexFilter (line 162) | class MultiChoiceRegexFilter(ExtendedRegexFilter):
    method __init__ (line 163) | def __init__(self, *args, **kwargs):
    method apply (line 175) | def apply(self, resps, docs):

FILE: lm_eval/tasks/bbh/zeroshot/utils.py
  class ExtendedRegexFilter (line 9) | class ExtendedRegexFilter(RegexFilter):
    method __init__ (line 14) | def __init__(
    method filter_ignores (line 28) | def filter_ignores(self, st):
    method find_match (line 41) | def find_match(self, regex, resp, convert_dict={}):
  class MapRegexFilter (line 53) | class MapRegexFilter(ExtendedRegexFilter):
    method __init__ (line 54) | def __init__(
    method apply (line 82) | def apply(self, resps, docs):
  class NumberParseRegexFilter (line 109) | class NumberParseRegexFilter(ExtendedRegexFilter):
    method apply (line 110) | def apply(self, resps, docs):
  class WordSortFilter (line 140) | class WordSortFilter(Filter):
    method apply (line 143) | def apply(self, resps, docs):
  class MultiChoiceRegexFilter (line 162) | class MultiChoiceRegexFilter(ExtendedRegexFilter):
    method __init__ (line 163) | def __init__(self, *args, **kwargs):
    method apply (line 175) | def apply(self, resps, docs):

FILE: lm_eval/tasks/bbq/utils.py
  function agg_accuracy_amb (line 33) | def agg_accuracy_amb(arr):
  function agg_accuracy_disamb (line 42) | def agg_accuracy_disamb(arr):
  function agg_disamb_bias_scores (line 51) | def agg_disamb_bias_scores(arr):
  function agg_amb_bias_scores (line 84) | def agg_amb_bias_scores(arr):
  function _process_results (line 110) | def _process_results(doc, answer: int):
  function _clean_answer (line 193) | def _clean_answer(answer: str):
  function _check_unk_answer (line 204) | def _check_unk_answer(answer: str):
  function process_results_generate_until (line 212) | def process_results_generate_until(doc, results):
  function process_results_multiple_choice (line 242) | def process_results_multiple_choice(doc, results):
  function doc_to_biased_answer (line 255) | def doc_to_biased_answer(doc):
  function _process_groups_in_answers (line 265) | def _process_groups_in_answers(string):
  function process_docs (line 300) | def process_docs(dataset: datasets.Dataset):
  function filter_dataset_context (line 362) | def filter_dataset_context(dataset: datasets.Dataset, context: str) -> d...
  function process_docs_ambig (line 368) | def process_docs_ambig(dataset: datasets.Dataset):
  function process_docs_disambig (line 372) | def process_docs_disambig(dataset: datasets.Dataset):
  function doc_to_choice (line 376) | def doc_to_choice(doc):
  function _doc_to_choice_groups (line 385) | def _doc_to_choice_groups(doc):
  function doc_to_targets (line 397) | def doc_to_targets(doc):
  function doc_to_target (line 412) | def doc_to_target(doc):
  function filter_dataset (line 417) | def filter_dataset(dataset: datasets.Dataset, bias_type: str) -> dataset...
  function filter_race_color (line 421) | def filter_race_color(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/belebele/_generate_configs.py
  function parse_args (line 18) | def parse_args():
  function query (line 41) | def query():

FILE: lm_eval/tasks/bigbench/generate_tasks.py
  function main (line 183) | def main() -> None:

FILE: lm_eval/tasks/blimp/generate_configs.py
  function main (line 75) | def main() -> None:

FILE: lm_eval/tasks/c4/preprocess_c4.py
  function c4_detokenizer (line 4) | def c4_detokenizer(doc):
  function process_results (line 39) | def process_results(doc, results):

FILE: lm_eval/tasks/cabbq/utils.py
  function _model_answer (line 6) | def _model_answer(lls):
  function _model_answer_type (line 25) | def _model_answer_type(doc, model_answer):
  function process_results (line 75) | def process_results(doc, results):
  function acc_ambig_agg (line 137) | def acc_ambig_agg(results):
  function acc_disambig_agg (line 159) | def acc_disambig_agg(results):
  function bias_score_ambig_agg (line 181) | def bias_score_ambig_agg(results):
  function bias_score_disambig_agg (line 212) | def bias_score_disambig_agg(results):

FILE: lm_eval/tasks/careqa/utils.py
  function doc_to_text (line 1) | def doc_to_text(doc) -> str:
  function doc_to_target (line 39) | def doc_to_target(doc) -> int:

FILE: lm_eval/tasks/careqa/utils_open.py
  function doc_eval (line 22) | def doc_eval(pred, refs):
  function doc_to_text (line 65) | def doc_to_text(doc) -> str:
  function doc_to_target (line 69) | def doc_to_target(doc) -> str:
  function process_results_gen (line 73) | def process_results_gen(doc, results):
  function process_results_gen_w_repeats (line 98) | def process_results_gen_w_repeats(doc, results):

FILE: lm_eval/tasks/careqa/utils_perplexity.py
  function doc_to_target (line 5) | def doc_to_target(doc) -> str:
  function process_results (line 9) | def process_results(doc, results):

FILE: lm_eval/tasks/catalan_bench/flores_ca/create_yamls_flores_ca.py
  function code_to_language_name (line 246) | def code_to_language_name(code):
  function code_to_short_name (line 250) | def code_to_short_name(code):
  function jinja_var (line 254) | def jinja_var(s):
  function doc_to_text (line 258) | def doc_to_text(src: str, tgt: str) -> str:
  function doc_to_target (line 266) | def doc_to_target(tgt: str) -> str:
  function gen_lang_yamls (line 273) | def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
  function main (line 317) | def main() -> None:

FILE: lm_eval/tasks/catalan_bench/truthfulqa_va/utils.py
  function lowercase_first_letter (line 14) | def lowercase_first_letter(text):
  function process_summarization (line 18) | def process_summarization(dataset):
  function process_docs_paraphrases (line 28) | def process_docs_paraphrases(dataset):
  function process_docs_paws (line 56) | def process_docs_paws(dataset):
  function rouge1 (line 84) | def rouge1(items):
  function rouge1_agg (line 91) | def rouge1_agg(items):
  function process_results_mc2 (line 102) | def process_results_mc2(doc, results):
  function process_docs_gen (line 115) | def process_docs_gen(dataset: datasets.Dataset) -> datasets.Dataset:
  function preprocess_function_gen (line 119) | def preprocess_function_gen(examples):
  function process_doc_nli (line 143) | def process_doc_nli(dataset):
  function process_results_gen (line 170) | def process_results_gen(doc, results):
  function bleu (line 241) | def bleu(refs, preds):
  function rouge (line 263) | def rouge(refs, preds):

FILE: lm_eval/tasks/catalan_bench/utils.py
  function lowercase_first_letter (line 10) | def lowercase_first_letter(text):
  function process_doc_nli (line 14) | def process_doc_nli(dataset):
  function process_results_coqcat (line 38) | def process_results_coqcat(doc, results):
  function process_results_qa (line 72) | def process_results_qa(doc, results):
  function process_doc_cabreu (line 81) | def process_doc_cabreu(dataset):
  function process_docs_paraphrases (line 96) | def process_docs_paraphrases(dataset):
  function process_docs_copa_ca (line 119) | def process_docs_copa_ca(dataset):
  function rouge1 (line 128) | def rouge1(items):
  function rouge1_agg (line 135) | def rouge1_agg(items):

FILE: lm_eval/tasks/ceval/_generate_configs.py
  function parse_args (line 72) | def parse_args():

FILE: lm_eval/tasks/chartqa/utils.py
  function _normalize_string (line 6) | def _normalize_string(s):
  function _remove_end_punctuation (line 14) | def _remove_end_punctuation(unnormalized_string: str) -> str:
  class RelaxedCorrectness (line 27) | class RelaxedCorrectness:
    method _relaxed_correctness (line 39) | def _relaxed_correctness(
    method score (line 132) | def score(self, model_answer: str, reference_answer: str | list[str]) ...
  class ExplicitPromptRelaxedCorrectness (line 141) | class ExplicitPromptRelaxedCorrectness(RelaxedCorrectness):
    method name (line 145) | def name(self) -> str:
    method _get_final_answer (line 148) | def _get_final_answer(self, generation: str) -> str:
    method score (line 174) | def score(self, model_answer: str, reference_answer: str | list[str]) ...
  class AnywhereInAnswerRelaxedCorrectness (line 182) | class AnywhereInAnswerRelaxedCorrectness(ExplicitPromptRelaxedCorrectness):
    method name (line 189) | def name(self) -> str:
    method score (line 192) | def score(self, model_answer: str, reference_answer: str | list[str]) ...
  function exact_match (line 242) | def exact_match(references, predictions):
  function relaxed_accuracy (line 257) | def relaxed_accuracy(references, predictions):
  function anywhere_accuracy (line 268) | def anywhere_accuracy(references, predictions):

FILE: lm_eval/tasks/click/click_cul/utils.py
  function get_context (line 6) | def get_context(doc) -> str:
  function get_target (line 18) | def get_target(doc) -> str:
  function get_choices (line 25) | def get_choices(doc) -> List[str]:
  function extract_economy (line 31) | def extract_economy(dataset: Dataset) -> Dataset:
  function extract_geography (line 35) | def extract_geography(dataset: Dataset) -> Dataset:
  function extract_history (line 39) | def extract_history(dataset: Dataset) -> Dataset:
  function extract_law (line 45) | def extract_law(dataset: Dataset) -> Dataset:
  function extract_politics (line 51) | def extract_politics(dataset: Dataset) -> Dataset:
  function extract_kpop (line 55) | def extract_kpop(dataset: Dataset) -> Dataset:
  function extract_society (line 59) | def extract_society(dataset: Dataset) -> Dataset:
  function extract_tradition (line 63) | def extract_tradition(dataset: Dataset) -> Dataset:

FILE: lm_eval/tasks/click/click_lang/utils.py
  function get_context (line 6) | def get_context(doc) -> str:
  function get_target (line 18) | def get_target(doc) -> str:
  function get_choices (line 25) | def get_choices(doc) -> List[str]:
  function extract_text (line 31) | def extract_text(dataset: Dataset) -> Dataset:
  function extract_grammar (line 41) | def extract_grammar(dataset: Dataset) -> Dataset:
  function extract_function (line 65) | def extract_function(dataset: Dataset) -> Dataset:

FILE: lm_eval/tasks/cmmlu/_generate_configs.py
  function parse_args (line 87) | def parse_args():

FILE: lm_eval/tasks/cnn_dailymail/utils.py
  function normalize_text (line 27) | def normalize_text(text: str) -> str:
  function calculate_rouge_scores (line 44) | def calculate_rouge_scores(
  function calculate_bertscore (line 82) | def calculate_bertscore(
  function process_results (line 124) | def process_results(doc: Dict[str, Any], results: List[str]) -> Dict[str...
  function postprocess_generation (line 186) | def postprocess_generation(generation: str) -> str:
  function filter_long_articles (line 208) | def filter_long_articles(doc: Dict[str, Any]) -> bool:
  function doc_to_choice (line 224) | def doc_to_choice(doc: Dict[str, Any]) -> List[str]:
  function process_docs (line 237) | def process_docs(dataset):
  function calculate_summary_length (line 269) | def calculate_summary_length(generated: str) -> int:

FILE: lm_eval/tasks/code_x_glue/code-text/bleu.py
  function normalize (line 58) | def normalize(s):
  function count_ngrams (line 78) | def count_ngrams(words, n=4):
  function cook_refs (line 87) | def cook_refs(refs, n=4):
  function cook_test (line 101) | def cook_test(test, item, n=4):
  function score_cooked (line 132) | def score_cooked(allcomps, n=4, ground=0, smooth=1):
  function bleu (line 174) | def bleu(refs, candidate, ground=0, smooth=1):
  function splitPuncts (line 180) | def splitPuncts(line):
  function computeMaps (line 184) | def computeMaps(predictions, goldfile):
  function bleuFromMaps (line 210) | def bleuFromMaps(m1, m2):
  function smoothed_bleu_4 (line 222) | def smoothed_bleu_4(references, predictions, **kwargs):

FILE: lm_eval/tasks/code_x_glue/code-text/utils.py
  function doc_to_text (line 1) | def doc_to_text(doc):
  function doc_to_target (line 8) | def doc_to_target(doc):

FILE: lm_eval/tasks/common_voice/utils.py
  function doc_to_text (line 10) | def doc_to_text(doc: Dict[str, Any]) -> str:
  function doc_to_audio (line 14) | def doc_to_audio(doc: Dict[str, Any]) -> List[dict]:

FILE: lm_eval/tasks/copal_id/utils.py
  function convert_choice (line 4) | def convert_choice(choice):
  function doc_to_text (line 8) | def doc_to_text(doc, connector):
  function doc_to_choice (line 13) | def doc_to_choice(doc):

FILE: lm_eval/tasks/coqa/utils.py
  function doc_to_text (line 6) | def doc_to_text(doc):
  function doc_to_target (line 19) | def doc_to_target(doc):
  function em (line 37) | def em(gold_list, pred):
  function compute_scores (line 51) | def compute_scores(gold_list, pred):
  function process_results (line 72) | def process_results(doc, results):

FILE: lm_eval/tasks/crows_pairs/utils.py
  function process_results (line 4) | def process_results(doc, results):
  function doc_to_choice (line 19) | def doc_to_choice(doc):
  function filter_dataset (line 23) | def filter_dataset(dataset: datasets.Dataset, bias_type: str) -> dataset...
  function filter_race_color (line 27) | def filter_race_color(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_socio (line 31) | def filter_socio(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_gender (line 35) | def filter_gender(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_age (line 39) | def filter_age(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_religion (line 43) | def filter_religion(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_disability (line 47) | def filter_disability(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_orientation (line 51) | def filter_orientation(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_nationality (line 55) | def filter_nationality(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_appearance (line 59) | def filter_appearance(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_autre (line 63) | def filter_autre(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/csatqa/_generate_configs.py
  function parse_args (line 19) | def parse_args():

FILE: lm_eval/tasks/csatqa/utils.py
  function process_docs (line 4) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/darija_bench/darija_sentiment/utils.py
  function doc_to_text (line 9) | def doc_to_text(doc):
  function doc_to_choice_3 (line 21) | def doc_to_choice_3(doc):
  function doc_to_choice_2 (line 25) | def doc_to_choice_2(doc):
  function doc_to_target (line 29) | def doc_to_target(doc):

FILE: lm_eval/tasks/darija_bench/darija_summarization/utils.py
  function strip (line 5) | def strip(resps, docs):
  function doc_to_text (line 12) | def doc_to_text(doc):
  function doc_to_target (line 19) | def doc_to_target(doc):
  function bert (line 23) | def bert(items):
  function Average (line 27) | def Average(lst):
  function darijabert (line 31) | def darijabert(items):
  function rouge1 (line 44) | def rouge1(items):
  function rougeL (line 48) | def rougeL(items):
  function rouge2 (line 52) | def rouge2(items):
  function rougeLsum (line 56) | def rougeLsum(items):
  function agg_rougelsum (line 60) | def agg_rougelsum(items):
  function agg_rouge1 (line 66) | def agg_rouge1(items):
  function agg_rouge2 (line 72) | def agg_rouge2(items):
  function agg_rougel (line 78) | def agg_rougel(items):

FILE: lm_eval/tasks/darija_bench/darija_translation/utils.py
  function strip (line 5) | def strip(resps, docs):
  function dr_fr (line 12) | def dr_fr(dataset: datasets.Dataset):
  function dr_en (line 16) | def dr_en(dataset: datasets.Dataset):
  function dr_msa (line 20) | def dr_msa(dataset: datasets.Dataset):
  function fr_dr (line 24) | def fr_dr(dataset: datasets.Dataset):
  function en_dr (line 28) | def en_dr(dataset: datasets.Dataset):
  function msa_dr (line 32) | def msa_dr(dataset: datasets.Dataset):
  function doc_to_text (line 46) | def doc_to_text(doc):
  function doc_to_target (line 51) | def doc_to_target(doc):
  function bert (line 55) | def bert(items):
  function Average (line 59) | def Average(lst):
  function camembert (line 63) | def camembert(items):
  function darijabert (line 76) | def darijabert(items):
  function arabert (line 89) | def arabert(items):
  function bertbase (line 102) | def bertbase(items):
  function mbert (line 115) | def mbert(items):

FILE: lm_eval/tasks/darija_bench/darija_transliteration/utils.py
  function strip (line 5) | def strip(resps, docs):
  function dr_ar (line 12) | def dr_ar(dataset: datasets.Dataset):
  function ar_dr (line 16) | def ar_dr(dataset: datasets.Dataset):
  function doc_to_text (line 20) | def doc_to_text(doc):
  function doc_to_target (line 25) | def doc_to_target(doc):
  function bert (line 29) | def bert(items):
  function Average (line 33) | def Average(lst):
  function arabizibert (line 37) | def arabizibert(items):
  function darijabert (line 50) | def darijabert(items):
  function mbert (line 63) | def mbert(items):

FILE: lm_eval/tasks/darijahellaswag/utils.py
  function process_docs (line 4) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/darijammlu/_generate_configs.py
  function parse_args (line 73) | def parse_args():

FILE: lm_eval/tasks/darijammlu/utils.py
  function doc_to_text (line 7) | def doc_to_text(doc):
  function doc_to_choice (line 24) | def doc_to_choice(doc):

FILE: lm_eval/tasks/discrim_eval/utils.py
  function _logit (line 8) | def _logit(p: float) -> float:
  function process_results (line 30) | def process_results(
  function agg_demographic_bias_regression (line 63) | def agg_demographic_bias_regression(items: List[BiasTuple]) -> float:

FILE: lm_eval/tasks/drop/utils.py
  function process_docs (line 10) | def process_docs(dataset):
  function get_answers (line 22) | def get_answers(doc):
  function parse_answer (line 51) | def parse_answer(answer):
  function process_results (line 64) | def process_results(doc, results):
  function get_metrics (line 76) | def get_metrics(predicted, gold):
  function _answer_to_bags (line 100) | def _answer_to_bags(answer):
  function _align_bags (line 114) | def _align_bags(predicted, gold):
  function _compute_f1 (line 134) | def _compute_f1(predicted_bag, gold_bag):
  function _match_numbers_if_present (line 152) | def _match_numbers_if_present(gold_bag, predicted_bag):
  function _is_number (line 166) | def _is_number(text):
  function _remove_articles (line 174) | def _remove_articles(text):
  function _white_space_fix (line 178) | def _white_space_fix(text):
  function _remove_punc (line 182) | def _remove_punc(text):
  function _fix_number (line 190) | def _fix_number(text):
  function _tokenize (line 194) | def _tokenize(text):
  function _normalize (line 198) | def _normalize(answer):

FILE: lm_eval/tasks/e2lmc/mmlu_early_training/custom_metrics.py
  function loglikelihood_diff (line 4) | def loglikelihood_diff(items):

FILE: lm_eval/tasks/e2lmc/noor/_generate_configs.py
  function parse_args (line 78) | def parse_args():

FILE: lm_eval/tasks/egyhellaswag/utils.py
  function process_docs (line 4) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/egymmlu/_generate_configs.py
  function parse_args (line 74) | def parse_args():

FILE: lm_eval/tasks/egymmlu/utils.py
  function doc_to_text (line 7) | def doc_to_text(doc):
  function doc_to_choice (line 24) | def doc_to_choice(doc):

FILE: lm_eval/tasks/eq_bench/multilingual/utils.py
  function calculate_score_fullscale (line 6) | def calculate_score_fullscale(docs, results):

FILE: lm_eval/tasks/eq_bench/utils.py
  function calculate_score_fullscale (line 6) | def calculate_score_fullscale(docs, results):

FILE: lm_eval/tasks/esbbq/utils.py
  function _model_answer (line 6) | def _model_answer(lls):
  function _model_answer_type (line 25) | def _model_answer_type(doc, model_answer):
  function process_results (line 75) | def process_results(doc, results):
  function acc_ambig_agg (line 137) | def acc_ambig_agg(results):
  function acc_disambig_agg (line 159) | def acc_disambig_agg(results):
  function bias_score_ambig_agg (line 181) | def bias_score_ambig_agg(results):
  function bias_score_disambig_agg (line 212) | def bias_score_disambig_agg(results):

FILE: lm_eval/tasks/eus_exams/configs.py
  function gen_config_yamls (line 16) | def gen_config_yamls(output_dir: str, overwrite: bool) -> None:
  function main (line 49) | def main() -> None:

FILE: lm_eval/tasks/eus_exams/utils.py
  function process_docs (line 4) | def process_docs(dataset: datasets.Dataset):

FILE: lm_eval/tasks/eus_reading/utils.py
  function doc_to_text_context (line 7) | def doc_to_text_context(doc) -> str:
  function doc_to_choice (line 28) | def doc_to_choice(doc) -> List[str]:

FILE: lm_eval/tasks/eus_trivia/utils.py
  function doc_to_text (line 7) | def doc_to_text(doc) -> str:
  function doc_to_choice (line 28) | def doc_to_choice(doc) -> List[str]:

FILE: lm_eval/tasks/evalita_llm/metrics.py
  function _aggreg_ls (line 10) | def _aggreg_ls(predictions):
  function _aggreg_sa_v2 (line 37) | def _aggreg_sa_v2(predictions):
  function _aggreg_sa (line 49) | def _aggreg_sa(predictions):
  function _aggreg_ner (line 124) | def _aggreg_ner(predictions):
  function _aggreg_rel (line 143) | def _aggreg_rel(predictions):
  function _aggreg_dd (line 160) | def _aggreg_dd(items):

FILE: lm_eval/tasks/evalita_llm/sum_utils.py
  function rouge1_score (line 7) | def rouge1_score(references, predictions, **kwargs):
  function process_results_sum (line 16) | def process_results_sum(doc, results):

FILE: lm_eval/tasks/evalita_llm/utils.py
  function sa_doc_to_target (line 11) | def sa_doc_to_target(x):
  function sa_doc_to_target_v2 (line 30) | def sa_doc_to_target_v2(x):
  function sa_doc_to_choice (line 49) | def sa_doc_to_choice(x):
  function _ls_gold_to_target (line 60) | def _ls_gold_to_target(x):
  function ls_doc_to_target (line 77) | def ls_doc_to_target(x):
  function _ls_split_gold (line 91) | def _ls_split_gold(x):
  function ls_process_results (line 112) | def ls_process_results(doc, results):
  function _ner_gold_to_target (line 163) | def _ner_gold_to_target(x: list) -> list:
  function _ner_gold_to_target_v2 (line 171) | def _ner_gold_to_target_v2(x: list) -> list:
  function ner_doc_to_target (line 179) | def ner_doc_to_target(doc):
  function ner_process_results (line 193) | def ner_process_results(doc, results):
  function ner_process_results_v2 (line 246) | def ner_process_results_v2(doc, results):
  function _ner_process_raw_output (line 313) | def _ner_process_raw_output(llm_result: str) -> list[tuple]:
  function _ner_process_raw_output_v2 (line 337) | def _ner_process_raw_output_v2(llm_result: str) -> list[tuple]:
  function _rel_process_raw_output (line 364) | def _rel_process_raw_output(llm_result: str) -> list[str]:
  function re_doc_to_target (line 391) | def re_doc_to_target(doc):
  function _rel_gold_to_target (line 403) | def _rel_gold_to_target(x: list) -> list:
  function rel_doc_to_target (line 410) | def rel_doc_to_target(doc):
  function _extract_relations (line 422) | def _extract_relations(results):
  function rel_process_results_v3 (line 439) | def rel_process_results_v3(doc, results):
  function split_text_with_regex (line 498) | def split_text_with_regex(text, pattern):
  function faq_doc_to_target (line 526) | def faq_doc_to_target(x):
  function ht_doc_to_target (line 541) | def ht_doc_to_target(x):

FILE: lm_eval/tasks/fda/task.py
  class FDA (line 10) | class FDA(ConfigurableTask):
    method __init__ (line 15) | def __init__(self, **kwargs):
    method has_training_docs (line 18) | def has_training_docs(self):
    method has_validation_docs (line 21) | def has_validation_docs(self):
    method has_test_docs (line 24) | def has_test_docs(self):
    method validation_docs (line 27) | def validation_docs(self):
    method doc_to_text (line 30) | def doc_to_text(self, doc):
    method doc_to_target (line 33) | def doc_to_target(self, doc):
    method construct_requests (line 36) | def construct_requests(
    method process_results (line 60) | def process_results(self, doc, results):
    method aggregation (line 75) | def aggregation(self):
    method higher_is_better (line 85) | def higher_is_better(self):
  function contains_score (line 96) | def contains_score(prediction: str, labels: List[str]):

FILE: lm_eval/tasks/french_bench/preprocess_wikitext.py
  function wikitext_detokenizer (line 4) | def wikitext_detokenizer(doc):
  function process_results (line 39) | def process_results(doc, results):

FILE: lm_eval/tasks/french_bench/utils.py
  function normalize_answer (line 9) | def normalize_answer(s):
  function get_tokens (line 29) | def get_tokens(s):
  function exact (line 36) | def exact(predictions, references):
  function f1 (line 41) | def f1(predictions, references):
  function rouge1 (line 57) | def rouge1(items):
  function rouge1_agg (line 64) | def rouge1_agg(items):
  function is_included (line 74) | def is_included(items):
  function preprocess (line 83) | def preprocess(text):
  function process_docs (line 92) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/galician_bench/flores_gl/create_yamls_flores_gl.py
  function doc_to_text (line 257) | def doc_to_text(src: str, tgt: str) -> str:
  function doc_to_target (line 265) | def doc_to_target(tgt: str) -> str:
  function gen_lang_yamls (line 272) | def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
  function main (line 316) | def main() -> None:

FILE: lm_eval/tasks/galician_bench/utils.py
  function lowercase_first_letter (line 14) | def lowercase_first_letter(text):
  function process_summarization (line 18) | def process_summarization(dataset):
  function process_docs_paraphrases (line 28) | def process_docs_paraphrases(dataset):
  function process_docs_paws (line 56) | def process_docs_paws(dataset):
  function rouge1 (line 84) | def rouge1(items):
  function rouge1_agg (line 91) | def rouge1_agg(items):
  function process_results_mc2 (line 102) | def process_results_mc2(doc, results):
  function process_docs_gen (line 115) | def process_docs_gen(dataset: datasets.Dataset) -> datasets.Dataset:
  function preprocess_function_gen (line 119) | def preprocess_function_gen(examples):
  function process_doc_nli (line 143) | def process_doc_nli(dataset):
  function process_results_gen (line 170) | def process_results_gen(doc, results):
  function bleu (line 241) | def bleu(refs, preds):
  function rouge (line 264) | def rouge(refs, preds):

FILE: lm_eval/tasks/glianorex/preprocess_glianorex.py
  function doc_to_text (line 4) | def doc_to_text(doc) -> str:
  function doc_to_target (line 10) | def doc_to_target(doc) -> str:
  function filter_dataset (line 15) | def filter_dataset(dataset: datasets.Dataset, lang: str) -> datasets.Dat...
  function filter_french (line 19) | def filter_french(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_english (line 23) | def filter_english(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/global_mmlu/default/ar/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/default/bn/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/default/de/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/default/en/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/default/es/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/default/fr/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/default/hi/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/default/id/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/default/it/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/default/ja/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/default/ko/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/default/pt/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/default/sw/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/default/yo/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/default/zh/utils.py
  function process_docs (line 7) | def process_docs(dataset, category):

FILE: lm_eval/tasks/global_mmlu/full/am/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/ar/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/bn/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/cs/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/de/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/el/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/en/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/es/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/fa/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/fil/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/fr/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/ha/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/he/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/hi/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/id/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/ig/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/it/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/ja/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/ko/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/ky/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/lt/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/mg/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/ms/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/ne/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/nl/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/ny/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/pl/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/pt/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/ro/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/ru/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/si/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/sn/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/so/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/sr/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/sv/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/sw/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/te/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/tr/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/uk/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/vi/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/yo/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_mmlu/full/zh/utils.py
  function process_docs (line 65) | def process_docs(dataset, subject):

FILE: lm_eval/tasks/global_piqa/completions/_generate_config.py
  class IndentedDumper (line 7) | class IndentedDumper(yaml.Dumper):
    method increase_indent (line 8) | def increase_indent(self, flow=False, indentless=False):
  function format_subset (line 15) | def format_subset(subset: str, preface: str = PREFACE) -> str:

FILE: lm_eval/tasks/global_piqa/prompted/_generate_config.py
  class IndentedDumper (line 7) | class IndentedDumper(yaml.Dumper):
    method increase_indent (line 8) | def increase_indent(self, flow=False, indentless=False):
  function format_subset (line 15) | def format_subset(subset: str, preface: str = PREFACE) -> str:

FILE: lm_eval/tasks/glue/mnli/utils.py
  function doc_to_text (line 1) | def doc_to_text(doc) -> str:

FILE: lm_eval/tasks/gpqa/cot_n_shot/_generate_configs.py
  function main (line 5) | def main() -> None:

FILE: lm_eval/tasks/gpqa/cot_n_shot/utils.py
  function preprocess (line 7) | def preprocess(text):
  function process_docs (line 17) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/gpqa/cot_zeroshot/_generate_configs.py
  function main (line 5) | def main() -> None:

FILE: lm_eval/tasks/gpqa/cot_zeroshot/utils.py
  function preprocess (line 7) | def preprocess(text):
  function process_docs (line 17) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/gpqa/generative/_generate_configs.py
  function main (line 5) | def main() -> None:

FILE: lm_eval/tasks/gpqa/generative/utils.py
  function preprocess (line 7) | def preprocess(text):
  function process_docs (line 17) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/gpqa/n_shot/_generate_configs.py
  function main (line 5) | def main() -> None:

FILE: lm_eval/tasks/gpqa/n_shot/utils.py
  function preprocess (line 7) | def preprocess(text):
  function process_docs (line 20) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/gpqa/zeroshot/_generate_configs.py
  function main (line 5) | def main() -> None:

FILE: lm_eval/tasks/gpqa/zeroshot/utils.py
  function preprocess (line 7) | def preprocess(text):
  function process_docs (line 17) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/graphwalks/utils.py
  function load_dataset (line 7) | def load_dataset(**kwargs):
  function extract_answer_list (line 27) | def extract_answer_list(response: str) -> Tuple[List[str], bool]:
  function extract_answer_list_flexible (line 65) | def extract_answer_list_flexible(response: str) -> Tuple[List[str], bool]:
  function process_results (line 100) | def process_results(doc, results):

FILE: lm_eval/tasks/groundcocoa/utils.py
  function process_docs (line 6) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/hellaswag/utils.py
  function preprocess (line 6) | def preprocess(text):
  function process_docs (line 15) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/hendrycks_ethics/utils.py
  function _preproc_doc (line 5) | def _preproc_doc(doc):
  function doc_to_text (line 18) | def doc_to_text(doc) -> str:
  function doc_to_target (line 23) | def doc_to_target(doc):

FILE: lm_eval/tasks/hendrycks_math/utils.py
  function process_docs (line 6) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
  function process_results (line 18) | def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
  function is_equiv (line 36) | def is_equiv(str1, str2, verbose=False):
  function remove_boxed (line 53) | def remove_boxed(s):
  function last_boxed_only_string (line 67) | def last_boxed_only_string(string):
  function fix_fracs (line 97) | def fix_fracs(string):
  function fix_a_slash_b (line 129) | def fix_a_slash_b(string):
  function remove_right_units (line 144) | def remove_right_units(string):
  function fix_sqrt (line 154) | def fix_sqrt(string):
  function strip_string (line 169) | def strip_string(string):

FILE: lm_eval/tasks/histoires_morales/utils.py
  function process_docs (line 4) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: lm_eval/tasks/hrm8k/default/utils.py
  function doc_to_text (line 5) | def doc_to_text(doc):
  function doc_to_text_mmmlu (line 14) | def doc_to_text_mmmlu(doc):
  function doc_to_target (line 23) | def doc_to_target(doc):
  function postprocess (line 27) | def postprocess(s):
  function process_results (line 36) | def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
  function is_equiv (line 54) | def is_equiv(str1, str2, verbose=False):
  function parse_math_answer (line 74) | def parse_math_answer(raw_string):
  function _fix_fracs (line 146) | def _fix_fracs(string):
  function _fix_a_slash_b (line 178) | def _fix_a_slash_b(string):
  function _remove_right_units (line 193) | def _remove_right_units(string):
  function _fix_sqrt (line 203) | def _fix_sqrt(string):
  function _strip_string (line 218) | def _strip_string(string):

FILE: lm_eval/tasks/hrm8k/en/utils.py
  function doc_to_text (line 5) | def doc_to_text(doc):
  function doc_to_text_mmmlu (line 14) | def doc_to_text_mmmlu(doc):
  function doc_to_target (line 23) | def doc_to_target(doc):
  function postprocess (line 27) | def postprocess(s):
  function process_results (line 36) | def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
  function is_equiv (line 54) | def is_equiv(str1, str2, verbose=False):
  function parse_math_answer (line 74) | def parse_math_answer(raw_string):
  function _fix_fracs (line 146) | def _fix_fracs(string):
  function _fix_a_slash_b (line 178) | def _fix_a_slash_b(string):
  function _remove_right_units (line 193) | def _remove_right_units(string):
  function _fix_sqrt (line 203) | def _fix_sqrt(string):
  function _strip_string (line 218) | def _strip_string(string):

FILE: lm_eval/tasks/humaneval/utils.py
  function pass_at_k (line 13) | def pass_at_k(references: list[str], predictions: list[list[str]], k: li...
  function build_predictions (line 26) | def build_predictions(resps: list[list[str]], docs: list[dict]) -> list[...
  function build_predictions_instruct (line 30) | def build_predictions_instruct(

FILE: lm_eval/tasks/humaneval_infilling/utils.py
  function pass_at_k (line 13) | def pass_at_k(references: list[str], predictions: list[list[str]], k: li...
  function build_predictions (line 26) | def build_predictions(resps: list[list[str]], docs: list[dict]) -> list[...

FILE: lm_eval/tasks/icelandic_winogrande/preprocess_winogrande.py
  function doc_to_text (line 1) | def doc_to_text(doc):
  function doc_to_target (line 6) | def doc_to_target(doc):
  function doc_to_choice (line 14) | def doc_to_choice(doc):

FILE: lm_eval/tasks/ifeval/instructions.py
  class Instruction (line 110) | class Instruction:
    method __init__ (line 113) | def __init__(self, instruction_id):
    method build_description (line 116) | def build_description(self, **kwargs):
    method get_instruction_args (line 119) | def get_instruction_args(self):
    method get_instruction_args_keys (line 122) | def get_instruction_args_keys(self):
    method check_following (line 125) | def check_following(self, value):
  class ResponseLanguageChecker (line 129) | class ResponseLanguageChecker(Instruction):
    method build_description (line 132) | def build_description(self, *, language=None):
    method get_instruction_args (line 155) | def get_instruction_args(self):
    method get_instruction_args_keys (line 159) | def get_instruction_args_keys(self):
    method check_following (line 163) | def check_following(self, value):
  class NumberOfSentences (line 184) | class NumberOfSentences(Instruction):
    method build_description (line 187) | def build_description(self, *, num_sentences=None, relation=None):
    method get_instruction_args (line 225) | def get_instruction_args(self):
    method get_instruction_args_keys (line 232) | def get_instruction_args_keys(self):
    method check_following (line 236) | def check_following(self, value):
  class PlaceholderChecker (line 256) | class PlaceholderChecker(Instruction):
    method build_description (line 259) | def build_description(self, *, num_placeholders=None):
    method get_instruction_args (line 278) | def get_instruction_args(self):
    method get_instruction_args_keys (line 282) | def get_instruction_args_keys(self):
    method check_following (line 286) | def check_following(self, value):
  class BulletListChecker (line 301) | class BulletListChecker(Instruction):
    method build_description (line 304) | def build_description(self, *, num_bullets=None):
    method get_instruction_args (line 325) | def get_instruction_args(self):
    method get_instruction_args_keys (line 329) | def get_instruction_args_keys(self):
    method check_following (line 333) | def check_following(self, value):
  class ConstrainedResponseChecker (line 350) | class ConstrainedResponseChecker(Instruction):
    method build_description (line 353) | def build_description(self):
    method get_instruction_args (line 364) | def get_instruction_args(self):
    method get_instruction_args_keys (line 368) | def get_instruction_args_keys(self):
    method check_following (line 372) | def check_following(self, value):
  class ConstrainedStartChecker (line 389) | class ConstrainedStartChecker(Instruction):
    method build_description (line 392) | def build_description(self, *, starter=None):
    method get_instruction_args (line 411) | def get_instruction_args(self):
    method get_instruction_args_keys (line 415) | def get_instruction_args_keys(self):
    method check_following (line 419) | def check_following(self, value):
  class HighlightSectionChecker (line 436) | class HighlightSectionChecker(Instruction):
    method build_description (line 439) | def build_description(self, *, num_highlights=None):
    method get_instruction_args (line 460) | def get_instruction_args(self):
    method get_instruction_args_keys (line 464) | def get_instruction_args_keys(self):
    method check_following (line 468) | def check_following(self, value):
  class SectionChecker (line 492) | class SectionChecker(Instruction):
    method build_description (line 495) | def build_description(self, *, section_spliter=None, num_sections=None):
    method get_instruction_args (line 531) | def get_instruction_args(self):
    method get_instruction_args_keys (line 538) | def get_instruction_args_keys(self):
    method check_following (line 542) | def check_following(self, value):
  class ParagraphChecker (line 561) | class ParagraphChecker(Instruction):
    method build_description (line 564) | def build_description(self, *, num_paragraphs=None):
    method get_instruction_args (line 584) | def get_instruction_args(self):
    method get_instruction_args_keys (line 588) | def get_instruction_args_keys(self):
    method check_following (line 592) | def check_following(self, value):
  class PostscriptChecker (line 616) | class PostscriptChecker(Instruction):
    method build_description (line 619) | def build_description(self, *, postscript_marker=None):
    method get_instruction_args (line 644) | def get_instruction_args(self):
    method get_instruction_args_keys (line 648) | def get_instruction_args_keys(self):
    method check_following (line 652) | def check_following(self, value):
  class RephraseChecker (line 674) | class RephraseChecker(Instruction):
    method build_description (line 677) | def build_description(self, *, original_message):
    method get_instruction_args (line 703) | def get_instruction_args(self):
    method get_instruction_args_keys (line 707) | def get_instruction_args_keys(self):
    method check_following (line 711) | def check_following(self, value):
    method is_change (line 733) | def is_change(self, response):
    method strip_changes (line 737) | def strip_changes(self, response):
  class KeywordChecker (line 742) | class KeywordChecker(Instruction):
    method build_description (line 745) | def build_description(self, *, keywords=None):
    method get_instruction_args (line 768) | def get_instruction_args(self):
    method get_instruction_args_keys (line 772) | def get_instruction_args_keys(self):
    method check_following (line 776) | def check_following(self, value):
  class KeywordFrequencyChecker (line 784) | class KeywordFrequencyChecker(Instruction):
    method build_description (line 787) | def build_description(self, *, keyword=None, frequency=None, relation=...
    method get_instruction_args (line 833) | def get_instruction_args(self):
    method get_instruction_args_keys (line 841) | def get_instruction_args_keys(self):
    method check_following (line 845) | def check_following(self, value):
  class NumberOfWords (line 855) | class NumberOfWords(Instruction):
    method build_description (line 858) | def build_description(self, *, num_words=None, relation=None):
    method get_instruction_args (line 896) | def get_instruction_args(self):
    method get_instruction_args_keys (line 900) | def get_instruction_args_keys(self):
    method check_following (line 904) | def check_following(self, value):
  class JsonFormat (line 914) | class JsonFormat(Instruction):
    method build_description (line 917) | def build_description(self):
    method get_instruction_args (line 924) | def get_instruction_args(self):
    method get_instruction_args_keys (line 928) | def get_instruction_args_keys(self):
    method check_following (line 932) | def check_following(self, value):
  class ParagraphFirstWordCheck (line 949) | class ParagraphFirstWordCheck(Instruction):
    method build_description (line 952) | def build_description(
    method get_instruction_args (line 998) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1006) | def get_instruction_args_keys(self):
    method check_following (line 1010) | def check_following(self, value):
  class KeySentenceChecker (line 1056) | class KeySentenceChecker(Instruction):
    method build_description (line 1059) | def build_description(self, key_sentences=None, num_sentences=None):
    method get_instruction_args (line 1091) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1098) | def get_instruction_args_keys(self):
    method check_following (line 1102) | def check_following(self, value):
  class ForbiddenWords (line 1113) | class ForbiddenWords(Instruction):
    method build_description (line 1116) | def build_description(self, forbidden_words=None):
    method get_instruction_args (line 1140) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1144) | def get_instruction_args_keys(self):
    method check_following (line 1148) | def check_following(self, value):
  class RephraseParagraph (line 1156) | class RephraseParagraph(Instruction):
    method build_description (line 1159) | def build_description(self, *, original_paragraph, low, high):
    method get_instruction_args (line 1190) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1198) | def get_instruction_args_keys(self):
    method check_following (line 1202) | def check_following(self, value):
  class TwoResponsesChecker (line 1216) | class TwoResponsesChecker(Instruction):
    method build_description (line 1219) | def build_description(self):
    method get_instruction_args (line 1227) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1231) | def get_instruction_args_keys(self):
    method check_following (line 1235) | def check_following(self, value):
  class RepeatPromptThenAnswer (line 1258) | class RepeatPromptThenAnswer(Instruction):
    method build_description (line 1261) | def build_description(self, *, prompt_to_repeat=None):
    method get_instruction_args (line 1282) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1285) | def get_instruction_args_keys(self):
    method check_following (line 1289) | def check_following(self, value):
  class EndChecker (line 1295) | class EndChecker(Instruction):
    method build_description (line 1298) | def build_description(self, *, end_phrase=None):
    method get_instruction_args (line 1318) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1321) | def get_instruction_args_keys(self):
    method check_following (line 1325) | def check_following(self, value):
  class TitleChecker (line 1332) | class TitleChecker(Instruction):
    method build_description (line 1335) | def build_description(self):
    method get_instruction_args (line 1343) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1346) | def get_instruction_args_keys(self):
    method check_following (line 1350) | def check_following(self, value):
  class LetterFrequencyChecker (line 1362) | class LetterFrequencyChecker(Instruction):
    method build_description (line 1365) | def build_description(self, *, letter=None, let_frequency=None, let_re...
    method get_instruction_args (line 1417) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1425) | def get_instruction_args_keys(self):
    method check_following (line 1429) | def check_following(self, value):
  class CapitalLettersEnglishChecker (line 1440) | class CapitalLettersEnglishChecker(Instruction):
    method build_description (line 1443) | def build_description(self):
    method get_instruction_args (line 1450) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1453) | def get_instruction_args_keys(self):
    method check_following (line 1457) | def check_following(self, value):
  class LowercaseLettersEnglishChecker (line 1471) | class LowercaseLettersEnglishChecker(Instruction):
    method build_description (line 1474) | def build_description(self):
    method get_instruction_args (line 1482) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1485) | def get_instruction_args_keys(self):
    method check_following (line 1489) | def check_following(self, value):
  class CommaChecker (line 1503) | class CommaChecker(Instruction):
    method build_description (line 1506) | def build_description(self):
    method get_instruction_args (line 1513) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1516) | def get_instruction_args_keys(self):
    method check_following (line 1520) | def check_following(self, value):
  class CapitalWordFrequencyChecker (line 1525) | class CapitalWordFrequencyChecker(Instruction):
    method build_description (line 1528) | def build_description(
    method get_instruction_args (line 1566) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1573) | def get_instruction_args_keys(self):
    method check_following (line 1577) | def check_following(self, value):
  class QuotationChecker (line 1591) | class QuotationChecker(Instruction):
    method build_description (line 1594) | def build_description(self):
    method get_instruction_args (line 1601) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1605) | def get_instruction_args_keys(self):
    method check_following (line 1609) | def check_following(self, value):

FILE: lm_eval/tasks/ifeval/instructions_registry.py
  function conflict_make (line 153) | def conflict_make(conflicts):

FILE: lm_eval/tasks/ifeval/instructions_util.py
  function download_nltk_resources (line 36) | def download_nltk_resources():
  function split_into_sentences (line 1628) | def split_into_sentences(text):
  function count_words (line 1679) | def count_words(text):
  function _get_sentence_tokenizer (line 1688) | def _get_sentence_tokenizer():
  function count_sentences (line 1692) | def count_sentences(text):
  function generate_keywords (line 1699) | def generate_keywords(num_keywords):

FILE: lm_eval/tasks/ifeval/multilingual/instruction_utils/ca_instructions_util.py
  function lang_code_to_name (line 32) | def lang_code_to_name(lang_code: str):
  function split_into_sentences (line 46) | def split_into_sentences(text):
  function count_words (line 98) | def count_words(text):
  function tokenize_words (line 105) | def tokenize_words(text):
  function count_sentences (line 113) | def count_sentences(text):
  function generate_keywords (line 120) | def generate_keywords(num_keywords):

FILE: lm_eval/tasks/ifeval/multilingual/instruction_utils/es_instructions_util.py
  function lang_code_to_name (line 32) | def lang_code_to_name(lang_code: str):
  function split_into_sentences (line 46) | def split_into_sentences(text):
  function count_words (line 98) | def count_words(text):
  function tokenize_words (line 105) | def tokenize_words(text):
  function count_sentences (line 113) | def count_sentences(text):
  function generate_keywords (line 120) | def generate_keywords(num_keywords):

FILE: lm_eval/tasks/ifeval/multilingual/instructions/ca_instructions.py
  class Instruction (line 90) | class Instruction:
    method __init__ (line 93) | def __init__(self, instruction_id):
    method build_description (line 96) | def build_description(self, **kwargs):
    method get_instruction_args (line 99) | def get_instruction_args(self):
    method get_instruction_args_keys (line 102) | def get_instruction_args_keys(self):
    method check_following (line 105) | def check_following(self, value):
  class ResponseLanguageChecker (line 109) | class ResponseLanguageChecker(Instruction):
    method build_description (line 112) | def build_description(self, *, language = None):
    method get_instruction_args (line 133) | def get_instruction_args(self):
    method get_instruction_args_keys (line 137) | def get_instruction_args_keys(self):
    method check_following (line 141) | def check_following(self, value):
  class NumberOfSentences (line 162) | class NumberOfSentences(Instruction):
    method build_description (line 165) | def build_description(self, *, num_sentences = None,
    method get_instruction_args (line 201) | def get_instruction_args(self):
    method get_instruction_args_keys (line 206) | def get_instruction_args_keys(self):
    method check_following (line 210) | def check_following(self, value):
  class PlaceholderChecker (line 248) | class PlaceholderChecker(Instruction):
    method build_description (line 251) | def build_description(self, *, num_placeholders = None,
    method get_instruction_args (line 284) | def get_instruction_args(self):
    method get_instruction_args_keys (line 289) | def get_instruction_args_keys(self):
    method check_following (line 293) | def check_following(self, value):
  class BulletListChecker (line 312) | class BulletListChecker(Instruction):
    method build_description (line 315) | def build_description(self, *, num_bullets = None):
    method get_instruction_args (line 336) | def get_instruction_args(self):
    method get_instruction_args_keys (line 340) | def get_instruction_args_keys(self):
    method check_following (line 344) | def check_following(self, value):
  class ConstrainedResponseChecker (line 360) | class ConstrainedResponseChecker(Instruction):
    method build_description (line 363) | def build_description(self):
    method get_instruction_args (line 372) | def get_instruction_args(self):
    method get_instruction_args_keys (line 376) | def get_instruction_args_keys(self):
    method check_following (line 380) | def check_following(self, value):
  class ConstrainedStartChecker (line 398) | class ConstrainedStartChecker(Instruction):
    method build_description (line 401) | def build_description(self, *, starter = None):
    method get_instruction_args (line 419) | def get_instruction_args(self):
    method get_instruction_args_keys (line 423) | def get_instruction_args_keys(self):
    method check_following (line 427) | def check_following(self, value):
  class HighlightSectionChecker (line 443) | class HighlightSectionChecker(Instruction):
    method build_description (line 446) | def build_description(self, *, num_highlights = None,
    method get_instruction_args (line 479) | def get_instruction_args(self):
    method get_instruction_args_keys (line 484) | def get_instruction_args_keys(self):
    method check_following (line 488) | def check_following(self, value):
  class SectionChecker (line 516) | class SectionChecker(Instruction):
    method build_description (line 519) | def build_description(self, *, section_spliter = None,
    method get_instruction_args (line 563) | def get_instruction_args(self):
    method get_instruction_args_keys (line 569) | def get_instruction_args_keys(self):
    method check_following (line 573) | def check_following(self, value):
  class ParagraphChecker (line 596) | class ParagraphChecker(Instruction):
    method build_description (line 599) | def build_description(self, *, num_paragraphs = None):
    method get_instruction_args (line 618) | def get_instruction_args(self):
    method get_instruction_args_keys (line 622) | def get_instruction_args_keys(self):
    method check_following (line 626) | def check_following(self, value):
  class PostscriptChecker (line 650) | class PostscriptChecker(Instruction):
    method build_description (line 653) | def build_description(self, *, postscript_marker = None
    method get_instruction_args (line 675) | def get_instruction_args(self):
    method get_instruction_args_keys (line 679) | def get_instruction_args_keys(self):
    method check_following (line 683) | def check_following(self, value):
  class RephraseChecker (line 706) | class RephraseChecker(Instruction):
    method build_description (line 709) | def build_description(self, *, original_message):
    method get_instruction_args (line 731) | def get_instruction_args(self):
    method get_instruction_args_keys (line 735) | def get_instruction_args_keys(self):
    method check_following (line 739) | def check_following(self, value):
    method is_change (line 761) | def is_change(self, response):
    method strip_changes (line 765) | def strip_changes(self, response):
  class KeywordChecker (line 770) | class KeywordChecker(Instruction):
    method build_description (line 773) | def build_description(self, *, keywords = None
    method get_instruction_args (line 796) | def get_instruction_args(self):
    method get_instruction_args_keys (line 800) | def get_instruction_args_keys(self):
    method check_following (line 804) | def check_following(self, value):
  class KeywordFrequencyChecker (line 812) | class KeywordFrequencyChecker(Instruction):
    method build_description (line 815) | def build_description(self, *, keyword = None,
    method get_instruction_args (line 859) | def get_instruction_args(self):
    method get_instruction_args_keys (line 865) | def get_instruction_args_keys(self):
    method check_following (line 869) | def check_following(self, value):
  class NumberOfWords (line 880) | class NumberOfWords(Instruction):
    method build_description (line 883) | def build_description(self, *, num_words = None,
    method get_instruction_args (line 921) | def get_instruction_args(self):
    method get_instruction_args_keys (line 926) | def get_instruction_args_keys(self):
    method check_following (line 930) | def check_following(self, value):
  class JsonFormat (line 946) | class JsonFormat(Instruction):
    method build_description (line 949) | def build_description(self):
    method get_instruction_args (line 955) | def get_instruction_args(self):
    method get_instruction_args_keys (line 959) | def get_instruction_args_keys(self):
    method check_following (line 966) | def check_following(self, value):
  class ParagraphFirstWordCheck (line 983) | class ParagraphFirstWordCheck(Instruction):
    method build_description (line 986) | def build_description(self, num_paragraphs = None,
    method get_instruction_args (line 1030) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1036) | def get_instruction_args_keys(self):
    method check_following (line 1040) | def check_following(self, value):
  class KeySentenceChecker (line 1095) | class KeySentenceChecker(Instruction):
    method build_description (line 1098) | def build_description(self, key_sentences = None,
    method get_instruction_args (line 1131) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1136) | def get_instruction_args_keys(self):
    method check_following (line 1140) | def check_following(self, value):
  class ForbiddenWords (line 1151) | class ForbiddenWords(Instruction):
    method build_description (line 1154) | def build_description(self, forbidden_words = None
    method get_instruction_args (line 1180) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1184) | def get_instruction_args_keys(self):
    method check_following (line 1188) | def check_following(self, value):
  class RephraseParagraph (line 1197) | class RephraseParagraph(Instruction):
    method build_description (line 1200) | def build_description(self, *, original_paragraph, low, high
    method get_instruction_args (line 1229) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1235) | def get_instruction_args_keys(self):
    method check_following (line 1239) | def check_following(self, value):
  class TwoResponsesChecker (line 1253) | class TwoResponsesChecker(Instruction):
    method build_description (line 1256) | def build_description(self):
    method get_instruction_args (line 1264) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1268) | def get_instruction_args_keys(self):
    method check_following (line 1272) | def check_following(self, value):
  class RepeatPromptThenAnswer (line 1295) | class RepeatPromptThenAnswer(Instruction):
    method build_description (line 1298) | def build_description(self, *, prompt_to_repeat = None):
    method get_instruction_args (line 1318) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1321) | def get_instruction_args_keys(self):
    method check_following (line 1325) | def check_following(self, value):
  class EndChecker (line 1331) | class EndChecker(Instruction):
    method build_description (line 1334) | def build_description(self, *, end_phrase = None):
    method get_instruction_args (line 1353) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1356) | def get_instruction_args_keys(self):
    method check_following (line 1360) | def check_following(self, value):
  class TitleChecker (line 1371) | class TitleChecker(Instruction):
    method build_description (line 1374) | def build_description(self):
    method get_instruction_args (line 1382) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1385) | def get_instruction_args_keys(self):
    method check_following (line 1389) | def check_following(self, value):
  class LetterFrequencyChecker (line 1401) | class LetterFrequencyChecker(Instruction):
    method build_description (line 1404) | def build_description(self, *, letter = None,
    method get_instruction_args (line 1458) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1464) | def get_instruction_args_keys(self):
    method check_following (line 1468) | def check_following(self, value):
  class CapitalLettersCatalanChecker (line 1479) | class CapitalLettersCatalanChecker(Instruction):
    method build_description (line 1482) | def build_description(self):
    method get_instruction_args (line 1489) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1492) | def get_instruction_args_keys(self):
    method check_following (line 1496) | def check_following(self, value):
  class LowercaseLettersCatalanChecker (line 1521) | class LowercaseLettersCatalanChecker(Instruction):
    method build_description (line 1524) | def build_description(self):
    method get_instruction_args (line 1532) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1535) | def get_instruction_args_keys(self):
    method check_following (line 1539) | def check_following(self, value):
  class CommaChecker (line 1553) | class CommaChecker(Instruction):
    method build_description (line 1556) | def build_description(self):
    method get_instruction_args (line 1563) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1566) | def get_instruction_args_keys(self):
    method check_following (line 1570) | def check_following(self, value):
  class CapitalWordFrequencyChecker (line 1575) | class CapitalWordFrequencyChecker(Instruction):
    method build_description (line 1578) | def build_description(
    method get_instruction_args (line 1616) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1623) | def get_instruction_args_keys(self):
    method check_following (line 1627) | def check_following(self, value):
  class QuotationChecker (line 1641) | class QuotationChecker(Instruction):
    method build_description (line 1644) | def build_description(self):
    method get_instruction_args (line 1651) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1655) | def get_instruction_args_keys(self):
    method check_following (line 1659) | def check_following(self, value):
  class QuestionMarkChecker (line 1665) | class QuestionMarkChecker(Instruction):
    method build_description (line 1668) | def build_description(self):
    method get_instruction_args (line 1675) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1679) | def get_instruction_args_keys(self):
    method check_following (line 1683) | def check_following(self, value):
  class ExclamationMarkChecker (line 1695) | class ExclamationMarkChecker(Instruction):
    method build_description (line 1698) | def build_description(self):
    method get_instruction_args (line 1705) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1709) | def get_instruction_args_keys(self):
    method check_following (line 1713) | def check_following(self, value):
  class EnieChecker (line 1725) | class EnieChecker(Instruction):
    method build_description (line 1728) | def build_description(
    method get_instruction_args (line 1754) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1760) | def get_instruction_args_keys(self):
    method check_following (line 1764) | def check_following(self, value):
  class DieresisChecker (line 1778) | class DieresisChecker(Instruction):
    method build_description (line 1781) | def build_description(
    method get_instruction_args (line 1807) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1813) | def get_instruction_args_keys(self):
    method check_following (line 1817) | def check_following(self, value):
  class TildesChecker (line 1831) | class TildesChecker(Instruction):
    method build_description (line 1834) | def build_description(self, *, num_words = None,
    method get_instruction_args (line 1872) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1877) | def get_instruction_args_keys(self):
    method check_following (line 1881) | def check_following(self, value):

FILE: lm_eval/tasks/ifeval/multilingual/instructions/es_instructions.py
  class Instruction (line 92) | class Instruction:
    method __init__ (line 95) | def __init__(self, instruction_id):
    method build_description (line 98) | def build_description(self, **kwargs):
    method get_instruction_args (line 101) | def get_instruction_args(self):
    method get_instruction_args_keys (line 104) | def get_instruction_args_keys(self):
    method check_following (line 107) | def check_following(self, value):
  class ResponseLanguageChecker (line 111) | class ResponseLanguageChecker(Instruction):
    method build_description (line 114) | def build_description(self, *, language = None):
    method get_instruction_args (line 135) | def get_instruction_args(self):
    method get_instruction_args_keys (line 139) | def get_instruction_args_keys(self):
    method check_following (line 143) | def check_following(self, value):
  class NumberOfSentences (line 164) | class NumberOfSentences(Instruction):
    method build_description (line 167) | def build_description(self, *, num_sentences = None,
    method get_instruction_args (line 203) | def get_instruction_args(self):
    method get_instruction_args_keys (line 208) | def get_instruction_args_keys(self):
    method check_following (line 212) | def check_following(self, value):
  class PlaceholderChecker (line 250) | class PlaceholderChecker(Instruction):
    method build_description (line 253) | def build_description(self, *, num_placeholders = None,
    method get_instruction_args (line 286) | def get_instruction_args(self):
    method get_instruction_args_keys (line 291) | def get_instruction_args_keys(self):
    method check_following (line 295) | def check_following(self, value):
  class BulletListChecker (line 314) | class BulletListChecker(Instruction):
    method build_description (line 317) | def build_description(self, *, num_bullets = None):
    method get_instruction_args (line 338) | def get_instruction_args(self):
    method get_instruction_args_keys (line 342) | def get_instruction_args_keys(self):
    method check_following (line 346) | def check_following(self, value):
  class ConstrainedResponseChecker (line 362) | class ConstrainedResponseChecker(Instruction):
    method build_description (line 365) | def build_description(self):
    method get_instruction_args (line 374) | def get_instruction_args(self):
    method get_instruction_args_keys (line 378) | def get_instruction_args_keys(self):
    method check_following (line 382) | def check_following(self, value):
  class ConstrainedStartChecker (line 400) | class ConstrainedStartChecker(Instruction):
    method build_description (line 403) | def build_description(self, *, starter = None):
    method get_instruction_args (line 421) | def get_instruction_args(self):
    method get_instruction_args_keys (line 425) | def get_instruction_args_keys(self):
    method check_following (line 429) | def check_following(self, value):
  class HighlightSectionChecker (line 445) | class HighlightSectionChecker(Instruction):
    method build_description (line 448) | def build_description(self, *, num_highlights = None,
    method get_instruction_args (line 481) | def get_instruction_args(self):
    method get_instruction_args_keys (line 486) | def get_instruction_args_keys(self):
    method check_following (line 490) | def check_following(self, value):
  class SectionChecker (line 518) | class SectionChecker(Instruction):
    method build_description (line 521) | def build_description(self, *, section_spliter = None,
    method get_instruction_args (line 565) | def get_instruction_args(self):
    method get_instruction_args_keys (line 571) | def get_instruction_args_keys(self):
    method check_following (line 575) | def check_following(self, value):
  class ParagraphChecker (line 598) | class ParagraphChecker(Instruction):
    method build_description (line 601) | def build_description(self, *, num_paragraphs = None):
    method get_instruction_args (line 620) | def get_instruction_args(self):
    method get_instruction_args_keys (line 624) | def get_instruction_args_keys(self):
    method check_following (line 628) | def check_following(self, value):
  class PostscriptChecker (line 652) | class PostscriptChecker(Instruction):
    method build_description (line 655) | def build_description(self, *, postscript_marker = None
    method get_instruction_args (line 677) | def get_instruction_args(self):
    method get_instruction_args_keys (line 681) | def get_instruction_args_keys(self):
    method check_following (line 685) | def check_following(self, value):
  class RephraseChecker (line 708) | class RephraseChecker(Instruction):
    method build_description (line 711) | def build_description(self, *, original_message):
    method get_instruction_args (line 733) | def get_instruction_args(self):
    method get_instruction_args_keys (line 737) | def get_instruction_args_keys(self):
    method check_following (line 741) | def check_following(self, value):
    method is_change (line 763) | def is_change(self, response):
    method strip_changes (line 767) | def strip_changes(self, response):
  class KeywordChecker (line 772) | class KeywordChecker(Instruction):
    method build_description (line 775) | def build_description(self, *, keywords = None
    method get_instruction_args (line 798) | def get_instruction_args(self):
    method get_instruction_args_keys (line 802) | def get_instruction_args_keys(self):
    method check_following (line 806) | def check_following(self, value):
  class KeywordFrequencyChecker (line 814) | class KeywordFrequencyChecker(Instruction):
    method build_description (line 817) | def build_description(self, *, keyword = None,
    method get_instruction_args (line 861) | def get_instruction_args(self):
    method get_instruction_args_keys (line 867) | def get_instruction_args_keys(self):
    method check_following (line 871) | def check_following(self, value):
  class NumberOfWords (line 882) | class NumberOfWords(Instruction):
    method build_description (line 885) | def build_description(self, *, num_words = None,
    method get_instruction_args (line 923) | def get_instruction_args(self):
    method get_instruction_args_keys (line 928) | def get_instruction_args_keys(self):
    method check_following (line 932) | def check_following(self, value):
  class JsonFormat (line 948) | class JsonFormat(Instruction):
    method build_description (line 951) | def build_description(self):
    method get_instruction_args (line 957) | def get_instruction_args(self):
    method get_instruction_args_keys (line 961) | def get_instruction_args_keys(self):
    method check_following (line 968) | def check_following(self, value):
  class ParagraphFirstWordCheck (line 985) | class ParagraphFirstWordCheck(Instruction):
    method build_description (line 988) | def build_description(self, num_paragraphs = None,
    method get_instruction_args (line 1032) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1038) | def get_instruction_args_keys(self):
    method check_following (line 1042) | def check_following(self, value):
  class KeySentenceChecker (line 1097) | class KeySentenceChecker(Instruction):
    method build_description (line 1100) | def build_description(self, key_sentences = None,
    method get_instruction_args (line 1133) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1138) | def get_instruction_args_keys(self):
    method check_following (line 1142) | def check_following(self, value):
  class ForbiddenWords (line 1153) | class ForbiddenWords(Instruction):
    method build_description (line 1156) | def build_description(self, forbidden_words = None
    method get_instruction_args (line 1182) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1186) | def get_instruction_args_keys(self):
    method check_following (line 1190) | def check_following(self, value):
  class RephraseParagraph (line 1199) | class RephraseParagraph(Instruction):
    method build_description (line 1202) | def build_description(self, *, original_paragraph, low, high
    method get_instruction_args (line 1231) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1237) | def get_instruction_args_keys(self):
    method check_following (line 1241) | def check_following(self, value):
  class TwoResponsesChecker (line 1255) | class TwoResponsesChecker(Instruction):
    method build_description (line 1258) | def build_description(self):
    method get_instruction_args (line 1266) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1270) | def get_instruction_args_keys(self):
    method check_following (line 1274) | def check_following(self, value):
  class RepeatPromptThenAnswer (line 1297) | class RepeatPromptThenAnswer(Instruction):
    method build_description (line 1300) | def build_description(self, *, prompt_to_repeat = None):
    method get_instruction_args (line 1320) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1323) | def get_instruction_args_keys(self):
    method check_following (line 1327) | def check_following(self, value):
  class EndChecker (line 1333) | class EndChecker(Instruction):
    method build_description (line 1336) | def build_description(self, *, end_phrase = None):
    method get_instruction_args (line 1355) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1358) | def get_instruction_args_keys(self):
    method check_following (line 1362) | def check_following(self, value):
  class TitleChecker (line 1373) | class TitleChecker(Instruction):
    method build_description (line 1376) | def build_description(self):
    method get_instruction_args (line 1384) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1387) | def get_instruction_args_keys(self):
    method check_following (line 1391) | def check_following(self, value):
  class LetterFrequencyChecker (line 1403) | class LetterFrequencyChecker(Instruction):
    method build_description (line 1406) | def build_description(self, *, letter = None,
    method get_instruction_args (line 1460) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1466) | def get_instruction_args_keys(self):
    method check_following (line 1470) | def check_following(self, value):
  class CapitalLettersSpanishChecker (line 1481) | class CapitalLettersSpanishChecker(Instruction):
    method build_description (line 1484) | def build_description(self):
    method get_instruction_args (line 1491) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1494) | def get_instruction_args_keys(self):
    method check_following (line 1498) | def check_following(self, value):
  class LowercaseLettersSpanishChecker (line 1523) | class LowercaseLettersSpanishChecker(Instruction):
    method build_description (line 1526) | def build_description(self):
    method get_instruction_args (line 1534) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1537) | def get_instruction_args_keys(self):
    method check_following (line 1541) | def check_following(self, value):
  class CommaChecker (line 1555) | class CommaChecker(Instruction):
    method build_description (line 1558) | def build_description(self):
    method get_instruction_args (line 1565) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1568) | def get_instruction_args_keys(self):
    method check_following (line 1572) | def check_following(self, value):
  class CapitalWordFrequencyChecker (line 1577) | class CapitalWordFrequencyChecker(Instruction):
    method build_description (line 1580) | def build_description(
    method get_instruction_args (line 1618) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1625) | def get_instruction_args_keys(self):
    method check_following (line 1629) | def check_following(self, value):
  class QuotationChecker (line 1643) | class QuotationChecker(Instruction):
    method build_description (line 1646) | def build_description(self):
    method get_instruction_args (line 1653) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1657) | def get_instruction_args_keys(self):
    method check_following (line 1661) | def check_following(self, value):
  class QuestionMarkChecker (line 1667) | class QuestionMarkChecker(Instruction):
    method build_description (line 1670) | def build_description(self):
    method get_instruction_args (line 1677) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1681) | def get_instruction_args_keys(self):
    method check_following (line 1685) | def check_following(self, value):
  class ExclamationMarkChecker (line 1697) | class ExclamationMarkChecker(Instruction):
    method build_description (line 1700) | def build_description(self):
    method get_instruction_args (line 1707) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1711) | def get_instruction_args_keys(self):
    method check_following (line 1715) | def check_following(self, value):
  class EnieChecker (line 1727) | class EnieChecker(Instruction):
    method build_description (line 1730) | def build_description(
    method get_instruction_args (line 1756) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1762) | def get_instruction_args_keys(self):
    method check_following (line 1766) | def check_following(self, value):
  class DieresisChecker (line 1780) | class DieresisChecker(Instruction):
    method build_description (line 1783) | def build_description(
    method get_instruction_args (line 1809) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1815) | def get_instruction_args_keys(self):
    method check_following (line 1819) | def check_following(self, value):
  class TildesChecker (line 1833) | class TildesChecker(Instruction):
    method build_description (line 1836) | def build_description(self, *, num_words = None,
    method get_instruction_args (line 1874) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1879) | def get_instruction_args_keys(self):
    method check_following (line 1883) | def check_following(self, value):

FILE: lm_eval/tasks/ifeval/multilingual/utils.py
  class InputExample (line 7) | class InputExample:
  class OutputE

[
  {
    "path": ".github/workflows/new_tasks.yml",
    "chars": 2729,
    "preview": "name: Tasks Modified\n\non:\n  push:\n    branches:\n      - 'main'\n  pull_request:\n    branches:\n      - 'main'\n  workflow_d"
  },
  {
    "path": ".github/workflows/publish.yml",
    "chars": 2125,
    "preview": "name: Publish Python distribution to PyPI\n\non:\n  push:\n    tags:\n      - '*'\n\njobs:\n  build:\n    name: Build distributio"
  },
  {
    "path": ".github/workflows/unit_tests.yml",
    "chars": 3881,
    "preview": "# This workflow will install Python dependencies, run tests and lint with a variety of Python versions\n# For more inform"
  },
  {
    "path": ".gitignore",
    "chars": 487,
    "preview": "# macOS system files\n.DS_Store\n\n# Virtual environments\n.venv/\nvenv/\nENV/\nenv/\n*.env\n\n# Python bytecode and build artifac"
  },
  {
    "path": ".pre-commit-config.yaml",
    "chars": 1562,
    "preview": "# Ignore test linting to avoid conflicting changes to version stability.\nexclude: ^tests/testdata/\nrepos:\n  - repo: http"
  },
  {
    "path": "CITATION.bib",
    "chars": 744,
    "preview": "@misc{eval-harness,\n  author       = {Gao, Leo and Tow, Jonathan and Abbasi, Baber and Biderman, Stella and Black, Sid a"
  },
  {
    "path": "CODEOWNERS",
    "chars": 21,
    "preview": "* @baberabb\n* @0xSMT\n"
  },
  {
    "path": "LICENSE.md",
    "chars": 1067,
    "preview": "MIT License\n\nCopyright (c) 2020 EleutherAI\n\nPermission is hereby granted, free of charge, to any person obtaining a copy"
  },
  {
    "path": "MANIFEST.in",
    "chars": 24,
    "preview": "recursive-include tests\n"
  },
  {
    "path": "README.md",
    "chars": 55737,
    "preview": "# Language Model Evaluation Harness\n\n[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10256836.svg)](https://doi.org/"
  },
  {
    "path": "docs/API_guide.md",
    "chars": 8173,
    "preview": "# TemplateAPI Usage Guide\n\nThe `TemplateAPI` class is a versatile superclass designed to facilitate the integration of v"
  },
  {
    "path": "docs/CONTRIBUTING.md",
    "chars": 6148,
    "preview": "# Contributing to LM Evaluation Harness\n\nWelcome and thank you for your interest in the LM Evaluation Harness! We welcom"
  },
  {
    "path": "docs/README.md",
    "chars": 859,
    "preview": "# Eval Harness Documentation\n\nWelcome to the docs for the LM Evaluation Harness!\n\n## Table of Contents\n\n* To learn about"
  },
  {
    "path": "docs/chat-template-readme.md",
    "chars": 1331,
    "preview": "# Chat Template Delimiter Handling Update\n\n## Overview\n\nThis change modifies how delimiters are handled when applying ch"
  },
  {
    "path": "docs/config_files.md",
    "chars": 3620,
    "preview": "# Configuration Guide\n\nThis guide explains how to use YAML configuration files with `lm-eval` to define reusable evaluat"
  },
  {
    "path": "docs/decontamination.md",
    "chars": 3506,
    "preview": "# Decontamination\n\n## Usage\n\nThe provided directory should contain\nthe ngram files and info.json produced in \"Pile Ngram"
  },
  {
    "path": "docs/footguns.md",
    "chars": 1827,
    "preview": "# Common Pitfalls and Troubleshooting Guide\n\nThis document highlights common pitfalls and troubleshooting tips when usin"
  },
  {
    "path": "docs/interface.md",
    "chars": 9997,
    "preview": "# User Guide\n\nThis document details the interface exposed by `lm-eval` and provides details on what flags are available "
  },
  {
    "path": "docs/model_guide.md",
    "chars": 11334,
    "preview": "# New Model Guide\n\nThis guide may be of special interest to users who are using the library outside of the repository, v"
  },
  {
    "path": "docs/new_task_guide.md",
    "chars": 27171,
    "preview": "# New Task Guide\n\n`lm-evaluation-harness` is a framework that strives to support a wide range of zero- and few-shot eval"
  },
  {
    "path": "docs/python-api.md",
    "chars": 7379,
    "preview": "# Python API\n\nThis guide covers programmatic usage of the evaluation harness in Python scripts and applications.\n\n## Ove"
  },
  {
    "path": "docs/task_guide.md",
    "chars": 20974,
    "preview": "# Task Configuration\n\nThe `lm-evaluation-harness` is meant to be an extensible and flexible framework within which many "
  },
  {
    "path": "examples/lm-eval-overview.ipynb",
    "chars": 50295,
    "preview": "{\n \"cells\": [\n  {\n   \"cell_type\": \"markdown\",\n   \"metadata\": {\n    \"id\": \"Qw83KAePAhaS\"\n   },\n   \"source\": [\n    \"# Rele"
  },
  {
    "path": "examples/transformer-lens.py",
    "chars": 1985,
    "preview": "import warnings\n\nimport torch\nimport torch.nn as nn\nfrom transformer_lens import HookedTransformer\nfrom transformers imp"
  },
  {
    "path": "examples/visualize-wandb.ipynb",
    "chars": 5374,
    "preview": "{\n \"cells\": [\n  {\n   \"cell_type\": \"markdown\",\n   \"id\": \"fc477b96-adee-4829-a9d7-a5eb990df358\",\n   \"metadata\": {},\n   \"so"
  },
  {
    "path": "examples/visualize-zeno.ipynb",
    "chars": 3563,
    "preview": "{\n \"cells\": [\n  {\n   \"cell_type\": \"markdown\",\n   \"metadata\": {},\n   \"source\": [\n    \"# Visualizing Results in Zeno\\n\",\n "
  },
  {
    "path": "ignore.txt",
    "chars": 45,
    "preview": "ROUGE\nrouge\nnin\nmaka\nmor\nte\nond\nextraversion\n"
  },
  {
    "path": "lm_eval/__init__.py",
    "chars": 805,
    "preview": "import importlib.metadata\nimport logging\nimport os\nfrom importlib.util import find_spec\n\n\n__version__ = importlib.metada"
  },
  {
    "path": "lm_eval/__main__.py",
    "chars": 288,
    "preview": "from lm_eval._cli import HarnessCLI\nfrom lm_eval.utils import setup_logging\n\n\ndef cli_evaluate() -> None:\n    \"\"\"Main CL"
  },
  {
    "path": "lm_eval/_cli/__init__.py",
    "chars": 110,
    "preview": "\"\"\"\nCLI subcommands to run from the terminal.\n\"\"\"\n\nfrom .harness import HarnessCLI\n\n\n__all__ = [\"HarnessCLI\"]\n"
  },
  {
    "path": "lm_eval/_cli/harness.py",
    "chars": 2416,
    "preview": "import argparse\nimport sys\nimport textwrap\n\nfrom lm_eval._cli.ls import List\nfrom lm_eval._cli.run import Run\nfrom lm_ev"
  },
  {
    "path": "lm_eval/_cli/ls.py",
    "chars": 3394,
    "preview": "import argparse\nimport textwrap\n\nfrom lm_eval._cli.subcommand import SubCommand\n\n\nclass List(SubCommand):\n    \"\"\"Command"
  },
  {
    "path": "lm_eval/_cli/run.py",
    "chars": 16917,
    "preview": "import argparse\nimport json\nimport logging\nimport os\nimport textwrap\nfrom functools import partial\n\nfrom lm_eval._cli.su"
  },
  {
    "path": "lm_eval/_cli/subcommand.py",
    "chars": 480,
    "preview": "import argparse\nfrom abc import ABC, abstractmethod\n\n\nclass SubCommand(ABC):\n    \"\"\"Base class for all subcommands.\"\"\"\n\n"
  },
  {
    "path": "lm_eval/_cli/utils.py",
    "chars": 5608,
    "preview": "import argparse\nimport ast\nimport json\nimport logging\nfrom collections.abc import Sequence\nfrom typing import Any\n\n\neval"
  },
  {
    "path": "lm_eval/_cli/validate.py",
    "chars": 4937,
    "preview": "import argparse\nimport sys\nimport textwrap\n\nfrom lm_eval._cli.subcommand import SubCommand\n\n\nclass Validate(SubCommand):"
  },
  {
    "path": "lm_eval/api/__init__.py",
    "chars": 0,
    "preview": ""
  },
  {
    "path": "lm_eval/api/filter.py",
    "chars": 2149,
    "preview": "from abc import ABC, abstractmethod\nfrom dataclasses import dataclass\nfrom typing import Callable, Iterable, List, Union"
  },
  {
    "path": "lm_eval/api/group.py",
    "chars": 14078,
    "preview": "\"\"\"\nGroup model for organizing tasks into hierarchical collections.\n\nA Group is a container for Tasks and/or sub-Groups "
  },
  {
    "path": "lm_eval/api/instance.py",
    "chars": 1060,
    "preview": "from dataclasses import dataclass, field\nfrom typing import Literal, Optional, Tuple\n\n\nOutputType = Literal[\n    \"loglik"
  },
  {
    "path": "lm_eval/api/metrics.py",
    "chars": 19150,
    "preview": "import logging\nimport math\nimport os\nimport random\nimport re\nimport string\nfrom collections.abc import Iterable\nfrom typ"
  },
  {
    "path": "lm_eval/api/model.py",
    "chars": 20799,
    "preview": "import abc\nimport hashlib\nimport json\nimport logging\nimport os\nfrom collections.abc import Iterable\nfrom typing import T"
  },
  {
    "path": "lm_eval/api/registry.py",
    "chars": 22271,
    "preview": "\"\"\"Registry system for lm_eval components.\n\nThis module provides a centralized registration system for models, tasks, me"
  },
  {
    "path": "lm_eval/api/samplers.py",
    "chars": 3971,
    "preview": "from __future__ import annotations\n\nimport logging\nfrom random import Random\nfrom typing import TYPE_CHECKING\n\n\nif TYPE_"
  },
  {
    "path": "lm_eval/api/task.py",
    "chars": 71922,
    "preview": "from __future__ import annotations\n\nimport abc\nimport ast\nimport logging\nimport random\nimport re\nfrom collections.abc im"
  },
  {
    "path": "lm_eval/api/utils.py",
    "chars": 3465,
    "preview": "from __future__ import annotations\n\nfrom dataclasses import dataclass\nfrom typing import Any\n\n\ndef maybe_delimit(prefix:"
  },
  {
    "path": "lm_eval/caching/__init__.py",
    "chars": 0,
    "preview": ""
  },
  {
    "path": "lm_eval/caching/cache.py",
    "chars": 1424,
    "preview": "import hashlib\nimport logging\nimport os\n\nimport dill\n\n\neval_logger = logging.getLogger(__name__)\n\n\nMODULE_DIR = os.path."
  },
  {
    "path": "lm_eval/config/__init__.py",
    "chars": 166,
    "preview": "from .evaluate_config import EvaluatorConfig\nfrom .group import GroupConfig\nfrom .task import TaskConfig\n\n\n__all__ = [\"E"
  },
  {
    "path": "lm_eval/config/evaluate_config.py",
    "chars": 15162,
    "preview": "import json\nimport logging\nimport textwrap\nfrom argparse import Namespace\nfrom dataclasses import asdict, dataclass, fie"
  },
  {
    "path": "lm_eval/config/group.py",
    "chars": 4631,
    "preview": "from collections.abc import Callable\nfrom dataclasses import dataclass\nfrom typing import Any\n\n\n@dataclass\nclass AggMetr"
  },
  {
    "path": "lm_eval/config/task.py",
    "chars": 8251,
    "preview": "from __future__ import annotations\n\nimport logging\nfrom dataclasses import asdict, dataclass\nfrom inspect import getsour"
  },
  {
    "path": "lm_eval/decontamination/__init__.py",
    "chars": 0,
    "preview": ""
  },
  {
    "path": "lm_eval/decontamination/archiver.py",
    "chars": 5789,
    "preview": "import datetime\nimport io\nimport json\nimport mmap\nimport os\nfrom pathlib import Path\nfrom typing import Any\n\nimport json"
  },
  {
    "path": "lm_eval/decontamination/decontaminate.py",
    "chars": 6821,
    "preview": "import collections\nimport glob\nimport json\nimport os\nimport pickle\nimport random\nimport time\n\nfrom .archiver import ZStd"
  },
  {
    "path": "lm_eval/decontamination/janitor.py",
    "chars": 13039,
    "preview": "import pickle\nimport re\nimport string\nimport traceback\nfrom typing import Iterator, List, Sequence, Tuple, TypeVar\n\n\n# T"
  },
  {
    "path": "lm_eval/defaults.py",
    "chars": 1351,
    "preview": "import os\nfrom typing import Any\n\n\nDEFAULT_MAX_LENGTH = 2048\nDEFAULT_MAX_GEN_TOKS = 256\nDEFAULT_RANDOM_SEED = 0\nDEFAULT_"
  },
  {
    "path": "lm_eval/evaluator.py",
    "chars": 29713,
    "preview": "from __future__ import annotations\n\nimport itertools\nimport json\nimport logging\nimport random\nimport time\nfrom collectio"
  },
  {
    "path": "lm_eval/evaluator_utils.py",
    "chars": 17288,
    "preview": "from __future__ import annotations\n\nimport logging\nimport math\nimport pathlib\nimport sys\nfrom dataclasses import datacla"
  },
  {
    "path": "lm_eval/filters/__init__.py",
    "chars": 803,
    "preview": "from __future__ import annotations\n\nfrom functools import partial\n\nfrom lm_eval.api.filter import FilterEnsemble\nfrom lm"
  },
  {
    "path": "lm_eval/filters/custom.py",
    "chars": 453,
    "preview": "from lm_eval.api.filter import Filter\nfrom lm_eval.api.registry import register_filter\n\n\n@register_filter(\"custom\")\nclas"
  },
  {
    "path": "lm_eval/filters/decontamination.py",
    "chars": 712,
    "preview": "from lm_eval.api.filter import Filter\nfrom lm_eval.api.registry import register_filter\n\n\n@register_filter(\"decontaminate"
  },
  {
    "path": "lm_eval/filters/extraction.py",
    "chars": 8679,
    "preview": "import re\nimport sys\nimport unicodedata\n\nfrom lm_eval.api.filter import Filter\nfrom lm_eval.api.registry import register"
  },
  {
    "path": "lm_eval/filters/selection.py",
    "chars": 2105,
    "preview": "from collections import Counter\n\nfrom lm_eval.api.filter import Filter\nfrom lm_eval.api.registry import register_filter\n"
  },
  {
    "path": "lm_eval/filters/transformation.py",
    "chars": 3971,
    "preview": "import re\n\nfrom lm_eval.api.filter import Filter\nfrom lm_eval.api.registry import register_filter\n\n\n@register_filter(\"lo"
  },
  {
    "path": "lm_eval/loggers/__init__.py",
    "chars": 88,
    "preview": "from .evaluation_tracker import EvaluationTracker\nfrom .wandb_logger import WandbLogger\n"
  },
  {
    "path": "lm_eval/loggers/evaluation_tracker.py",
    "chars": 26111,
    "preview": "import json\nimport logging\nimport os\nimport re\nimport time\nfrom collections import defaultdict\nfrom dataclasses import a"
  },
  {
    "path": "lm_eval/loggers/utils.py",
    "chars": 5600,
    "preview": "import logging\nimport os\nimport re\nimport subprocess\nfrom importlib.metadata import version\nfrom pathlib import Path\nfro"
  },
  {
    "path": "lm_eval/loggers/wandb_logger.py",
    "chars": 13844,
    "preview": "import copy\nimport json\nimport logging\nfrom typing import Any, Dict, List, Literal, Tuple\n\nimport numpy as np\nimport pan"
  },
  {
    "path": "lm_eval/models/__init__.py",
    "chars": 3108,
    "preview": "\"\"\"Model implementations for lm_eval.\n\nModels are lazily loaded via the registry system to improve startup performance.\n"
  },
  {
    "path": "lm_eval/models/anthropic_llms.py",
    "chars": 12816,
    "preview": "import logging\nimport os\nfrom functools import cached_property\nfrom typing import Any, Dict, List, Tuple, Union\n\nfrom tq"
  },
  {
    "path": "lm_eval/models/api_models.py",
    "chars": 34542,
    "preview": "import abc\nimport asyncio\nimport copy\nimport itertools\nimport json\nimport logging\nfrom functools import cached_property\n"
  },
  {
    "path": "lm_eval/models/dummy.py",
    "chars": 1991,
    "preview": "import random\nfrom functools import cached_property\n\nfrom tqdm import tqdm\n\nfrom lm_eval.api.model import LM\nfrom lm_eva"
  },
  {
    "path": "lm_eval/models/gguf.py",
    "chars": 4815,
    "preview": "import logging\nimport time\n\nimport requests\nfrom requests.exceptions import RequestException\nfrom tqdm import tqdm\n\nfrom"
  },
  {
    "path": "lm_eval/models/hf_audiolm.py",
    "chars": 11704,
    "preview": "import copy\n\nimport torch\nimport transformers\nfrom tqdm import tqdm\nfrom transformers import BatchEncoding\n\nfrom lm_eval"
  },
  {
    "path": "lm_eval/models/hf_steered.py",
    "chars": 10010,
    "preview": "from __future__ import annotations\n\nfrom contextlib import contextmanager\nfrom functools import partial\nfrom pathlib imp"
  },
  {
    "path": "lm_eval/models/hf_vlms.py",
    "chars": 31685,
    "preview": "import copy\nimport logging\n\nimport torch\nimport torch.nn.functional as F\nimport transformers\nfrom tqdm import tqdm\nfrom "
  },
  {
    "path": "lm_eval/models/huggingface.py",
    "chars": 73991,
    "preview": "from __future__ import annotations\n\nimport logging\nimport os\nfrom datetime import timedelta\nfrom pathlib import Path\nfro"
  },
  {
    "path": "lm_eval/models/ibm_watsonx_ai.py",
    "chars": 18489,
    "preview": "import copy\nimport json\nimport logging\nimport os\nimport warnings\nfrom functools import cache\nfrom typing import NamedTup"
  },
  {
    "path": "lm_eval/models/mamba_lm.py",
    "chars": 6237,
    "preview": "import torch\n\nimport lm_eval.models.utils\nimport lm_eval.models.utils_hf\nfrom lm_eval.api.registry import register_model"
  },
  {
    "path": "lm_eval/models/megatron_lm.py",
    "chars": 54119,
    "preview": "r\"\"\"\nMegatron-LM backend for lm-evaluation-harness.\n\nThis module provides support for evaluating Megatron-LM models, inc"
  },
  {
    "path": "lm_eval/models/mistral3.py",
    "chars": 4197,
    "preview": "\"\"\"\nMistral3 model adapter for lm-evaluation-harness.\n\nThis adapter enables evaluation of Ministral-3 models (3B, 8B, 14"
  },
  {
    "path": "lm_eval/models/nemo_lm.py",
    "chars": 19598,
    "preview": "# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the "
  },
  {
    "path": "lm_eval/models/neuron_optimum.py",
    "chars": 28840,
    "preview": "import copy\nimport logging\nfrom collections import defaultdict\nfrom typing import TYPE_CHECKING, Optional\n\nimport torch\n"
  },
  {
    "path": "lm_eval/models/openai_completions.py",
    "chars": 13516,
    "preview": "import logging\nimport os\nfrom functools import cached_property\nfrom operator import itemgetter\nfrom typing import Any, D"
  },
  {
    "path": "lm_eval/models/optimum_habana.py",
    "chars": 6930,
    "preview": "import logging\nimport os\nfrom importlib.util import find_spec\nfrom typing import Any\n\nimport torch\nimport torch.nn.funct"
  },
  {
    "path": "lm_eval/models/optimum_ipex.py",
    "chars": 2342,
    "preview": "import logging\nfrom importlib.util import find_spec\n\nfrom lm_eval.api.registry import register_model\nfrom lm_eval.models"
  },
  {
    "path": "lm_eval/models/optimum_lm.py",
    "chars": 3051,
    "preview": "import json\nimport logging\nfrom importlib.util import find_spec\nfrom pathlib import Path\n\nfrom lm_eval.api.registry impo"
  },
  {
    "path": "lm_eval/models/sglang_causallms.py",
    "chars": 21266,
    "preview": "import copy\nimport logging\nfrom importlib.util import find_spec\nfrom typing import TYPE_CHECKING, Dict, List, Optional, "
  },
  {
    "path": "lm_eval/models/sglang_generate_API.py",
    "chars": 3360,
    "preview": "from typing import Dict, List, Optional, Tuple, Union\n\nfrom lm_eval.api.registry import register_model\nfrom lm_eval.mode"
  },
  {
    "path": "lm_eval/models/textsynth.py",
    "chars": 5877,
    "preview": "\"\"\"TextSynth API\nImplementation provided by Fabrice Bellard:\n    https://github.com/EleutherAI/lm-evaluation-harness/iss"
  },
  {
    "path": "lm_eval/models/utils.py",
    "chars": 35903,
    "preview": "from __future__ import annotations\n\nimport collections\nimport fnmatch\nimport itertools\nimport logging\nimport time\nfrom f"
  },
  {
    "path": "lm_eval/models/utils_hf.py",
    "chars": 4805,
    "preview": "import gc\nfrom typing import Literal\n\nimport torch\nimport transformers\n\n\ndef pad_and_concat(\n    max_length: int,\n    te"
  },
  {
    "path": "lm_eval/models/vllm_causallms.py",
    "chars": 35158,
    "preview": "from __future__ import annotations\n\nimport gc\nimport logging\nimport os\nfrom importlib.metadata import version\nfrom impor"
  },
  {
    "path": "lm_eval/models/vllm_vlms.py",
    "chars": 12270,
    "preview": "import logging\nfrom typing import TYPE_CHECKING, Any\n\nimport ray\nimport transformers\nfrom more_itertools import distribu"
  },
  {
    "path": "lm_eval/models/winml.py",
    "chars": 28691,
    "preview": "\"\"\"\nWinML backend for lm-eval-harness with NPU/GPU/CPU support.\n\nThis backend leverages Windows Machine Learning (WinML)"
  },
  {
    "path": "lm_eval/prompts/__init__.py",
    "chars": 4575,
    "preview": "import ast\nimport logging\nimport os\nfrom typing import Dict\n\nfrom lm_eval import utils\n\n\neval_logger = logging.getLogger"
  },
  {
    "path": "lm_eval/result_schema.py",
    "chars": 7851,
    "preview": "from typing import Any, Generic, TypeVar\n\nfrom typing_extensions import NotRequired, TypedDict\n\n\nT = TypeVar(\"T\", bound="
  },
  {
    "path": "lm_eval/tasks/README.md",
    "chars": 131803,
    "preview": "# Tasks\n\nA list of supported tasks and task groupings can be viewed with `lm-eval ls tasks`.\n\nFor more information, incl"
  },
  {
    "path": "lm_eval/tasks/__init__.py",
    "chars": 6321,
    "preview": "\"\"\"Task management for lm-evaluation-harness.\n\nThis module provides:\n- TaskManager: Main class for discovering and loadi"
  },
  {
    "path": "lm_eval/tasks/_factory.py",
    "chars": 10675,
    "preview": "from __future__ import annotations\n\nimport inspect\nimport logging\nfrom copy import deepcopy\nfrom dataclasses import fiel"
  },
  {
    "path": "lm_eval/tasks/_index.py",
    "chars": 6605,
    "preview": "from __future__ import annotations\n\nimport logging\nfrom dataclasses import dataclass, field\nfrom enum import Enum, auto\n"
  },
  {
    "path": "lm_eval/tasks/_yaml_loader.py",
    "chars": 7311,
    "preview": "from __future__ import annotations\n\nimport importlib.util\nimport sys\nfrom pathlib import Path\nfrom typing import Any\n\nim"
  },
  {
    "path": "lm_eval/tasks/aclue/README.md",
    "chars": 2288,
    "preview": "# ACLUE\n\n### Paper\n\nCan Large Language Model Comprehend Ancient Chinese? A Preliminary Test on ACLUE\nhttps://arxiv.org/a"
  },
  {
    "path": "lm_eval/tasks/aclue/_aclue.yaml",
    "chars": 684,
    "preview": "group: aclue\ntask:\n  - aclue_ancient_chinese_culture\n  - aclue_ancient_literature\n  - aclue_ancient_medical\n  - aclue_an"
  },
  {
    "path": "lm_eval/tasks/aclue/_default_template_yaml",
    "chars": 475,
    "preview": "dataset_path: tyouisen/aclue\ntest_split: test\nfewshot_split: dev\nfewshot_config:\n  sampler: first_n\noutput_type: multipl"
  },
  {
    "path": "lm_eval/tasks/aclue/_generate_configs.py",
    "chars": 2552,
    "preview": "\"\"\"\nTake in a YAML, and output all other splits with this YAML\n\"\"\"\n\nimport argparse\nimport logging\nimport os\n\nimport yam"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_ancient_chinese_culture.yaml",
    "chars": 169,
    "preview": "\"dataset_name\": \"ancient_chinese_culture\"\n\"description\": \"以下是关于国学常识的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_default_templa"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_ancient_literature.yaml",
    "chars": 161,
    "preview": "\"dataset_name\": \"ancient_literature\"\n\"description\": \"以下是关于古代文学知识的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_default_template_"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_ancient_medical.yaml",
    "chars": 152,
    "preview": "\"dataset_name\": \"ancient_medical\"\n\"description\": \"以下是关于医古文的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_default_template_yaml\"\n"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_ancient_phonetics.yaml",
    "chars": 156,
    "preview": "\"dataset_name\": \"ancient_phonetics\"\n\"description\": \"以下是关于古音学的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_default_template_yaml"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_basic_ancient_chinese.yaml",
    "chars": 166,
    "preview": "\"dataset_name\": \"basic_ancient_chinese\"\n\"description\": \"以下是关于古汉语知识的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_default_templat"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_couplet_prediction.yaml",
    "chars": 157,
    "preview": "\"dataset_name\": \"couplet_prediction\"\n\"description\": \"以下是关于对联的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_default_template_yaml"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_homographic_character_resolution.yaml",
    "chars": 186,
    "preview": "\"dataset_name\": \"homographic_character_resolution\"\n\"description\": \"以下是关于通假字的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_defaul"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_named_entity_recognition.yaml",
    "chars": 175,
    "preview": "\"dataset_name\": \"named_entity_recognition\"\n\"description\": \"以下是关于古汉语命名体识别的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_default_t"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_poetry_appreciate.yaml",
    "chars": 159,
    "preview": "\"dataset_name\": \"poetry_appreciate\"\n\"description\": \"以下是关于古诗词曲鉴赏的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_default_template_y"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_poetry_context_prediction.yaml",
    "chars": 177,
    "preview": "\"dataset_name\": \"poetry_context_prediction\"\n\"description\": \"以下是关于古诗词上下句预测的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_default_"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_poetry_quality_assessment.yaml",
    "chars": 176,
    "preview": "\"dataset_name\": \"poetry_quality_assessment\"\n\"description\": \"以下是关于古诗词质量评估的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_default_t"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_poetry_sentiment_analysis.yaml",
    "chars": 175,
    "preview": "\"dataset_name\": \"poetry_sentiment_analysis\"\n\"description\": \"以下是关于诗词情感分类的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_default_te"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_polysemy_resolution.yaml",
    "chars": 163,
    "preview": "\"dataset_name\": \"polysemy_resolution\"\n\"description\": \"以下是关于古文单字多义的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_default_template"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_reading_comprehension.yaml",
    "chars": 167,
    "preview": "\"dataset_name\": \"reading_comprehension\"\n\"description\": \"以下是关于古文阅读理解的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_default_templa"
  },
  {
    "path": "lm_eval/tasks/aclue/aclue_sentence_segmentation.yaml",
    "chars": 165,
    "preview": "\"dataset_name\": \"sentence_segmentation\"\n\"description\": \"以下是关于古文断句的单项选择题，请直接给出正确答案的选项。\\n\\n\"\n\"include\": \"_default_template"
  },
  {
    "path": "lm_eval/tasks/acpbench/README.md",
    "chars": 5770,
    "preview": "# ACPBench\n\n**Homepage:** https://ibm.github.io/ACPBench/\n\n### Papers\n\n**Title:** ACPBench: Reasoning About Action, Chan"
  },
  {
    "path": "lm_eval/tasks/acpbench/boolq_cot_2shot/_boolq_cot_2shot_yaml",
    "chars": 920,
    "preview": "tag:\n  - acp_bool_cot_2shot\n  - acp_bench\noutput_type: generate_until\ndataset_path: ibm-research/acp_bench\ntest_split: t"
  },
  {
    "path": "lm_eval/tasks/acpbench/boolq_cot_2shot/act_reach.yaml",
    "chars": 2393,
    "preview": "task: acp_areach_bool\ndataset_name: acp_areach_bool\ninclude: _boolq_cot_2shot_yaml\nfewshot_config:\n  sampler: first_n\n  "
  },
  {
    "path": "lm_eval/tasks/acpbench/boolq_cot_2shot/app.yaml",
    "chars": 2109,
    "preview": "task: acp_app_bool\ndataset_name: acp_app_bool\ninclude: _boolq_cot_2shot_yaml\nfewshot_config:\n  sampler: first_n\n  sample"
  },
  {
    "path": "lm_eval/tasks/acpbench/boolq_cot_2shot/just.yaml",
    "chars": 3967,
    "preview": "task: acp_just_bool\ndataset_name: acp_just_bool\ninclude: _boolq_cot_2shot_yaml\nfewshot_config:\n  sampler: first_n\n  samp"
  },
  {
    "path": "lm_eval/tasks/acpbench/boolq_cot_2shot/land.yaml",
    "chars": 2898,
    "preview": "task: acp_land_bool\ndataset_name: acp_land_bool\ninclude: _boolq_cot_2shot_yaml\nfewshot_config:\n  sampler: first_n\n  samp"
  },
  {
    "path": "lm_eval/tasks/acpbench/boolq_cot_2shot/prog.yaml",
    "chars": 2532,
    "preview": "task: acp_prog_bool\ndataset_name: acp_prog_bool\ninclude: _boolq_cot_2shot_yaml\nfewshot_config:\n  sampler: first_n\n  samp"
  },
  {
    "path": "lm_eval/tasks/acpbench/boolq_cot_2shot/reach.yaml",
    "chars": 2637,
    "preview": "task: acp_reach_bool\ndataset_name: acp_reach_bool\ninclude: _boolq_cot_2shot_yaml\nfewshot_config:\n  sampler: first_n\n  sa"
  },
  {
    "path": "lm_eval/tasks/acpbench/boolq_cot_2shot/val.yaml",
    "chars": 3070,
    "preview": "task: acp_val_bool\ndataset_name: acp_val_bool\ninclude: _boolq_cot_2shot_yaml\nfewshot_config:\n  sampler: first_n\n  sample"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot/_gen_yaml_2shot",
    "chars": 503,
    "preview": "tag:\n  - acp_gen_2shot\n  - acp_bench_hard\ndataset_path: ibm-research/acp_bench\ntest_split: test\ndoc_to_target: \"{{answer"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot/acp_grammar.lark",
    "chars": 383,
    "preview": "NAME: /[a-zA-Z][a-zA-Z0-9-_]*/\nLPAR : \"(\"\nRPAR : \")\"\nLSPAR: \"[\"\nRSPAR: \"]\"\nCOMMA: \",\"\nWS: /[ \\n]/\n\naction_none : \"None\"\n"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot/acp_utils.py",
    "chars": 37056,
    "preview": "import json\nimport os\nfrom abc import ABC, abstractmethod\nfrom collections import defaultdict\nfrom pathlib import Path\n\n"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot/act_reach.yaml",
    "chars": 3462,
    "preview": "task: acp_areach_gen\ndataset_name: acp_areach_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  samples:"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot/app.yaml",
    "chars": 3713,
    "preview": "task: acp_app_gen\ndataset_name: acp_app_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  samples:\n  - c"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot/just.yaml",
    "chars": 4868,
    "preview": "task: acp_just_gen\ndataset_name: acp_just_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  samples:\n  -"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot/land.yaml",
    "chars": 2692,
    "preview": "task: acp_land_gen\ndataset_name: acp_land_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  samples:\n  -"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot/next_act.yaml",
    "chars": 3553,
    "preview": "task: acp_nexta_gen\ndataset_name: acp_nexta_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  samples:\n "
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot/prog.yaml",
    "chars": 3145,
    "preview": "task: acp_prog_gen\ndataset_name: acp_prog_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  samples:\n  -"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot/reach.yaml",
    "chars": 2373,
    "preview": "task: acp_reach_gen\ndataset_name: acp_reach_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  samples:\n "
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot/val.yaml",
    "chars": 4131,
    "preview": "task: acp_val_gen\ndataset_name: acp_val_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  samples:\n  - c"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot_with_pddl/_gen_yaml_2shot",
    "chars": 690,
    "preview": "tag:\n  - acp_gen_2shot_with_pddl\n  - acp_bench_hard_with_pddl\ndataset_path: ibm-research/acp_bench\ntest_split: test\ndesc"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot_with_pddl/acp_grammar.lark",
    "chars": 383,
    "preview": "NAME: /[a-zA-Z][a-zA-Z0-9-_]*/\nLPAR : \"(\"\nRPAR : \")\"\nLSPAR: \"[\"\nRSPAR: \"]\"\nCOMMA: \",\"\nWS: /[ \\n]/\n\naction_none : \"None\"\n"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot_with_pddl/acp_utils.py",
    "chars": 37056,
    "preview": "import json\nimport os\nfrom abc import ABC, abstractmethod\nfrom collections import defaultdict\nfrom pathlib import Path\n\n"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot_with_pddl/act_reach.yaml",
    "chars": 10049,
    "preview": "task: acp_areach_gen_with_pddl\ndataset_name: acp_areach_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot_with_pddl/app.yaml",
    "chars": 10370,
    "preview": "task: acp_app_gen_with_pddl\ndataset_name: acp_app_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  samp"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot_with_pddl/just.yaml",
    "chars": 11547,
    "preview": "task: acp_just_gen_with_pddl\ndataset_name: acp_just_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  sa"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot_with_pddl/land.yaml",
    "chars": 9311,
    "preview": "task: acp_land_gen_with_pddl\ndataset_name: acp_land_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  sa"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot_with_pddl/next_act.yaml",
    "chars": 10202,
    "preview": "task: acp_nexta_gen_with_pddl\ndataset_name: acp_nexta_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  "
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot_with_pddl/prog.yaml",
    "chars": 9871,
    "preview": "task: acp_prog_gen_with_pddl\ndataset_name: acp_prog_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  sa"
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot_with_pddl/reach.yaml",
    "chars": 8996,
    "preview": "task: acp_reach_gen_with_pddl\ndataset_name: acp_reach_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  "
  },
  {
    "path": "lm_eval/tasks/acpbench/gen_2shot_with_pddl/val.yaml",
    "chars": 10729,
    "preview": "task: acp_val_gen_with_pddl\ndataset_name: acp_val_gen\ninclude: _gen_yaml_2shot\nfewshot_config:\n  sampler: first_n\n  samp"
  },
  {
    "path": "lm_eval/tasks/acpbench/mcq_cot_2shot/_mcq_cot_2shot_yaml",
    "chars": 997,
    "preview": "tag:\n  - acp_mcq_cot_2shot\n  - acp_bench\noutput_type: generate_until\ndataset_path: ibm-research/acp_bench\ntest_split: te"
  },
  {
    "path": "lm_eval/tasks/acpbench/mcq_cot_2shot/act_reach.yaml",
    "chars": 2836,
    "preview": "task: acp_areach_mcq\ndataset_name: acp_areach_mcq\ninclude: _mcq_cot_2shot_yaml\nfewshot_config:\n  sampler: first_n\n  samp"
  },
  {
    "path": "lm_eval/tasks/acpbench/mcq_cot_2shot/app.yaml",
    "chars": 2545,
    "preview": "task: acp_app_mcq\ndataset_name: acp_app_mcq\ninclude: _mcq_cot_2shot_yaml\nfewshot_config:\n  sampler: first_n\n  samples:\n "
  },
  {
    "path": "lm_eval/tasks/acpbench/mcq_cot_2shot/just.yaml",
    "chars": 4556,
    "preview": "task: acp_just_mcq\ndataset_name: acp_just_mcq\ninclude: _mcq_cot_2shot_yaml\nfewshot_config:\n  sampler: first_n\n  samples:"
  },
  {
    "path": "lm_eval/tasks/acpbench/mcq_cot_2shot/land.yaml",
    "chars": 2394,
    "preview": "task: acp_land_mcq\ndataset_name: acp_land_mcq\ninclude: _mcq_cot_2shot_yaml\nfewshot_config:\n  sampler: first_n\n  samples:"
  },
  {
    "path": "lm_eval/tasks/acpbench/mcq_cot_2shot/prog.yaml",
    "chars": 2572,
    "preview": "task: acp_prog_mcq\ndataset_name: acp_prog_mcq\ninclude: _mcq_cot_2shot_yaml\nfewshot_config:\n  sampler: first_n\n  samples:"
  },
  {
    "path": "lm_eval/tasks/acpbench/mcq_cot_2shot/reach.yaml",
    "chars": 2712,
    "preview": "task: acp_reach_mcq\ndataset_name: acp_reach_mcq\ninclude: _mcq_cot_2shot_yaml\nfewshot_config:\n  sampler: first_n\n  sample"
  },
  {
    "path": "lm_eval/tasks/acpbench/mcq_cot_2shot/val.yaml",
    "chars": 4240,
    "preview": "task: acp_val_mcq\ndataset_name: acp_val_mcq\ninclude: _mcq_cot_2shot_yaml\nfewshot_config:\n  sampler: first_n\n  samples:\n "
  },
  {
    "path": "lm_eval/tasks/aexams/README.md",
    "chars": 1896,
    "preview": "# Arabic EXAMS\n\n### Paper\n\nEXAMS: a resource specialized in multilingual high school exam questions.\nThe original paper "
  },
  {
    "path": "lm_eval/tasks/aexams/_aexams.yaml",
    "chars": 300,
    "preview": "group: aexams\ntask:\n  - aexams_Biology\n  - aexams_IslamicStudies\n  - aexams_Physics\n  - aexams_Science\n  - aexams_Social"
  },
  {
    "path": "lm_eval/tasks/aexams/_default_template_yaml",
    "chars": 479,
    "preview": "dataset_path: Hennara/aexams\ntest_split: test\nfewshot_split: dev\nfewshot_config:\n  sampler: first_n\noutput_type: multipl"
  },
  {
    "path": "lm_eval/tasks/aexams/aexams_Biology.yaml",
    "chars": 153,
    "preview": "\"dataset_name\": \"Biology\"\n\"description\": \"قم بالإجابة على مايلي في مجال العلوم الحيوية\\n\\n\"\n\"include\": \"_default_templat"
  },
  {
    "path": "lm_eval/tasks/aexams/aexams_IslamicStudies.yaml",
    "chars": 170,
    "preview": "\"dataset_name\": \"IslamicStudies\"\n\"description\": \"قم بالإجابة على مايلي في مجال العلوم الإسلامية \\n\\n\"\n\"include\": \"_defau"
  },
  {
    "path": "lm_eval/tasks/aexams/aexams_Physics.yaml",
    "chars": 148,
    "preview": "\"dataset_name\": \"Physics\"\n\"description\": \"قم بالإجابة على مايلي في مجال الفيزياء \\n\\n\"\n\"include\": \"_default_template_yam"
  },
  {
    "path": "lm_eval/tasks/aexams/aexams_Science.yaml",
    "chars": 146,
    "preview": "\"dataset_name\": \"Science\"\n\"description\": \"قم بالإجابة على مايلي في مجال العلوم \\n\\n\"\n\"include\": \"_default_template_yaml\""
  },
  {
    "path": "lm_eval/tasks/aexams/aexams_Social.yaml",
    "chars": 155,
    "preview": "\"dataset_name\": \"Social\"\n\"description\": \"قم بالإجابة على مايلي في مجال العلوم الإجتماعية \\n\\n\"\n\"include\": \"_default_temp"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/README.md",
    "chars": 2587,
    "preview": "# MathQA\n\n### Paper\n\nIrokoBench: A New Benchmark for African Languages in the Age of Large Language Models\nhttps://arxiv"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/afrimgsm.yaml",
    "chars": 282,
    "preview": "group: afrimgsm-irokobench\ntask:\n  - afrimgsm_tasks_prompt_1\n  - afrimgsm_tasks_prompt_2\n  - afrimgsm_tasks_prompt_3\n  -"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_amh.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: amh\ninclude: afrimgsm_yaml\ntask: afrimgsm_amh_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_eng.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: eng\ninclude: afrimgsm_yaml\ntask: afrimgsm_eng_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_ewe.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: ewe\ninclude: afrimgsm_yaml\ntask: afrimgsm_ewe_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_fra.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: fra\ninclude: afrimgsm_yaml\ntask: afrimgsm_fra_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_hau.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: hau\ninclude: afrimgsm_yaml\ntask: afrimgsm_hau_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_ibo.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: ibo\ninclude: afrimgsm_yaml\ntask: afrimgsm_ibo_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_kin.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: kin\ninclude: afrimgsm_yaml\ntask: afrimgsm_kin_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_lin.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: lin\ninclude: afrimgsm_yaml\ntask: afrimgsm_lin_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_lug.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: lug\ninclude: afrimgsm_yaml\ntask: afrimgsm_lug_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_orm.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: orm\ninclude: afrimgsm_yaml\ntask: afrimgsm_orm_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_sna.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: sna\ninclude: afrimgsm_yaml\ntask: afrimgsm_sna_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_sot.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: sot\ninclude: afrimgsm_yaml\ntask: afrimgsm_sot_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_swa.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: swa\ninclude: afrimgsm_yaml\ntask: afrimgsm_swa_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_twi.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: twi\ninclude: afrimgsm_yaml\ntask: afrimgsm_twi_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_vai.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: vai\ninclude: afrimgsm_yaml\ntask: afrimgsm_vai_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_wol.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: wol\ninclude: afrimgsm_yaml\ntask: afrimgsm_wol_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_xho.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: xho\ninclude: afrimgsm_yaml\ntask: afrimgsm_xho_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_yaml",
    "chars": 974,
    "preview": "tag:\n    - afrimgsm_tasks\n    - afrimgsm_tasks_prompt_1\ndataset_path: masakhane/afrimgsm\ndataset_name: null  # Overridde"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_yor.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: yor\ninclude: afrimgsm_yaml\ntask: afrimgsm_yor_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_zul.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: zul\ninclude: afrimgsm_yaml\ntask: afrimgsm_zul_prompt_1\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_amh.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: amh\ninclude: afrimgsm_yaml\ntask: afrimgsm_amh_prompt_2\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_eng.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: eng\ninclude: afrimgsm_yaml\ntask: afrimgsm_eng_prompt_2\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_ewe.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: ewe\ninclude: afrimgsm_yaml\ntask: afrimgsm_ewe_prompt_2\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_fra.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: fra\ninclude: afrimgsm_yaml\ntask: afrimgsm_fra_prompt_2\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_hau.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: hau\ninclude: afrimgsm_yaml\ntask: afrimgsm_hau_prompt_2\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_ibo.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: ibo\ninclude: afrimgsm_yaml\ntask: afrimgsm_ibo_prompt_2\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_kin.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: kin\ninclude: afrimgsm_yaml\ntask: afrimgsm_kin_prompt_2\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_lin.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: lin\ninclude: afrimgsm_yaml\ntask: afrimgsm_lin_prompt_2\n"
  },
  {
    "path": "lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_lug.yaml",
    "chars": 93,
    "preview": "# Generated by utils.py\ndataset_name: lug\ninclude: afrimgsm_yaml\ntask: afrimgsm_lug_prompt_2\n"
  }
]

// ... and 15534 more files (download for full content)