SYMBOL INDEX (3360 symbols across 473 files)

FILE: dataset_explorer_demo/visualize_similar.py
  function load_jsonl (line 27) | def load_jsonl(file_path):
  function render_latex (line 33) | def render_latex(text):
  function display_entry (line 98) | def display_entry(index, test_set):
  function random_entry (line 137) | def random_entry(data):
  function load_test_sets (line 142) | def load_test_sets(test_set):
  function update_test_set (line 201) | def update_test_set(test_set):
  function display_entry_wrapper (line 217) | def display_entry_wrapper(index, current_test_set):
  function random_entry_wrapper (line 223) | def random_entry_wrapper(current_test_set):

FILE: dockerfiles/sandbox/block_network.c
  function socket (line 41) | int socket(int domain, int type, int protocol) {

FILE: nemo_skills/_cli_stub.py
  function main (line 18) | def main():

FILE: nemo_skills/code_execution/local_sandbox/local_sandbox_server.py
  function shell_worker (line 78) | def shell_worker(conn):
  class ShellManager (line 174) | class ShellManager:
    method __init__ (line 175) | def __init__(self):
    method start_shell (line 182) | def start_shell(self, shell_id):
    method stop_shell (line 197) | def stop_shell(self, shell_id):
    method _finish_restart (line 215) | def _finish_restart(self, shell_id):
    method _cleanup_shell_resources (line 229) | def _cleanup_shell_resources(self, proc, conn):
    method run_cell (line 247) | def run_cell(self, shell_id, code, timeout=1.0, grace=2.0, traceback_v...
  function log_session_count (line 405) | def log_session_count(prefix: str = "") -> None:
  function cleanup_expired_sessions (line 419) | def cleanup_expired_sessions():
  function postprocess_output (line 440) | def postprocess_output(output, traceback_verbosity):
  function cleanup_session (line 464) | def cleanup_session(session_id):
  function execute_ipython_session (line 470) | def execute_ipython_session(generated_code, session_id, timeout=30, trac...
  function _after_log_session_count (line 547) | def _after_log_session_count(response):
  function kill_process_tree (line 561) | def kill_process_tree(proc):
  function set_limits (line 596) | def set_limits(mem_bytes: int = MEM_LIMIT_BYTES) -> None:
  function execute_python (line 607) | def execute_python(generated_code, std_input, timeout, language):
  function execute_lean4 (line 631) | def execute_lean4(generated_code, timeout):
  function execute_shell (line 688) | def execute_shell(command, timeout):
  function execute (line 716) | def execute():
  function list_sessions (line 747) | def list_sessions():
  function delete_session (line 771) | def delete_session(session_id):
  function health (line 787) | def health():

FILE: nemo_skills/code_execution/proof_utils.py
  class ProofBuildConfig (line 30) | class ProofBuildConfig:
  function extract_proof_only (line 39) | def extract_proof_only(lean_code: str) -> str:
  function build_lean4_proof (line 97) | def build_lean4_proof(
  function determine_proof_status (line 140) | def determine_proof_status(compiler_output: Dict[str, Any]) -> str:
  function prepare_predicted_proof_from_line_dict (line 169) | def prepare_predicted_proof_from_line_dict(
  function remove_comments (line 207) | def remove_comments(text):
  function move_imports_to_beginning (line 223) | def move_imports_to_beginning(input_string):
  function return_theorem_to_prove (line 230) | def return_theorem_to_prove(text):
  function return_theorem_to_replace (line 237) | def return_theorem_to_replace(text):
  function replace_statement_in_proof (line 244) | def replace_statement_in_proof(statement, proof):
  function refine_by_sorry (line 260) | def refine_by_sorry(text):
  function extract_code (line 281) | def extract_code(inputs):
  function parse_error (line 300) | def parse_error(log_string):
  function get_error_str (line 322) | def get_error_str(code, errors, error_thres=True):

FILE: nemo_skills/code_execution/sandbox.py
  class Sandbox (line 36) | class Sandbox(abc.ABC):
    method __init__ (line 55) | def __init__(
    method close (line 76) | async def close(self):
    method _send_request (line 80) | async def _send_request(self, request, timeout):
    method _parse_request_output (line 115) | def _parse_request_output(self, output):
    method _get_execute_url (line 119) | def _get_execute_url(self):
    method _prepare_request (line 123) | def _prepare_request(
    method delete_session (line 135) | async def delete_session(self, session_id: str) -> None:
    method execute_code (line 139) | async def execute_code(
    method is_proof_correct (line 279) | async def is_proof_correct(self, pred_output, timeout=30.0):
    method _check_ready (line 290) | def _check_ready(self, timeout: float = 5.0) -> bool:
    method wait_for_sandbox (line 307) | def wait_for_sandbox(self, wait_timeout: int = 240, http_timeout: int ...
  class LocalSandbox (line 315) | class LocalSandbox(Sandbox):
    method _get_execute_url (line 318) | def _get_execute_url(self):
    method _parse_request_output (line 321) | def _parse_request_output(self, output):
    method _prepare_request (line 328) | def _prepare_request(
    method delete_session (line 346) | async def delete_session(self, session_id: str) -> None:
  function get_sandbox (line 396) | def get_sandbox(sandbox_type: str = "local", **kwargs):
  function sandbox_params (line 402) | def sandbox_params():

FILE: nemo_skills/code_execution/utils.py
  function format_code_output (line 24) | def format_code_output(
  function _extract_between_separators (line 70) | def _extract_between_separators(generation: str, separators: Tuple[str, ...
  function extract_code_to_execute (line 82) | def extract_code_to_execute(generation: str, code_begin: str, code_end: ...
  function extract_code_output (line 86) | def extract_code_output(generation: str, code_output_begin: str, code_ou...
  function extract_code_block (line 90) | def extract_code_block(text: str, languages=None, extract_code_mode: str...
  function clean_formal_generation (line 101) | def clean_formal_generation(

FILE: nemo_skills/conversion/hf_to_nemo_llama.py
  function get_args (line 39) | def get_args():
  function load_config (line 65) | def load_config(llama_config):
  function load_state_dict_helper (line 119) | def load_state_dict_helper(cls, cfg, trainer: Trainer, state_dict):
  function convert (line 138) | def convert(args):

FILE: nemo_skills/conversion/hf_to_nemo_qwen.py
  function get_args (line 38) | def get_args():
  function load_config (line 61) | def load_config(args, qwen_config):
  function convert (line 89) | def convert(args):

FILE: nemo_skills/conversion/nemo_to_hf_llama.py
  function get_args (line 33) | def get_args():
  function create_hf_config (line 68) | def create_hf_config(hf_model_name, nemo_config):
  function convert (line 95) | def convert(

FILE: nemo_skills/conversion/nemo_to_hf_qwen.py
  function get_args (line 30) | def get_args():
  function convert (line 66) | def convert(

FILE: nemo_skills/dataset/aai/aai_score.py
  function compute_score (line 18) | def compute_score(metrics: dict):

FILE: nemo_skills/dataset/aalcr/prepare.py
  function construct_prompt (line 66) | def construct_prompt(docs, question, prompt_template=prompt_template):
  function count_n_tokens (line 72) | def count_n_tokens(prompt: str, tokenizer_name: str) -> int:
  function find_actual_file (line 80) | def find_actual_file(base_path, target_filename):
  function write_data_to_file (line 153) | def write_data_to_file(output_file, data, txt_file_folder, max_context_w...
  function prepare_aalcr_data (line 206) | def prepare_aalcr_data(max_context_window, setup, tokenizer_name):

FILE: nemo_skills/dataset/aime24-x/prepare.py
  function _load_utils (line 24) | def _load_utils():
  function format_entry (line 38) | def format_entry(entry, lang, prompt_language):
  function main (line 49) | def main(args):

FILE: nemo_skills/dataset/aime25-x/prepare.py
  function _load_utils (line 24) | def _load_utils():
  function format_entry (line 38) | def format_entry(entry, lang, prompt_language):
  function main (line 49) | def main(args):

FILE: nemo_skills/dataset/aime26/prepare.py
  function format_entry (line 23) | def format_entry(entry):
  function write_data_to_file (line 31) | def write_data_to_file(output_file, data):
  function main (line 38) | def main(args):

FILE: nemo_skills/dataset/apex-shortlist/prepare.py
  function write_data_to_file (line 22) | def write_data_to_file(output_file, data):

FILE: nemo_skills/dataset/arena-hard-v2/prepare.py
  function extract_answer_text (line 31) | def extract_answer_text(data):

FILE: nemo_skills/dataset/asr-leaderboard/prepare.py
  function save_audio_and_format_entry (line 55) | def save_audio_and_format_entry(
  function prepare_dataset (line 99) | def prepare_dataset(dataset_name, output_dir, with_audio=True):
  function main (line 139) | def main():

FILE: nemo_skills/dataset/audiobench/prepare.py
  function get_audio_duration (line 109) | def get_audio_duration(audio_array: np.ndarray, sampling_rate: int) -> f...
  function save_audio_file (line 116) | def save_audio_file(audio_array: np.ndarray, sampling_rate: int, output_...
  function extract_audio_dict (line 122) | def extract_audio_dict(sample: Dict) -> Dict | None:
  function create_manifest_entry (line 136) | def create_manifest_entry(
  function process_dataset (line 206) | def process_dataset(
  function main (line 485) | def main():

FILE: nemo_skills/dataset/beyond-aime/prepare.py
  function save_data (line 22) | def save_data():

FILE: nemo_skills/dataset/bfcl_v3/bfcl_score.py
  function calculate_combined_accuracy (line 52) | def calculate_combined_accuracy(accuracy_dict_list: list[dict], weighted...
  function get_accuracy_dict (line 77) | def get_accuracy_dict(metrics, category):
  function calculate_non_live_single_turn_accuracy (line 120) | def calculate_non_live_single_turn_accuracy(metrics):
  function calculate_live_single_turn_accuracy (line 145) | def calculate_live_single_turn_accuracy(metrics):
  function calculate_multi_turn_accuracy (line 164) | def calculate_multi_turn_accuracy(metrics):
  function compute_score (line 173) | def compute_score(metrics: dict):

FILE: nemo_skills/dataset/bfcl_v3/prepare.py
  function ensure_bfcl_eval_installed (line 33) | def ensure_bfcl_eval_installed():
  function process_multi_turn_test_case (line 92) | def process_multi_turn_test_case(instance):
  function load_dataset_entry (line 106) | def load_dataset_entry(
  function download_and_process_bfcl_data (line 156) | def download_and_process_bfcl_data(repo_url, subfolder_path, output_dir,...
  function main (line 205) | def main(args):

FILE: nemo_skills/dataset/bfcl_v3/utils.py
  function _get_language_specific_hint (line 39) | def _get_language_specific_hint(test_category):
  function func_doc_language_specific_pre_processing (line 48) | def func_doc_language_specific_pre_processing(function, test_category):
  function _cast_to_openai_type (line 102) | def _cast_to_openai_type(properties, mapping):
  function convert_to_tool (line 137) | def convert_to_tool(functions):

FILE: nemo_skills/dataset/bfcl_v4/bfcl_score.py
  function calculate_non_live_single_turn_accuracy (line 69) | def calculate_non_live_single_turn_accuracy(metrics):
  function calculate_live_single_turn_accuracy (line 86) | def calculate_live_single_turn_accuracy(metrics):
  function calculate_agentic_accuracy (line 98) | def calculate_agentic_accuracy(metrics):
  function calculate_hallucination_measurement (line 115) | def calculate_hallucination_measurement(metrics):
  function compute_score (line 124) | def compute_score(metrics: dict):

FILE: nemo_skills/dataset/bfcl_v4/prepare.py
  function main (line 32) | def main():

FILE: nemo_skills/dataset/bigcodebench/prepare.py
  function parse_data (line 26) | def parse_data(split="hard"):
  function extract_prefix (line 32) | def extract_prefix(text: str, delimiter: str) -> str:
  function clean_data (line 38) | def clean_data(dataset, subset):
  function wrap_in_code_tag (line 74) | def wrap_in_code_tag(text):

FILE: nemo_skills/dataset/birdbench/prepare.py
  function download_data (line 26) | def download_data(data_dir):
  function read_tables_file (line 45) | def read_tables_file(base_dir):
  function format_entries (line 82) | def format_entries(file_path, tables_info, out_file):
  function main (line 105) | def main():

FILE: nemo_skills/dataset/brumo25/prepare.py
  function write_data_to_file (line 22) | def write_data_to_file(output_file, data):

FILE: nemo_skills/dataset/challenge19/prepare.py
  function process_row (line 21) | def process_row(row, source):
  function load_jsonl_problems (line 29) | def load_jsonl_problems(file_path, target_ids):
  function load_ids_from_file (line 62) | def load_ids_from_file(file_path):
  function main (line 68) | def main():

FILE: nemo_skills/dataset/compute-eval/prepare.py
  function _fence_for_path (line 29) | def _fence_for_path(path: str) -> str:
  function _format_context_files_block (line 43) | def _format_context_files_block(context_files: list[dict[str, str]]) -> ...

FILE: nemo_skills/dataset/contextasr-bench/contextasr_score.py
  function compute_score (line 16) | def compute_score(combined_metrics: dict) -> dict:

FILE: nemo_skills/dataset/contextasr-bench/prepare.py
  function download_dataset (line 59) | def download_dataset(download_dir):
  function build_messages (line 131) | def build_messages(prompt_text, audio_path, duration):
  function format_entry (line 145) | def format_entry(sample, mode, audio_prefix):
  function main (line 173) | def main():

FILE: nemo_skills/dataset/covost2/prepare.py
  function load_tsv (line 86) | def load_tsv(path: Path) -> list[dict]:
  function download_covost_tsv (line 91) | def download_covost_tsv(src_lang: str, tgt_lang: str, local_dir: Path) -...
  function load_validated_sentences (line 105) | def load_validated_sentences(path: Path) -> dict:
  function load_covost2 (line 112) | def load_covost2(
  function get_audio_duration (line 142) | def get_audio_duration(audio_file: str) -> float:
  function get_container_audio_path (line 147) | def get_container_audio_path(src_lang: str, split: str, audio_id: str) -...
  function copy_audio_file (line 151) | def copy_audio_file(src_wav: Path, audio_dir: Path, src_lang: str, split...
  function get_ast_instruction (line 159) | def get_ast_instruction(target_lang: str) -> str:
  function get_asr_instruction (line 164) | def get_asr_instruction() -> str:
  function _build_record (line 168) | def _build_record(
  function prepare_covost2 (line 199) | def prepare_covost2(
  function main (line 286) | def main():

FILE: nemo_skills/dataset/dsbench_da/prepare.py
  function read_excel_to_text (line 23) | def read_excel_to_text(excel_path: Path) -> str:
  function format_paths_for_prompt (line 42) | def format_paths_for_prompt(paths: list[Path], actual_root: Path, displa...
  function save_data (line 65) | def save_data(split: str, data_dir: str | Path, display_root: str | Path...

FILE: nemo_skills/dataset/fleurs/prepare.py
  function load_fleurs_module (line 29) | def load_fleurs_module():
  function parse_tsv (line 56) | def parse_tsv(tsv_path: str) -> dict[str, dict]:
  function load_fleurs (line 74) | def load_fleurs(locale: str, split: str, local_dir: str) -> list[dict]:
  function index_by_id (line 106) | def index_by_id(rows: list[dict]) -> dict[int, dict]:
  function build_translation_pairs (line 110) | def build_translation_pairs(languages: list[str]) -> list[tuple[str, str]]:
  function prepare_audio (line 121) | def prepare_audio(item: dict) -> tuple[np.ndarray, int, float]:
  function get_container_audio_path (line 128) | def get_container_audio_path(locale: str, wav_filename: str) -> str:
  function save_audio (line 132) | def save_audio(y: np.ndarray, sr: int, wav_path: Path) -> None:
  function get_ast_instruction (line 137) | def get_ast_instruction(target_locale: str) -> str:
  function get_asr_instruction (line 142) | def get_asr_instruction() -> str:
  function _build_record (line 146) | def _build_record(
  function prepare_fleurs (line 177) | def prepare_fleurs(data_dir: Path, split: str, languages: list[str], no_...
  function main (line 276) | def main():

FILE: nemo_skills/dataset/flores200/prepare.py
  function write_data_to_file (line 23) | def write_data_to_file(output_file, datasets, src_languages, tgt_languag...
  function main (line 41) | def main(args):

FILE: nemo_skills/dataset/frontierscience-olympiad/prepare.py
  function format_entry (line 32) | def format_entry(entry, problem_index):
  function write_data_to_file (line 49) | def write_data_to_file(output_file, data, subject_filter=None):

FILE: nemo_skills/dataset/global_piqa/global_piqa_utils.py
  function supported_languages (line 18) | def supported_languages() -> list[str]:
  function load_global_piqa_datasets (line 22) | def load_global_piqa_datasets(languages: list[str], split: str = "test")...
  function digit_to_letter (line 26) | def digit_to_letter(digit: int) -> str:
  class Schema (line 30) | class Schema:
  function get_mcq_fields (line 65) | def get_mcq_fields(entry: dict) -> dict:

FILE: nemo_skills/dataset/global_piqa/prepare.py
  function format_entry (line 29) | def format_entry(entry: dict, language: str) -> dict:
  function main (line 40) | def main(args):

FILE: nemo_skills/dataset/gpqa-x/prepare.py
  function _load_utils (line 24) | def _load_utils():
  function format_entry (line 39) | def format_entry(entry, lang, prompt_language):
  function main (line 57) | def main(args):

FILE: nemo_skills/dataset/gpqa/prepare.py
  function preprocess (line 31) | def preprocess(text):
  function format_entry (line 40) | def format_entry(entry):
  function write_data_to_file (line 63) | def write_data_to_file(output_file, data):
  function save_data (line 70) | def save_data(split, random_seed):

FILE: nemo_skills/dataset/gsm8k/prepare.py
  function save_data (line 41) | def save_data(split):

FILE: nemo_skills/dataset/hendrycks_math/fix_ref_solns.py
  function _post_fix (line 18) | def _post_fix(problem_id, soln_string):
  function _post_fix_multi_answer (line 53) | def _post_fix_multi_answer(problem_id, results):
  function _fix_solution (line 149) | def _fix_solution(problem_id, ref_soln):

FILE: nemo_skills/dataset/hle/prepare.py
  function format_entry (line 37) | def format_entry(entry):
  function write_data_to_file (line 51) | def write_data_to_file(output_file, data, split):

FILE: nemo_skills/dataset/hle_verified/prepare.py
  function load_dataset_from_hub (line 48) | def load_dataset_from_hub():
  function format_entry (line 63) | def format_entry(entry):
  function write_data_to_file (line 78) | def write_data_to_file(output_file, data, split):

FILE: nemo_skills/dataset/hmmt_feb25/prepare.py
  function write_data_to_file (line 22) | def write_data_to_file(output_file, data):

FILE: nemo_skills/dataset/hmmt_nov25/prepare.py
  function write_data_to_file (line 22) | def write_data_to_file(output_file, data):

FILE: nemo_skills/dataset/hotpotqa/prepare_utils.py
  function format_context (line 28) | def format_context(context: dict) -> str:
  function format_entry (line 48) | def format_entry(entry: dict) -> dict:
  function prepare_validation (line 63) | def prepare_validation(output_path: Path) -> int:

FILE: nemo_skills/dataset/human-eval-infilling/prepare.py
  function parse_data (line 28) | def parse_data(split):
  function clean_data (line 33) | def clean_data(dataset, split):

FILE: nemo_skills/dataset/librispeech-pc/prepare.py
  function download_with_progress (line 37) | def download_with_progress(url: str, output_path: Path, desc: str):
  function download_manifests (line 60) | def download_manifests(output_dir: Path) -> Path:
  function download_audio (line 86) | def download_audio(split: str, audio_dir: Path):
  function process_split (line 103) | def process_split(split: str, data_dir: Path, audio_dir: Path, with_audi...
  function main (line 165) | def main():

FILE: nemo_skills/dataset/livebench-coding/prepare.py
  function parse_data (line 23) | def parse_data():
  function clean_data (line 32) | def clean_data(dataset):

FILE: nemo_skills/dataset/livecodebench-cpp/prepare.py
  class PromptConstants (line 22) | class PromptConstants:
  function parse_data (line 35) | def parse_data(split):
  function clean_data (line 53) | def clean_data(dataset, keep_all_columns=False):
  function prepare (line 89) | def prepare(output_dir, split):

FILE: nemo_skills/dataset/livecodebench-pro/prepare.py
  function download_testcases (line 38) | def download_testcases(local_dir, token):
  function process_problem_splits (line 51) | def process_problem_splits(output_dir, token):

FILE: nemo_skills/dataset/livecodebench-x/prepare.py
  function _load_utils (line 24) | def _load_utils():
  function format_entry (line 40) | def format_entry(entry, lang, prompt_language):
  function main (line 52) | def main(args):

FILE: nemo_skills/dataset/livecodebench/prepare.py
  class PromptConstants (line 25) | class PromptConstants:
  function parse_data (line 33) | def parse_data(release_version="release_latest"):
  function get_first_last_day (line 57) | def get_first_last_day(year_month_str):
  function parse_month_range (line 67) | def parse_month_range(start_date, end_date):
  function clean_data (line 76) | def clean_data(dataset, keep_all_columns=False):
  function prepare (line 111) | def prepare(start_date, end_date, release_version, output_dir, keep_all_...

FILE: nemo_skills/dataset/longbench-v2/prepare.py
  function count_n_tokens (line 56) | def count_n_tokens(prompt: str, tokenizer_name: str) -> int:
  function write_data_to_file (line 71) | def write_data_to_file(output_file: Path, data, difficulty, length, toke...
  function prepare_longbenchv2_data (line 106) | def prepare_longbenchv2_data(setup: str, difficulty, length, tokenizer_n...

FILE: nemo_skills/dataset/longcodebench/prepare.py
  function count_n_tokens (line 26) | def count_n_tokens(prompt: str, tokenizer_name: str) -> int:
  function write_data_to_file (line 41) | def write_data_to_file(output_file, data, tokenizer_name):
  function prepare_longcodebench_data (line 57) | def prepare_longcodebench_data(setup, tokenizer_name):

FILE: nemo_skills/dataset/m-arena-hard-v2/prepare.py
  function format_entry (line 25) | def format_entry(row: dict, language: str) -> dict:
  function main (line 38) | def main(args):

FILE: nemo_skills/dataset/m-arena-hard/prepare.py
  function format_entry (line 25) | def format_entry(row: dict, language: str) -> dict:
  function main (line 40) | def main(args):

FILE: nemo_skills/dataset/math-odyssey/prepare.py
  function identify_label (line 23) | def identify_label(answer_endings, answer):

FILE: nemo_skills/dataset/minif2f/prepare.py
  function download_dataset (line 25) | def download_dataset(output_path):
  function _ensure_header_ends_with_by (line 30) | def _ensure_header_ends_with_by(text: str) -> str:
  function clean_lean_snippet (line 39) | def clean_lean_snippet(text: str | None) -> str | None:
  function _split_header_and_theorem (line 49) | def _split_header_and_theorem(text: str) -> tuple[str, str]:
  function process_entry (line 72) | def process_entry(entry: dict) -> dict:
  function split_data (line 96) | def split_data(input_file):
  function save_data (line 117) | def save_data(data, output_file):
  function delete_file (line 123) | def delete_file(file_path):
  function main (line 128) | def main(split):

FILE: nemo_skills/dataset/mmau-pro/mmau_pro_score.py
  function compute_score (line 16) | def compute_score(combined_metrics: dict) -> dict:

FILE: nemo_skills/dataset/mmau-pro/prepare.py
  function download_mmau_data (line 28) | def download_mmau_data(download_dir, hf_token):
  function format_entry (line 59) | def format_entry(entry, with_audio=False):
  function main (line 97) | def main():

FILE: nemo_skills/dataset/mmlu-pro/prepare.py
  function format_entry (line 27) | def format_entry(entry):
  function write_data_to_file (line 38) | def write_data_to_file(output_file, data):
  function main (line 45) | def main(args):

FILE: nemo_skills/dataset/mmlu-prox/prepare.py
  function download_and_parse_lang_libs (line 28) | def download_and_parse_lang_libs():
  function format_entry (line 84) | def format_entry(entry, language, lang_libs, lang_subjects):
  function write_data_to_file (line 120) | def write_data_to_file(output_file, datasets, languages, lang_libs, lang...
  function main (line 131) | def main(args):

FILE: nemo_skills/dataset/mmlu-redux/prepare.py
  function format_entry (line 87) | def format_entry(entry, category):
  function write_data_to_file (line 105) | def write_data_to_file(output_file, data, category):
  function main (line 113) | def main(args):

FILE: nemo_skills/dataset/mmlu/prepare.py
  function read_csv_files_from_tar (line 90) | def read_csv_files_from_tar(tar_file_path, split):
  function save_data (line 132) | def save_data(split):

FILE: nemo_skills/dataset/mmmlu/mmmlu_utils.py
  class Schema (line 158) | class Schema:
  function download_mmmlu_datasets (line 165) | def download_mmmlu_datasets(languages: list[str]) -> dict[str, list[dict]]:
  function format_multichoice_question (line 186) | def format_multichoice_question(row):
  function get_mcq_fields (line 190) | def get_mcq_fields(entry: dict):

FILE: nemo_skills/dataset/mmmlu/prepare.py
  function format_entry (line 30) | def format_entry(entry: dict, language: str) -> dict:
  function main (line 50) | def main(args):

FILE: nemo_skills/dataset/mmmu-pro/prepare.py
  function format_entry (line 26) | def format_entry(entry, images_dir: Path) -> dict | None:
  function save_data (line 48) | def save_data(split: str):

FILE: nemo_skills/dataset/mobench/prepare.py
  function download_dataset (line 24) | def download_dataset(output_path: str):
  function load_jsonl (line 29) | def load_jsonl(path: str):
  function write_jsonl (line 36) | def write_jsonl(path: str, rows):
  function strip_trailing_sorry (line 42) | def strip_trailing_sorry(text: str) -> str:
  function split_prelude_and_theorem (line 52) | def split_prelude_and_theorem(code: str):
  function extract_theorem_by (line 65) | def extract_theorem_by(theorem_block: str) -> str:
  function ensure_fields (line 83) | def ensure_fields(entry: dict, lean_header: str) -> dict:
  function get_lean4_header (line 118) | def get_lean4_header() -> str:
  function main (line 123) | def main():

FILE: nemo_skills/dataset/mrcr/prepare.py
  function count_n_tokens (line 33) | def count_n_tokens(messages: list[dict]) -> int:
  function write_data_to_file (line 42) | def write_data_to_file(output_file, data, max_context_window, needles_su...
  function get_mrcr_data (line 65) | def get_mrcr_data(needles_subset, setup, max_context_window):

FILE: nemo_skills/dataset/musan/prepare.py
  function download_from_kaggle (line 55) | def download_from_kaggle(output_dir: Path) -> Path:
  function download_from_openslr (line 72) | def download_from_openslr(output_dir: Path) -> Path:
  function load_dataset_from_source (line 110) | def load_dataset_from_source(source: str, output_dir: Path):
  function get_audio_duration (line 150) | def get_audio_duration(audio_array: np.ndarray, sampling_rate: int) -> f...
  function save_audio_file (line 157) | def save_audio_file(audio_array: np.ndarray, sampling_rate: int, output_...
  function create_manifest_entry (line 163) | def create_manifest_entry(
  function process_category_from_files (line 203) | def process_category_from_files(
  function process_category (line 278) | def process_category(
  function main (line 397) | def main():

FILE: nemo_skills/dataset/numb3rs/prepare.py
  function build_messages_with_prompt (line 64) | def build_messages_with_prompt(audio_metadata, prompt_text):
  function save_audio_and_format_entry (line 75) | def save_audio_and_format_entry(entry, category, audio_dir, sample_idx, ...
  function prepare_category (line 140) | def prepare_category(category, dataset, output_dir, with_audio=True, aud...
  function main (line 220) | def main():

FILE: nemo_skills/dataset/omniscience/prepare.py
  function parse_args (line 32) | def parse_args() -> argparse.Namespace:
  function format_entry (line 44) | def format_entry(entry) -> dict:
  function write_jsonl (line 54) | def write_jsonl(data: list[dict], path: str):

FILE: nemo_skills/dataset/open-proof-corpus-judge/prepare.py
  function load_jsonl (line 23) | def load_jsonl(file_path):
  function prepare_bon_binary_data (line 32) | def prepare_bon_binary_data(output_path):

FILE: nemo_skills/dataset/physics/prepare.py
  function strip_boxed (line 22) | def strip_boxed(s):
  function process_answer (line 29) | def process_answer(answer):
  function format_entry (line 35) | def format_entry(entry):
  function write_data_to_file (line 47) | def write_data_to_file(output_file, data):
  function save_data (line 54) | def save_data(split_data, split_name):

FILE: nemo_skills/dataset/polymath/prepare.py
  function _load_instructions (line 25) | def _load_instructions(url: str) -> tuple[dict, dict, dict]:
  function format_entry (line 46) | def format_entry(entry: dict, language: str, difficulty: str, language_c...
  function main (line 62) | def main(args):

FILE: nemo_skills/dataset/prepare.py
  function parse_prepare_cli_arguments (line 23) | def parse_prepare_cli_arguments(args=None, datasets_nargs="+"):
  function prepare_datasets (line 41) | def prepare_datasets(

FILE: nemo_skills/dataset/proof-arena-judge/prepare.py
  function prepare_data (line 40) | def prepare_data(output_path):
  function load_jsonl (line 87) | def load_jsonl(file_path):
  function grading_scheme_to_rubric (line 96) | def grading_scheme_to_rubric(grading_scheme, desc_key="grading_scheme_de...
  function load_openai_imo_proofs (line 104) | def load_openai_imo_proofs():
  function load_gemini_imo_proofs (line 136) | def load_gemini_imo_proofs():
  function process_imo_usamo_data (line 168) | def process_imo_usamo_data(raw_data, source):
  function process_imc_data (line 201) | def process_imc_data(raw_data):

FILE: nemo_skills/dataset/proof-bench-judge/prepare.py
  function prepare_verification_data (line 31) | def prepare_verification_data(output_path):
  function prepare_bon_binary_data (line 63) | def prepare_bon_binary_data(output_path):
  function load_hf_data (line 103) | def load_hf_data(split: str):

FILE: nemo_skills/dataset/proofnet/prepare.py
  function download_dataset (line 24) | def download_dataset(output_path):
  function split_data (line 29) | def split_data(input_file):
  function save_data (line 44) | def save_data(data, output_file):
  function delete_file (line 50) | def delete_file(file_path):
  function main (line 55) | def main(split):

FILE: nemo_skills/dataset/putnam-bench/prepare.py
  function parse_lean_file (line 36) | def parse_lean_file(path: Path) -> dict:
  function download_dataset_and_process (line 82) | def download_dataset_and_process(output_path):
  function delete_file (line 124) | def delete_file(file_path):
  function main (line 131) | def main():

FILE: nemo_skills/dataset/ruler/prepare.py
  function prepare_task_for_ns (line 45) | def prepare_task_for_ns(task, data_dir, setup, data_format):
  function get_ruler_data (line 79) | def get_ruler_data(tasks, setup, template_tokens, max_seq_length, data_f...

FILE: nemo_skills/dataset/ruler/ruler_score.py
  function compute_score (line 16) | def compute_score(metrics: dict):

FILE: nemo_skills/dataset/ruler2/prepare.py
  function prepare_mk_niah_basic (line 31) | def prepare_mk_niah_basic(output_folder, tokenizer_type, tokenizer_path,...
  function prepare_mk_niah_easy (line 68) | def prepare_mk_niah_easy(output_folder, tokenizer_type, tokenizer_path, ...
  function prepare_mk_niah_medium (line 103) | def prepare_mk_niah_medium(output_folder, tokenizer_type, tokenizer_path...
  function prepare_mk_niah_hard (line 138) | def prepare_mk_niah_hard(output_folder, tokenizer_type, tokenizer_path, ...
  function prepare_mv_niah_basic (line 173) | def prepare_mv_niah_basic(output_folder, tokenizer_type, tokenizer_path,...
  function prepare_mv_niah_easy (line 210) | def prepare_mv_niah_easy(output_folder, tokenizer_type, tokenizer_path, ...
  function prepare_mv_niah_medium (line 245) | def prepare_mv_niah_medium(output_folder, tokenizer_type, tokenizer_path...
  function prepare_mv_niah_hard (line 280) | def prepare_mv_niah_hard(output_folder, tokenizer_type, tokenizer_path, ...
  function prepare_qa_basic (line 315) | def prepare_qa_basic(output_folder, tokenizer_type, tokenizer_path, leng...
  function prepare_qa_easy (line 348) | def prepare_qa_easy(output_folder, tokenizer_type, tokenizer_path, lengt...
  function prepare_qa_medium (line 381) | def prepare_qa_medium(output_folder, tokenizer_type, tokenizer_path, len...
  function prepare_qa_hard (line 414) | def prepare_qa_hard(output_folder, tokenizer_type, tokenizer_path, lengt...
  function prepare_task_for_ns (line 447) | def prepare_task_for_ns(output_folder, task):
  function prepare_dataset (line 468) | def prepare_dataset(tasks, setup, max_seq_length, tokenizer_type, tokeni...

FILE: nemo_skills/dataset/ruler2/prepare_mmlu.py
  function generate_random_number (line 287) | def generate_random_number(num_digits=7):
  function generate_input_output (line 293) | def generate_input_output(index, num_qs):
  function generate_samples (line 400) | def generate_samples(max_seq_length: int, incremental: int = 10):
  function main (line 466) | def main():

FILE: nemo_skills/dataset/ruler2/prepare_niah.py
  function generate_random_number (line 91) | def generate_random_number(num_digits=7):
  function generate_random_word (line 97) | def generate_random_word():
  function generate_random_uuid (line 102) | def generate_random_uuid():
  function generate_random (line 106) | def generate_random(type_needle: str, digits: int | None = None):
  function generate_input_output (line 119) | def generate_input_output(num_haystack):
  function generate_samples (line 193) | def generate_samples(num_samples: int, max_seq_length: int, incremental:...
  function main (line 263) | def main():

FILE: nemo_skills/dataset/ruler2/prepare_qa.py
  function read_squad (line 97) | def read_squad():
  function read_hotpotqa (line 123) | def read_hotpotqa():
  function read_musique (line 152) | def read_musique():
  function generate_random_number (line 189) | def generate_random_number(num_digits=7):
  function generate_input_output (line 195) | def generate_input_output(index, num_docs):
  function generate_samples (line 309) | def generate_samples(num_samples: int, max_seq_length: int, incremental:...
  function main (line 374) | def main():

FILE: nemo_skills/dataset/ruler2/ruler2_score.py
  function compute_score (line 16) | def compute_score(metrics: dict):

FILE: nemo_skills/dataset/ruler2/tokenizer.py
  function select_tokenizer (line 27) | def select_tokenizer(tokenizer_type, tokenizer_path):
  class HFTokenizer (line 38) | class HFTokenizer:
    method __init__ (line 43) | def __init__(self, model_path) -> None:
    method text_to_tokens (line 48) | def text_to_tokens(self, text: str) -> List[str]:
    method tokens_to_text (line 52) | def tokens_to_text(self, tokens: List[int]) -> str:
  class OpenAITokenizer (line 57) | class OpenAITokenizer:
    method __init__ (line 62) | def __init__(self, model_path="cl100k_base") -> None:
    method text_to_tokens (line 67) | def text_to_tokens(self, text: str) -> List[int]:
    method tokens_to_text (line 71) | def tokens_to_text(self, tokens: List[int]) -> str:
  class GeminiTokenizer (line 76) | class GeminiTokenizer:
    method __init__ (line 81) | def __init__(self, model_path="gemini-1.5-pro-latest") -> None:
    method text_to_tokens (line 88) | def text_to_tokens(self, text: str) -> List[int]:
    method tokens_to_text (line 92) | def tokens_to_text(self, tokens: List[int]) -> str:

FILE: nemo_skills/dataset/simpleqa/prepare.py
  function format_entry (line 27) | def format_entry(entry: dict, idx: int) -> dict:
  function format_entry_verified (line 37) | def format_entry_verified(entry: dict, idx: int) -> dict:
  function write_data_to_file (line 47) | def write_data_to_file(output_file, examples: List[dict]):

FILE: nemo_skills/dataset/speed-bench/prepare.py
  class BenchmarkDataset (line 35) | class BenchmarkDataset(str, Enum):
  function _get_external_dataset (line 118) | def _get_external_dataset(dataset_name: str, config_name: str = "default"):
  function _generate_stackselect_prompt (line 130) | def _generate_stackselect_prompt(question: str, answers: list[str], answ...
  function _generate_textsort_prompt (line 210) | def _generate_textsort_prompt(prompt: str) -> str:
  function _generate_writing_prompt (line 265) | def _generate_writing_prompt(contents: list[str]) -> str:
  function _pad_or_truncate_prompt (line 281) | def _pad_or_truncate_prompt(prompt: str, target_num_tokens: int, padding...
  function _generate_bamboo_prompt (line 305) | def _generate_bamboo_prompt(external_dataset: "Dataset", num_tokens: int...
  function _generate_chatrag_bench_prompt (line 310) | def _generate_chatrag_bench_prompt(external_dataset: "Dataset") -> str:
  function _generate_coser_prompt (line 320) | def _generate_coser_prompt(external_dataset: "Dataset") -> str:
  function _generate_mmlu_pro_prompt (line 367) | def _generate_mmlu_pro_prompt(external_dataset: "Dataset", subject: str)...
  function _generate_hle_prompt (line 384) | def _generate_hle_prompt(
  function _get_num_tokens_from_config (line 407) | def _get_num_tokens_from_config(speed_config: DATASET_CONFIG | str) -> int:
  function _fetch_all_turns_data (line 415) | def _fetch_all_turns_data(example: dict[str, Any], speed_config: DATASET...
  function _resolve_external_data (line 573) | def _resolve_external_data(dataset: Dataset, speed_config: DATASET_CONFI...
  function prepare_data (line 592) | def prepare_data(args: argparse.Namespace) -> None:

FILE: nemo_skills/dataset/supergpqa/prepare.py
  function preprocess (line 30) | def preprocess(text):
  function format_entry (line 38) | def format_entry(entry):
  function write_data_to_file (line 76) | def write_data_to_file(output_file, data):
  function save_data (line 83) | def save_data(split, random_seed):

FILE: nemo_skills/dataset/swe-bench-multilingual/prepare.py
  function get_language (line 69) | def get_language(row):

FILE: nemo_skills/dataset/swe-bench/dump_images.py
  function read_container_names (line 23) | def read_container_names(jsonl_file):
  function convert_to_sif (line 40) | def convert_to_sif(container_name, output_dir):
  function main (line 72) | def main():

FILE: nemo_skills/dataset/swe-bench/dump_repos.py
  function read_repos (line 24) | def read_repos(jsonl_file):
  function clone_repo (line 36) | def clone_repo(repo, output_dir, force):
  function main (line 69) | def main():

FILE: nemo_skills/dataset/swe-rebench/prepare.py
  function get_date_range (line 22) | def get_date_range(start_str, end_str):

FILE: nemo_skills/dataset/ugphysics/prepare.py
  function get_prompt_sentence (line 50) | def get_prompt_sentence(answer_type, is_multiple_answer):
  function get_boxed_answer_example (line 64) | def get_boxed_answer_example(is_multiple_answer):
  function format_entry (line 71) | def format_entry(entry):
  function load_data (line 88) | def load_data(lang_split):
  function save_data (line 96) | def save_data(data, output_path):

FILE: nemo_skills/dataset/utils.py
  function locate (line 30) | def locate(path):
  function add_rounding_instruction (line 56) | def add_rounding_instruction(data: Dict) -> Dict:
  function import_from_path (line 73) | def import_from_path(file_path, module_name=None):
  function add_to_path (line 84) | def add_to_path(p):
  function get_dataset_name (line 94) | def get_dataset_name(dataset):
  function get_dataset_path (line 101) | def get_dataset_path(dataset, extra_benchmark_map=None):
  function get_extra_benchmark_map (line 122) | def get_extra_benchmark_map(extra_benchmark_map=None):
  function _load_external_dataset (line 150) | def _load_external_dataset(dataset_path):
  function get_default_dataset_module (line 162) | def get_default_dataset_module(dataset):
  function get_dataset_module (line 169) | def get_dataset_module(dataset, data_dir=None, extra_benchmark_map=None):
  function get_lean4_header (line 234) | def get_lean4_header():
  function download_with_retries (line 239) | def download_with_retries(url, output_file, max_retries=3, retry_delay=1):
  function save_data_from_qwen (line 252) | def save_data_from_qwen(dataset, split="test"):
  function get_mcq_fields (line 295) | def get_mcq_fields(question, choices):
  function get_question_hash (line 306) | def get_question_hash(question, options=None):
  function load_subset_ids (line 317) | def load_subset_ids(ids_file):
  function filter_by_subset (line 323) | def filter_by_subset(dataset, subset_ids, question_key="question", optio...

FILE: nemo_skills/dataset/wmt24pp/prepare.py
  function write_data_to_file (line 23) | def write_data_to_file(output_file, datasets, tgt_languages):
  function main (line 39) | def main(args):

FILE: nemo_skills/evaluation/aggregate_answers.py
  class ProcessTopAnswerConfig (line 34) | class ProcessTopAnswerConfig:
    method __post_init__ (line 73) | def __post_init__(self):
  function map_to_output_path (line 86) | def map_to_output_path(file_path, input_dir, output_dir):
  class ProcessMode (line 103) | class ProcessMode(Enum):
  class TopAnswerProcessor (line 108) | class TopAnswerProcessor:
    method __init__ (line 109) | def __init__(self, cfg: ProcessTopAnswerConfig):
    method _validate_cfg (line 113) | def _validate_cfg(self):
    method __enter__ (line 134) | def __enter__(self):
    method __exit__ (line 179) | def __exit__(self, exc_type, exc_val, exc_tb):
    method process (line 186) | def process(self):
    method _read_predictions (line 191) | def _read_predictions(self) -> Tuple[List, List]:
    method _write_results (line 251) | def _write_results(self, all_predictions: List, new_answers: List):
    method _write_results_fill (line 258) | def _write_results_fill(self, all_predictions: List, new_answers: List):
    method _write_results_extract (line 301) | def _write_results_extract(self, all_predictions: List, new_answers: L...
  function process_top_answer (line 319) | def process_top_answer(cfg: ProcessTopAnswerConfig):

FILE: nemo_skills/evaluation/compute_group_score.py
  function load_metric_files (line 22) | def load_metric_files(metric_files: List[str]) -> Dict[str, Any]:
  function import_score_module (line 34) | def import_score_module(score_module: str):
  function main (line 48) | def main():

FILE: nemo_skills/evaluation/evaluator/__init__.py
  function _resolve (line 76) | def _resolve(dotted: str):
  function _get_evaluator_fn (line 83) | def _get_evaluator_fn(eval_type: str) -> Callable:
  function _get_evaluator_cls (line 89) | def _get_evaluator_cls(eval_type: str) -> type:
  function _resolve_eval_type (line 104) | def _resolve_eval_type(eval_type: str):
  function is_evaluator_registered (line 127) | def is_evaluator_registered(eval_type: str):
  function register_evaluator (line 132) | def register_evaluator(eval_type: str, eval_fn: Callable[[Dict[str, Any]...
  function get_evaluator_class (line 142) | def get_evaluator_class(eval_type: str, config: Dict[str, Any]) -> BaseE...
  function supports_single_eval (line 156) | def supports_single_eval(eval_type: str, config: Dict[str, Any]) -> bool:
  function evaluate (line 166) | def evaluate(eval_type, eval_config):

FILE: nemo_skills/evaluation/evaluator/arena.py
  function compute_mle_elo (line 35) | def compute_mle_elo(df, SCALE=400, BASE=10, INIT_RATING=1000):
  function get_bootstrap_result (line 69) | def get_bootstrap_result(battles, func_compute_elo, num_round):
  function predict_win_rate (line 80) | def predict_win_rate(elo_ratings, SCALE=400, BASE=10, INIT_RATING=1000):
  function get_win_rate_column (line 97) | def get_win_rate_column(df, column):
  function get_battles_from_judgment (line 103) | def get_battles_from_judgment(scores, WEIGHT=3):
  function get_aggregate_score (line 161) | def get_aggregate_score(scores, weight=3):

FILE: nemo_skills/evaluation/evaluator/audio.py
  class AudioEvaluatorConfig (line 32) | class AudioEvaluatorConfig(BaseEvaluatorConfig):
  function remove_symbols_and_diacritics (line 69) | def remove_symbols_and_diacritics(s: str, keep: str = ""):
  function remove_symbols (line 92) | def remove_symbols(s: str):
  function normalize_compound_pairs (line 99) | def normalize_compound_pairs(ref_text: str, pred_text: str) -> tuple[str...
  class MultilingualTextNormalizer (line 129) | class MultilingualTextNormalizer:
    method __init__ (line 136) | def __init__(self, remove_diacritics: bool = True):
    method _normalize_numbers (line 139) | def _normalize_numbers(self, text, lang):
    method __call__ (line 154) | def __call__(self, s: str, lang=None):
  function extract_asr_text (line 179) | def extract_asr_text(generation: str) -> str:
  function strip_helpful_prefixes (line 194) | def strip_helpful_prefixes(text: str) -> str:
  function normalize_whitespace (line 234) | def normalize_whitespace(text: str) -> str:
  function split_tokens (line 239) | def split_tokens(text: str) -> list[str]:
  function extract_punctuation (line 244) | def extract_punctuation(text: str) -> list[str]:
  function calculate_per (line 249) | def calculate_per(reference: str, hypothesis: str) -> float:
  function evaluate_asr_pc (line 286) | def evaluate_asr_pc(
  function _normalize_digits_to_words (line 328) | def _normalize_digits_to_words(text: str) -> str:
  function _expand_contractions (line 365) | def _expand_contractions(text: str) -> str:
  function _remove_non_speech_elements (line 396) | def _remove_non_speech_elements(text: str) -> str:
  function resolve_asr_normalization_mode (line 406) | def resolve_asr_normalization_mode(config: AudioEvaluatorConfig) -> str:
  function preprocess_asr_text (line 417) | def preprocess_asr_text(text: str, mode: str = "standard", **kwargs) -> ...
  function _wer_with_counts (line 487) | def _wer_with_counts(ref: str, hyp: str) -> dict[str, Any]:
  function _cer_with_counts (line 506) | def _cer_with_counts(ref: str, hyp: str, key_prefix: str = "cer") -> dic...
  function evaluate_asr (line 527) | def evaluate_asr(
  function resolve_bleu_tokenize (line 570) | def resolve_bleu_tokenize(tgt_lang: str | None) -> str:
  function evaluate_translation (line 578) | def evaluate_translation(
  function evaluate_cer (line 611) | def evaluate_cer(
  function evaluate_hallucination (line 634) | def evaluate_hallucination(reference: str, hypothesis: str, audio_contex...
  function evaluate_pc_rate (line 668) | def evaluate_pc_rate(reference: str, hypothesis: str) -> dict[str, Any]:
  class AudioEvaluator (line 717) | class AudioEvaluator(BaseEvaluator):
    method __init__ (line 720) | def __init__(self, config: dict, num_parallel_requests=10):
    method eval_single (line 724) | async def eval_single(self, data_point: dict[str, any]) -> dict[str, a...
  function eval_audio (line 732) | def eval_audio(cfg):
  function evaluate_sample (line 738) | def evaluate_sample(sample: dict[str, Any], config: AudioEvaluatorConfig...

FILE: nemo_skills/evaluation/evaluator/base.py
  class BaseEvaluatorConfig (line 27) | class BaseEvaluatorConfig:
  class BaseEvaluator (line 34) | class BaseEvaluator(ABC):
    method __init__ (line 37) | def __init__(self, config: Dict[str, Any], num_parallel_requests=10):
    method eval_full (line 42) | async def eval_full(self) -> None:
    method eval_single (line 74) | async def eval_single(self, data_point: Dict[str, Any]) -> Dict[str, A...
    method supports_single_eval (line 89) | def supports_single_eval(self) -> bool:

FILE: nemo_skills/evaluation/evaluator/bfcl.py
  class BFCLEvaluatorConfig (line 38) | class BFCLEvaluatorConfig(BaseEvaluatorConfig):
  function eval_bfcl (line 44) | def eval_bfcl(cfg):
  function _convert_to_bfcl_format (line 103) | def _convert_to_bfcl_format(jsonl_file, output_dir, test_category):
  function _merge_bfcl_results (line 124) | def _merge_bfcl_results(generation_file, bfcl_fmted_file, score_file):

FILE: nemo_skills/evaluation/evaluator/bird.py
  function execute_sql (line 53) | def execute_sql(predicted_sql, ground_truth, db_path):
  class BirdEvaluatorConfig (line 71) | class BirdEvaluatorConfig(BaseEvaluatorConfig):
  class BirdEvaluator (line 81) | class BirdEvaluator(BaseEvaluator):
    method __init__ (line 82) | def __init__(self, config: dict, num_parallel_requests=10):
    method _extract_answer (line 88) | def _extract_answer(self, text):
    method eval_single (line 129) | async def eval_single(self, data_point: dict):

FILE: nemo_skills/evaluation/evaluator/ccc.py
  class CCCEvaluatorConfig (line 19) | class CCCEvaluatorConfig(BaseEvaluatorConfig):
  function _sandbox_exec_sync (line 33) | def _sandbox_exec_sync(sandbox: LocalSandbox, cmd: str, *, language: str...
  function _test_exec_sync (line 42) | def _test_exec_sync(sandbox: LocalSandbox, cmd: str, *, language: str = ...
  function _get_thread_test_sandbox (line 51) | def _get_thread_test_sandbox() -> LocalSandbox:
  function wait_for_sandbox (line 60) | def wait_for_sandbox(sandbox, timeout: int = 240, poll: float = 1.0):
  function _precompile_problem (line 74) | def _precompile_problem(problem_id: str, grader_files, compile_code: str...
  function run_test_case (line 102) | def run_test_case(task_args: dict, worker_id: int) -> dict:
  function extract_final_cpp_block (line 162) | def extract_final_cpp_block(text):
  function extract_final_text_block (line 169) | def extract_final_text_block(text):
  function extract_task_config (line 176) | def extract_task_config(problem_metadata: dict) -> dict:
  function add_includes (line 187) | def add_includes(code: str, problem_header_include: str | None = None, p...
  class CCCEvaluator (line 209) | class CCCEvaluator(BaseEvaluator):
    method __init__ (line 212) | def __init__(self, config: dict, num_parallel_requests: int = 10):
    method _initialize_runtime (line 221) | async def _initialize_runtime(self):
    method _get_precompiled_dir (line 240) | def _get_precompiled_dir(self, problem_id: str, problem_metadata: dict):
    method _build_test_task (line 256) | def _build_test_task(
    method _aggregate_subtask_score (line 270) | def _aggregate_subtask_score(self, subtask_meta: dict, outputs: list[d...
    method _evaluate_entry (line 286) | async def _evaluate_entry(self, entry: dict) -> dict:
    method eval_full (line 369) | async def eval_full(self):  # type: ignore[override]
    method eval_single (line 398) | async def eval_single(self, data_point: dict):

FILE: nemo_skills/evaluation/evaluator/code.py
  class CodeExecEvaluatorConfig (line 39) | class CodeExecEvaluatorConfig:
  class CodeExecEvaluator (line 47) | class CodeExecEvaluator(BaseEvaluator):
    method __init__ (line 48) | def __init__(self, config: dict, num_parallel_requests: int = 12):
    method eval_single (line 58) | async def eval_single(self, data: dict):
    method eval_full (line 95) | async def eval_full(self):  # type: ignore[override]
  function preprocess_code (line 118) | def preprocess_code(generation_dict: dict, language: str = "python", str...
  function install_from_git (line 176) | def install_from_git(git_url):
  class EvalPlusEvaluatorConfig (line 185) | class EvalPlusEvaluatorConfig(BaseEvaluatorConfig):
  function eval_evalplus (line 190) | def eval_evalplus(cfg):
  function install_requirements (line 231) | def install_requirements(url):
  class LiveCodeBenchProEvaluatorConfig (line 240) | class LiveCodeBenchProEvaluatorConfig(BaseEvaluatorConfig):
  function eval_livecodebench_pro (line 249) | def eval_livecodebench_pro(cfg):
  function eval_livebench_coding (line 297) | def eval_livebench_coding(cfg):
  function install_or_upgrade_package (line 349) | def install_or_upgrade_package(package_name):
  function eval_bigcodebench (line 358) | def eval_bigcodebench(cfg):
  function eval_human_eval_infilling (line 415) | def eval_human_eval_infilling(cfg):

FILE: nemo_skills/evaluation/evaluator/comet.py
  function load_comet_model (line 37) | def load_comet_model(model_path: str):
  function process_file (line 49) | def process_file(
  function main (line 106) | def main():

FILE: nemo_skills/evaluation/evaluator/compute_eval.py
  class ComputeEvalEvaluator (line 31) | class ComputeEvalEvaluator(BaseEvaluator):
    method __init__ (line 32) | def __init__(self, config: dict, num_parallel_requests=10):
    method eval_single (line 40) | async def eval_single(self, data_point: dict[str, Any]) -> dict[str, A...

FILE: nemo_skills/evaluation/evaluator/contextasr.py
  function _merge_single_letters (line 50) | def _merge_single_letters(text):
  function simple_tokenize (line 78) | def simple_tokenize(text):
  function extract_entities (line 103) | def extract_entities(text, entities_list, entity2count=None):
  function extract_entities_fuzzy (line 127) | def extract_entities_fuzzy(text, entities_list):
  function calculate_wer (line 174) | def calculate_wer(hyp_tokens, ref_tokens):
  function evaluate_contextasr_sample (line 223) | def evaluate_contextasr_sample(data_point):
  class ContextASREvaluatorConfig (line 314) | class ContextASREvaluatorConfig(BaseEvaluatorConfig):
  class ContextASREvaluator (line 320) | class ContextASREvaluator(BaseEvaluator):
    method __init__ (line 323) | def __init__(self, config: dict, num_parallel_requests=10):
    method eval_single (line 327) | async def eval_single(self, data_point: dict) -> dict:

FILE: nemo_skills/evaluation/evaluator/critpt.py
  class CritPtEvaluatorConfig (line 31) | class CritPtEvaluatorConfig(BaseEvaluatorConfig):
  class CritPtEvaluator (line 43) | class CritPtEvaluator(BaseEvaluator):
    method __init__ (line 53) | def __init__(self, config: dict, num_parallel_requests: int = 10):
    method _extract_code_from_generation (line 65) | def _extract_code_from_generation(self, generation: str) -> str:
    method _format_submission (line 81) | def _format_submission(self, data_point: dict) -> dict:
    method eval_full (line 105) | async def eval_full(self) -> None:
    method _submit_to_api (line 190) | def _submit_to_api(self, submissions: list[dict]) -> dict:

FILE: nemo_skills/evaluation/evaluator/dsbench.py
  function relaxed_equal (line 29) | def relaxed_equal(gt_answer: Any, predicted_answer: Any) -> bool:
  class DSBenchEvaluator (line 82) | class DSBenchEvaluator(MathEvaluator):
    method __init__ (line 83) | def __init__(self, config: dict, num_parallel_requests=10):
    method eval_single (line 87) | async def eval_single(self, data_point: dict[str, Any]) -> dict[str, A...

FILE: nemo_skills/evaluation/evaluator/icpc.py
  function sha256_hex (line 31) | def sha256_hex(text: str) -> str:
  class ICPCEvaluatorConfig (line 36) | class ICPCEvaluatorConfig(BaseEvaluatorConfig):
  function _sandbox_exec_sync (line 48) | def _sandbox_exec_sync(sandbox: LocalSandbox, cmd: str, *, language: str...
  function init_worker (line 65) | def init_worker():
  function _precompile_grader (line 73) | def _precompile_grader(
  function run_test_case (line 117) | def run_test_case(task_args: dict, worker_id: int) -> dict:
  function run_input_case (line 193) | def run_input_case(task_args: dict, worker_id: int) -> dict:
  function extract_final_cpp_block (line 267) | def extract_final_cpp_block(text):
  function add_includes (line 273) | def add_includes(code: str, problem_id: str) -> str:
  class ICPCEvaluator (line 289) | class ICPCEvaluator(BaseEvaluator):
    method __init__ (line 290) | def __init__(self, config: dict, num_parallel_requests: int = 10):
    method _initialize_runtime (line 300) | async def _initialize_runtime(self):
    method _evaluate_entry (line 338) | async def _evaluate_entry(self, entry: dict) -> dict:
    method eval_full (line 448) | async def eval_full(self, input_files):  # type: ignore[override]
    method eval_single (line 467) | async def eval_single(self, data_point: dict):

FILE: nemo_skills/evaluation/evaluator/ifbench.py
  function eval_ifbench (line 27) | def eval_ifbench(cfg):

FILE: nemo_skills/evaluation/evaluator/ifeval.py
  function eval_if (line 27) | def eval_if(cfg):

FILE: nemo_skills/evaluation/evaluator/ioi.py
  class IOIEvaluatorConfig (line 31) | class IOIEvaluatorConfig(BaseEvaluatorConfig):
  function sha256_hex (line 45) | def sha256_hex(text: str) -> str:
  function _sandbox_exec_sync (line 49) | def _sandbox_exec_sync(sandbox: LocalSandbox, cmd: str, *, language: str...
  function wait_for_sandbox (line 66) | def wait_for_sandbox(sandbox, timeout: int = 240, poll: float = 1.0):
  function init_worker (line 79) | def init_worker():
  function _precompile_grader (line 87) | def _precompile_grader(
  function run_test_case (line 127) | def run_test_case(task_args: dict, worker_id: int) -> dict:
  function run_input_case (line 202) | def run_input_case(task_args: dict, worker_id: int) -> dict:
  function extract_final_cpp_block (line 273) | def extract_final_cpp_block(text):
  function add_includes (line 279) | def add_includes(code: str, problem_id: str) -> str:
  class IOIEvaluator (line 306) | class IOIEvaluator(BaseEvaluator):
    method __init__ (line 307) | def __init__(self, config: dict, num_parallel_requests: int = 10):
    method _initialize_runtime (line 318) | async def _initialize_runtime(self):
    method _evaluate_entry (line 357) | async def _evaluate_entry(self, entry: dict) -> dict:
    method eval_full (line 476) | async def eval_full(self, input_files):  # type: ignore[override]
    method eval_single (line 494) | async def eval_single(self, data_point: dict):

FILE: nemo_skills/evaluation/evaluator/livecodebench.py
  class LiveCodeBenchEvaluatorConfig (line 40) | class LiveCodeBenchEvaluatorConfig(BaseEvaluatorConfig):
  function sandbox_context (line 52) | async def sandbox_context(config: dict):
  function execute_in_sandbox_with_retries (line 62) | async def execute_in_sandbox_with_retries(
  function is_sandbox_available (line 93) | async def is_sandbox_available(sandbox_config: dict) -> bool:
  function _preprocess_and_validate_file (line 124) | def _preprocess_and_validate_file(jsonl_file: str, language: str) -> Tup...
  function _postprocess_results (line 153) | def _postprocess_results(jsonl_file: str, samples: List[Dict[str, Any]]):
  function _install_packages_in_sandbox (line 172) | async def _install_packages_in_sandbox(sandbox: Sandbox, eval_config: Li...
  function _install_packages_locally (line 189) | def _install_packages_locally(interpreter: str):
  function eval_livecodebench_async (line 210) | async def eval_livecodebench_async(eval_config: LiveCodeBenchEvaluatorCo...
  function eval_livecodebench_without_sandbox (line 255) | def eval_livecodebench_without_sandbox(eval_config: LiveCodeBenchEvaluat...
  function eval_livecodebench (line 284) | def eval_livecodebench(cfg):

FILE: nemo_skills/evaluation/evaluator/math.py
  class MathEvaluatorConfig (line 32) | class MathEvaluatorConfig(BaseEvaluatorConfig):
  class LeanEvaluatorConfig (line 47) | class LeanEvaluatorConfig(BaseEvaluatorConfig):
  class MathEvaluator (line 57) | class MathEvaluator(BaseEvaluator):
    method __init__ (line 58) | def __init__(self, config: dict, num_parallel_requests=10):
    method eval_single (line 62) | async def eval_single(self, data_point: dict[str, any]) -> dict[str, a...
  class Lean4ProofEvaluator (line 90) | class Lean4ProofEvaluator(BaseEvaluator):
    method __init__ (line 93) | def __init__(self, config: dict, num_parallel_requests=10):
    method eval_single (line 99) | async def eval_single(self, data_point: dict[str, any]) -> dict[str, a...

FILE: nemo_skills/evaluation/evaluator/mcq.py
  function normalize_extracted_answer (line 28) | def normalize_extracted_answer(extracted_answer: str) -> str:
  class MCQEvaluatorConfig (line 50) | class MCQEvaluatorConfig(BaseEvaluatorConfig):
  function eval_mcq (line 62) | def eval_mcq(cfg):

FILE: nemo_skills/evaluation/evaluator/mmau_pro.py
  function eval_mmau_pro (line 28) | def eval_mmau_pro(cfg):
  function evaluate_instruction_following_sample (line 57) | def evaluate_instruction_following_sample(sample: dict[str, Any]) -> dic...
  function evaluate_aif_constraints (line 73) | def evaluate_aif_constraints(

FILE: nemo_skills/evaluation/evaluator/mrcr.py
  function eval_mrcr (line 27) | def eval_mrcr(cfg):

FILE: nemo_skills/evaluation/evaluator/nvembed_judge.py
  function install_packages (line 41) | def install_packages():
  function load_nvembed_model (line 60) | def load_nvembed_model(model_name: str = "nvidia/NV-Embed-v2"):
  function evaluate_with_nvembed_similarity (line 86) | def evaluate_with_nvembed_similarity(
  function evaluate_sample_with_nvembed (line 116) | def evaluate_sample_with_nvembed(sample: dict[str, Any], model_name: str...
  function process_file (line 150) | def process_file(input_file: Path, output_file: Path, model_name: str = ...
  function main (line 193) | def main():

FILE: nemo_skills/evaluation/evaluator/ruler.py
  class RulerEvaluatorConfig (line 30) | class RulerEvaluatorConfig(BaseEvaluatorConfig):
  function eval_ruler (line 35) | def eval_ruler(cfg):
  function eval_ruler2 (line 87) | def eval_ruler2(cfg):

FILE: nemo_skills/evaluation/evaluator/scicode.py
  class ScicodeEvaluatorConfig (line 30) | class ScicodeEvaluatorConfig(BaseEvaluatorConfig):
  function _execute_single_test (line 36) | async def _execute_single_test(args):
  function test_code (line 74) | def test_code(eval_config, scicode_data):
  function eval_scicode (line 111) | def eval_scicode(cfg):

FILE: nemo_skills/evaluation/evaluator/specdec.py
  class SpecdecEvaluatorConfig (line 27) | class SpecdecEvaluatorConfig(BaseEvaluatorConfig):
    method __post_init__ (line 39) | def __post_init__(self):
  function eval_specdec (line 44) | def eval_specdec(cfg: dict[str, Any]) -> None:

FILE: nemo_skills/evaluation/math_grader.py
  function _additional_normalization (line 26) | def _additional_normalization(expr):
  function math_equal (line 37) | def math_equal(gt_answer, predicted_answer, take_modulo: int | None = No...
  function extract_answer (line 102) | def extract_answer(
  function search_regex (line 117) | def search_regex(string: str, regex: str):
  function search_boxed (line 124) | def search_boxed(string: str):

FILE: nemo_skills/evaluation/metrics/aalcr_metrics.py
  class AALCRMetrics (line 20) | class AALCRMetrics(BaseMetrics):
    method __init__ (line 27) | def __init__(self):
    method reset (line 40) | def reset(self):
    method is_aalcr_correct (line 48) | def is_aalcr_correct(judgement: str) -> bool:
    method _get_score_dict (line 58) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method _get_token_bucket (line 72) | def _get_token_bucket(self, input_tokens: int) -> str:
    method _update_token_bucket_metrics (line 85) | def _update_token_bucket_metrics(self, prediction: dict, score_dict: d...
    method get_incorrect_sample (line 101) | def get_incorrect_sample(cls, prediction: dict) -> dict:
    method _update_category_metrics (line 108) | def _update_category_metrics(self, prediction: dict, score_dict: dict):
    method _update_token_stats (line 117) | def _update_token_stats(self, prediction: dict):
    method update (line 124) | def update(self, predictions):
    method get_metrics (line 148) | def get_metrics(self):
    method _print_category_table (line 188) | def _print_category_table(self, category_results):
    method _print_token_length_analysis (line 233) | def _print_token_length_analysis(self):
    method evaluations_to_print (line 303) | def evaluations_to_print(self):
    method metrics_to_print (line 310) | def metrics_to_print(self):

FILE: nemo_skills/evaluation/metrics/answer_judgement_metrics.py
  class AnswerJudgementMetrics (line 24) | class AnswerJudgementMetrics(BaseMetrics):
    method __init__ (line 25) | def __init__(self):
    method reset (line 31) | def reset(self):
    method _get_score_dict (line 35) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method get_incorrect_sample (line 41) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method _store_individual_metrics (line 49) | def _store_individual_metrics(self, agg_key, pred_judgement, gt_judgem...
    method _update_fp_fn (line 66) | def _update_fp_fn(self, metrics_dict, pred_judgement, gt_judgement, di...
    method _update_score_metrics_for_majority (line 73) | def _update_score_metrics_for_majority(
    method _update_score_metrics_for_pass (line 90) | def _update_score_metrics_for_pass(
    method update (line 121) | def update(self, predictions):
    method _compute_precision_recall_f1 (line 134) | def _compute_precision_recall_f1(self, datapoint_metrics):
    method get_metrics (line 182) | def get_metrics(self):

FILE: nemo_skills/evaluation/metrics/arena_metrics.py
  class ArenaMetrics (line 21) | class ArenaMetrics(BaseMetrics):
    method __init__ (line 22) | def __init__(self):
    method _get_judge_score (line 25) | def _get_judge_score(self, judgment):
    method get_incorrect_sample (line 37) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method update (line 43) | def update(self, predictions):
    method get_metrics (line 92) | def get_metrics(self):
    method reset (line 119) | def reset(self):

FILE: nemo_skills/evaluation/metrics/audio_metrics.py
  function compute_corpus_bleu (line 43) | def compute_corpus_bleu(
  class AudioMetrics (line 76) | class AudioMetrics(BaseMetrics):
    method __init__ (line 85) | def __init__(self, compute_no_answer: bool = True, max_k: int = 1):
    method _extract_judge_result (line 127) | def _extract_judge_result(self, judgement_text: str) -> tuple[bool, fl...
    method _get_score_dict (line 162) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method get_incorrect_sample (line 192) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method update_common_metrics (line 210) | def update_common_metrics(self, agg_dict):
    method update (line 223) | def update(self, predictions):
    method get_metrics (line 304) | def get_metrics(self):
    method evaluations_to_print (line 380) | def evaluations_to_print(self):
    method metrics_to_print (line 391) | def metrics_to_print(self):
  function compute_score (line 457) | def compute_score(combined_metrics: dict) -> dict:

FILE: nemo_skills/evaluation/metrics/base.py
  class BaseMetrics (line 23) | class BaseMetrics(abc.ABC):
    method __init__ (line 24) | def __init__(self, compute_no_answer: bool = True):
    method update_common_metrics (line 28) | def update_common_metrics(self, agg_dict):
    method get_metrics (line 35) | def get_metrics(self):
    method _add_std_metrics (line 49) | def _add_std_metrics(self, metrics_dict):
    method _get_score_dict (line 124) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method update (line 145) | def update(self, predictions):
    method reset (line 191) | def reset(self):
    method get_incorrect_sample (line 200) | def get_incorrect_sample(self, predictions: list[dict]) -> list[dict]:
    method _update_score_metrics_for_majority (line 208) | def _update_score_metrics_for_majority(
    method _update_metrics_for_majority (line 228) | def _update_metrics_for_majority(
    method _compute_majority_at_k (line 246) | def _compute_majority_at_k(
    method _update_score_metrics_for_pass (line 315) | def _update_score_metrics_for_pass(
    method _update_metrics_for_pass (line 334) | def _update_metrics_for_pass(
    method _compute_pass_at_k (line 352) | def _compute_pass_at_k(
    method setup (line 425) | def setup(self, input_files):
    method metrics_to_print (line 428) | def metrics_to_print(self):
    method evaluations_to_print (line 432) | def evaluations_to_print(self):
  function as_percentage (line 437) | def as_percentage(metric_key: str, metric_value: float, all_metrics: dict):
  function as_int (line 443) | def as_int(metric_key: str, metric_value: float, all_metrics: dict):
  function as_float (line 449) | def as_float(metric_key: str, metric_value: float, all_metrics: dict):
  function default_formatting (line 454) | def default_formatting(metric_key: str, metric_value, all_metrics: dict)...

FILE: nemo_skills/evaluation/metrics/bfcl_metrics.py
  class BFCLMetrics (line 18) | class BFCLMetrics(BaseMetrics):
    method _get_score_dict (line 24) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method update (line 27) | def update(self, predictions):

FILE: nemo_skills/evaluation/metrics/bird_metrics.py
  class BirdMetrics (line 18) | class BirdMetrics(BaseMetrics):
    method __init__ (line 21) | def __init__(self):
    method reset (line 25) | def reset(self):
    method update (line 33) | def update(self, predictions):
    method get_metrics (line 47) | def get_metrics(self):
    method evaluations_to_print (line 67) | def evaluations_to_print(self):
    method metrics_to_print (line 70) | def metrics_to_print(self):

FILE: nemo_skills/evaluation/metrics/ccc_metrics.py
  class CCCMetrics (line 11) | class CCCMetrics(BaseMetrics):
    method __init__ (line 14) | def __init__(self, **kwargs):
    method reset (line 21) | def reset(self):
    method setup (line 27) | def setup(self, input_files):
    method update (line 38) | def update(self, predictions):
    method _get_score_dict (line 55) | def _get_score_dict(self, submission):
    method _aggregate_row_group (line 64) | def _aggregate_row_group(self, submissions, mode: str, subtask_name: s...
    method _build_problem_reports (line 165) | def _build_problem_reports(self, mode: str):
    method _select_minimal_solutions (line 338) | def _select_minimal_solutions(self, problem_id: str, problem_name: str...
    method _sanitize_filename_component (line 429) | def _sanitize_filename_component(value):
    method _extract_solution_code (line 435) | def _extract_solution_code(solution_text: str) -> str:
    method _write_selected_solutions (line 442) | def _write_selected_solutions(self, report: dict):
    method get_metrics (line 499) | def get_metrics(self):
    method evaluations_to_print (line 594) | def evaluations_to_print(self):

FILE: nemo_skills/evaluation/metrics/code_metrics.py
  class EvalPlusMetrics (line 18) | class EvalPlusMetrics(BaseMetrics):
    method _get_score_dict (line 19) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method get_incorrect_sample (line 25) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method update (line 28) | def update(self, predictions):
  class LiveCodeBenchMetrics (line 33) | class LiveCodeBenchMetrics(BaseMetrics):
    method _get_score_dict (line 34) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method get_incorrect_sample (line 39) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method update (line 42) | def update(self, predictions):
  class SweBenchMetrics (line 47) | class SweBenchMetrics(BaseMetrics):
    method _get_score_dict (line 48) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method get_incorrect_sample (line 55) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method update (line 58) | def update(self, predictions):
  class SciCodeMetrics (line 63) | class SciCodeMetrics(BaseMetrics):
    method _get_score_dict (line 64) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method get_incorrect_sample (line 72) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method update (line 80) | def update(self, predictions):
    method get_metrics (line 85) | def get_metrics(self):
    method reset (line 95) | def reset(self):
  class BigCodeBenchMetrics (line 100) | class BigCodeBenchMetrics(BaseMetrics):
    method _get_score_dict (line 101) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method get_incorrect_sample (line 106) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method update (line 109) | def update(self, predictions):
  class HumanEvalInfillingMetrics (line 114) | class HumanEvalInfillingMetrics(BaseMetrics):
    method _get_score_dict (line 115) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method get_incorrect_sample (line 118) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method update (line 121) | def update(self, predictions):
  class ComputeEvalMetrics (line 126) | class ComputeEvalMetrics(BaseMetrics):
    method _get_score_dict (line 127) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method get_incorrect_sample (line 130) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method update (line 133) | def update(self, predictions):

FILE: nemo_skills/evaluation/metrics/compute_metrics.py
  class ComputeMetrics (line 24) | class ComputeMetrics:
    method __init__ (line 25) | def __init__(
    method get_metrics_calculator (line 44) | def get_metrics_calculator(self):
    method compute_metrics (line 49) | def compute_metrics(self, input_files):
    method metrics_to_print (line 89) | def metrics_to_print(self):
    method evaluations_to_print (line 92) | def evaluations_to_print(self):

FILE: nemo_skills/evaluation/metrics/contextasr_metrics.py
  class ContextASRMetrics (line 26) | class ContextASRMetrics(BaseMetrics):
    method __init__ (line 29) | def __init__(self, compute_no_answer: bool = True, max_k: int = 1):
    method _get_score_dict (line 43) | def _get_score_dict(self, prediction):
    method get_incorrect_sample (line 52) | def get_incorrect_sample(self, prediction):
    method update_common_metrics (line 58) | def update_common_metrics(self, agg_dict):
    method update (line 65) | def update(self, predictions):
    method get_metrics (line 96) | def get_metrics(self):
    method evaluations_to_print (line 115) | def evaluations_to_print(self):
    method metrics_to_print (line 122) | def metrics_to_print(self):

FILE: nemo_skills/evaluation/metrics/critpt_metrics.py
  class CritPtMetrics (line 23) | class CritPtMetrics(BaseMetrics):
    method _get_score_dict (line 33) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method update (line 42) | def update(self, predictions):
    method metrics_to_print (line 51) | def metrics_to_print(self):

FILE: nemo_skills/evaluation/metrics/gradingbench_metrics.py
  class GradingBenchMetrics (line 24) | class GradingBenchMetrics(BaseMetrics):
    method __init__ (line 54) | def __init__(self):
    method _extract_grade (line 58) | def _extract_grade(self, text: str) -> str | None:
    method _get_grades (line 89) | def _get_grades(self, prediction: dict) -> tuple[str | None, str | None]:
    method _get_score_dict (line 108) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method update (line 123) | def update(self, predictions):
    method get_metrics (line 137) | def get_metrics(self):
    method reset (line 149) | def reset(self):
    method metrics_to_print (line 154) | def metrics_to_print(self):
    method evaluations_to_print (line 162) | def evaluations_to_print(self):

FILE: nemo_skills/evaluation/metrics/hleaa_metrics.py
  class HLEAAMetrics (line 24) | class HLEAAMetrics(MathMetrics):
    method _postprocess_judgement (line 27) | def _postprocess_judgement(self, prediction: dict) -> dict:
    method update (line 37) | def update(self, predictions):

FILE: nemo_skills/evaluation/metrics/hotpotqa_filtering.py
  function _normalize_unicode (line 109) | def _normalize_unicode(s: str) -> str:
  function _gt_alternatives (line 122) | def _gt_alternatives(gt: str) -> tuple[list[str], list[str]]:
  function _is_multi_word_name (line 207) | def _is_multi_word_name(gt: str) -> bool:
  function _should_remove (line 219) | def _should_remove(gt: str) -> tuple[bool, str]:
  function normalize_gt (line 228) | def normalize_gt(gt_answer: str) -> dict:
  function is_correct (line 252) | def is_correct(alternatives: list[str], model_answer: str) -> bool:
  function is_correct_strict (line 263) | def is_correct_strict(alternatives: list[str], model_answer: str) -> bool:

FILE: nemo_skills/evaluation/metrics/hotpotqa_metrics.py
  function normalize_answer (line 36) | def normalize_answer(s: str) -> str:
  function answer_f1_score (line 55) | def answer_f1_score(prediction: str, ground_truth: str) -> tuple[float, ...
  function answer_exact_match (line 82) | def answer_exact_match(prediction: str, ground_truth: str) -> float:
  function sp_scores (line 87) | def sp_scores(prediction: list, gold: list) -> tuple[float, float, float...
  function _try_parse_answer_json (line 113) | def _try_parse_answer_json(text: str) -> tuple[str, list] | None:
  function _extract_json_candidates (line 135) | def _extract_json_candidates(text: str) -> list[str]:
  function parse_generation (line 158) | def parse_generation(generation: str) -> tuple[str, list]:
  class HotpotQAMetrics (line 187) | class HotpotQAMetrics(BaseMetrics):
    method __init__ (line 203) | def __init__(self, compute_no_answer: bool = False, closed_book: bool ...
    method reset (line 207) | def reset(self):
    method _get_score_dict (line 214) | def _get_score_dict(self, prediction: dict) -> dict[str, float]:
    method _update_score_metrics_for_pass (line 254) | def _update_score_metrics_for_pass(
    method update (line 272) | def update(self, predictions):
    method get_metrics (line 284) | def get_metrics(self):
    method evaluations_to_print (line 300) | def evaluations_to_print(self):
    method metrics_to_print (line 306) | def metrics_to_print(self):

FILE: nemo_skills/evaluation/metrics/icpc_metrics.py
  function extract_final_cpp_block (line 24) | def extract_final_cpp_block(text):
  class ICPCMetrics (line 30) | class ICPCMetrics(BaseMetrics):
    method __init__ (line 31) | def __init__(self, **kwargs):
    method update (line 37) | def update(self, predictions):
    method _get_score_dict (line 43) | def _get_score_dict(self, p):
    method get_problem_score (line 46) | def get_problem_score(self, submissions) -> bool:
    method get_problem_sample_score (line 52) | def get_problem_sample_score(self, submissions) -> bool:
    method extract_info (line 58) | def extract_info(self, submission) -> dict:
    method get_clusters (line 66) | def get_clusters(self, submissions) -> dict:
    method get_metrics (line 100) | def get_metrics(self):
    method evaluations_to_print (line 157) | def evaluations_to_print(self):
    method metrics_to_print (line 161) | def metrics_to_print(self):
    method reset (line 171) | def reset(self):
    method print_problem_scores (line 176) | def print_problem_scores(self):

FILE: nemo_skills/evaluation/metrics/if_metrics.py
  class IFMetrics (line 20) | class IFMetrics(BaseMetrics):
    method _get_score_dict (line 24) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method get_incorrect_sample (line 30) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method update (line 35) | def update(self, predictions):
    method get_metrics (line 50) | def get_metrics(self):
    method reset (line 70) | def reset(self):

FILE: nemo_skills/evaluation/metrics/ioi_metrics.py
  function extract_final_cpp_block (line 22) | def extract_final_cpp_block(text):
  class IOIMetrics (line 28) | class IOIMetrics(BaseMetrics):
    method __init__ (line 29) | def __init__(self, **kwargs):
    method update (line 35) | def update(self, predictions):
    method _get_score_dict (line 41) | def _get_score_dict(self, p):
    method extract_info (line 44) | def extract_info(self, submission) -> dict:
    method get_clusters (line 53) | def get_clusters(self, submissions) -> dict:
    method get_problem_score (line 92) | def get_problem_score(self, submissions) -> float:
    method get_metrics (line 107) | def get_metrics(self):
    method reset (line 158) | def reset(self):
    method evaluations_to_print (line 164) | def evaluations_to_print(self):
    method print_problem_scores (line 167) | def print_problem_scores(self):

FILE: nemo_skills/evaluation/metrics/lean4_metrics.py
  class Lean4Metrics (line 19) | class Lean4Metrics(BaseMetrics):
    method __init__ (line 20) | def __init__(self):
    method _get_score_dict (line 23) | def _get_score_dict(self, prediction):
    method get_incorrect_sample (line 26) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method _update_score_metrics_for_pass (line 31) | def _update_score_metrics_for_pass(
    method update (line 46) | def update(self, predictions):

FILE: nemo_skills/evaluation/metrics/map_metrics.py
  function get_metrics (line 109) | def get_metrics(metric_type: str, **kwargs):

FILE: nemo_skills/evaluation/metrics/math_metrics.py
  class MathMetrics (line 25) | class MathMetrics(BaseMetrics):
    method __init__ (line 28) | def __init__(
    method _compute_reward_at_k (line 35) | def _compute_reward_at_k(self, predictions: list[dict]):
    method _get_score_dict (line 70) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method is_correct_judgement (line 84) | def is_correct_judgement(self, judgement: str) -> bool:
    method get_incorrect_sample (line 88) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method update (line 97) | def update(self, predictions):
    method evaluations_to_print (line 129) | def evaluations_to_print(self):
    method metrics_to_print (line 139) | def metrics_to_print(self):

FILE: nemo_skills/evaluation/metrics/mcq_multilingual_metrics.py
  class MCQMultilingualMetrics (line 41) | class MCQMultilingualMetrics(MathMetrics):
    method __init__ (line 42) | def __init__(
    method _get_score_dict (line 50) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method metrics_to_print (line 99) | def metrics_to_print(self):
    method _detect_language (line 104) | def _detect_language(self, text):

FILE: nemo_skills/evaluation/metrics/mmau_pro_metrics.py
  function extract_multicriteria_scores (line 26) | def extract_multicriteria_scores(judgement_text: str) -> dict[str, float]:
  class MMAUProMetrics (line 68) | class MMAUProMetrics(BaseMetrics):
    method __init__ (line 71) | def __init__(self, compute_no_answer: bool = True, max_k: int = 1):
    method _get_score_dict (line 84) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method get_incorrect_sample (line 100) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method update (line 110) | def update(self, predictions):
    method get_metrics (line 125) | def get_metrics(self):
    method metrics_to_print (line 170) | def metrics_to_print(self):

FILE: nemo_skills/evaluation/metrics/mrcr_metrics.py
  class MRCRMetrics (line 18) | class MRCRMetrics(BaseMetrics):
    method _get_score_dict (line 21) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method update (line 24) | def update(self, predictions):

FILE: nemo_skills/evaluation/metrics/omni_metrics.py
  class OmniMetrics (line 20) | class OmniMetrics(BaseMetrics):
    method __init__ (line 21) | def __init__(self, compute_no_answer: bool = True, answer_key: str = "...
    method _compute_reward_at_k (line 26) | def _compute_reward_at_k(self, predictions: list[dict]):
    method _get_score_dict (line 61) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method get_metrics (line 76) | def get_metrics(self):
    method get_incorrect_sample (line 107) | def get_incorrect_sample(self, prediction: dict) -> dict:
    method update (line 118) | def update(self, predictions):
    method evaluations_to_print (line 125) | def evaluations_to_print(self):
    method metrics_to_print (line 131) | def metrics_to_print(self):

FILE: nemo_skills/evaluation/metrics/physics_metrics.py
  class PhysicsMetrics (line 24) | class PhysicsMetrics(MathMetrics):
    method __init__ (line 25) | def __init__(self, compute_no_answer: bool = False, answer_key: str = ...
    method is_correct_judgement (line 29) | def is_correct_judgement(self, judgement: str, return_none: bool = Fal...
    method get_incorrect_sample (line 41) | def get_incorrect_sample(self, prediction: dict) -> dict:

FILE: nemo_skills/evaluation/metrics/ruler2_metrics.py
  class Ruler2Metrics (line 18) | class Ruler2Metrics(BaseMetrics):
    method _get_score_dict (line 28) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method update (line 33) | def update(self, predictions):
    method get_incorrect_sample (line 37) | def get_incorrect_sample(self, prediction: dict) -> dict:

FILE: nemo_skills/evaluation/metrics/ruler_metrics.py
  class RulerMetrics (line 18) | class RulerMetrics(BaseMetrics):
    method _get_score_dict (line 19) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method update (line 22) | def update(self, predictions):
    method get_incorrect_sample (line 26) | def get_incorrect_sample(self, prediction: dict) -> dict:

FILE: nemo_skills/evaluation/metrics/simpleqa_metrics.py
  function is_correct_judgement_label_matching (line 24) | def is_correct_judgement_label_matching(judgement: str, correct_label: s...
  class SimpleQAMetrics (line 38) | class SimpleQAMetrics(BaseMetrics):
    method __init__ (line 41) | def __init__(self, compute_no_answer: bool = False, answer_key: str = ...
    method update (line 45) | def update(self, predictions):
    method _get_score_dict (line 60) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method _to_bool_or_none (line 90) | def _to_bool_or_none(j):
    method get_metrics (line 103) | def get_metrics(self):

FILE: nemo_skills/evaluation/metrics/specdec_metrics.py
  class SpecdecMetrics (line 23) | class SpecdecMetrics(BaseMetrics):
    method __init__ (line 42) | def __init__(self):
    method _get_score_dict (line 45) | def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | ...
    method update (line 54) | def update(self, predictions: list[dict]) -> None:
    method get_metrics (line 67) | def get_metrics(self) -> dict:
    method metrics_to_print (line 87) | def metrics_to_print(self) -> dict:

FILE: nemo_skills/evaluation/metrics/translation_metrics.py
  function install_packages (line 24) | def install_packages(lang):
  class TranslationMetrics (line 34) | class TranslationMetrics(BaseMetrics):
    method get_metrics (line 37) | def get_metrics(self):
    method _add_std_metrics (line 89) | def _add_std_metrics(self, metrics_dict):
    method update (line 101) | def update(self, predictions):
    method reset (line 130) | def reset(self):
    method evaluations_to_print (line 136) | def evaluations_to_print(self):
    method metrics_to_print (line 140) | def metrics_to_print(self):

FILE: nemo_skills/evaluation/metrics/ugphysics_metrics.py
  class UGPhysicsMetrics (line 24) | class UGPhysicsMetrics(MathMetrics):
    method __init__ (line 25) | def __init__(self, compute_no_answer: bool = False, answer_key: str = ...
    method is_correct_judgement (line 29) | def is_correct_judgement(self, judgement: str, return_none: bool = Fal...
    method get_incorrect_sample (line 44) | def get_incorrect_sample(self, prediction: dict) -> dict:

FILE: nemo_skills/evaluation/metrics/utils.py
  function read_predictions (line 24) | def read_predictions(predictions, line_idx, file_handles):
  function is_correct_judgement (line 37) | def is_correct_judgement(judgement, return_none=False) -> Union[bool, No...

FILE: nemo_skills/evaluation/metrics/weighted_math_metrics.py
  class WeightedMathMetrics (line 24) | class WeightedMathMetrics(MathMetrics):
    method reset (line 27) | def reset(self) -> None:
    method _get_sample_weight (line 33) | def _get_sample_weight(self, prediction: dict) -> float:
    method _update_pass1_avg_of_k (line 37) | def _update_pass1_avg_of_k(self, score_method: str, attempt_scores: li...
    method _update_pass_at_k (line 43) | def _update_pass_at_k(self, score_method: str, attempt_scores: list[bo...
    method _update_majority_at_k (line 54) | def _update_majority_at_k(
    method update (line 70) | def update(self, predictions: list[dict]) -> None:
    method _add_weighted_std_metrics (line 89) | def _add_weighted_std_metrics(self, metrics_dict: dict) -> None:
    method get_metrics (line 116) | def get_metrics(self) -> dict:
    method metrics_to_print (line 127) | def metrics_to_print(self) -> dict:

FILE: nemo_skills/evaluation/utils.py
  function load_config (line 20) | def load_config(config: str, config_dir: str | None = None) -> dict:
  function get_eval_group (line 48) | def get_eval_group(eval_config: str | dict, eval_group_dir: str | None =...

FILE: nemo_skills/file_utils.py
  function unroll_files (line 21) | def unroll_files(input_files, parent_dir: str | None = None):
  function _make_w_io_base (line 35) | def _make_w_io_base(f, mode: str):
  function _make_r_io_base (line 53) | def _make_r_io_base(f, mode: str):
  function jdump (line 68) | def jdump(obj, f, mode="w", indent=None, default=str):
  function jload (line 103) | def jload(filepath, mode="r", verbose=False):
  function count_newlines (line 141) | def count_newlines(fname, verbose: bool = False):
  function calculate_chunk_indices (line 172) | def calculate_chunk_indices(num_samples: int, num_chunks: int, chunk_id:...
  function jload_chunk (line 214) | def jload_chunk(filepath, num_chunks: int, chunk_id: int, mode="r", verb...

FILE: nemo_skills/inference/autoformalize.py
  class AutoformalizeConfig (line 49) | class AutoformalizeConfig(GenerationTaskConfig):
  class AutoformalizeTask (line 72) | class AutoformalizeTask(GenerationTask):
    method __init__ (line 73) | def __init__(self, cfg: AutoformalizeConfig):
    method setup_llm (line 87) | def setup_llm(self):
    method setup_refine_prompt (line 101) | def setup_refine_prompt(self):
    method setup_judge_prompt (line 116) | def setup_judge_prompt(self):
    method _extract_code_sync (line 126) | def _extract_code_sync(self, completion: str):
    method _extract_code (line 140) | async def _extract_code(self, completion: str):
    method _backtranslate_code (line 144) | async def _backtranslate_code(self, code: str) -> str:
    method _judge_backtranslation (line 149) | async def _judge_backtranslation(self, backtranslation_result: str, da...
    method _judge_code (line 159) | async def _judge_code(self, code: str | None, data_point) -> dict:
    method _construct_refine_prompt (line 212) | def _construct_refine_prompt(self, results_dict):
    method _generate_single_completion (line 226) | async def _generate_single_completion(self, prompt: List[str]):
    method _single_data_point_generate (line 267) | async def _single_data_point_generate(self, data_point, data):
    method process_single_datapoint (line 308) | async def process_single_datapoint(self, data_point, all_data, prompt_...
  function generate (line 319) | def generate(cfg: AutoformalizeConfig):

FILE: nemo_skills/inference/chat_interface/chat_service.py
  class ChatService (line 26) | class ChatService:
    method __init__ (line 29) | def __init__(self, loader: ModelLoader, prompts: PromptManager):
    method stream_chat (line 33) | def stream_chat(
  class AppContext (line 77) | class AppContext:
    method __init__ (line 80) | def __init__(self, cfg: AppConfig):

FILE: nemo_skills/inference/chat_interface/core.py
  class AppConfig (line 42) | class AppConfig:
    method __post_init__ (line 71) | def __post_init__(self):
  class CodeExecStatus (line 116) | class CodeExecStatus(Enum):
  class PromptManager (line 129) | class PromptManager:
    method __init__ (line 132) | def __init__(self, cfg: AppConfig):
    method get (line 136) | def get(self, use_code: bool, prompt_config_override: str | None = Non...
  class ModelLoader (line 159) | class ModelLoader:
    method __init__ (line 162) | def __init__(self, cfg: AppConfig):
    method generic_llm (line 169) | def generic_llm(self) -> Any | None:  # noqa: D401
    method code_llm (line 173) | def code_llm(self) -> Any | None:  # noqa: D401
    method sandbox (line 177) | def sandbox(self):  # noqa: D401
    method cfg (line 181) | def cfg(self):  # noqa: D401
    method load_generic (line 184) | def load_generic(self) -> Tuple[bool, str]:
    method load_code_and_sandbox (line 206) | def load_code_and_sandbox(self) -> Tuple[bool, str]:
    method get_code_execution_status (line 246) | def get_code_execution_status(self, requested: bool) -> CodeExecStatus:
    method _is_sandbox_alive (line 256) | def _is_sandbox_alive(self) -> bool:
    method supports_code_toggle (line 267) | def supports_code_toggle(self) -> bool:

FILE: nemo_skills/inference/chat_interface/launch.py
  function launch (line 33) | def launch(cfg: AppConfig):

FILE: nemo_skills/inference/chat_interface/ui.py
  function _format_output (line 48) | def _format_output(text: str) -> str:
  class ChatUI (line 67) | class ChatUI:
    method __init__ (line 70) | def __init__(self, ctx: AppContext):
    method _get_default_prompt_config (line 104) | def _get_default_prompt_config(self, use_code: bool) -> str:
    method _get_current_prompt_config (line 108) | def _get_current_prompt_config(self, use_code: bool) -> str:
    method _build_chat_panel (line 117) | def _build_chat_panel(self):
    method on_prompt_config_change (line 195) | def on_prompt_config_change(self, prompt_config_value: str):
    method on_toggle_code_exec (line 210) | def on_toggle_code_exec(self, checkbox_val: bool):
    method on_cancel (line 253) | def on_cancel(self):
    method on_clear_chat (line 267) | def on_clear_chat(self):
    method on_reset_params (line 273) | def on_reset_params(self):
    method handle_chat_submit (line 326) | def handle_chat_submit(self, user_msg: str, max_tokens: int, temperatu...
    method launch (line 422) | def launch(self):
    method _banner_from_code_status (line 425) | def _banner_from_code_status(self, code_status: CodeExecStatus):

FILE: nemo_skills/inference/check_contamination.py
  class CheckContaminationConfig (line 40) | class CheckContaminationConfig(GenerationTaskConfig):
    method _get_disallowed_params (line 62) | def _get_disallowed_params(self):
  class CheckContaminationTask (line 74) | class CheckContaminationTask(GenerationTask):
    method __init__ (line 75) | def __init__(self, cfg: CheckContaminationConfig):
    method load_data (line 78) | def load_data(self):
    method log_example_prompt (line 88) | def log_example_prompt(self, data):
    method _create_query_data (line 102) | def _create_query_data(self, data_point):
    method prefill_generation (line 123) | def prefill_generation(self, data_point):
    method process_single_datapoint (line 130) | async def process_single_datapoint(self, data_point, all_data, prompt_...
    method postprocess (line 152) | def postprocess(self):
  function check_contamination (line 171) | def check_contamination(cfg: CheckContaminationConfig):

FILE: nemo_skills/inference/eval/arena_judge.py
  function sanitize_generation (line 42) | def sanitize_generation(generation: str) -> str:
  class ArenaJudgeConfig (line 51) | class ArenaJudgeConfig(GenerationTaskConfig):
  class ArenaJudgeTask (line 79) | class ArenaJudgeTask(GenerationTask):
    method __init__ (line 80) | def __init__(self, cfg: ArenaJudgeConfig):
    method setup_prompt (line 83) | def setup_prompt(self):
    method fill_prompt (line 113) | def fill_prompt(self, data_point, data, prompt_format=None):
    method log_example_prompt (line 141) | def log_example_prompt(self, all_data):
    method process_single_datapoint (line 159) | async def process_single_datapoint(self, data_point, all_data, prompt_...
  function generate (line 196) | def generate(cfg: ArenaJudgeConfig):

FILE: nemo_skills/inference/eval/bfcl.py
  class BFCLGenerationConfig (line 89) | class BFCLGenerationConfig(GenerationTaskConfig):
    method _post_init_validate_params (line 100) | def _post_init_validate_params(self):
    method _get_disallowed_params (line 113) | def _get_disallowed_params(self):
  class ClientMessageParser (line 124) | class ClientMessageParser:
    method __init__ (line 127) | def __init__(self, cfg: BFCLGenerationConfig):
    method _validate_and_setup_client_parsing (line 131) | def _validate_and_setup_client_parsing(self):
    method create_response_parser (line 172) | def create_response_parser(self, native_response_parser):
    method construct_input_dict (line 203) | def construct_input_dict(self, messages: list[dict], tools: list[dict]):
    method parse_output_dict (line 221) | def parse_output_dict(self, output_dict: dict):
    method get_response_text (line 263) | def get_response_text(self, message):
    method set_response_text (line 266) | def set_response_text(self, message, response_text):
  class ServerMessageParser (line 270) | class ServerMessageParser:
    method __init__ (line 273) | def __init__(self, cfg: BFCLGenerationConfig):
    method construct_input_dict (line 276) | def construct_input_dict(self, messages: list[dict], tools: list[dict]):
    method parse_output_dict (line 284) | def parse_output_dict(self, output_dict: dict):
    method get_response_text (line 324) | def get_response_text(self, message):
    method set_response_text (line 329) | def set_response_text(self, message, response_text):
  class BFCLGenerationTask (line 336) | class BFCLGenerationTask(GenerationTask):
    method get_generation_requirements (line 338) | def get_generation_requirements(cls) -> list[str] | None:
    method __init__ (line 341) | def __init__(self, cfg: BFCLGenerationConfig):
    method log_example_prompt (line 348) | def log_example_prompt(self, data):
    method setup_prompt (line 352) | def setup_prompt(self):
    method load_data (line 355) | def load_data(self):
    method _generate_single_assistant_turn (line 382) | async def _generate_single_assistant_turn(self, inference_state_dict):
    method _generate_single_data_point_single_turn (line 418) | async def _generate_single_data_point_single_turn(self, data_point):
    method _generate_single_data_point_multi_turn (line 437) | async def _generate_single_data_point_multi_turn(self, data_point):
    method _parse_reasoning_from_message_content (line 600) | def _parse_reasoning_from_message_content(self, model_response_text: s...
    method process_single_datapoint (line 611) | async def process_single_datapoint(self, data_point, all_data, prompt_...
  function bfcl_generation (line 624) | def bfcl_generation(cfg: BFCLGenerationConfig):

FILE: nemo_skills/inference/eval/bfcl_utils.py
  function convert_to_function_call (line 79) | def convert_to_function_call(function_call_list):
  function execute_multi_turn_func_call (line 93) | def execute_multi_turn_func_call(
  function is_empty_execute_response (line 195) | def is_empty_execute_response(input_list: list):
  function _process_method_calls (line 203) | def _process_method_calls(function_call_string: str, instance_mapping: d...

FILE: nemo_skills/inference/eval/bfcl_web_search.py
  class WebSearchBackendUnavailable (line 44) | class WebSearchBackendUnavailable(RuntimeError):
  class WebSearchAPI (line 48) | class WebSearchAPI:
    method __init__ (line 49) | def __init__(self):
    method _load_scenario (line 55) | def _load_scenario(self, initial_config: dict, long_context: bool = Fa...
    method _get_serp_api_key (line 63) | def _get_serp_api_key() -> Optional[str]:
    method _has_module (line 70) | def _has_module(module_name: str) -> bool:
    method _validate_backends_available (line 74) | def _validate_backends_available(self):
    method _warn_no_serp_api_key_once (line 105) | def _warn_no_serp_api_key_once(self):
    method _format_results (line 121) | def _format_results(self, results: list[dict]) -> list[dict]:
    method _search_with_serpapi_duckduckgo (line 131) | def _search_with_serpapi_duckduckgo(
    method _search_with_ddgs (line 215) | def _search_with_ddgs(self, *, keywords: str, max_results: int, region...
    method search_engine_query (line 242) | def search_engine_query(
    method fetch_url_content (line 399) | def fetch_url_content(self, url: str, mode: str = "raw") -> str:

FILE: nemo_skills/inference/eval/compute_eval.py
  class ComputeEvalGenerationTask (line 30) | class ComputeEvalGenerationTask(GenerationTask):
    method __init__ (line 31) | def __init__(self, cfg: GenerationTaskConfig):
    method process_single_datapoint (line 34) | async def process_single_datapoint(self, data_point, data, prompt_form...
  function run_compute_eval (line 65) | def run_compute_eval(cfg: GenerationTaskConfig):

FILE: nemo_skills/inference/eval/critpt.py
  class CritPtInferenceConfig (line 41) | class CritPtInferenceConfig:
  class CritPtGenerationConfig (line 57) | class CritPtGenerationConfig(GenerationTaskConfig):
  class CritPtGenerationTask (line 77) | class CritPtGenerationTask(GenerationTask):
    method __init__ (line 80) | def __init__(self, cfg: GenerationTaskConfig):
    method fill_prompt (line 89) | def fill_prompt(self, data_point, data, prompt_format=None):
    method process_single_datapoint (line 100) | async def process_single_datapoint(self, data_point, all_data):
  function generate (line 146) | def generate(cfg: CritPtGenerationConfig):

FILE: nemo_skills/inference/eval/scicode.py
  class SciCodeGenerationConfig (line 45) | class SciCodeGenerationConfig(GenerationTaskConfig):
  class SciCodeGenerationTask (line 63) | class SciCodeGenerationTask(GenerationTask):
    method log_example_prompt (line 64) | def log_example_prompt(self, data):
    method process_single_datapoint (line 68) | async def process_single_datapoint(self, data_point, all_data, prompt_...
  function scicode_generation (line 138) | def scicode_generation(cfg: SciCodeGenerationConfig):

FILE: nemo_skills/inference/eval/scicode_utils.py
  function process_problem_code (line 25) | def process_problem_code(prob_data: dict, num_steps: int) -> str:
  function process_problem_steps (line 32) | def process_problem_steps(problem_data: dict, num_steps: int, previous_l...
  function extract_python_script (line 63) | def extract_python_script(response: str):

FILE: nemo_skills/inference/eval/specdec.py
  class SpecDecodeMetricsError (line 37) | class SpecDecodeMetricsError(Exception):
    method __init__ (line 40) | def __init__(self, message: str):
    method __str__ (line 44) | def __str__(self):
  class SpecDecodeMetrics (line 54) | class SpecDecodeMetrics:
  function _fetch_metrics_text (line 70) | def _fetch_metrics_text(base_url: str) -> str | None:
  function fetch_vllm_spec_decode_metrics (line 84) | def fetch_vllm_spec_decode_metrics(base_url: str) -> SpecDecodeMetrics:
  function find_sglang_metrics_file (line 144) | def find_sglang_metrics_file(metrics_dir: str) -> str | None:
  function fetch_sglang_spec_decode_metrics (line 171) | def fetch_sglang_spec_decode_metrics(base_url: str) -> SpecDecodeMetrics:
  function _build_specdec_stats (line 221) | def _build_specdec_stats(
  function _compute_weighted_delta (line 241) | def _compute_weighted_delta(
  function compute_sglang_spec_decode_delta (line 259) | def compute_sglang_spec_decode_delta(
  function compute_vllm_spec_decode_delta (line 330) | def compute_vllm_spec_decode_delta(
  class SpecdecGenerationConfig (line 396) | class SpecdecGenerationConfig(GenerationTaskConfig):
    method _post_init_validate_server (line 414) | def _post_init_validate_server(self):
  class SpecdecGenerationTask (line 425) | class SpecdecGenerationTask(GenerationTask):
    method __init__ (line 436) | def __init__(self, cfg: SpecdecGenerationConfig):
    method _ensure_sglang_metrics_dir (line 441) | def _ensure_sglang_metrics_dir(cls) -> str:
    method get_generation_default_args (line 451) | def get_generation_default_args(cls) -> str:
    method get_server_command_fn (line 462) | def get_server_command_fn(cls) -> callable:
    method inject_sglang_metrics (line 500) | def inject_sglang_metrics(
    method process_single_datapoint (line 601) | async def process_single_datapoint(self, data_point, all_data, prompt_...
    method _get_server_base_address (line 635) | def _get_server_base_address(self) -> str:
    method wait_for_server (line 643) | def wait_for_server(self):
    method run_batch_evaluation (line 684) | def run_batch_evaluation(self):
  function specdec_generation (line 766) | def specdec_generation(cfg: SpecdecGenerationConfig):

FILE: nemo_skills/inference/eval/swebench.py
  class SupportedAgentFrameworks (line 45) | class SupportedAgentFrameworks(str, Enum):
  class SupportedDatasetTypes (line 52) | class SupportedDatasetTypes(str, Enum):
  class SweBenchInferenceConfig (line 60) | class SweBenchInferenceConfig:
  class SweBenchGenerationConfig (line 105) | class SweBenchGenerationConfig:
  class SweBenchGenerationTask (line 192) | class SweBenchGenerationTask(GenerationTask):
    method __init__ (line 193) | def __init__(self, cfg: SweBenchGenerationConfig):
    method log_example_prompt (line 383) | def log_example_prompt(self, data):
    method setup_prompt (line 386) | def setup_prompt(self):
    method setup_llm (line 389) | def setup_llm(self):
    method setup_litellm_cache (line 392) | def setup_litellm_cache(self):
    method cleanup_litellm_cache (line 395) | def cleanup_litellm_cache(self):
    method evaluate_single_datapoint (line 398) | async def evaluate_single_datapoint(self, data_point):
    method _execute_local_command (line 402) | async def _execute_local_command(self, command, timeout=None):
    method _execute_container_command (line 436) | async def _execute_container_command(self, data_point, command, expect...
    method _run_swe_agent (line 589) | async def _run_swe_agent(self, data_point, api_base):
    method _run_mini_swe_agent (line 661) | async def _run_mini_swe_agent(self, data_point, api_base):
    method _run_openhands (line 757) | async def _run_openhands(self, data_point, api_base):
    method _get_gold_patch (line 891) | async def _get_gold_patch(self, data_point):
    method process_single_datapoint (line 910) | async def process_single_datapoint(self, data_point, data, prompt_form...
    method _process_single_datapoint_impl (line 915) | async def _process_single_datapoint_impl(self, data_point, data):
  function swebench_generation (line 1035) | def swebench_generation(cfg: SweBenchGenerationConfig):

FILE: nemo_skills/inference/factory.py
  class GenerationType (line 18) | class GenerationType(str, Enum):

FILE: nemo_skills/inference/generate.py
  class InferenceConfig (line 66) | class InferenceConfig:
  class GenerationTaskConfig (line 90) | class GenerationTaskConfig:
    method __post_init__ (line 229) | def __post_init__(self):
    method _post_init_validate_data (line 235) | def _post_init_validate_data(self):
    method _post_init_validate_server (line 247) | def _post_init_validate_server(self):
    method _post_init_validate_params (line 251) | def _post_init_validate_params(self):
    method _post_init_deprecated_params (line 263) | def _post_init_deprecated_params(self):
    method _get_disallowed_params (line 267) | def _get_disallowed_params(self):
  class GenerationTask (line 276) | class GenerationTask:
    method get_generation_default_args (line 278) | def get_generation_default_args(cls) -> str:
    method get_server_command_fn (line 289) | def get_server_command_fn(cls) -> callable:
    method get_generation_requirements (line 302) | def get_generation_requirements(cls) -> list[str] | None:
    method __init__ (line 306) | def __init__(self, cfg: GenerationTaskConfig):
    method setup_prompt (line 413) | def setup_prompt(self):
    method setup_llm (line 430) | def setup_llm(self):
    method log_example_prompt (line 519) | def log_example_prompt(self, data):
    method load_data (line 524) | def load_data(self):
    method preprocess_data (line 543) | def preprocess_data(self, data):
    method postprocess (line 547) | def postprocess(self):
    method run_batch_evaluation (line 554) | def run_batch_evaluation(self):
    method skip_completed_samples (line 559) | def skip_completed_samples(self, data):
    method _merge_audio_from_data (line 591) | def _merge_audio_from_data(self, template_filled_messages, data_point):
    method _set_message_text_content (line 619) | def _set_message_text_content(message: dict, text: str) -> None:
    method _append_message_text_suffix (line 635) | def _append_message_text_suffix(message: dict, suffix: str) -> None:
    method fill_prompt (line 651) | def fill_prompt(self, data_point, data, prompt_format=None):
    method dump_outputs (line 717) | def dump_outputs(self, outputs, data_points, fout):
    method drop_fields_from_messages (line 721) | def drop_fields_from_messages(self, output):
    method postprocess_single_output (line 740) | async def postprocess_single_output(self, output, original_data_point):
    method prefill_generation (line 776) | def prefill_generation(self, data_point) -> dict | None:
    method process_single_datapoint (line 781) | async def process_single_datapoint(self, data_point, all_data, prompt_...
    method generate_with_semaphore (line 811) | async def generate_with_semaphore(self, **generation_params):
    method evaluate_single_datapoint (line 832) | async def evaluate_single_datapoint(self, data_point):
    method _generate_and_save_datapoint (line 840) | async def _generate_and_save_datapoint(self, data_point, all_data, fou...
    method async_loop (line 863) | async def async_loop(self, data):
    method restore_async_order (line 910) | def restore_async_order(self):
    method wait_for_server (line 927) | def wait_for_server(self):
    method wait_for_sandbox (line 939) | def wait_for_sandbox(self):
    method setup_litellm_cache (line 943) | def setup_litellm_cache(self):
    method cleanup_litellm_cache (line 952) | def cleanup_litellm_cache(self):
    method generate (line 957) | def generate(self):
  function generate (line 994) | def generate(cfg: GenerationTaskConfig):

FILE: nemo_skills/inference/litellm_hybrid_cache.py
  class HybridCache (line 36) | class HybridCache:
    method __init__ (line 37) | def __init__(
    method _check_no_ttl (line 56) | def _check_no_ttl(self, **kwargs):
    method _load_from_disk (line 61) | def _load_from_disk(self):
    method _save_to_disk (line 68) | def _save_to_disk(self):
    method _start_background_save_thread (line 84) | def _start_background_save_thread(self):
    method _shutdown (line 94) | def _shutdown(self):
    method set_cache (line 101) | def set_cache(self, key, value, **kwargs):
    method async_set_cache (line 108) | async def async_set_cache(self, key, value, **kwargs):
    method async_set_cache_pipeline (line 112) | async def async_set_cache_pipeline(self, cache_list, **kwargs):
    method get_cache (line 117) | def get_cache(self, key, **kwargs):
    method async_get_cache (line 130) | async def async_get_cache(self, key, **kwargs):
    method batch_get_cache (line 134) | def batch_get_cache(self, keys: list, **kwargs):
    method async_batch_get_cache (line 138) | async def async_batch_get_cache(self, keys: list, **kwargs):
    method increment_cache (line 142) | def increment_cache(self, key, value: int, **kwargs) -> int:
    method async_increment (line 150) | async def async_increment(self, key, value: float, **kwargs) -> float:
    method flush_cache (line 154) | def flush_cache(self):
    method delete_cache (line 160) | def delete_cache(self, key):
    method disconnect (line 166) | async def disconnect(self):
    method async_set_cache_sadd (line 170) | async def async_set_cache_sadd(self, key, value: List):
    method force_save (line 179) | def force_save(self):
  class StableLiteLLMCache (line 185) | class StableLiteLLMCache(LiteLLMCache):
    method __init__ (line 194) | def __init__(self, cache_file_path: str, save_interval_seconds: float ...
    method _stable_str (line 201) | def _stable_str(self, value) -> str:
    method get_cache_key (line 207) | def get_cache_key(self, **kwargs) -> str:

FILE: nemo_skills/inference/llm_math_judge.py
  class LlmMathJudgeConfig (line 40) | class LlmMathJudgeConfig(GenerationTaskConfig):
  class LLMMathJudgeTask (line 61) | class LLMMathJudgeTask(GenerationTask):
    method __init__ (line 62) | def __init__(self, cfg: LlmMathJudgeConfig):
    method preprocess_data (line 65) | def preprocess_data(self, data):
    method prefill_generation (line 73) | def prefill_generation(self, data_point):
  function generate (line 87) | def generate(cfg: LlmMathJudgeConfig):

FILE: nemo_skills/inference/log_samples_wandb.py
  function _process_and_log_samples (line 25) | def _process_and_log_samples(jsonl_file, num_samples, output_name, tmpdi...
  function log_random_samples (line 44) | def log_random_samples(jsonl_file, num_samples, project, name, group=None):

FILE: nemo_skills/inference/merge_chunks.py
  function unescape_shell_command (line 23) | def unescape_shell_command(command: str) -> str:

FILE: nemo_skills/inference/model/__init__.py
  function get_model (line 72) | def get_model(server_type, tokenizer=None, model_class: str | None = Non...
  function get_code_execution_model (line 95) | def get_code_execution_model(server_type, tokenizer=None, code_execution...
  function get_parallel_thinking_model (line 104) | def get_parallel_thinking_model(
  function get_tool_calling_model (line 131) | def get_tool_calling_model(
  function server_params (line 153) | def server_params():

FILE: nemo_skills/inference/model/asr_nim.py
  class ASRNIMModel (line 44) | class ASRNIMModel:
    method __init__ (line 69) | def __init__(
    method generate_async (line 129) | async def generate_async(self, prompt: str, **kwargs):
    method _generate_single (line 149) | def _generate_single(
    method __del__ (line 301) | def __del__(self):

FILE: nemo_skills/inference/model/audio_utils.py
  function audio_file_to_base64 (line 30) | def audio_file_to_base64(audio_file_path: str) -> str:
  function load_audio_file (line 44) | def load_audio_file(audio_file_path: str):
  function chunk_audio (line 59) | def chunk_audio(audio_array, sampling_rate, chunk_duration_sec=30, min_c...
  function save_audio_chunk_to_base64 (line 99) | def save_audio_chunk_to_base64(audio_chunk, sampling_rate) -> str:
  function make_audio_content_block (line 133) | def make_audio_content_block(base64_audio: str, audio_format: str = "aud...

FILE: nemo_skills/inference/model/azure.py
  class AzureOpenAIModel (line 20) | class AzureOpenAIModel(OpenAIModel):
    method __init__ (line 23) | def __init__(
    method _get_api_key (line 32) | def _get_api_key(self, api_key: str | None, api_key_env_var: str | Non...

FILE: nemo_skills/inference/model/base.py
  class EndpointType (line 53) | class EndpointType(str, Enum):
  class BaseModel (line 59) | class BaseModel:
    method __init__ (line 76) | def __init__(
    method _get_api_key (line 174) | def _get_api_key(self, api_key: str | None, api_key_env_var: str | Non...
    method __del__ (line 187) | def __del__(self):
    method _maybe_apply_stop_phrase_removal (line 191) | def _maybe_apply_stop_phrase_removal(
    method _get_tokenizer (line 197) | def _get_tokenizer(self, tokenizer: str | None) -> Union[ServerTokeniz...
    method _get_tokenizer_endpoint (line 217) | def _get_tokenizer_endpoint(self) -> str | None:
    method _initialize_tokenizer (line 221) | def _initialize_tokenizer(self, tokenizer: str | None) -> WrapperAutoT...
    method _build_chat_request_params (line 232) | def _build_chat_request_params(self, **kwargs) -> dict:
    method _build_completion_request_params (line 236) | def _build_completion_request_params(self, **kwargs) -> dict:
    method _build_responses_request_params (line 239) | def _build_responses_request_params(self, **kwargs) -> dict:
    method generate_async (line 243) | async def generate_async(
    method _parse_completion_response (line 354) | def _parse_completion_response(
    method _parse_chat_completion_response (line 387) | def _parse_chat_completion_response(self, response, include_response: ...
    method _process_completion_chunk (line 428) | def _process_completion_chunk(self, chunk, emitted_so_far: list):
    method _process_chat_chunk (line 462) | def _process_chat_chunk(self, chunk):
    method _stream_completion_chunks_async (line 495) | async def _stream_completion_chunks_async(self, response):
    method _parse_responses_completion_response (line 502) | def _parse_responses_completion_response(self, response, include_respo...
    method _serialize_output (line 548) | def _serialize_output(self, response):
    method _stream_chat_chunks_async (line 562) | async def _stream_chat_chunks_async(self, response):

FILE: nemo_skills/inference/model/code_execution.py
  class CodeExecutionConfig (line 31) | class CodeExecutionConfig:
  class CodeExecutionWrapper (line 41) | class CodeExecutionWrapper:
    method __init__ (line 42) | def __init__(self, model: BaseModel, sandbox: Sandbox, config: CodeExe...
    method _generate_single (line 47) | async def _generate_single(
    method execute_generated_code (line 234) | async def execute_generated_code(self, input_prompt, code_begin, code_...
    method generate_async (line 250) | async def generate_async(
    method _stream_single (line 321) | async def _stream_single(

FILE: nemo_skills/inference/model/context_retry.py
  function parse_context_window_exceeded_error (line 30) | def parse_context_window_exceeded_error(error) -> Union[Dict[str, int], ...
  class ContextLimitRetryConfig (line 114) | class ContextLimitRetryConfig:
    method __post_init__ (line 123) | def __post_init__(self):
    method reduce_generate_tokens (line 133) | def reduce_generate_tokens(self):
    method reduce_prompt_from_start (line 142) | def reduce_prompt_from_start(self):
    method reduce_prompt_from_end (line 151) | def reduce_prompt_from_end(self):
  function with_context_retry (line 160) | def with_context_retry(func: Callable) -> Callable:
  function handle_context_retries_async (line 183) | async def handle_context_retries_async(
  function handle_context_retries_sync (line 217) | def handle_context_retries_sync(
  function _prepare_context_error_retry (line 251) | def _prepare_context_error_retry(
  function _try_reduce_generation_tokens (line 291) | def _try_reduce_generation_tokens(
  function _try_reduce_prompt_tokens (line 320) | def _try_reduce_prompt_tokens(
  function _trim_string_prompt (line 365) | def _trim_string_prompt(
  function _trim_list_prompt (line 385) | def _trim_list_prompt(
  function _trim_messages_from_end (line 411) | def _trim_messages_from_end(
  function _trim_messages_from_start (line 453) | def _trim_messages_from_start(
  function get_trimmed_content (line 503) | def get_trimmed_content(
  function return_empty_generation_with_error (line 525) | def return_empty_generation_with_error(detailed_error: str, error_reason...

FILE: nemo_skills/inference/model/gemini.py
  class GeminiModel (line 20) | class GeminiModel(BaseModel):
    method __init__ (line 23) | def __init__(self, base_url: str | None = None, *args, **kwargs):
    method _get_api_key (line 34) | def _get_api_key(self, api_key: str | None, api_key_env_var: str | Non...
    method _build_chat_request_params (line 43) | def _build_chat_request_params(

FILE: nemo_skills/inference/model/megatron.py
  class MegatronModel (line 20) | class MegatronModel(BaseModel):
    method __init__ (line 21) | def __init__(self, **kwargs):
    method _build_chat_request_params (line 25) | def _build_chat_request_params(
    method _build_completion_request_params (line 74) | def _build_completion_request_params(
    method _parse_completion_response (line 122) | def _parse_completion_response(
    method _parse_chat_completion_response (line 158) | def _parse_chat_completion_response(

FILE: nemo_skills/inference/model/nim_utils.py
  class TTSExtraConfig (line 26) | class TTSExtraConfig:
  class ASRExtraConfig (line 49) | class ASRExtraConfig:
  function setup_ssh_tunnel (line 84) | def setup_ssh_tunnel(
  function validate_unsupported_params (line 141) | def validate_unsupported_params(kwargs: dict, model_name: str = "NIM mod...

FILE: nemo_skills/inference/model/openai.py
  class OpenAIModel (line 22) | class OpenAIModel(BaseModel):
    method __init__ (line 23) | def __init__(
    method _get_api_key (line 47) | def _get_api_key(self, api_key: str | None, api_key_env_var: str | Non...
    method _is_reasoning_model (line 61) | def _is_reasoning_model(self, model_name: str) -> bool:
    method _build_completion_request_params (line 66) | def _build_completion_request_params(self, **kwargs) -> dict:
    method _build_chat_request_params (line 91) | def _build_chat_request_params(
    method _build_responses_request_params (line 168) | def _build_responses_request_params(self, input, **kwargs) -> dict:

FILE: nemo_skills/inference/model/parallel_thinking.py
  class GenSelectSpecificConfig (line 38) | class GenSelectSpecificConfig:
  class GenSynthesisSpecificConfig (line 44) | class GenSynthesisSpecificConfig:
  class ParallelThinkingConfig (line 50) | class ParallelThinkingConfig:
  class ParallelThinkingTask (line 82) | class ParallelThinkingTask:
    method __init__ (line 88) | def __init__(self, model: BaseModel, tokenizer: str | None, orig_promp...
    method hash_prompt (line 126) | def hash_prompt(cls, prompt: Union[str, List[dict]]) -> str:
    method generate_solutions (line 130) | async def generate_solutions(
    method _load_solutions (line 182) | def _load_solutions(self, input_dir: str) -> Dict[str, List[Dict]]:
    method _get_multiple_solutions (line 230) | async def _get_multiple_solutions(
    method _generate_parallel_thinking_contraction (line 267) | async def _generate_parallel_thinking_contraction(self, prompt: str, s...
    method _extract_selected_solution (line 316) | def _extract_selected_solution(self, generation: str, max_idx: int) ->...
    method _extract_synthesized_solution (line 333) | def _extract_synthesized_solution(self, generation: str) -> str:
    method _run_genselect (line 341) | async def _run_genselect(
    method _run_gensynthesis (line 365) | async def _run_gensynthesis(
    method generate_async (line 389) | async def generate_async(self, prompt: Union[str, List], **kwargs):

FILE: nemo_skills/inference/model/sglang.py
  class SGLangModel (line 18) | class SGLangModel(VLLMModel):
    method _build_chat_request_params (line 25) | def _build_chat_request_params(

FILE: nemo_skills/inference/model/tool_call.py
  class ToolCallingWrapper (line 37) | class ToolCallingWrapper:
    method __init__ (line 44) | def __init__(
    method _execute_tool_call (line 67) | async def _execute_tool_call(self, tool_call, request_id: str, endpoin...
    method _execute_tool_calls (line 100) | async def _execute_tool_calls(self, tool_calls: List, request_id: str,...
    method _count_tool_response_tokens (line 111) | def _count_tool_response_tokens(self, tool_response_messages: list) ->...
    method _coerce_tool_call_dict (line 133) | def _coerce_tool_call_dict(self, tool_call: object) -> dict:
    method _duplicate_reasoning_content_keys (line 142) | def _duplicate_reasoning_content_keys(self, value):
    method _merge_tool_call_delta (line 155) | def _merge_tool_call_delta(self, tool_call_delta: object, tool_call_ac...
    method _finalize_tool_calls (line 183) | def _finalize_tool_calls(self, tool_call_accumulator: dict) -> list[di...
    method generate_async (line 201) | async def generate_async(
    method _stream_single (line 309) | async def _stream_single(

FILE: nemo_skills/inference/model/tts_nim.py
  class TTSNIMModel (line 29) | class TTSNIMModel:
    method __init__ (line 37) | def __init__(
    method _get_available_voices (line 100) | def _get_available_voices(self):
    method _generate_audio_filename (line 125) | def _generate_audio_filename(self, text: str, voice: str, idx: int) ->...
    method _save_audio (line 135) | def _save_audio(self, audio_data: bytes, output_file: Path, sample_rat...
    method generate_async (line 148) | async def generate_async(self, prompt: str, **kwargs):
    method _generate_single (line 168) | def _generate_single(
    method __del__ (line 285) | def __del__(self):

FILE: nemo_skills/inference/model/utils.py
  function trim_after_stop_phrases (line 27) | def trim_after_stop_phrases(text: str, stop_phrases: list[str]) -> str:
  function is_context_window_exceeded_error (line 36) | def is_context_window_exceeded_error(error: Exception) -> bool:
  class ServerTokenizer (line 54) | class ServerTokenizer:
    method __init__ (line 57) | def __init__(self, url):
    method encode (line 61) | def encode(self, prompt: str | list[dict], tools=None) -> list[int]:
    method decode (line 76) | def decode(self, tokens: list) -> str:
  class WrapperAutoTokenizer (line 86) | class WrapperAutoTokenizer:
    method __init__ (line 89) | def __init__(self, model_name: str):
    method encode (line 93) | def encode(self, prompt: str | list[dict], tools=None) -> list[int]:
    method decode (line 104) | def decode(self, tokens: list[int]) -> str:
  class RequestException (line 109) | class RequestException(RuntimeError):

FILE: nemo_skills/inference/model/vllm.py
  function encode_image_to_base64 (line 31) | def encode_image_to_base64(image_path: str) -> str:
  function process_image_content (line 48) | def process_image_content(content: list | str | None, data_dir: str = ""...
  class VLLMModel (line 93) | class VLLMModel(BaseModel):
    method __init__ (line 94) | def __init__(self, **kwargs):
    method _get_tokenizer_endpoint (line 97) | def _get_tokenizer_endpoint(self):
    method _build_request_body (line 114) | def _build_request_body(self, top_k, min_p, repetition_penalty, extra_...
    method _build_completion_request_params (line 129) | def _build_completion_request_params(
    method _build_chat_request_params (line 172) | def _build_chat_request_params(
    method _build_responses_request_params (line 224) | def _build_responses_request_params(self, input, **kwargs) -> dict:

FILE: nemo_skills/inference/model/vllm_multimodal.py
  class VLLMMultimodalModel (line 47) | class VLLMMultimodalModel(VLLMModel):
    method __init__ (line 72) | def __init__(
    method _is_local_url (line 116) | def _is_local_url(self, base_url: str | None) -> bool:
    method _get_api_key (line 130) | def _get_api_key(self, api_key: str | None, api_key_env_var: str | Non...
    method _build_request_body (line 183) | def _build_request_body(self, top_k, min_p, repetition_penalty, extra_...
    method _parse_chat_completion_response (line 218) | def _parse_chat_completion_response(self, response, include_response: ...
    method _process_audio_response (line 251) | def _process_audio_response(self, audio_data, response_id: str) -> dict:
    method _preprocess_messages_for_model (line 287) | def _preprocess_messages_for_model(self, messages: list[dict]) -> list...
    method content_text_to_list (line 300) | def content_text_to_list(self, message: dict) -> dict:
    method _needs_audio_chunking (line 345) | def _needs_audio_chunking(self, messages: list[dict], task_type: str =...
    method _generate_with_chunking (line 388) | async def _generate_with_chunking(
    method generate_async (line 471) | async def generate_async(

FILE: nemo_skills/inference/patch_litellm_logging.py
  class NoOpLoggingWorker (line 28) | class NoOpLoggingWorker:
    method __init__ (line 31) | def __init__(self, *args, **kwargs):
    method _ensure_queue (line 34) | def _ensure_queue(self) -> None:
    method start (line 37) | def start(self) -> None:
    method _worker_loop (line 40) | async def _worker_loop(self) -> None:
    method enqueue (line 43) | def enqueue(self, coroutine: Coroutine) -> None:
    method ensure_initialized_and_enqueue (line 47) | def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine):
    method stop (line 51) | async def stop(self) -> None:
    method flush (line 54) | async def flush(self) -> None:
    method clear_queue (line 57) | async def clear_queue(self):
  function patch_litellm_logging_worker (line 61) | def patch_litellm_logging_worker():

FILE: nemo_skills/inference/prover.py
  class ProverConfig (line 55) | class ProverConfig(GenerationTaskConfig):
    method _post_init_validate_params (line 72) | def _post_init_validate_params(self):
  class ProverTask (line 97) | class ProverTask(GenerationTask):
    method __init__ (line 98) | def __init__(self, cfg: ProverConfig):
    method log_example_prompt (line 118) | def log_example_prompt(self, data):
    method setup_llm (line 121) | def setup_llm(self):
    method setup_refine_prompt (line 126) | def setup_refine_prompt(self):
    method _generate_single_completion (line 138) | async def _generate_single_completion(self, prompt: str, **kwargs):
    method _extract_and_replace_code (line 184) | async def _extract_and_replace_code(self, formal_statement, generation):
    method _transform_for_nemotron_refinement (line 189) | def _transform_for_nemotron_refinement(self, proof_attempt: str, error...
    method _parse_gpt_oss_output (line 198) | def _parse_gpt_oss_output(self, content: str) -> tuple[str, str | None]:
    method _make_assistant_message (line 236) | def _make_assistant_message(self, content: str, reasoning_content: str...
    method _single_data_point_generate (line 254) | async def _single_data_point_generate(self, data_point, data):
    method pass_at_N (line 440) | async def pass_at_N(self, data_point, data, N=None):
    method process_single_datapoint (line 457) | async def process_single_datapoint(self, data_point, all_data, prompt_...
  function generate (line 469) | def generate(cfg: ProverConfig):

FILE: nemo_skills/inference/retrieve_similar.py
  function top_k_similarity (line 33) | def top_k_similarity(from_emb, to_emb, top_k, chunk_size):
  function encode (line 51) | def encode(model, data, batch_size):
  function read_data (line 55) | def read_data(file_paths, retrieve_key) -> list:
  class RetrieveSimilarConfig (line 64) | class RetrieveSimilarConfig:
    method __post_init__ (line 86) | def __post_init__(self):
  function retrieve_similar (line 105) | def retrieve_similar(cfg: RetrieveSimilarConfig):

FILE: nemo_skills/inference/server/serve_riva_nim.py
  function main (line 20) | def main():

FILE: nemo_skills/inference/server/serve_sglang.py
  function main (line 20) | def main():

FILE: nemo_skills/inference/server/serve_unified.py
  function setup_pythonpath (line 68) | def setup_pythonpath(code_path: Optional[str] = None):
  function apply_safetensors_patch (line 103) | def apply_safetensors_patch(hack_path: Optional[str]):
  function load_yaml_config (line 119) | def load_yaml_config(config_path: str) -> dict:
  function _coerce_value (line 127) | def _coerce_value(value: str):
  function parse_extra_args (line 144) | def parse_extra_args(extra_args: list) -> dict:
  function main (line 186) | def main():

FILE: nemo_skills/inference/server/serve_vllm.py
  function main (line 20) | def main():

FILE: nemo_skills/inference/server/serve_vllm_dp_ray.py
  function _apply_vllm_patches (line 98) | def _apply_vllm_patches() -> None:
  function _reserve_head_placement_group (line 280) | def _reserve_head_placement_group(
  function _patch_signal_for_thread_safety (line 337) | def _patch_signal_for_thread_safety() -> None:
  function _build_vllm_argv (line 353) | def _build_vllm_argv(args: argparse.Namespace, extra: Sequence[str]) -> ...
  function main (line 411) | def main() -> None:

FILE: nemo_skills/inference/structured_outputs.py
  class HLEJudgeAAResponseFormat (line 20) | class HLEJudgeAAResponseFormat(BaseModel):

FILE: nemo_skills/inference/tournament_utils.py
  class KnockoutTournamentManager (line 25) | class KnockoutTournamentManager:
    method __init__ (line 26) | def __init__(
    method load_prompt_template (line 40) | def load_prompt_template(self, prompt_config_path: str) -> str:
    method _llm_call (line 45) | async def _llm_call(self, prompt: str, req_seed: int) -> Tuple[str, int]:
    method format_participants (line 57) | def format_participants(self, participants: List[Tuple[int, str]], com...
    method extract_winner_from_result (line 61) | def extract_winner_from_result(
    method validate_participant (line 67) | def validate_participant(self, participant: str) -> bool:
    method run_single_game (line 71) | async def run_single_game(
    method run_tournament (line 103) | async def run_tournament(
  class ProofKnockoutTournamentManager (line 186) | class ProofKnockoutTournamentManager(KnockoutTournamentManager):
    method format_participants (line 192) | def format_participants(self, participants: List[Tuple[int, str]], com...
    method extract_winner_from_result (line 198) | def extract_winner_from_result(
    method validate_participant (line 227) | def validate_participant(self, participant: str) -> bool:

FILE: nemo_skills/mcp/adapters.py
  class ToolSchemaAdapter (line 29) | class ToolSchemaAdapter(ABC):
    method convert (line 31) | def convert(self, tools: list[dict]) -> list[dict]:
  class ToolCallInterpreter (line 36) | class ToolCallInterpreter(ABC):
    method parse (line 38) | def parse(self, raw_call: dict) -> dict:
  class ToolResponseFormatter (line 42) | class ToolResponseFormatter(ABC):
    method format (line 44) | def format(self, tool_call: ChatCompletionMessageToolCall, result: dic...
  function load_schema_overrides (line 54) | def load_schema_overrides(schema_overrides: dict | None) -> Dict[str, Di...
  function apply_schema_overrides (line 92) | def apply_schema_overrides(
  function remap_tool_call (line 131) | def remap_tool_call(tool_name: str, args: dict, mappings: dict) -> tuple...
  function format_tool_list_by_endpoint_type (line 139) | def format_tool_list_by_endpoint_type(
  class OpenAICallInterpreter (line 198) | class OpenAICallInterpreter(ToolCallInterpreter):
    method parse (line 199) | def parse(self, tool_call):
  class CompletionResponseFormatter (line 205) | class CompletionResponseFormatter(ToolResponseFormatter):
    method format (line 207) | def format(self, tool_call: ChatCompletionMessageToolCall, result):
  function format_tool_response_by_endpoint_type (line 215) | def format_tool_response_by_endpoint_type(tool_call, result, endpoint_ty...
  function get_tool_details_by_endpoint_type (line 233) | def get_tool_details_by_endpoint_type(tool_call, endpoint_type: Endpoint...

FILE: nemo_skills/mcp/clients.py
  function _process_hide_args (line 31) | def _process_hide_args(result, hide_args):
  function _filter_tools (line 49) | def _filter_tools(result, disabled_tools, enabled_tools):
  function async_wrapper (line 76) | def async_wrapper(method):
  function _sanitize_input_args_for_tool (line 95) | def _sanitize_input_args_for_tool(args_dict, tool_name, hide_args):
  function _extract_item (line 109) | def _extract_item(item) -> Any:
  function _extract_tool_result (line 124) | def _extract_tool_result(result) -> Any:
  function _wrap_call_tool_output_formatter (line 156) | def _wrap_call_tool_output_formatter(method):
  function inject_hide_args (line 185) | def inject_hide_args(init_func):
  class MCPClientMeta (line 217) | class MCPClientMeta(type):
    method __new__ (line 269) | def __new__(mcls, name, bases, namespace):
    method __call__ (line 286) | def __call__(cls, *args, **kwargs):
  class MCPClient (line 299) | class MCPClient(metaclass=MCPClientMeta):
    method sanitize (line 333) | def sanitize(self, tool: str, args: dict) -> dict:
    method list_tools (line 338) | async def list_tools(self):
    method call_tool (line 342) | async def call_tool(self, tool: str, args: dict) -> Any:
    method _assert_tool_allowed (line 346) | def _assert_tool_allowed(self, tool: str):
  class MCPStreamableHttpClient (line 354) | class MCPStreamableHttpClient(MCPClient):
    method __init__ (line 378) | def __init__(self, base_url: str):
    method list_tools (line 382) | async def list_tools(self):
    method call_tool (line 404) | async def call_tool(self, tool: str, args: dict) -> Any:
  class MCPStdioClient (line 413) | class MCPStdioClient(MCPClient):
    method __init__ (line 437) | def __init__(self, command: str, args: list[str] | None = None):
    method list_tools (line 444) | async def list_tools(self):
    method call_tool (line 464) | async def call_tool(self, tool: str, args: dict) -> Any:

FILE: nemo_skills/mcp/config.py
  class MCPAdaptersConfig (line 37) | class MCPAdaptersConfig:
  class MCPClientParamsBase (line 44) | class MCPClientParamsBase:
  class MCPStdioClientParams (line 53) | class MCPStdioClientParams(MCPClientParamsBase):
  class MCPStreamableHttpClientParams (line 59) | class MCPStreamableHttpClientParams(MCPClientParamsBase):
  class MCPToolConfig (line 64) | class MCPToolConfig:
  class MCPConfig (line 71) | class MCPConfig:
  function _is_locate_mapping (line 83) | def _is_locate_mapping(value: Any) -> bool:
  function _resolve_special (line 94) | def _resolve_special(value: Any, full_cfg: DictConfig) -> Any:
  function _resolve_locate_mapping (line 100) | def _resolve_locate_mapping(spec: Mapping, full_cfg: DictConfig) -> Any:
  function resolve_value (line 109) | def resolve_value(value: Any, full_cfg: DictConfig) -> Any:
  function resolve_adapters (line 115) | def resolve_adapters(cfg: DictConfig):

FILE: nemo_skills/mcp/servers/chemistry/periodictable_tool.py
  function _resolve_element (line 37) | def _resolve_element(name_or_symbol: str):
  function element_info (line 50) | def element_info(
  function isotope_info (line 86) | def isotope_info(
  class PeriodictableTool (line 120) | class PeriodictableTool(Tool):
    method __init__ (line 121) | def __init__(self) -> None:
    method default_config (line 124) | def default_config(self) -> dict[str, Any]:
    method configure (line 127) | def configure(self, overrides: dict[str, Any] | None = None, context: ...
    method list_tools (line 131) | async def list_tools(self) -> list[dict[str, Any]]:
    method execute (line 158) | async def execute(self, tool_name: str, arguments: dict[str, Any], ext...

FILE: nemo_skills/mcp/servers/exa_tool.py
  class ExecutionResult (line 30) | class ExecutionResult:
  function exa_websearch (line 42) | async def exa_websearch(
  function main (line 67) | def main():
  class ExaTool (line 86) | class ExaTool(MCPClientTool):
    method __init__ (line 87) | def __init__(self) -> None:
  class ExaMCPTool (line 103) | class ExaMCPTool(MCPClientTool):
    method __init__ (line 104) | def __init__(self) -> None:

FILE: nemo_skills/mcp/servers/physics/coolprop_tool.py
  function fluid_property (line 54) | def fluid_property(
  function fluid_list (line 86) | def fluid_list() -> str:
  class CoolPropTool (line 94) | class CoolPropTool(Tool):
    method __init__ (line 95) | def __init__(self) -> None:
    method default_config (line 98) | def default_config(self) -> dict[str, Any]:
    method configure (line 101) | def configure(self, overrides: dict[str, Any] | None = None, context: ...
    method list_tools (line 105) | async def list_tools(self) -> list[dict[str, Any]]:
    method execute (line 131) | async def execute(self, tool_name: str, arguments: dict[str, Any], ext...

FILE: nemo_skills/mcp/servers/physics/particle_tool.py
  function _format_particle (line 39) | def _format_particle(p) -> str:
  function particle_lookup (line 63) | def particle_lookup(
  function particle_search (line 91) | def particle_search(
  class ParticleTool (line 112) | class ParticleTool(Tool):
    method __init__ (line 113) | def __init__(self) -> None:
    method default_config (line 116) | def default_config(self) -> dict[str, Any]:
    method configure (line 119) | def configure(self, overrides: dict[str, Any] | None = None, context: ...
    method list_tools (line 124) | async def list_tools(self) -> list[dict[str, Any]]:
    method execute (line 146) | async def execute(self, tool_name: str, arguments: dict[str, Any], ext...

FILE: nemo_skills/mcp/servers/physics/radioactivedecay_tool.py
  function nuclide_info (line 40) | def nuclide_info(
  function decay_chain (line 81) | def decay_chain(
  class RadioactivedecayTool (line 116) | class RadioactivedecayTool(Tool):
    method __init__ (line 117) | def __init__(self) -> None:
    method default_config (line 120) | def default_config(self) -> dict[str, Any]:
    method configure (line 123) | def configure(self, overrides: dict[str, Any] | None = None, context: ...
    method list_tools (line 137) | async def list_tools(self) -> list[dict[str, Any]]:
    method execute (line 164) | async def execute(self, tool_name: str, arguments: dict[str, Any], ext...

FILE: nemo_skills/mcp/servers/python_tool.py
  class ExecutionResult (line 36) | class ExecutionResult:
  function stateful_python_code_exec (line 54) | async def stateful_python_code_exec(
  function main (line 71) | def main():
  class PythonTool (line 107) | class PythonTool(MCPClientTool):
    method __init__ (line 108) | def __init__(self) -> None:
    method execute (line 128) | async def execute(self, tool_name: str, arguments: Dict[str, Any], ext...
    method shutdown (line 143) | async def shutdown(self) -> None:
  class DirectPythonTool (line 147) | class DirectPythonTool(Tool):
    method __init__ (line 162) | def __init__(self) -> None:
    method default_config (line 173) | def default_config(self) -> Dict[str, Any]:
    method configure (line 176) | def configure(self, overrides: Dict[str, Any] | None = None, context: ...
    method list_tools (line 192) | async def list_tools(self) -> List[Dict[str, Any]]:
    method execute (line 207) | async def execute(
    method shutdown (line 258) | async def shutdown(self) -> None:
    method cleanup_request (line 274) | async def cleanup_request(self, request_id: str) -> None:

FILE: nemo_skills/mcp/servers/tavily_search_tool.py
  class ExecutionResult (line 33) | class ExecutionResult:
  function answer (line 61) | async def answer(
  function _parse_exclude_domains (line 128) | def _parse_exclude_domains(exclude_config: dict) -> list[str]:
  class TavilySearchTool (line 139) | class TavilySearchTool(MCPClientTool):
    method __init__ (line 140) | def __init__(self) -> None:
    method post_configure (line 156) | def post_configure(self) -> None:
    method execute (line 165) | async def execute(self, tool_name: str, arguments: dict[str, Any], ext...
  function main (line 183) | def main():

FILE: nemo_skills/mcp/servers/web/arxiv_tool.py
  function _cache_key (line 92) | def _cache_key(*args: Any) -> str:
  function _cache_get (line 97) | def _cache_get(key: str) -> str | None:
  function _cache_set (line 102) | def _cache_set(key: str, value: str) -> None:
  function _paper_cache_get (line 110) | def _paper_cache_get(key: str) -> tuple[str, str] | None:
  function _paper_cache_set (line 118) | def _paper_cache_set(key: str, value: tuple[str, str]) -> None:
  function _reconstruct_abstract (line 126) | def _reconstruct_abstract(inv_idx: dict[str, list[int]] | None) -> str:
  function _truncate (line 139) | def _truncate(text: str, limit: int = ABSTRACT_LIMIT) -> str:
  class _ArxivHTMLTextParser (line 148) | class _ArxivHTMLTextParser(HTMLParser):
    method __init__ (line 155) | def __init__(self) -> None:
    method handle_starttag (line 162) | def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]...
    method handle_endtag (line 176) | def handle_endtag(self, tag: str) -> None:
    method handle_data (line 193) | def handle_data(self, data: str) -> None:
    method text (line 203) | def text(self) -> str:
  function _normalize_id (line 210) | def _normalize_id(paper_id: str) -> str:
  function _extract_arxiv_id (line 234) | def _extract_arxiv_id(paper_id: str) -> str | None:
  function _fetch_paper_text (line 251) | async def _fetch_paper_text(paper_id: str) -> tuple[str, str]:
  function _section_offsets (line 287) | def _section_offsets(text: str) -> list[tuple[int, int, str]]:
  function _format_openalex_work (line 297) | def _format_openalex_work(work: dict[str, Any], include_abstract: bool =...
  function _format_arxiv_entry (line 336) | def _format_arxiv_entry(entry: dict[str, Any], include_abstract: bool = ...
  function _parse_arxiv_atom (line 368) | def _parse_arxiv_atom(feed_text: str) -> list[dict[str, Any]]:
  function _arxiv_api_search (line 416) | async def _arxiv_api_search(query: str, max_results: int) -> str:
  function _http_get_json (line 436) | async def _http_get_json(client: httpx.AsyncClient, url: str, params: di...
  function _arxiv_rate_limit (line 485) | async def _arxiv_rate_limit() -> None:
  function arxiv_search (line 542) | async def arxiv_search(
  function arxiv_get (line 578) | async def arxiv_get(
  function arxiv_sections (line 625) | async def arxiv_sections(
  function arxiv_read_chunk (line 667) | async def arxiv_read_chunk(
  function _arxiv_api_get (line 717) | async def _arxiv_api_get(arxiv_id: str) -> str:
  class ArxivSearchTool (line 756) | class ArxivSearchTool(Tool):
    method __init__ (line 759) | def __init__(self) -> None:
    method default_config (line 766) | def default_config(self) -> dict[str, Any]:
    method configure (line 769) | def configure(self, overrides: dict[str, Any] | None = None, context: ...
    method list_tools (line 785) | async def list_tools(self) -> list[dict[str, Any]]:
    method execute (line 829) | async def execute(self, tool_name: str, arguments: dict[str, Any], ext...

FILE: nemo_skills/mcp/servers/web/wikipedia_tool.py
  function _cache_key (line 84) | def _cache_key(*args: Any) -> str:
  function _cache_get (line 89) | def _cache_get(key: str) -> str | None:
  function _cache_set (line 94) | def _cache_set(key: str, value: str) -> None:
  function _strip_html (line 101) | def _strip_html(s: str) -> str:
  function _truncate (line 109) | def _truncate(text: str, limit: int) -> str:
  function _page_url (line 118) | def _page_url(title: str) -> str:
  function _sentence_split (line 123) | def _sentence_split(text: str) -> list[str]:
  function _page_extract (line 131) | async def _page_extract(title: str) -> tuple[str, str, str] | tuple[None...
  function _http_get_json (line 165) | async def _http_get_json(client: httpx.AsyncClient, url: str, params: di...
  function _retry_after_seconds (line 195) | def _retry_after_seconds(response: httpx.Response) -> float | None:
  function _rate_limit (line 206) | async def _rate_limit() -> None:
  function wikipedia_search (line 242) | async def wikipedia_search(
  function wikipedia_page (line 297) | async def wikipedia_page(
  function wikipedia_summary (line 357) | async def wikipedia_summary(
  function wikipedia_sections (line 401) | async def wikipedia_sections(
  function wikipedia_query_summary (line 447) | async def wikipedia_query_summary(
  function wikipedia_key_facts (line 484) | async def wikipedia_key_facts(
  function wikipedia_section (line 519) | async def wikipedia_section(
  function _suggest_titles (line 606) | async def _suggest_titles(query: str, n: int = 5) -> list[str]:
  class WikipediaSearchTool (line 630) | class WikipediaSearchTool(Tool):
    method __init__ (line 633) | def __init__(self) -> None:
    method default_config (line 641) | def default_config(self) -> dict[str, Any]:
    method configure (line 644) | def configure(self, overrides: dict[str, Any] | None = None, context: ...
    method list_tools (line 664) | async def list_tools(self) -> list[dict[str, Any]]:
    method execute (line 737) | async def execute(self, tool_name: str, arguments: dict[str, Any], ext...

FILE: nemo_skills/mcp/tool_manager.py
  class FatalToolError (line 34) | class FatalToolError(Exception):
  class Tool (line 44) | class Tool(ABC):
    method default_config (line 53) | def default_config(self) -> Dict[str, Any]:
    method configure (line 57) | def configure(self, overrides: Dict[str, Any] | None = None, context: ...
    method list_tools (line 61) | async def list_tools(self) -> List[Dict[str, Any]]:
    method execute (line 65) | async def execute(
    method cleanup_request (line 70) | async def cleanup_request(self, request_id: str) -> None:  # Optional ...
    method shutdown (line 73) | async def shutdown(self) -> None:  # Optional hook
    method post_configure (line 76) | def post_configure(self) -> None:
  class ToolManager (line 80) | class ToolManager:
    method __init__ (line 89) | def __init__(
    method shutdown (line 120) | async def shutdown(self) -> None:
    method cleanup_request (line 128) | async def cleanup_request(self, request_id: str) -> None:
    method list_all_tools (line 132) | async def list_all_tools(self, use_cache: bool = True) -> List[Dict[st...
    method _resolve (line 174) | def _resolve(self, qualified_name: str) -> tuple[Tool, str]:
    method execute_tool (line 183) | async def execute_tool(self, raw_name: str, args: Dict[str, Any], extr...

FILE: nemo_skills/mcp/tool_providers.py
  class MCPClientTool (line 26) | class MCPClientTool(Tool):
    method __init__ (line 40) | def __init__(self) -> None:
    method apply_config_updates (line 54) | def apply_config_updates(self, updates: Dict[str, Any] | None) -> None:
    method default_config (line 60) | def default_config(self) -> Dict[str, Any]:
    method _resolve_maybe_callable (line 63) | def _resolve_maybe_callable(self, value: Any):
    method post_configure (line 74) | def post_configure(self) -> None:
    method configure (line 77) | def configure(self, overrides: Dict[str, Any] | None = None, context: ...
    method list_tools (line 123) | async def list_tools(self) -> List[Dict[str, Any]]:
    method execute (line 126) | async def execute(self, tool_name: str, arguments: Dict[str, Any], ext...

FILE: nemo_skills/mcp/utils.py
  function exa_auth_connector (line 34) | def exa_auth_connector(client: MCPStreamableHttpClient):
  function exa_stdio_connector (line 38) | def exa_stdio_connector(client: MCPStdioClient):
  function exa_output_formatter (line 45) | def exa_output_formatter(result: CallToolResult):
  function hydra_config_connector_factory (line 52) | def hydra_config_connector_factory(config_obj):
  function load_mcp_config (line 73) | def load_mcp_config(
  function add_config_args (line 107) | def add_config_args(parser):

FILE: nemo_skills/pipeline/app.py
  function typer_unpacker (line 25) | def typer_unpacker(f: Callable):

FILE: nemo_skills/pipeline/cli.py
  function wrap_arguments (line 44) | def wrap_arguments(arguments: str):

FILE: nemo_skills/pipeline/convert.py
  function get_hf_to_trtllm_cmd (line 37) | def get_hf_to_trtllm_cmd(
  function get_hf_to_megatron_cmd (line 99) | def get_hf_to_megatron_cmd(
  class SupportedTypes (line 126) | class SupportedTypes(str, Enum):
  class SupportedFormatsTo (line 132) | class SupportedFormatsTo(str, Enum):
  class SupportedFormatsFrom (line 138) | class SupportedFormatsFrom(str, Enum):
  class SupportedDtypes (line 142) | class SupportedDtypes(str, Enum):
  function convert (line 151) | def convert(

FILE: nemo_skills/pipeline/dataset.py
  function _get_dataset_module_from_cluster (line 36) | def _get_dataset_module_from_cluster(cluster_config, mounted_path):
  function get_dataset_module (line 50) | def get_dataset_module(dataset, data_dir=None, cluster_config=None, extr...

FILE: nemo_skills/pipeline/eval.py
  class SingleNodeMode (line 44) | class SingleNodeMode(str, enum.Enum):
  function _resolve_child_sbatch_kwargs (line 49) | def _resolve_child_sbatch_kwargs(sbatch_kwargs, child_sbatch_kwargs):
  function _create_llm_judge_tasks (line 55) | def _create_llm_judge_tasks(
  function eval (line 136) | def eval(

FILE: nemo_skills/pipeline/generate.py
  function _create_job_unified (line 50) | def _create_job_unified(
  function generate (line 216) | def generate(

FILE: nemo_skills/pipeline/judges/comet_judge.py
  function create_judge_tasks (line 26) | def create_judge_tasks(

FILE: nemo_skills/pipeline/judges/nvembed_judge.py
  function create_judge_tasks (line 26) | def create_judge_tasks(

FILE: nemo_skills/pipeline/megatron_lm/train.py
  function get_training_cmd (line 38) | def get_training_cmd(
  function train_megatron_lm (line 96) | def train_megatron_lm(

FILE: nemo_skills/pipeline/nemo_evaluator.py
  function nemo_evaluator (line 113) | def nemo_evaluator(
  function _create_serving_command_obj (line 439) | def _create_serving_command_obj(
  class _TaskCreationContext (line 509) | class _TaskCreationContext:
  function _hardware_for_group (line 560) | def _hardware_for_group(
  function _build_main_server_if_needed (line 594) | def _build_main_server_if_needed(ctx: _TaskCreationContext) -> Optional[...
  function _build_judge_server_if_needed (line 619) | def _build_judge_server_if_needed(ctx: _TaskCreationContext) -> Optional...
  function _build_client_command (line 644) | def _build_client_command(
  function _build_task_cmd (line 671) | def _build_task_cmd(
  class EvaluatorClientScript (line 749) | class EvaluatorClientScript(BaseJobScript):
    method __post_init__ (line 757) | def __post_init__(self):

FILE: nemo_skills/pipeline/nemo_gym_rollouts.py
  function nemo_gym_rollouts (line 77) | def nemo_gym_rollouts(

FILE: nemo_skills/pipeline/nemo_rl/average_checkpoints.py
  class SupportedBackends (line 29) | class SupportedBackends(str, Enum):
  function list_candidate_model_dirs (line 34) | def list_candidate_model_dirs(checkpoint_dir, steps):
  function find_index_json (line 46) | def find_index_json(model_dir):
  function build_key_to_shard_map (line 54) | def build_key_to_shard_map(model_dir):
  function copy_side_files (line 103) | def copy_side_files(src_model_dir, dst_dir):
  function convert_fsdp_bin_to_safetensors (line 121) | def convert_fsdp_bin_to_safetensors(model_dir):
  function main (line 170) | def main():

FILE: nemo_skills/pipeline/nemo_rl/grpo.py
  class SupportedBackends (line 52) | class SupportedBackends(str, Enum):
  class NemoRLTask (line 58) | class NemoRLTask:
    method format_train_args (line 76) | def format_train_args(self):
    method format_data_args (line 93) | def format_data_args(self):
    method format_wandb_args (line 99) | def format_wandb_args(self):
    method get_cmd (line 135) | def get_cmd(self):
  function get_training_cmd (line 152) | def get_training_cmd(
  function get_checkpoint_convert_cmd (line 195) | def get_checkpoint_convert_cmd(output_dir, final_hf_path, step, backend,...
  function get_checkpoint_average_cmd (line 220) | def get_checkpoint_average_cmd(output_dir, average_steps, backend, remov...
  function grpo_nemo_rl (line 242) | def grpo_nemo_rl(

FILE: nemo_skills/pipeline/nemo_rl/sft.py
  class SupportedBackends (line 49) | class SupportedBackends(str, Enum):
  class NemoRLTask (line 55) | class NemoRLTask:
    method format_train_args (line 73) | def format_train_args(self):
    method format_data_args (line 88) | def format_data_args(self):
    method format_wandb_args (line 94) | def format_wandb_args(self):
    method get_cmd (line 114) | def get_cmd(self):
  function get_training_cmd (line 131) | def get_training_cmd(
  function get_checkpoint_convert_cmd (line 174) | def get_checkpoint_convert_cmd(output_dir, final_hf_path, step, backend,...
  function get_checkpoint_average_cmd (line 199) | def get_checkpoint_average_cmd(output_dir, average_steps, backend, remov...
  function sft_nemo_rl (line 221) | def sft_nemo_rl(

FILE: nemo_skills/pipeline/prepare_data.py
  function _parse_prepare_cli_arguments (line 44) | def _parse_prepare_cli_arguments(args: list[str]) -> tuple[list[str], li...
  function _is_external_dataset (line 52) | def _is_external_dataset(dataset: str, extra_benchmark_map: dict[str, st...
  function _get_container_dataset_path (line 56) | def _get_container_dataset_path(dataset: str, extra_benchmark_map: dict[...
  function _build_command (line 62) | def _build_command(
  function prepare_data (line 111) | def prepare_data(

FILE: nemo_skills/pipeline/robust_eval.py
  class PromptConfig (line 33) | class PromptConfig:
  function robust_eval (line 40) | def robust_eval(

FILE: nemo_skills/pipeline/run_cmd.py
  function get_cmd (line 34) | def get_cmd(command):
  function run_cmd (line 46) | def run_cmd(

FILE: nemo_skills/pipeline/setup.py
  function is_docker_available (line 29) | def is_docker_available():
  function pull_docker_containers (line 38) | def pull_docker_containers(containers):
  function setup (line 57) | def setup():

FILE: nemo_skills/pipeline/start_server.py
  function get_gradio_chat_cmd (line 42) | def get_gradio_chat_cmd(model, server_type, extra_args):
  function create_job_tunnel (line 52) | def create_job_tunnel(
  function launch_server (line 112) | def launch_server(
  function stop_server (line 199) | def stop_server(exp):
  function start_server (line 207) | def start_server(

FILE: nemo_skills/pipeline/summarize_results.py
  function get_subset_name (line 43) | def get_subset_name(benchmark: str, subset: str) -> str:
  function _set_asr_leaderboard_macro_wer (line 50) | def _set_asr_leaderboard_macro_wer(metrics: dict):
  function add_benchmark_groups (line 61) | def add_benchmark_groups(results, metrics_to_print, evaluations_to_print):
  function summarize_results (line 148) | def summarize_results(

FILE: nemo_skills/pipeline/summarize_robustness.py
  function get_metrics (line 43) | def get_metrics(prediction_files: List[str]) -> List[float] | List[float]:
  function summarize_robustness (line 81) | def summarize_robustness(

FILE: nemo_skills/pipeline/utils/cluster.py
  function _parse_slurm_timeout (line 43) | def _parse_slurm_timeout(value: str) -> timedelta:
  function _get_timeout (line 77) | def _get_timeout(cluster_config, partition, with_save_delay: bool = True...
  function get_slurm_timeout_str (line 93) | def get_slurm_timeout_str(cluster_config, partition, with_save_delay: bo...
  function get_timeout_str (line 102) | def get_timeout_str(cluster_config, partition, with_save_delay: bool = T...
  function kwargs_to_string (line 109) | def kwargs_to_string(kwargs: str | dict) -> dict:
  function parse_kwargs (line 121) | def parse_kwargs(kwargs: str | dict | None, **extra_kwargs) -> dict | None:
  function get_env_variables (line 163) | def get_env_variables(cluster_config):
  function temporary_env_update (line 281) | def temporary_env_update(cluster_config, updates):
  function read_config (line 293) | def read_config(config_file):
  function get_cluster_config (line 315) | def get_cluster_config(cluster=None, config_dir=None):
  function update_ssh_tunnel_config (line 372) | def update_ssh_tunnel_config(cluster_config: dict):
  function _get_tunnel_cached (line 416) | def _get_tunnel_cached(
  function tunnel_hash (line 446) | def tunnel_hash(tunnel):
  function get_tunnel (line 452) | def get_tunnel(cluster_config):
  class OutputWatcher (line 461) | class OutputWatcher(StreamWatcher):
    method submit (line 464) | def submit(self, stream):
  function progress_callback (line 470) | def progress_callback(transferred: int, total: int) -> None:
  function cluster_download_file (line 481) | def cluster_download_file(cluster_config: dict, remote_file: str, local_...
  function cluster_path_exists (line 486) | def cluster_path_exists(cluster_config: dict, remote_path: str):
  function cluster_download_dir (line 492) | def cluster_download_dir(
  function cluster_upload (line 566) | def cluster_upload(cluster_config: dict, local_file: str, remote_dir: st...

FILE: nemo_skills/pipeline/utils/commands.py
  function vllm_server_command (line 28) | def vllm_server_command(
  function sandbox_command (line 77) | def sandbox_command(cluster_config: Dict, port: int, **kwargs) -> Tuple[...
  function wrap_command (line 114) | def wrap_command(command: str, working_dir: str = "/nemo_run/code", env_...

FILE: nemo_skills/pipeline/utils/declarative.py
  class Command (line 212) | class Command:
    method prepare_for_execution (line 240) | def prepare_for_execution(self, cluster_config: Dict) -> Tuple[run.Scr...
    method get_name (line 328) | def get_name(self) -> str:
  class HardwareConfig (line 333) | class HardwareConfig:
  class CommandGroup (line 344) | class CommandGroup:
    method __init__ (line 347) | def __init__(
  class Pipeline (line 360) | class Pipeline:
    method __init__ (line 370) | def __init__(
    method _validate (line 398) | def _validate(self):
    method run (line 427) | def run(self, dry_run: bool = False, log_dir: Optional[str] = None, _r...
    method _prepare_command (line 566) | def _prepare_command(self, command, cluster_config: Dict) -> Tuple[run...
    method _rewrite_local_paths (line 581) | def _rewrite_local_paths(self, script: run.Script) -> run.Script:
    method _resolve_container (line 610) | def _resolve_container(self, exec_config: Dict, command, cluster_confi...
    method _create_executor (line 617) | def _create_executor(
    method _plan_and_add_job (line 719) | def _plan_and_add_job(
    method _add_single_group_job (line 938) | def _add_single_group_job(
    method _add_multi_group_job (line 959) | def _add_multi_group_job(

FILE: nemo_skills/pipeline/utils/docker_images.py
  function _sanitize_image_component (line 29) | def _sanitize_image_component(value: str) -> str:
  function _resolve_dockerfile_path (line 34) | def _resolve_dockerfile_path(dockerfile_path_str: str) -> Path:
  function _build_local_docker_image (line 55) | def _build_local_docker_image(dockerfile_spec: str) -> str:
  function resolve_container_image (line 102) | def resolve_container_image(container: str, cluster_config: dict) -> str:

FILE: nemo_skills/pipeline/utils/eval.py
  class BenchmarkArgs (line 34) | class BenchmarkArgs:
    method requires_judge (line 55) | def requires_judge(self):
  class EvalGenerationUnit (line 60) | class EvalGenerationUnit:
  function get_arg_from_module_or_dict (line 79) | def get_arg_from_module_or_dict(module, arg_name, default_value=None, ov...
  function get_benchmark_args_from_module (line 90) | def get_benchmark_args_from_module(
  function _resolve_data_path (line 217) | def _resolve_data_path(data_path):
  function add_default_args (line 226) | def add_default_args(
  function prepare_eval_commands (line 297) | def prepare_eval_commands(

FILE: nemo_skills/pipeline/utils/exp.py
  function get_exp_handles (line 70) | def get_exp_handles(expname: str, ignore_finished=True, ignore_exp_not_e...
  function get_sandbox_command (line 118) | def get_sandbox_command(cluster_config):
  class CustomJobDetails (line 125) | class CustomJobDetails(SlurmJobDetails):
    method stdout (line 131) | def stdout(self) -> Path:
    method srun_stdout (line 135) | def srun_stdout(self) -> Path:
    method stderr (line 139) | def stderr(self) -> Path:
    method srun_stderr (line 143) | def srun_stderr(self) -> Path:
    method ls_term (line 147) | def ls_term(self) -> str:
  class CustomJobDetailsRay (line 157) | class CustomJobDetailsRay(CustomJobDetails):
    method ls_term (line 162) | def ls_term(self) -> str:
  function get_executor (line 167) | def get_executor(
  function install_packages_wrap (line 421) | def install_packages_wrap(cmd, installation_command: str | None = None):
  function add_task (line 469) | def add_task(
  function run_exp (line 889) | def run_exp(exp, cluster_config, sequential=False, dry_run=False):
  function get_exp (line 937) | def get_exp(expname, cluster_config, _reuse_exp=None):
  function get_nsight_cmd (line 956) | def get_nsight_cmd(profile_step_range):

FILE: nemo_skills/pipeline/utils/generation.py
  function normalize_models_config (line 32) | def normalize_models_config(
  function normalize_parameter (line 64) | def normalize_parameter(
  function build_requirements_venv_cmd (line 107) | def build_requirements_venv_cmd(requirements: list[str]) -> str:
  function get_chunked_rs_filename (line 152) | def get_chunked_rs_filename(
  function get_expected_done_files (line 171) | def get_expected_done_files(output_dir, random_seeds, chunk_ids):
  function get_remaining_jobs (line 183) | def get_remaining_jobs(cluster_config, output_dir, random_seeds, chunk_i...
  function separate_hydra_args (line 301) | def separate_hydra_args(extra_arguments: str) -> tuple[str, str]:
  function get_generation_cmd (line 407) | def get_generation_cmd(
  function wrap_cmd (line 551) | def wrap_cmd(cmd, preprocess_cmd, postprocess_cmd, random_seed=None, wan...
  function configure_client (line 573) | def configure_client(

FILE: nemo_skills/pipeline/utils/mounts.py
  function is_mounted_filepath (line 27) | def is_mounted_filepath(cluster_config: dict | None, path: str, mounts: ...
  function check_if_mounted (line 49) | def check_if_mounted(cluster_config, path_to_check):
  function _resolve_path_placeholders (line 59) | def _resolve_path_placeholders(path: str) -> str:
  function check_mounts (line 71) | def check_mounts(
  function get_mounted_path (line 165) | def get_mounted_path(cluster_config: dict, path: str):
  function get_unmounted_path (line 213) | def get_unmounted_path(cluster_config: dict, path: str):
  function add_mount_path (line 261) | def add_mount_path(mount_source: str, mount_dest: str, cluster_config):
  function create_remote_directory (line 284) | def create_remote_directory(directory: str | list, cluster_config: dict):
  function resolve_mount_paths (line 317) | def resolve_mount_paths(cluster_config: dict, mount_paths: str | list | ...
  function check_remote_mount_directories (line 362) | def check_remote_mount_directories(directories: list, cluster_config: di...
  function normalize_mounts_list (line 399) | def normalize_mounts_list(mounts: list[str], allow_rw_mode: bool = False):
  function get_mounts_from_config (line 473) | def get_mounts_from_config(cluster_config: dict):

FILE: nemo_skills/pipeline/utils/packager.py
  class RepoMetadata (line 30) | class RepoMetadata:
    method __post_init__ (line 36) | def __post_init__(self):
  function register_external_repo (line 52) | def register_external_repo(metadata: RepoMetadata, ignore_if_registered:...
  function get_registered_external_repo (line 67) | def get_registered_external_repo(name: str) -> Optional[RepoMetadata]:
  function resolve_external_data_path (line 82) | def resolve_external_data_path(local_data_path: str | Path) -> str:
  function get_git_repo_path (line 132) | def get_git_repo_path(path: str | Path = None):
  function get_packager (line 164) | def get_packager(extra_package_dirs: tuple[str] | None = None):

FILE: nemo_skills/pipeline/utils/ray_executor.py
  function _import_ray (line 71) | def _import_ray():
  class RayJobConfig (line 87) | class RayJobConfig:
  class RayJobClient (line 101) | class RayJobClient:
    method __init__ (line 104) | def __init__(self, ray_address: str = "auto", namespace: str = "nemo"):
    method _connect (line 117) | def _connect(self):
    method submit_job (line 138) | def submit_job(self, config: RayJobConfig) -> str:
    method _wait_for_dependencies (line 195) | def _wait_for_dependencies(self, job_ids: List[str], poll_interval: in...
    method get_job_status (line 242) | def get_job_status(self, job_id: str) -> str:
    method get_job_logs (line 246) | def get_job_logs(self, job_id: str) -> str:
    method cancel_job (line 254) | def cancel_job(self, job_id: str):
    method list_jobs (line 262) | def list_jobs(self) -> List[Dict[str, Any]]:
  function get_ray_client (line 271) | def get_ray_client(cluster_config: Dict[str, Any]) -> RayJobClient:
  class RayExecutor (line 283) | class RayExecutor(Executor):
    method assign (line 328) | def assign(
    method nnodes (line 345) | def nnodes(self) -> int:
    method nproc_per_node (line 349) | def nproc_per_node(self) -> int:

FILE: nemo_skills/pipeline/utils/scripts/base.py
  class BaseJobScript (line 26) | class BaseJobScript(run.Script):
    method __post_init__ (line 54) | def __post_init__(self):
    method set_inline (line 73) | def set_inline(self, command: Union[str, Callable, run.Script]) -> None:
    method hostname_ref (line 77) | def hostname_ref(self) -> str:

FILE: nemo_skills/pipeline/utils/scripts/eval.py
  function _combine_cmds (line 24) | def _combine_cmds(cmds: List[str], single_node_mode: str) -> str:
  function _inject_if_missing (line 38) | def _inject_if_missing(extra_arguments: str, needle: str, insertion: str...
  function _inject_single_server_overrides (line 45) | def _inject_single_server_overrides(
  class EvalClientScript (line 76) | class EvalClientScript(BaseJobScript):
    method __post_init__ (line 98) | def __post_init__(self):

FILE: nemo_skills/pipeline/utils/scripts/generation.py
  class GenerationClientScript (line 26) | class GenerationClientScript(BaseJobScript):
    method __post_init__ (line 79) | def __post_init__(self):

FILE: nemo_skills/pipeline/utils/scripts/nemo_gym.py
  class NemoGymRolloutsScript (line 26) | class NemoGymRolloutsScript(BaseJobScript):
    method __post_init__ (line 62) | def __post_init__(self):

FILE: nemo_skills/pipeline/utils/scripts/server.py
  class ServerScript (line 30) | class ServerScript(BaseJobScript):
    method __post_init__ (line 79) | def __post_init__(self):
    method get_address (line 100) | def get_address(self) -> str:
  class SandboxScript (line 106) | class SandboxScript(BaseJobScript):
    method __post_init__ (line 128) | def __post_init__(self):

FILE: nemo_skills/pipeline/utils/server.py
  class SupportedServersSelfHosted (line 25) | class SupportedServersSelfHosted(str, Enum):
  class SupportedServers (line 35) | class SupportedServers(str, Enum):
  function get_free_port (line 48) | def get_free_port(exclude: list[int] | None = None, strategy: int | str ...
  function should_get_random_port (line 67) | def should_get_random_port(server_gpus, exclusive):
  function wrap_python_path (line 71) | def wrap_python_path(cmd):
  function set_python_path_and_wait_for_server (line 75) | def set_python_path_and_wait_for_server(server_address, generation_comma...
  function _parse_last_flag (line 85) | def _parse_last_flag(tokens: list[str], *names: str) -> str | None:
  function _compute_vllm_dp_ray_serving_nodes (line 114) | def _compute_vllm_dp_ray_serving_nodes(server_args: str, num_gpus: int, ...
  function get_ray_server_cmd (line 151) | def get_ray_server_cmd(start_cmd, serving_nodes: int | None = None, num_...
  function get_server_command (line 229) | def get_server_command(

FILE: nemo_skills/pipeline/verl/ppo.py
  class PPOVerlTask (line 38) | class PPOVerlTask:
    method get_ray_launch_cmd (line 55) | def get_ray_launch_cmd(self):
    method format_train_args (line 59) | def format_train_args(self):
    method format_data_args (line 116) | def format_data_args(self):
    method format_wandb_args (line 125) | def format_wandb_args(self, disable_wandb, wandb_project, expname):
    method get_preamble_cmd (line 139) | def get_preamble_cmd(self):
    method get_script_module (line 143) | def get_script_module(self):
    method get_job_cmd (line 146) | def get_job_cmd(self):
    method get_cmd (line 158) | def get_cmd(self):
  function get_training_cmd (line 178) | def get_training_cmd(
  class SupportedServers (line 225) | class SupportedServers(str, Enum):
  function ppo_verl (line 234) | def ppo_verl(

FILE: nemo_skills/prompt/utils.py
  class BM25Retriever (line 34) | class BM25Retriever:
    method __init__ (line 35) | def __init__(self, data_path: str, field: str):
    method retrieve (line 45) | def retrieve(self, query: str, top_k: int = 1):
  class FewShotExamplesConfig (line 51) | class FewShotExamplesConfig:
    method __post_init__ (line 67) | def __post_init__(self):
  class CodeTags (line 85) | class CodeTags:
  class PromptConfig (line 99) | class PromptConfig:
  class Prompt (line 114) | class Prompt:
    method __init__ (line 115) | def __init__(self, config, tokenizer):
    method build_filled_example (line 125) | def build_filled_example(self, example_dict: Dict[str, Any]) -> str:
    method build_examples_dict (line 154) | def build_examples_dict(self, input_dict):
    method build_user_message (line 192) | def build_user_message(self, input_dict: Dict[str, str]) -> str:
    method get_code_execution_args (line 204) | def get_code_execution_args(self):
    method format_assistant_response (line 218) | def format_assistant_response(
    method fill (line 250) | def fill(
    method __str__ (line 351) | def __str__(self):
  function get_token_count (line 355) | def get_token_count(
  function get_config_path (line 423) | def get_config_path(config: str, config_dir: str | None = None, config_e...
  function load_config (line 439) | def load_config(config: str, config_dir: str | None = None) -> dict:
  function get_prompt (line 458) | def get_prompt(

FILE: nemo_skills/training/data_preparation_utils/arithmetic_utils.py
  function get_eval_func (line 40) | def get_eval_func(op):
  function get_op_counts (line 45) | def get_op_counts(counter):
  function extract_expressions (line 49) | def extract_expressions(text: str):
  function tokenize (line 84) | def tokenize(expression):
  function infix_to_postfix (line 95) | def infix_to_postfix(tokens):
  function evaluate_postfix_once (line 120) | def evaluate_postfix_once(postfix):
  function solve_expression (line 141) | def solve_expression(expression):
  function merge_solution_steps (line 163) | def merge_solution_steps(solution_steps):

FILE: nemo_skills/training/data_preparation_utils/filters.py
  class BaseFilter (line 45) | class BaseFilter(BaseParallelProcessor):
    method __init__ (line 46) | def __init__(self, **kwargs):
    method finalize (line 55) | def finalize(self, metrics: List):
    method _chunk_manifest (line 69) | def _chunk_manifest(self):
  class DropIfRegexMatch (line 83) | class DropIfRegexMatch(BaseFilter):
    method __init__ (line 86) | def __init__(
    method process_dataset_entry (line 96) | def process_dataset_entry(self, data_entry) -> List:
  class DropIfRegexNotMatch (line 103) | class DropIfRegexNotMatch(BaseFilter):
    method __init__ (line 106) | def __init__(
    method process_dataset_entry (line 116) | def process_dataset_entry(self, data_entry) -> List:
  class DropIfEqual (line 123) | class DropIfEqual(BaseFilter):
    method __init__ (line 126) | def __init__(
    method process_dataset_entry (line 136) | def process_dataset_entry(self, data_entry) -> List:
  class DropMultiBoxed (line 143) | class DropMultiBoxed(BaseFilter):
    method __init__ (line 144) | def __init__(self, solution_key: str = "generation", **kwargs):
    method process_dataset_entry (line 148) | def process_dataset_entry(self, data_entry) -> List:
  class DropIncorrectCodeBlocks (line 154) | class DropIncorrectCodeBlocks(BaseFilter):
    method __init__ (line 155) | def __init__(self, solution_key: str = "generation", **kwargs):
    method process_dataset_entry (line 159) | def process_dataset_entry(self, data_entry) -> List:
  class AddCodeExecutionsCounts (line 165) | class AddCodeExecutionsCounts(BaseFilter):
    method __init__ (line 166) | def __init__(self, solution_key: str = "generation", ce_counter_key: s...
    method process_dataset_entry (line 171) | def process_dataset_entry(self, data_entry) -> List:
  class DropIncorrectArithmetic (line 185) | class DropIncorrectArithmetic(BaseFilter):
    method __init__ (line 186) | def __init__(self, solution_key: str = "generation", tolerance=1e-4, *...
    method process_dataset_entry (line 191) | def process_dataset_entry(self, data_entry: str) -> List:
  class MajorityFilter (line 214) | class MajorityFilter(BaseFilter):
    method __init__ (line 215) | def __init__(
    method process_dataset_entry (line 225) | def process_dataset_entry(self, data_entry) -> List:
  class RemoveContaminated (line 236) | class RemoveContaminated(BaseFilter):
    method __init__ (line 237) | def __init__(self, contamination_file, check_key="problem", **kwargs):
    method process_dataset_entry (line 249) | def process_dataset_entry(self, data_entry) -> List:
  class RemoveLenOutliers (line 256) | class RemoveLenOutliers(BaseFilter):
    method __init__ (line 259) | def __init__(
    method process_dataset_entry (line 280) | def process_dataset_entry(self, data_entry):
  class TrimPrefix (line 297) | class TrimPrefix(BaseFilter):
    method __init__ (line 300) | def __init__(self, solution_key: str = "generation", **kwargs):
    method process_dataset_entry (line 304) | def process_dataset_entry(self, data_entry) -> List:
  class TrimSolutions (line 312) | class TrimSolutions(BaseFilter):
    method __init__ (line 315) | def __init__(self, solution_key: str = "generation", **kwargs):
    method process_dataset_entry (line 319) | def process_dataset_entry(self, data_entry) -> List:
  class SplitArithmetic (line 333) | class SplitArithmetic(BaseFilter):
    method __init__ (line 334) | def __init__(self, solution_key: str = "generation", **kwargs):
    method process_dataset_entry (line 338) | def process_dataset_entry(self, data_entry: str) -> List:
  class CodeTextFilter (line 389) | class CodeTextFilter(BaseParallelProcessor):
    method __init__ (line 390) | def __init__(self, filter_type, code_tags, solution_key="generation", ...
    method process_dataset_entry (line 400) | def process_dataset_entry(self, grouped_samples: List, code_begin_toke...
    method process (line 435) | def process(self):
    method finalize (line 464) | def finalize(self, metrics: List):

FILE: nemo_skills/training/data_preparation_utils/merge_processor.py
  class MergeProcessor (line 26) | class MergeProcessor(BaseFilter):
    method __init__ (line 27) | def __init__(self, processor_configs: list[dict], **kwargs):
    method process_dataset_entry (line 41) | def process_dataset_entry(self, data_entry: dict) -> list[DataEntry]:
    method finalize (line 54) | def finalize(self, metrics: list):

FILE: nemo_skills/training/data_preparation_utils/preprocessing.py
  class ReadData (line 33) | class ReadData(BaseProcessor):
    method __init__ (line 34) | def __init__(
    method _read_preprocessed_data (line 89) | def _read_preprocessed_data(self, file_handle) -> int:
    method _parallel_read_file (line 103) | def _parallel_read_file(self, args):
    method _read_raw_data (line 109) | def _read_raw_data(self, file_handle) -> int:
    method _get_sample_hash (line 154) | def _get_sample_hash(self, sample):
    method _batch_deduplicate (line 159) | def _batch_deduplicate(self, batch):
    method _chunks (line 171) | def _chunks(self, lst, n):
    method process (line 176) | def process(self):
  class GroupSamples (line 222) | class GroupSamples(BaseProcessor):
    method __init__ (line 223) | def __init__(self, group_key="input", **kwargs):
    method process (line 227) | def process(self):
  class ShuffleAndDownsampleData (line 239) | class ShuffleAndDownsampleData(BaseProcessor):
    method __init__ (line 240) | def __init__(
    method process (line 265) | def process(self):
  class WriteFinalSftManifest (line 309) | class WriteFinalSftManifest(BaseProcessor):
    method __init__ (line 310) | def __init__(
    method process (line 359) | def process(self):
  class WriteFinalConversationManifest (line 417) | class WriteFinalConversationManifest(WriteFinalSftManifest):
    method process (line 418) | def process(self):
  class WriteFinalRLManifest (line 455) | class WriteFinalRLManifest(BaseProcessor):
    method __init__ (line 456) | def __init__(
    method process (line 504) | def process(self):

FILE: nemo_skills/training/nemo_rl/convert_dcp_to_hf.py
  function parse_args (line 28) | def parse_args():
  function find_max_step_folder (line 58) | def find_max_step_folder(training_folder, step_override=None):
  function is_safetensors_checkpoint (line 85) | def is_safetensors_checkpoint(weights_path):
  function copy_tokenizer_files (line 91) | def copy_tokenizer_files(tokenizer_path, hf_ckpt_path):
  function convert_safetensors_to_hf (line 114) | def convert_safetensors_to_hf(weights_path, hf_ckpt_path, model_name, to...
  function main (line 160) | def main():

FILE: nemo_skills/training/nemo_rl/convert_megatron_to_hf.py
  function parse_args (line 26) | def parse_args():
  function find_max_step_folder (line 62) | def find_max_step_folder(training_folder, step_override=None):
  function main (line 89) | def main():

FILE: nemo_skills/training/nemo_rl/environments/math_environment.py
  class MathEnvConfig (line 36) | class MathEnvConfig(TypedDict):
  function _mute_output (line 42) | def _mute_output():
  class HFVerifyWorker (line 52) | class HFVerifyWorker:
    method __init__ (line 53) | def __init__(self) -> None:
    method verify (line 56) | def verify(self, pred_responses: list[str], ground_truths: list[str]) ...
  class MathEnvironmentMetadata (line 83) | class MathEnvironmentMetadata(TypedDict):
  class MathEnvironment (line 88) | class MathEnvironment(EnvironmentInterface):
    method __init__ (line 89) | def __init__(self, cfg: MathEnvConfig):
    method shutdown (line 99) | def shutdown(self) -> None:
    method step (line 104) | def step(  # type: ignore[override]
    method global_post_process_and_metrics (line 173) | def global_post_process_and_metrics(

FILE: nemo_skills/training/nemo_rl/offline_hf_consolidation.py
  function copy_metadata_files (line 50) | def copy_metadata_files(input_dir, output_dir):
  function parse_args (line 63) | def parse_args() -> argparse.Namespace:
  function main (line 107) | def main() -> None:

FILE: nemo_skills/training/nemo_rl/start_grpo.py
  function parse_args (line 46) | def parse_args() -> tuple[argparse.Namespace, list[str]]:
  function load_jsonl_as_dataset (line 62) | def load_jsonl_as_dataset(
  function extract_dataset (line 89) | def extract_dataset(split, dataset_path):
  function format_passthrough (line 99) | def format_passthrough(data):
  function prepare_math_dataset (line 107) | def prepare_math_dataset(split_ds):
  class NeMoSkillsDataset (line 119) | class NeMoSkillsDataset:
    method __init__ (line 122) | def __init__(self, training_data, validation_data):
  class NSTaskDataSpec (line 143) | class NSTaskDataSpec(TaskDataSpec):
  function ns_data_processor (line 148) | def ns_data_processor(
  function setup_data (line 196) | def setup_data(
  function main (line 272) | def main() -> None:

FILE: nemo_skills/training/nemo_rl/start_sft.py
  function detect_data_format (line 45) | def detect_data_format(data_path: str) -> str:
  class PromptResponseDataset (line 82) | class PromptResponseDataset:
    method __init__ (line 83) | def __init__(
    method load_or_process_split (line 115) | def load_or_process_split(self, path: str, split_name: str) -> Dataset:
    method add_messages_key (line 149) | def add_messages_key(self, examples: dict[str, list[Any]]) -> dict[str...
  function parse_args (line 161) | def parse_args():
  function sft_preprocessor (line 175) | def sft_preprocessor(
  function setup_data (line 229) | def setup_data(tokenizer: AutoTokenizer, data_config: DataConfig):
  function main (line 278) | def main():

FILE: nemo_skills/training/prepare_data.py
  function main (line 22) | def main(cfg):

FILE: nemo_skills/training/train_redrafter.py
  class ReDrafterTrainer (line 64) | class ReDrafterTrainer(Trainer):
    method __init__ (line 65) | def __init__(self, *args, **kwargs):
    method compute_loss (line 69) | def compute_loss(self, model, inputs, return_outputs=False, **kwargs):
  class ModelArguments (line 105) | class ModelArguments:
  class TrainingArguments (line 111) | class TrainingArguments(transformers.TrainingArguments):
  function get_tokenizer (line 172) | def get_tokenizer(model_args, training_args):
  function generate_drafter_config_from_base (line 183) | def generate_drafter_config_from_base(llm, training_args):
  function get_compute_metrics (line 193) | def get_compute_metrics(training_args):
  function record_to_training_instance (line 207) | def record_to_training_instance(
  function train (line 245) | def train(model_args, training_args):
  function eval (line 299) | def eval(model_args, training_args):

FILE: nemo_skills/training/verl/prepare_data.py
  function parse_args (line 21) | def parse_args():
  function transform_data (line 32) | def transform_data(input_file, data_source, ability):
  function save_to_parquet (line 52) | def save_to_parquet(df, output_file):
  function main (line 56) | def main():

FILE: nemo_skills/utils.py
  function get_logger_name (line 37) | def get_logger_name(file):
  function parse_reasoning (line 47) | def parse_reasoning(sample: dict, generation_key: str = "generation", en...
  function nested_dataclass (line 67) | def nested_dataclass(*args, **kwargs):
  function setup_logging (line 103) | def setup_logging(disable_hydra_logs: bool = True, log_level: int = logg...
  function remove_handlers (line 141) | def remove_handlers():
  function get_skills_root_dir (line 148) | def get_skills_root_dir():
  function init_wandb (line 153) | def init_wandb(project, name, exp_dir=None, verbose=False):
  function validate_wandb_project_name (line 204) | def validate_wandb_project_name(wandb_project=None, wandb_name=None, wan...
  function extract_comments (line 232) | def extract_comments(code: str):
  function type_to_str (line 244) | def type_to_str(type_hint):
  function extract_comments_above_fields (line 270) | def extract_comments_above_fields(dataclass_obj, prefix: str = "", level...
  function get_fields_docstring (line 333) | def get_fields_docstring(dataclass_obj, **kwargs):
  function get_help_message (line 339) | def get_help_message(dataclass_obj, help_message="", **kwargs):
  function python_doc_to_cmd_help (line 362) | def python_doc_to_cmd_help(doc_class, docs_prefix="", arg_prefix=""):
  function get_chunked_filename (line 383) | def get_chunked_filename(chunk_id, output_filename):
  function chunk_data (line 388) | def chunk_data(data: List[Any], output_filename: str, chunk_id: Optional...
  function str_ids_to_list (line 426) | def str_ids_to_list(ids: str) -> list[int]:
  function compute_chunk_ids (line 454) | def compute_chunk_ids(chunk_ids: list[int] | str, num_chunks: int) -> li...
  function prefill_judgement (line 485) | def prefill_judgement(data_point: dict) -> str | None:
  function check_no_extra_args_fire (line 496) | def check_no_extra_args_fire():
  function resolve_python_module_from_file (line 557) | def resolve_python_module_from_file(py_filepath: str, root_module: str =...
  function maybe_get_env (line 580) | def maybe_get_env(value: Union[Any, List[Any]], env_name, default=None, ...
  function get_server_wait_cmd (line 621) | def get_server_wait_cmd(server_address):
  function setup_make_sequence_length_divisible_by (line 630) | def setup_make_sequence_length_divisible_by(tensor_model_parallel_size: ...

FILE: recipes/asr_tts/riva_generate.py
  class RivaGenerateConfig (line 36) | class RivaGenerateConfig(GenerationTaskConfig):
  class RivaGenerationTask (line 55) | class RivaGenerationTask(GenerationTask):
    method __init__ (line 56) | def __init__(self, cfg: RivaGenerateConfig):
    method wait_for_server (line 59) | def wait_for_server(self):
    method setup_llm (line 102) | def setup_llm(self):
    method setup_prompt (line 131) | def setup_prompt(self):
    method fill_prompt (line 134) | def fill_prompt(self, data_point, all_data, prompt_format=None):
    method log_example_prompt (line 140) | def log_example_prompt(self, data):
    method process_single_datapoint (line 144) | async def process_single_datapoint(self, data_point, all_data, prompt_...
  function generate (line 177) | def generate(cfg: RivaGenerateConfig):

FILE: recipes/data-integrity/model_comparison/analyses/length_analysis.py
  function analyze_response_lengths (line 28) | def analyze_response_lengths(df, subdirs):

FILE: recipes/data-integrity/model_comparison/analyses/similarity_analysis.py
  function analyze_semantic_similarity (line 32) | def analyze_semantic_similarity(df, subdirs, sentence_model=None):
  function _fallback_similarity_analysis (line 205) | def _fallback_similarity_analysis(df, subdirs):

FILE: recipes/data-integrity/model_comparison/analyses/umap_analysis.py
  function analyze_response_embeddings_umap (line 52) | def analyze_response_embeddings_umap(df, subdirs, sentence_model):
  function analyze_input_response_mapping_umap (line 154) | def analyze_input_response_mapping_umap(df, subdirs, sentence_model):
  function analyze_multimodal_space_umap (line 299) | def analyze_multimodal_space_umap(df, subdirs, sentence_model):

FILE: recipes/data-integrity/model_comparison/analyses/vocabulary_analysis.py
  function analyze_vocabulary_diversity (line 30) | def analyze_vocabulary_diversity(df, subdirs):

FILE: recipes/data-integrity/model_comparison/analyzer.py
  class OrganizedModelAnalyzer (line 56) | class OrganizedModelAnalyzer:
    method __init__ (line 59) | def __init__(self, json_file_path, results_base_dir="model_comparison_...
    method setup_results_directory (line 71) | def setup_results_directory(self):
    method load_data (line 96) | def load_data(self):
    method initialize_models (line 103) | def initialize_models(self):
    method generate_final_report (line 118) | def generate_final_report(self):

FILE: recipes/data-integrity/model_comparison/data_loader.py
  function load_json_data (line 26) | def load_json_data(json_file_path):
  function json_to_dataframe (line 33) | def json_to_dataframe(data):
  function load_and_prepare_data (line 56) | def load_and_prepare_data(json_file_path):

FILE: recipes/data-integrity/model_comparison/main.py
  function main (line 27) | def main():

FILE: recipes/data-integrity/model_comparison/report_generator.py
  function generate_analysis_report (line 26) | def generate_analysis_report(df, results_dir, subdirs, length_stats, div...
  function generate_index_file (line 106) | def generate_index_file(results_dir, subdirs, df):

FILE: recipes/data-integrity/model_comparison/setup.py
  function install_requirements (line 25) | def install_requirements():
  function download_nltk_data (line 31) | def download_nltk_data():
  function download_spacy_model (line 41) | def download_spacy_model():
  function verify_installation (line 48) | def verify_installation():
  function main (line 98) | def main():

FILE: recipes/data-integrity/model_comparison/utils/file_utils.py
  function get_model_comparison_name (line 28) | def get_model_comparison_name(df):
  function save_plot (line 53) | def save_plot(subdirs, df, filename_suffix, title=""):
  function save_data (line 64) | def save_data(subdirs, df, data, filename_suffix, format="csv"):

FILE: recipes/data-integrity/model_comparison/utils/model_utils.py
  function shorten_model_name (line 18) | def shorten_model_name(model_name):

FILE: recipes/data-integrity/model_comparison/utils/text_utils.py
  function calculate_rouge_l (line 32) | def calculate_rouge_l(text1, text2):
  function basic_rouge_l (line 46) | def basic_rouge_l(text1, text2):

FILE: recipes/data-integrity/model_comparison/visualization/interactive_plots.py
  function create_response_embeddings_umap (line 47) | def create_response_embeddings_umap(df, subdirs, sentence_model):
  function create_input_response_mapping_umap (line 57) | def create_input_response_mapping_umap(df, subdirs, sentence_model):
  function create_multimodal_space_umap (line 67) | def create_multimodal_space_umap(df, subdirs, sentence_model):
  function create_interactive_explorer (line 77) | def create_interactive_explorer(df, subdirs, sentence_model):

FILE: recipes/data-integrity/model_comparison/visualization/static_plots.py
  function plot_response_lengths (line 22) | def plot_response_lengths(df, subdirs):
  function plot_vocabulary_diversity (line 32) | def plot_vocabulary_diversity(df, subdirs):
  function plot_similarity_heatmaps (line 42) | def plot_similarity_heatmaps(df, subdirs, sentence_model=None):
  function plot_similarity_histograms (line 52) | def plot_similarity_histograms(df, subdirs, sentence_model=None):

FILE: recipes/data-integrity/postprocess_data.py
  function process_data (line 23) | def process_data(elem, target_model):

FILE: recipes/data-integrity/prepare_data.py
  function process_data (line 24) | def process_data(elem, split):
  function get_from_iterable (line 33) | def get_from_iterable(dataset: IterableDataset):

FILE: recipes/data-integrity/run_integrity_pipeline.py
  function download (line 24) | def download(workspace, cluster, num_gpus, expname_prefix, target_model,...
  function gen_answer (line 41) | def gen_answer(workspace, cluster, num_gpus, expname_prefix, target_mode...
  function postprocess (line 59) | def postprocess(workspace, cluster, num_gpus, expname_prefix, target_mod...
  function compare (line 75) | def compare(workspace, cluster, num_gpus, expname_prefix, target_model, ...

FILE: recipes/gencluster/pipeline/run_inter_tournament.py
  function tournament_schedule_file_exists (line 24) | def tournament_schedule_file_exists(
  function main (line 41) | def main():

FILE: recipes/gencluster/pipeline/run_intra_tournament.py
  function tournament_schedule_file_exists (line 24) | def tournament_schedule_file_exists(
  function main (line 41) | def main():

FILE: recipes/gencluster/pipeline/solution_generation.py
  function parse_generation_benchmark (line 23) | def parse_generation_benchmark(benchmark: str, split: str | None = None)...
  function main (line 48) | def main():

FILE: recipes/gencluster/pipeline/test_case_generation.py
  function main (line 20) | def main() -> None:

FILE: recipes/gencluster/scripts/compute_tournament_score.py
  function parse_tail_scores_and_winner (line 32) | def parse_tail_scores_and_winner(generation_text: str) -> Tuple[float, f...
  function try_get_numeric (line 62) | def try_get_numeric(value) -> Optional[float]:
  function extract_cluster_base_score (line 77) | def extract_cluster_base_score(obj: dict, side: str, explicit_key: Optio...
  function extract_cluster_grade (line 109) | def extract_cluster_grade(obj: dict, side: str, explicit_key: Optional[s...
  function main (line 135) | def main():

FILE: recipes/gencluster/scripts/extract_cpp_code.py
  function extract_final_cpp_block (line 29) | def extract_final_cpp_block(text):
  function wait_for_sandbox (line 36) | def wait_for_sandbox(sandbox, loop, timeout: int = 240, poll: float = 1.0):
  function compile_cpp_file (line 51) | def compile_cpp_file(cpp_file_path, binary_dir, sandbox, loop):
  function process_jsonl_file (line 73) | def process_jsonl_file(jsonl_path, output_dir, binary_dir, folder_name, ...
  function main (line 163) | def main():

FILE: recipes/gencluster/scripts/filter_clusters.py
  function filter_cluster (line 28) | def filter_cluster(cluster_data):
  function filter_file (line 68) | def filter_file(input_file, output_file):
  function main (line 115) | def main():

FILE: recipes/gencluster/scripts/generate_datasets_json.py
  function collect_datasets (line 22) | def collect_datasets(root_dir: Path):
  function main (line 69) | def main() -> None:

FILE: recipes/gencluster/scripts/generate_test_cases.py
  function _get_thread_context (line 35) | def _get_thread_context():
  function wait_for_sandbox (line 48) | def wait_for_sandbox(sandbox, loop, timeout: int = 240, poll: float = 1.0):
  function run_generator (line 62) | def run_generator(gen_binary_path, timeout=10, *, loop=None, sandbox: Lo...
  function run_generator_to_sandbox_file (line 95) | def run_generator_to_sandbox_file(gen_binary_path, timeout=10, *, loop=N...
  function run_validator (line 117) | def run_validator(
  function validate_dataset (line 173) | def validate_dataset(
  function generate_datasets_for_problem (line 207) | def generate_datasets_for_problem(
  function main (line 455) | def main():

FILE: recipes/gencluster/scripts/merge_tournament_scores.py
  function load_clusters (line 24) | def load_clusters(path: str) -> Dict[str, dict]:
  function write_clusters (line 30) | def write_clusters(path: str, clusters: Dict[str, dict]) -> None:
  function read_scores_by_problem (line 37) | def read_scores_by_problem(csv_path: str, include_solution: bool = False):
  function main (line 160) | def main():

FILE: recipes/gencluster/scripts/run_tournament_all.py
  function derive_output_path (line 26) | def derive_output_path(input_file: str, output_dir: str) -> str:
  function build_directed_edges (line 35) | def build_directed_edges(n: int, edges: List[Tuple[int, int]], k: int, r...
  function build_simple_schedule (line 85) | def build_simple_schedule(n: int, games_per_player: int, rng: random.Ran...
  function write_schedule_jsonl (line 122) | def write_schedule_jsonl(
  function write_intracluster_schedule_jsonl (line 169) | def write_intracluster_schedule_jsonl(
  function main (line 203) | def main():

FILE: recipes/gencluster/scripts/submission_ICPC.py
  function to_bool (line 39) | def to_bool(value: Any) -> bool:
  function to_int (line 50) | def to_int(value: Any, default: int = 0) -> int:
  function to_float (line 58) | def to_float(value: Any, default: float = 0.0) -> float:
  function load_clusters (line 66) | def load_clusters(path: Path) -> Dict[str, Any]:
  function extract_problem_number (line 76) | def extract_problem_number(filename: str) -> int:
  function any_solution_true (line 84) | def any_solution_true(clusters_payload: Dict[str, Any]) -> bool:
  function build_sorted_clusters (line 98) | def build_sorted_clusters(
  function compute_submission_count_for_problem (line 205) | def compute_submission_count_for_problem(
  function cluster_has_any_true (line 242) | def cluster_has_any_true(cluster_val: Dict[str, Any]) -> bool:
  function compute_oracle_inside_cluster_submission_count (line 253) | def compute_oracle_inside_cluster_submission_count(
  function main (line 272) | def main() -> int:

FILE: recipes/gencluster/scripts/submission_IOI.py
  function get_max_score_for_subtask (line 107) | def get_max_score_for_subtask(subtask_number, dataset="ioi24"):
  function get_grade_slice_for_problem (line 115) | def get_grade_slice_for_problem(problem_name, dataset="ioi24"):
  function load_cluster_data (line 126) | def load_cluster_data(filepath):
  function apply_blind_cluster_filtering (line 132) | def apply_blind_cluster_filtering(clusters, strategy="balanced"):
  function get_solution_iterator (line 164) | def get_solution_iterator(clusters):
  function run_submission (line 190) | def run_submission(
  function calculate_theoretical_max_score (line 527) | def calculate_theoretical_max_score(submission_scores=None, dataset="ioi...

FILE: recipes/gencluster/scripts/tournament_schedule.py
  function load_clusters (line 24) | def load_clusters(cluster_file: str) -> Dict[str, Any]:
  function remove_empty_output_clusters (line 30) | def remove_empty_output_clusters(clusters: Dict[str, Any]) -> Dict[str, ...
  function compute_cluster_representatives (line 44) | def compute_cluster_representatives(
  function extract_problem_number_from_cluster_path (line 106) | def extract_problem_number_from_cluster_path(cluster_path: str) -> int:
  function load_problem_metadata (line 118) | def load_problem_metadata(problem_number: int, meta_path: str) -> Dict[s...
  function generate_k_regular_simple_graph (line 149) | def generate_k_regular_simple_graph(num_nodes: int, k: int, rng: random....
  function generate_k_regular_fast (line 239) | def generate_k_regular_fast(num_nodes: int, k: int, rng: random.Random) ...
  function main (line 280) | def main():

FILE: recipes/libtrace/scripts/collect_generated_problems.py
  function _json_loads (line 43) | def _json_loads(s: str):
  function _json_dumps (line 46) | def _json_dumps(obj) -> str:
  function _json_loads (line 52) | def _json_loads(s: str):
  function _json_dumps (line 55) | def _json_dumps(obj) -> str:
  function parse_seed (line 59) | def parse_seed(filename: str) -> int | None:
  function read_jsonl (line 66) | def read_jsonl(path: Path, skip_invalid: bool) -> tuple[list[dict], int]:
  function resolve_input_files (line 87) | def resolve_input_files(input_dir: str | None, input_files: list[str] | ...
  function parse_args (line 106) | def parse_args() -> argparse.Namespace:

FILE: recipes/libtrace/scripts/filter_applicability_relevance.py
  function _json_loads (line 40) | def _json_loads(s: str):
  function _json_dumps (line 43) | def _json_dumps(obj) -> str:
  function _json_loads (line 50) | def _json_loads(s: str):
  function _json_dumps (line 53) | def _json_dumps(obj) -> str:
  function read_jsonl (line 69) | def read_jsonl(path: Path, skip_invalid: bool) -> tuple[list[dict], int]:
  function extract_generation_text (line 90) | def extract_generation_text(item: dict) -> str:
  function parse_scores (line 106) | def parse_scores(generation: str, label: str, path: str, line_num: int) ...
  function resolve_label (line 123) | def resolve_label(domain: str | None, label: str | None) -> str:
  function print_distribution (line 136) | def print_distribution(title: str, counter: Counter, keys: list[int] | N...
  function parse_args (line 150) | def parse_args() -> argparse.Namespace:

FILE: recipes/libtrace/scripts/gather_solutions.py
  function _json_loads (line 49) | def _json_loads(s: str):
  function _json_dumps (line 52) | def _json_dumps(obj) -> str:
  function _json_loads (line 58) | def _json_loads(s: str):
  function _json_dumps (line 61) | def _json_dumps(obj) -> str:
  function _progress (line 71) | def _progress(iterable, **kwargs):
  class Statistics (line 90) | class Statistics:
  function parse_messages (line 97) | def parse_messages(
  class UsedNamesExtractor (line 196) | class UsedNamesExtractor(ast.NodeVisitor):
    method __init__ (line 202) | def __init__(self):
    method visit_Import (line 208) | def visit_Import(self, node: ast.Import):
    method visit_ImportFrom (line 219) | def visit_ImportFrom(self, node: ast.ImportFrom):
    method _get_full_attribute_path (line 234) | def _get_full_attribute_path(self, node: ast.AST) -> str | None:
    method visit_Name (line 249) | def visit_Name(self, node: ast.Name):
    method visit_Attribute (line 254) | def visit_Attribute(self, node: ast.Attribute):
    method visit_Call (line 260) | def visit_Call(self, node: ast.Call):
  function _extract_used_names_from_code (line 267) | def _extract_used_names_from_code(code: str) -> tuple[set[str], set[str]]:
  function _check_name_in_extracted (line 278) | def _check_name_in_extracted(name: str, used_names: set[str], imported_f...
  function _check_any_name_in_extracted (line 294) | def _check_any_name_in_extracted(all_names: set[str], used_names: set[st...
  function _find_matching_names (line 312) | def _find_matching_names(all_names: set[str], used_names: set[str], impo...
  function _extract_all_code_from_messages (line 330) | def _extract_all_code_from_messages(messages: list, dump_json: bool) -> ...
  function _has_boxed_in_last_assistant (line 345) | def _has_boxed_in_last_assistant(messages: list) -> bool:
  function resolve_input_files (line 352) | def resolve_input_files(input_dir: str | None, input_files: list[str] | ...
  function _get_code_rounds (line 371) | def _get_code_rounds(data: dict, field: str, require_field: bool) -> int:
  function _weighted_sample_indices (line 382) | def _weighted_sample_indices(weights: list[float], k: int, rng: random.R...
  function _init_worker (line 408) | def _init_worker(
  function process_line (line 431) | def process_line(line: str) -> tuple[bool, bool, bool, bool]:
  function process_line_for_sampling (line 452) | def process_line_for_sampling(args: tuple[int, str]) -> tuple[int, bool,...
  function run_stats (line 475) | def run_stats(
  function save_sampled_messages (line 524) | def save_sampled_messages(
  function run_sample (line 549) | def run_sample(
  function parse_args (line 686) | def parse_args() -> argparse.Namespace:

FILE: recipes/libtrace/scripts/harvest_docs.py
  function _json_dumps (line 41) | def _json_dumps(obj) -> str:
  function _json_dumps (line 48) | def _json_dumps(obj) -> str:
  function safe_getmembers (line 52) | def safe_getmembers(obj) -> list[tuple[str, object]]:
  function is_function_or_method (line 168) | def is_function_or_method(obj) -> bool:
  function extract_docs_from_module (line 172) | def extract_docs_from_module(module, module_name: str, seen: set[str]) -...
  function write_jsonl (line 224) | def write_jsonl(path: Path, rows: list[dict]) -> None:
  function process_library (line 230) | def process_library(library_name: str, output_dir: Path) -> list[dict]:
  function parse_args (line 264) | def parse_args() -> argparse.Namespace:

FILE: recipes/libtrace/scripts/prepare_inference_jsonl.py
  function _json_loads (line 39) | def _json_loads(s: str):
  function _json_dumps (line 42) | def _json_dumps(obj) -> str:
  function _json_loads (line 48) | def _json_loads(s: str):
  function _json_dumps (line 51) | def _json_dumps(obj) -> str:
  function read_jsonl (line 66) | def read_jsonl(path: Path) -> list[dict]:
  function resolve_domain_label (line 80) | def resolve_domain_label(domain: str) -> str:
  function validate_row (line 84) | def validate_row(row: dict, path: Path, line_num: int, domain_label: str...
  function write_jsonl (line 112) | def write_jsonl(path: Path, rows: list[dict]) -> None:
  function parse_args (line 119) | def parse_args() -> argparse.Namespace:

FILE: recipes/multimodal/server/backends/__init__.py
  function list_backends (line 46) | def list_backends() -> list:
  function get_backend (line 51) | def get_backend(backend_name: str) -> type:

FILE: recipes/multimodal/server/backends/base.py
  class Modality (line 28) | class Modality(str, Enum):
  class BackendConfig (line 37) | class BackendConfig:
    method from_dict (line 58) | def from_dict(cls, d: Dict[str, Any]) -> "BackendConfig":
  class GenerationRequest (line 67) | class GenerationRequest:
  class GenerationResult (line 100) | class GenerationResult:
    method is_success (line 125) | def is_success(self) -> bool:
  class InferenceBackend (line 129) | class InferenceBackend(ABC):
    method __init__ (line 143) | def __init__(self, config: BackendConfig):
    method get_config_class (line 150) | def get_config_class(cls) -> type:
    method name (line 160) | def name(self) -> str:
    method supported_modalities (line 166) | def supported_modalities(self) -> Set[Modality]:
    method load_model (line 171) | def load_model(self) -> None:
    method generate (line 180) | def generate(self, requests: List[GenerationRequest]) -> List[Generati...
    method get_extra_routes (line 192) | def get_extra_routes(cls, backend_instance: "InferenceBackend") -> list:
    method is_loaded (line 204) | def is_loaded(self) -> bool:
    method health_check (line 208) | def health_check(self) -> Dict[str, Any]:
    method get_generation_params (line 218) | def get_generation_params(self, request: GenerationRequest) -> Dict[st...
    method validate_request (line 229) | def validate_request(self, request: GenerationRequest) -> Optional[str]:

FILE: recipes/multimodal/server/backends/magpie_tts_backend.py
  class MagpieTTSConfig (line 72) | class MagpieTTSConfig(BackendConfig):
    method from_dict (line 96) | def from_dict(cls, d: Dict[str, Any]) -> "MagpieTTSConfig":
  class MagpieTTSBackend (line 135) | class MagpieTTSBackend(InferenceBackend):
    method get_config_class (line 139) | def get_config_class(cls) -> type:
    method name (line 143) | def name(self) -> str:
    method supported_modalities (line 147) | def supported_modalities(self) -> Set[Modality]:
    method __init__ (line 150) | def __init__(self, config: BackendConfig):
    method _patch_hf_fsspec_loader (line 169) | def _patch_hf_fsspec_loader(self) -> None:
    method _resolve_context_audio_path (line 206) | def _resolve_context_audio_path(self, raw_path: str) -> str:
    method load_model (line 229) | def load_model(self) -> None:
    method _extract_json (line 347) | def _extract_json(self, text: str) -> dict:
    method generate (line 359) | def generate(self, requests: List[GenerationRequest]) -> List[Generati...
    method validate_request (line 477) | def validate_request(self, request: GenerationRequest) -> Optional[str]:
    method health_check (line 480) | def health_check(self) -> Dict[str, Any]:
    method __del__ (line 494) | def __del__(self):

FILE: recipes/multimodal/server/backends/nemo_asr_backend.py
  class NeMoASRConfig (line 30) | class NeMoASRConfig(BackendConfig):
    method from_dict (line 43) | def from_dict(cls, d: Dict[str, Any]) -> "NeMoASRConfig":
  class NeMoASRBackend (line 67) | class NeMoASRBackend(InferenceBackend):
    method get_config_class (line 71) | def get_config_class(cls) -> type:
    method name (line 75) | def name(self) -> str:
    method supported_modalities (line 79) | def supported_modalities(self) -> Set[Modality]:
    method __init__ (line 82) | def __init__(self, config: BackendConfig):
    method load_model (line 88) | def load_model(self) -> None:
    method _run_warmup (line 119) | def _run_warmup(self) -> None:
    method _transcribe_paths (line 135) | def _transcribe_paths(
    method _normalize_words (line 161) | def _normalize_words(words_obj: Any) -> List[Dict[str, Any]]:
    method _parse_single_hypothesis (line 214) | def _parse_single_hypothesis(self, hyp: Any) -> tuple[str, List[Dict[s...
    method _get_request_audio_bytes (line 244) | def _get_request_audio_bytes(self, request: GenerationRequest) -> bytes:
    method validate_request (line 253) | def validate_request(self, request: GenerationRequest) -> Optional[str]:
    method generate (line 261) | def generate(self, requests: List[GenerationRequest]) -> List[Generati...

FILE: recipes/multimodal/server/unified_server.py
  class PendingRequest (line 50) | class PendingRequest:
  class RequestBatcher (line 58) | class RequestBatcher:
    method __init__ (line 61) | def __init__(self, backend, batch_size: int, batch_timeout: float):
    method add_request (line 74) | async def add_request(self, request: GenerationRequest) -> GenerationR...
    method _timeout_handler (line 93) | async def _timeout_handler(self):
    method _process_batch (line 102) | async def _process_batch(self):
  function extract_audio_from_messages (line 151) | def extract_audio_from_messages(messages: List[Dict[str, Any]]) -> List[...
  function extract_text_from_messages (line 186) | def extract_text_from_messages(messages: List[Dict[str, Any]]) -> str:
  function extract_system_prompt (line 207) | def extract_system_prompt(messages: List[Dict[str, Any]]) -> Optional[str]:
  function create_app (line 222) | def create_app(

FILE: recipes/noc-reasoning-agent/scripts/create_agent_with_tools.py
  function ensure_parent_dir (line 29) | def ensure_parent_dir(path: Path):
  function load_processed_indices (line 33) | def load_processed_indices(output_path: Path):
  function append_jsonl (line 52) | def append_jsonl(output_path: Path, obj: dict):
  function main (line 57) | def main():

FILE: recipes/noc-reasoning-agent/scripts/create_agent_with_tools_batch.py
  function get_bound_tools (line 53) | def get_bound_tools(row: pd.Series) -> list:
  function ensure_parent_dir (line 73) | def ensure_parent_dir(path: Path):
  function load_processed_indices (line 77) | def load_processed_indices(output_path: Path) -> set:
  function append_jsonl_batch (line 94) | def append_jsonl_batch(output_path: Path, records: list):
  function _safe_str (line 100) | def _safe_str(val):
  function format_tool_response (line 107) | def format_tool_response(string: str) -> str:
  function _parse_tool_call_json (line 113) | def _parse_tool_call_json(raw: str):
  function main (line 128) | def main():

FILE: recipes/noc-reasoning-agent/scripts/evaluation/evaluation_with_judge.py
  function llm_judge_final_output (line 49) | def llm_judge_final_output(expected, generated):
  function llm_judge_reasoning (line 80) | def llm_judge_reasoning(expected, generated):

FILE: recipes/noc-reasoning-agent/scripts/evaluation/problem_code_evaluation.py
  function normalize_close_code (line 81) | def normalize_close_code(s: str) -> str:
  function _acceptable_phrases_for_expected (line 88) | def _acceptable_phrases_for_expected(expected_norm: str):
  function response_matches_expected (line 98) | def response_matches_expected(response_lower: str, expected_close_code: ...

FILE: recipes/noc-reasoning-agent/scripts/evaluation/score.py
  function main (line 20) | def main():

FILE: recipes/noc-reasoning-agent/scripts/filtering/filter_rows.py
  function _load_csv (line 120) | def _load_csv(path: str) -> pd.DataFrame:
  function filter_auto (line 128) | def filter_auto(input_csv, output_csv):
  function filter_soft_solve (line 148) | def filter_soft_solve(input_csv, output_csv):
  function filter_problem_codes (line 160) | def filter_problem_codes(input_csv, output_csv):
  function filter_close_codes (line 174) | def filter_close_codes(input_csv, output_csv):
  function main (line 188) | def main():

FILE: recipes/noc-reasoning-agent/scripts/filtering/match_keywords.py
  function get_close_codes (line 22) | def get_close_codes():
  function find_keyword_matches (line 268) | def find_keyword_matches(row, pattern, keywords):
  function categorize_incidents_by_close_code (line 278) | def categorize_incidents_by_close_code(df: pd.DataFrame) -> pd.DataFrame:

FILE: recipes/noc-reasoning-agent/scripts/ns_pipelines/generate_synthetic_data.py
  function generate_synthetic_data (line 21) | def generate_synthetic_data(args, cluster, num_gpus, step=None, input_fo...
  function generate_synthetic_data_oss_gpt (line 47) | def generate_synthetic_data_oss_gpt(args, cluster, num_gpus):
  function main (line 52) | def main():

FILE: recipes/noc-reasoning-agent/scripts/ns_pipelines/prepare_react_agent.py
  function extract_number_from_input (line 24) | def extract_number_from_input(input_text):
  function get_tools (line 36) | def get_tools(text):
  function main (line 70) | def main(file1_path, file2_path, prompt_config, output_path="output.json...

FILE: recipes/noc-reasoning-agent/scripts/tools.py
  function _safe_tool_val (line 18) | def _safe_tool_val(val, default: str = "NotApplicable") -> str:
  function query_alarm (line 27) | def query_alarm(row: pd.Series, site_or_element_id: str = "", **kwargs) ...
  function query_resource_health (line 38) | def query_resource_health(row: pd.Series, element_id: str = "", **kwargs...
  function query_performance (line 49) | def query_performance(row: pd.Series, metric_type: str = "", **kwargs) -...
  function query_topology (line 61) | def query_topology(row: pd.Series, element_id: str = "", **kwargs) -> str:
  function execute_remote_action (line 72) | def execute_remote_action(row: pd.Series, element_id: str = "", action: ...
  function apply_configuration (line 83) | def apply_configuration(row: pd.Series, element_id: str = "", config_typ...
  function run_diagnostics (line 94) | def run_diagnostics(row: pd.Series, diagnostic_type: str = "", **kwargs)...
  function inspect_logs (line 105) | def inspect_logs(row: pd.Series, log_type: str = "", **kwargs) -> str:
  function create_trouble_ticket (line 116) | def create_trouble_ticket(
  function verify_recovery (line 129) | def verify_recovery(row: pd.Series, element_id: str = "", **kwargs) -> str:
  function query_external_factors (line 140) | def query_external_factors(row: pd.Series, site_or_area: str = "", **kwa...
  function orchestrate_workload (line 151) | def orchestrate_workload(row: pd.Series, action: str, workload_type: str...
  function query_power_system (line 162) | def query_power_system(row: pd.Series, target: str = "", **kwargs) -> str:
  function query_rf_status (line 173) | def query_rf_status(row: pd.Series, sector_or_antenna_id: str = "", **kw...
  function query_container_status (line 184) | def query_container_status(row: pd.Series, container_type: str = "", **k...
  function verify_signaling_path (line 194) | def verify_signaling_path(row: pd.Series, interface: str = "", **kwargs)...
  function test_connectivity (line 205) | def test_connectivity(row: pd.Series, test_type: str = "", **kwargs) -> ...

FILE: recipes/noc-reasoning-agent/scripts/utils/create_input_jsonl_from_incidents.py
  function coerce_to_str (line 90) | def coerce_to_str(value) -> str:
  function extract_examples_by_problem_code (line 96) | def extract_examples_by_problem_code(df, num):
  function main (line 105) | def main():

FILE: recipes/noc-reasoning-agent/scripts/utils/format_reasoning_json.py
  function _incident_id (line 24) | def _incident_id(data):
  function _resolution_method (line 32) | def _resolution_method(data):
  function extract_formatted_json_steps (line 37) | def extract_formatted_json_steps(input_file):
  function extract_final_thinking_processes (line 89) | def extract_final_thinking_processes(input_file):
  function prepare_data_for_reasoning_traces (line 107) | def prepare_data_for_reasoning_traces(jsonl_file, input_file, output_file):
  function token_converting (line 155) | def token_converting(string, model):
  function merge_reasoning_steps (line 305) | def merge_reasoning_steps(steps_taken, reasoning_steps, model="qwen32"):
  function compute_prefix_len_for_dummy_user (line 331) | def compute_prefix_len_for_dummy_user(tokenizer):
  function qwen_token_converter (line 348) | def qwen_token_converter(data, full_reasoning_steps, tokenizer=None):
  function compile_reasoning (line 476) | def compile_reasoning(jsonl_file, input_file, output_dir, reasoning_json...
  function main (line 528) | def main(jsonl_file, input_file, output_file, parse_types, reasoning_jso...

FILE: recipes/noc-reasoning-agent/scripts/utils/reasoning_processes.py
  function get_reasoning_process_for_fault_category (line 2978) | def get_reasoning_process_for_fault_category(fault_category: str) -> str:

FILE: recipes/noc-reasoning-agent/scripts/utils/split_incident_data.py
  function split_generation_field (line 22) | def split_generation_field(

FILE: recipes/noc-reasoning-agent/scripts/utils/split_mocktools_answers.py
  function parse_generation (line 19) | def parse_generation(gen_str):
  function main (line 36) | def main(file1_path, file2_path, output_path=None):

FILE: recipes/noc-reasoning-agent/scripts/utils/token_usage.py
  function load_tokenizer (line 48) | def load_tokenizer(model_name: Optional[str]):
  function count_tokens (line 67) | def count_tokens(text: str, tokenizer) -> int:
  function read_yaml_prompt (line 80) | def read_yaml_prompt(path: Optional[str]) -> Optional[str]:
  function stream_jsonl_lengths (line 95) | def stream_jsonl_lengths(jsonl_path: str, tokenizer) -> pd.DataFrame:
  function describe_lengths (line 135) | def describe_lengths(df: pd.DataFrame, col: str = "token_length") -> pd....
  function plot_histogram (line 157) | def plot_histogram(df: pd.DataFrame, out_png: str, col: str = "token_len...
  function main (line 175) | def main():

FILE: recipes/noc-reasoning-agent/scripts/visualization/extract_representation_columns.py
  function enrich_row (line 46) | def enrich_row(row, lookup):
  function token_count (line 86) | def token_count(text: str) -> int:
  function length_bucket (line 99) | def length_bucket(text: str) -> str:
  function pick_samples (line 109) | def pick_samples(rows, n=5):
  function sanitize (line 155) | def sanitize(name: str) -> str:

FILE: recipes/noc-reasoning-agent/scripts/visualization/generate_trace_visualization.py
  function parse_steps (line 23) | def parse_steps(reasoning):
  function parse_final_reasoning (line 39) | def parse_final_reasoning(generation_text):
  function find_finish_action (line 66) | def find_finish_action(generation_text):
  function parse_generation (line 74) | def parse_generation(generation_text):
  function parse_work_notes (line 85) | def parse_work_notes(work_notes_text):
  function esc (line 109) | def esc(s):
  function render (line 114) | def render(incident_data):
  function main (line 203) | def main(input_file, output_file, max_incidents=15, selected_criteria=No...

FILE: recipes/opencodereasoning/pipeline/prepare_questions.py
  function prepare_questions (line 21) | def prepare_questions(cluster, expname, output_dir: str):

FILE: recipes/opencodereasoning/pipeline/prepare_solutions.py
  function get_stage_expname (line 23) | def get_stage_expname(base_expname, stage_name, suffix):
  function generate_solutions (line 27) | def generate_solutions(cluster, expname, run_after, stage_config, **kwar...
  function filter_solutions (line 49) | def filter_solutions(cluster, expname, run_after, stage_config, **kwargs):
  function get_available_configs (line 94) | def get_available_configs(config_dir):

FILE: recipes/opencodereasoning/scripts/functional_helpers.py
  function rename_files_to_json (line 36) | def rename_files_to_json(data_path: str):
  function filter_invalid_samples (line 60) | def filter_invalid_samples(
  function filter_code_samples (line 132) | def filter_code_samples(

FILE: recipes/opencodereasoning/scripts/output_processing.py
  function check_generation (line 18) | def check_generation(response, keep_explanations=False, do_ast_check=Fal...
  function post_process_generation (line 138) | def post_process_generation(response, keep_explanations=False, do_ast_ch...

FILE: recipes/opencodereasoning/scripts/prepare_questions.py
  function get_question (line 31) | def get_question(ds_name, split, index):

FILE: recipes/openmathreasoning/pipeline/genselect_generation.py
  function get_stage_expname (line 23) | def get_stage_expname(base_expname, stage_name, suffix):
  function prepare_labeling_data (line 27) | def prepare_labeling_data(cluster, expname, run_after, stage_config, **k...
  function label_data (line 50) | def label_data(cluster, expname, run_after, stage_config, **kwargs):
  function extract_judgment (line 68) | def extract_judgment(cluster, expname, run_after, stage_config, **kwargs):
  function generate_new_summaries (line 89) | def generate_new_summaries(cluster, expname, run_after, stage_config, **...
  function merge_new_summaries (line 106) | def merge_new_summaries(cluster, expname, run_after, stage_config, **kwa...
  function prepare_for_sft (line 130) | def prepare_for_sft(cluster, expname, run_after, stage_config, **kwargs):
  function get_available_configs (line 184) | def get_available_configs(config_dir):

FILE: recipes/openmathreasoning/pipeline/problem_generation.py
  function get_stage_expname (line 23) | def get_stage_expname(base_expname, stage_name, suffix):
  function extract_problems (line 27) | def extract_problems(cluster, expname, run_after, stage_config, **kwargs):
  function classify_problems (line 53) | def classify_problems(cluster, expname, run_after, stage_config, **kwargs):
  function extract_answers (line 91) | def extract_answers(cluster, expname, run_after, stage_config, **kwargs):
  function convert_proofs (line 117) | def convert_proofs(cluster, expname, run_after, stage_config, **kwargs):
  function merge_data (line 143) | def merge_data(cluster, expname, run_after, stage_config, **kwargs):
  function decontaminate (line 162) | def decontaminate(cluster, expname, run_after, stage_config, **kwargs):
  function get_available_configs (line 215) | def get_available_configs(config_dir):

FILE: recipes/openmathreasoning/pipeline/solution_generation.py
  function get_stage_expname (line 23) | def get_stage_expname(base_expname, stage_name, suffix):
  function generate_solutions (line 27) | def generate_solutions(cluster, expname, run_after, stage_config, **kwar...
  function fill_majority_answer (line 44) | def fill_majority_answer(cluster, expname, run_after, stage_config, **kw...
  function judge_answers (line 67) | def judge_answers(cluster, expname, run_after, stage_config, **kwargs):
  function postprocess_tir_generations (line 83) | def postprocess_tir_generations(cluster, expname, run_after, stage_confi...
  function extract_python_fragments (line 112) | def extract_python_fragments(cluster, expname, run_after, stage_config, ...
  function _run_fragment_judge (line 141) | def _run_fragment_judge(
  function judge_novelty (line 176) | def judge_novelty(cluster, expname, run_after, stage_config, **kwargs):
  function judge_significance (line 188) | def judge_significance(cluster, expname, run_after, stage_config, **kwar...
  function filter_fragments (line 200) | def filter_fragments(cluster, expname, run_after, stage_config, **kwargs):
  function generate_new_summaries (line 224) | def generate_new_summaries(cluster, expname, run_after, stage_config, **...
  function judge_new_summaries (line 255) | def judge_new_summaries(cluster, expname, run_after, stage_config, **kwa...
  function merge_new_summaries (line 288) | def merge_new_summaries(cluster, expname, run_after, stage_config, **kwa...
  function prepare_for_sft (line 333) | def prepare_for_sft(cluster, expname, run_after, stage_config, **kwargs):
  function get_available_configs (line 375) | def get_available_configs(config_dir):

FILE: recipes/openmathreasoning/scripts/extract_python_fragments.py
  function extract_python_blocks_with_context (line 22) | def extract_python_blocks_with_context(document: str, args) -> List[str]:
  function process_jsonl_file (line 61) | def process_jsonl_file(args) -> None:
  function main (line 96) | def main():

FILE: recipes/openmathreasoning/scripts/filter_novelty_significance.py
  function process_fragments (line 24) | def process_fragments(novelty_files: List[str], significance_files: List...
  function process_single_index (line 123) | def process_single_index(
  function main (line 289) | def main(args):

FILE: recipes/openmathreasoning/scripts/genselect/extract_judgment.py
  function assign_rm_values (line 25) | def assign_rm_values(input_file, output_dir):
  function main (line 82) | def main():

FILE: recipes/openmathreasoning/scripts/genselect/merge_new_summary.py
  function read_jsonl_file (line 25) | def read_jsonl_file(file_path, key=None):
  function is_valid_summary (line 38) | def is_valid_summary(reasoning_judgment, summary_generation):
  function select_best_summary (line 46) | def select_best_summary(valid_summaries):
  function format_reasoning_trace_with_summary (line 52) | def format_reasoning_trace_with_summary(reasoning_file, summary_dir):
  function main (line 99) | def main():

FILE: recipes/openmathreasoning/scripts/genselect/prepare_labeling_data.py
  function get_tokenizer (line 51) | def get_tokenizer():
  function read_data (line 58) | def read_data(file_paths):
  function hash_signature (line 75) | def hash_signature(problem, solutions):
  function process_problem_batch (line 80) | def process_problem_batch(problem_batch, max_instances_per_problem, max_...
  function prepare_data (line 134) | def prepare_data(
  function save_data (line 169) | def save_data(unique_instances, output_dir):

FILE: recipes/openmathreasoning/scripts/genselect/utils.py
  function _format_instance (line 26) | def _format_instance(instance, max_solutions=16):
  function _generate_random_count (line 58) | def _generate_random_count(min_val=2, max_val=16, peak=8):
  function extract_judgment (line 75) | def extract_judgment(text, max_idx=None):
  function extract_summary (line 100) | def extract_summary(reasoning_solution, just_true_summary=False):
  function segregate_instances (line 125) | def segregate_instances(all_instances):
  function create_comparison_instance (line 144) | def create_comparison_instance(correct_solutions, incorrect_solutions, m...

FILE: recipes/openmathreasoning/scripts/merge_new_summary.py
  function read_jsonl_file (line 30) | def read_jsonl_file(file_path: str, key: Optional[str] = None) -> List[D...
  function is_valid_summary (line 43) | def is_valid_summary(reasoning_instance: Dict, summary_instance: Dict) -...
  function select_best_summary (line 54) | def select_best_summary(valid_summaries):
  function trim_reasoning_generation (line 61) | def trim_reasoning_generation(reasoning_generation, start_tag, end_tag, ...
  function format_reasoning_trace_with_summary (line 105) | def format_reasoning_trace_with_summary(
  function main (line 168) | def main():

FILE: recipes/openmathreasoning/scripts/postprocess_classification.py
  function filter (line 19) | def filter(input_file, output_file_yes, output_file_no, mode):

FILE: recipes/openmathreasoning/scripts/postprocess_tir_generations.py
  function validate_code_execution (line 23) | def validate_code_execution(text, code_begin="```python", code_end="```"):
  function cut_final_answer_part (line 57) | def cut_final_answer_part(output):
  function replace_code_tags (line 78) | def replace_code_tags(text, args):
  function filter_code_solution (line 86) | def filter_code_solution(sample, args):
  function preprocess_code_judge (line 120) | def preprocess_code_judge(args):

FILE: recipes/openmathreasoning/scripts/prepare_raw_data.py
  function clean_quoted_text (line 25) | def clean_quoted_text(text):
  function process_element (line 45) | def process_element(elem, tokenizer, max_tokens):
  function init_worker (line 67) | def init_worker():
  function process_element_wrapper (line 73) | def process_element_wrapper(elem):

FILE: recipes/openmathreasoning/scripts/simplified_recipe.py
  function prepare (line 27) | def prepare(workspace, cluster, expname_prefix):
  function run_sdg (line 49) | def run_sdg(workspace, cluster, num_gpus, expname_prefix, wandb_params):
  function run_training (line 93) | def run_training(workspace, cluster, num_gpus, expname_prefix, backend, ...
  function final_eval (line 144) | def final_eval(workspace, cluster, num_gpus, expname_prefix, backend, wa...
  function initial_eval (line 163) | def initial_eval(workspace, cluster, num_gpus, expname_prefix, wandb_par...

FILE: recipes/openreasoning/eval.py
  function eval_aai (line 39) | def eval_aai(model_size):
  function eval_math (line 56) | def eval_math(model_size):
  function eval_code (line 97) | def eval_code(model_size):
  function eval_science (line 113) | def eval_science(model_size):

FILE: recipes/openreasoning/scripts/use_majority_if_no_answer.py
  function process_files (line 24) | def process_files(input_folder: str, output_folder: str) -> None:
  function main (line 102) | def main():

FILE: recipes/opensciencereasoning/openscience_dataset_collection/scripts/filter_mcq_solutions.py
  function extract_answer (line 37) | def extract_answer(
  function get_answer_after_think (line 88) | def get_answer_after_think(text):
  function has_thought (line 92) | def has_thought(gen: str) -> bool:
  function process_prediction_group (line 97) | def process_prediction_group(
  function main (line 131) | def main() -> None:

FILE: recipes/opensciencereasoning/sdg_pipeline/run_pipeline.py
  function to_remote_path (line 35) | def to_remote_path(path: str | Path, remote_repo_root: str | Path) -> str:
  function get_stage_expname (line 46) | def get_stage_expname(base_expname: str, stage_name: str, suffix: str):
  function resolve_config_path (line 50) | def resolve_config_path(raw_path: str, search_dir: Path) -> Path:
  function filter_problems (line 81) | def filter_problems(cluster: str, expname: str, run_after: str, stage_co...
  function decontaminate (line 173) | def decontaminate(cluster: str, expname: str, run_after: str, stage_conf...
  function topics_labeling (line 243) | def topics_labeling(cluster: str, expname: str, run_after: str, stage_co...
  function generate_solutions (line 327) | def generate_solutions(cluster, expname, run_after, stage_config, **kwar...
  function difficulty_estimation (line 413) | def difficulty_estimation(cluster, expname, run_after, stage_config, **k...
  function aggregate (line 488) | def aggregate(cluster, expname, run_after, stage_config, **kwargs):
  function filter_solutions (line 516) | def filter_solutions(cluster, expname, run_after, stage_config, **kwargs):
  function prepare_for_sft (line 572) | def prepare_for_sft(cluster, expname, run_after, stage_config, **kwargs):
  function process_messages_and_bucket (line 633) | def process_messages_and_bucket(cluster, expname, run_after, stage_confi...
  function validate (line 674) | def validate(

FILE: recipes/opensciencereasoning/sdg_pipeline/scripts/aggregate_difficulty.py
  function main (line 26) | def main():

FILE: recipes/opensciencereasoning/sdg_pipeline/scripts/aggregate_metadata.py
  function read_jsonl (line 24) | def read_jsonl(path: str) -> List[dict]:
  function merge_metadata (line 35) | def merge_metadata(metadata_files: List[str]) -> Dict[str, dict]:
  function collect_solutions (line 48) | def collect_solutions(solutions_path: str) -> List[dict]:
  function write (line 60) | def write(output_file: str, dataset: List[dict], metadata: Dict[str, dic...
  function main (line 69) | def main():

FILE: recipes/opensciencereasoning/sdg_pipeline/scripts/aggregate_solutions.py
  function is_correct_judgement (line 35) | def is_correct_judgement(judgement, return_none=False) -> Union[bool, No...
  function parse_args (line 59) | def parse_args() -> argparse.Namespace:
  function aggregate_samples (line 73) | def aggregate_samples(generation_files: Iterable[Path], judgement_files:...
  function main (line 174) | def main() -> None:

FILE: recipes/opensciencereasoning/sdg_pipeline/scripts/aggregate_topics.py
  function check_topic_structure (line 26) | def check_topic_structure(sample: dict, topics_structure: dict, names: l...
  function aggregate_topics (line 91) | def aggregate_topics(input_files: dict, output_file: str, topics_structu...
  function main (line 140) | def main():

FILE: recipes/opensciencereasoning/sdg_pipeline/scripts/decontaminate.py
  function main (line 21) | def main():

FILE: recipes/opensciencereasoning/sdg_pipeline/scripts/extract_predictions.py
  function parse_args (line 32) | def parse_args() -> argparse.Namespace:
  function collect_predictions (line 54) | def collect_predictions(
  function main (line 115) | def main() -> None:

FILE: recipes/opensciencereasoning/sdg_pipeline/scripts/filter_problems.py
  function _json_loads (line 28) | def _json_loads(s: str):
  function _json_loads (line 33) | def _json_loads(s: str):
  function _json_dumps (line 40) | def _json_dumps(obj) -> str:
  function extract_dataset_name (line 56) | def extract_dataset_name(input_path: str) -> str:
  function generate_id (line 60) | def generate_id(dataset_name: str, line_index: str) -> str:
  function count_options (line 65) | def count_options(problem):
  function match_option_format (line 86) | def match_option_format(text: str, pattern: str) -> bool:
  function contains_image (line 91) | def contains_image(problem: str) -> bool:
  function process_file (line 103) | def process_file(

FILE: recipes/opensciencereasoning/sdg_pipeline/scripts/filter_solutions.py
  function extract_python_calls (line 27) | def extract_python_calls(serialized_output):
  function extract_python_code (line 39) | def extract_python_code(call):
  function uses_internet (line 47) | def uses_internet(serialized_output):
  function record_passes_filters (line 65) | def record_passes_filters(
  function parse_args (line 120) | def parse_args() -> argparse.Namespace:
  function main (line 169) | def main() -> None:

FILE: recipes/opensciencereasoning/sdg_pipeline/scripts/map_diversity_prompts.py
  function load_diversity_prompts (line 29) | def load_diversity_prompts(
  function map_prompts_to_dataset (line 84) | def map_prompts_to_dataset(
  function main (line 138) | def main():

FILE: recipes/opensciencereasoning/sdg_pipeline/scripts/prepare_topics.py
  function prepare_examples (line 22) | def prepare_examples(prompt_examples: dict, generation_key: str):
  function prepare_topics (line 40) | def prepare_topics(

FILE: recipes/opensciencereasoning/sdg_pipeline/scripts/process_messages_and_bucket.py
  function _json_loads (line 29) | def _json_loads(s: str):
  function _json_loads (line 35) | def _json_loads(s: str):
  function _json_dumps (line 42) | def _json_dumps(obj) -> str:
  function messages_to_string (line 58) | def messages_to_string(
  function compute_token_length (line 113) | def compute_token_length(text: str, tokenizer: AutoTokenizer) -> int:
  function bucket_index (line 120) | def bucket_index(length: int, bucket_sizes: List[int]) -> int:
  function _parse_chat_template_kwargs_json (line 128) | def _parse_chat_template_kwargs_json(raw: str) -> Dict[str, Any]:
  function extract_input_output_from_messages (line 140) | def extract_input_output_from_messages(
  function process_jsonl (line 193) | def process_jsonl(

FILE: recipes/opensciencereasoning/sdg_pipeline/scripts/remove_redundant_fields.py
  function process_file (line 29) | def process_file(
  function parse_args (line 79) | def parse_args() -> argparse.Namespace:
  function main (line 100) | def main() -> None:

FILE: recipes/opensciencereasoning/sdg_pipeline/scripts/validate_pipeline.py
  function soft_assert (line 41) | def soft_assert(condition: bool, message: str):
  function assert_all (line 46) | def assert_all():
  function iter_jsonl (line 56) | def iter_jsonl(path: Path):
  function count_jsonl (line 64) | def count_jsonl(path: Path) -> int:
  function load_first_record (line 68) | def load_first_record(path: Path) -> Optional[dict]:
  function ensure_file (line 74) | def ensure_file(path: Path, description: str) -> bool:
  function check_no_expected_answers (line 80) | def check_no_expected_answers(path: Path):
  function check_has_expected_answers (line 88) | def check_has_expected_answers(path: Path):
  function check_required_fields (line 96) | def check_required_fields(record: dict, fields: set[str], stage: str, fi...
  function resolve_config_path (line 101) | def resolve_config_path(raw_path: str, search_dir: Path) -> Path:
  function apply_overrides (line 132) | def apply_overrides(config: OmegaConf, override_paths: list[str], dotlis...
  function collect_setting_labels (line 142) | def collect_setting_labels(paths: list[str]) -> set[str]:
  function main (line 151) | def main():

FILE: recipes/proof-gen-verification/pipeline/eval_judge.py
  function get_stage_expname (line 63) | def get_stage_expname(base_expname, stage_name, suffix):
  function get_server_kwargs (line 67) | def get_server_kwargs(model_config):
  function run_evals (line 78) | def run_evals(cluster, expname, run_after, stage_config, **kwargs):
  function eval_step_judge (line 116) | def eval_step_judge(cluster, expname, run_after, stage_config, **kwargs):
  function genselect_eval (line 183) | def genselect_eval(cluster, expname, run_after, stage_config, **kwargs):
  function make_final_answer_dataset (line 250) | def make_final_answer_dataset(cluster, expname, run_after, stage_config,...
  function run_end_to_end_eval (line 303) | def run_end_to_end_eval(cluster, expname, run_after, stage_config, **kwa...
  function generic_bon_eval (line 370) | def generic_bon_eval(cluster, expname, run_after, stage_config, **kwargs):
  function get_available_configs (line 459) | def get_available_configs(config_dir):

FILE: recipes/proof-gen-verification/scripts/build_final_ans_dataset.py
  function gather_problem_solutions (line 28) | def gather_problem_solutions(input_dir):
  function filter_problem_solutions (line 46) | def filter_problem_solutions(all_data, n_pos_neg, reference_model=None, ...
  function main (line 103) | def main():

FILE: recipes/proof-gen-verification/scripts/final_answer_qs.py
  function _llm_call (line 32) | async def _llm_call(llm: BaseModel, messages: list[dict], llm_kwargs: di...
  function extract_corrects_incorrects (line 46) | def extract_corrects_incorrects(batch_results: list[str], gt_answer: str...
  function process_single (line 64) | async def process_single(

FILE: recipes/proof-gen-verification/scripts/generate_generic_bon_dspy.py
  function process_single (line 37) | async def process_single(
  function _llm_call (line 102) | async def _llm_call(llm: BaseModel, prompt: str, llm_kwargs: dict, req_s...
  function extract_score_from_xml (line 116) | def extract_score_from_xml(text: str) -> float:
  function extract_binary_correctness (line 132) | def extract_binary_correctness(text: str) -> float:
  function run_llm_as_judge (line 146) | async def run_llm_as_judge(
  function run_genselect_pairwise (line 222) | async def run_genselect_pairwise(

FILE: recipes/proof-gen-verification/scripts/generate_generic_bon_generation.py
  function _llm_call (line 37) | async def _llm_call(llm: BaseModel, prompt: str, llm_kwargs: dict, req_s...
  function extract_score_from_xml (line 51) | def extract_score_from_xml(text: str) -> float:
  function extract_binary_correctness (line 67) | def extract_binary_correctness(text: str) -> float:
  function process_single (line 81) | async def process_single(
  function run_llm_as_judge (line 146) | async def run_llm_as_judge(
  function run_genselect_pairwise (line 222) | async def run_genselect_pairwise(

FILE: recipes/proof-gen-verification/scripts/generic_eval_bon.py
  function compute_metrics_for_seed (line 31) | def compute_metrics_for_seed(seed_idx: int, data_list: list[dict], eval_...
  function main (line 44) | def main():
  function load_evaluation_results_by_seed (line 89) | def load_evaluation_results_by_seed(input_dir: str) -> tuple[list[list[d...
  function expert_score_to_correctness (line 126) | def expert_score_to_correctness(score: float) -> int:
  function aggregate_dict_across_seeds (line 130) | def aggregate_dict_across_seeds(values_across_seeds: list) -> dict:
  function aggregate_metrics_across_seeds (line 151) | def aggregate_metrics_across_seeds(seed_metrics: list[dict]) -> dict:
  function compute_llm_as_judge_metrics (line 160) | def compute_llm_as_judge_metrics(data_list: list[dict], num_shuffles: in...
  function compute_genselect_metrics (line 267) | def compute_genselect_metrics(data_list: list[dict], num_shuffles: int) ...

FILE: recipes/proof-gen-verification/scripts/genselect_judge_generation.py
  function load_prompt_template (line 56) | def load_prompt_template(prompt_config_path):
  function _llm_call (line 64) | async def _llm_call(llm: BaseModel, prompt: str, llm_kwargs: dict, req_s...
  function extract_judgement_summary_result (line 77) | def extract_judgement_summary_result(judgement: str):
  function _create_tournament_prompt (line 92) | def _create_tournament_prompt(
  function _extract_winner_and_eliminate (line 112) | def _extract_winner_and_eliminate(tournament_result, tournament_judgemen...
  function _run_single_tournament (line 140) | async def _run_single_tournament(
  function run_judgement_tournament (line 175) | async def run_judgement_tournament(
  function process_single (line 250) | async def process_single(

FILE: recipes/proof-gen-verification/scripts/make_metrics_fa_qs.py
  function keep_only_digits (line 26) | def keep_only_digits(answer):
  function compute_majority_k (line 33) | def compute_majority_k(original_proofs_list, expected_answer):
  function compute_pass_at_1 (line 59) | def compute_pass_at_1(original_proofs_list, expected_answer):
  function main (line 82) | def main():

FILE: recipes/proof-gen-verification/scripts/make_rubric_generation.py
  function process_single (line 56) | async def process_single(
  function extract_rubric (line 80) | def extract_rubric(llm_response: str) -> str:

FILE: recipes/proof-gen-verification/scripts/script_generation.py
  class ScriptInferenceConfig (line 39) | class ScriptInferenceConfig(InferenceConfig):
  class ScriptGenerationConfig (line 44) | class ScriptGenerationConfig(GenerationTaskConfig):
    method _get_disallowed_params (line 52) | def _get_disallowed_params(self):
  class ScriptGenerationTask (line 65) | class ScriptGenerationTask(GenerationTask):
    method log_example_prompt (line 66) | def log_example_prompt(self, data):
    method setup_prompt (line 69) | def setup_prompt(self):
    method setup_llm (line 73) | def setup_llm(self):
    method process_single_datapoint (line 98) | async def process_single_datapoint(self, data_point, all_data, prompt_...
  function script_generation (line 120) | def script_generation(cfg: ScriptGenerationConfig):

FILE: recipes/proof-gen-verification/scripts/sol_selection_generation.py
  function process_single (line 34) | async def process_single(
  function _llm_call (line 99) | async def _llm_call(llm: BaseModel, prompt: str, llm_kwargs: dict, req_s...
  function generate_proofs (line 112) | async def generate_proofs(
  function run_proof_genselect (line 135) | async def run_proof_genselect(
  function process_single_proof_judgements (line 182) | async def process_single_proof_judgements(
  function compute_judgement_scores (line 222) | def compute_judgement_scores(judgements_list: list[str]) -> list[float]:

FILE: recipes/proof-gen-verification/scripts/step_judgement_generation.py
  function process_single (line 34) | async def process_single(
  function load_prompt_template (line 76) | def load_prompt_template(prompt_config_path):
  function _surround_with_step_index (line 83) | def _surround_with_step_index(proof_steps: list[str], target_step_slice:...
  function _llm_call (line 96) | async def _llm_call(llm: BaseModel, prompt: str, llm_kwargs: dict, req_s...
  function _parse_lemmas (line 117) | def _parse_lemmas(lemma_break_result: str):
  function _parse_truth_statements (line 152) | def _parse_truth_statements(truth_break_result: str):
  function _compute_majority_vote (line 187) | def _compute_majority_vote(step_results):
  function process_step_judgement (line 211) | async def process_step_judgement(
  function process_lemma_judgement (line 274) | async def process_lemma_judgement(
  function process_truth_judgement (line 342) | async def process_truth_judgement(

FILE: recipes/translation/translate_jsonl.py
  function is_line_translatable_content (line 34) | def is_line_translatable_content(line: str) -> bool:
  function _get_all_nested_fields (line 44) | def _get_all_nested_fields(obj, path: str) -> List[str]:
  function full_language_name (line 78) | def full_language_name(lang_code: str) -> str:
  class TranslationConfig (line 98) | class TranslationConfig(GenerationTaskConfig):
  class TranslationTask (line 115) | class TranslationTask(GenerationTask):
    method __init__ (line 116) | def __init__(self, cfg: TranslationConfig):
    method preprocess_data (line 120) | def preprocess_data(self, data):
    method log_example_prompt (line 238) | def log_example_prompt(self, data):
    method unwrap_translation_results (line 256) | def unwrap_translation_results(self, text):
    method postprocess (line 273) | def postprocess(self):
  function main (line 402) | def main(cfg: TranslationConfig):

FILE: tests/conftest.py
  function docker_run (line 22) | def docker_run(command, image_name=None, volume_paths=None):
  function docker_rm (line 64) | def docker_rm(paths):
  function docker_rm_and_mkdir (line 69) | def docker_rm_and_mkdir(file_):

FILE: tests/data/dummy_external_benchmark/my_benchmarks/evaluation/word_count.py
  class WordCountEvaluator (line 20) | class WordCountEvaluator(BaseEvaluator):
    method eval_single (line 21) | async def eval_single(self, data_point):

FILE: tests/data/dummy_external_benchmark/my_benchmarks/inference/word_count.py
  class WordCountGenerationConfig (line 22) | class WordCountGenerationConfig(GenerationTaskConfig):
  class WordCountGenerationTask (line 31) | class WordCountGenerationTask(GenerationTask):
    method process_single_datapoint (line 34) | async def process_single_datapoint(self, data_point, all_data, prompt_...
  function generate (line 66) | def generate(cfg: WordCountGenerationConfig):

FILE: tests/data/dummy_external_benchmark/my_benchmarks/metrics/word_count.py
  class WordCountMetrics (line 18) | class WordCountMetrics(BaseMetrics):
    method _get_score_dict (line 19) | def _get_score_dict(self, prediction):
    method get_incorrect_sample (line 22) | def get_incorrect_sample(self, prediction):
    method update (line 30) | def update(self, predictions):

FILE: tests/data/multi_model_eval_smoke.py
  function _normalize_multi_model_value (line 29) | def _normalize_multi_model_value(value):
  function _normalize_server_address (line 39) | def _normalize_server_address(address: str) -> str:
  class MultiModelEvalSmokeConfig (line 48) | class MultiModelEvalSmokeConfig(GenerationTaskConfig):
    method __post_init__ (line 51) | def __post_init__(self):
  class MultiModelEvalSmokeTask (line 84) | class MultiModelEvalSmokeTask(GenerationTask):
    method setup_prompt (line 87) | def setup_prompt(self):
    method log_example_prompt (line 90) | def log_example_prompt(self, data):
    method setup_llm (line 93) | def setup_llm(self):
    method wait_for_server (line 120) | def wait_for_server(self):
    method process_single_datapoint (line 123) | async def process_single_datapoint(self, data_point, all_data, prompt_...
  function generate (line 165) | def generate(cfg: MultiModelEvalSmokeConfig):

FILE: tests/gpu-tests/test_contamination.py
  function test_check_contamination (line 26) | def test_check_contamination():

FILE: tests/gpu-tests/test_context_retry.py
  class TestConfig (line 29) | class TestConfig:
  class TestEnvironment (line 40) | class TestEnvironment:
    method validate_environment (line 46) | def validate_environment(self):
  class CommandBuilder (line 54) | class CommandBuilder:
    method __init__ (line 57) | def __init__(self, env: TestEnvironment, config: TestConfig):
    method _build_base_cmd (line 62) | def _build_base_cmd(self, cmd_type: str, output_dir: str, server_type:...
    method build_eval_cmd (line 74) | def build_eval_cmd(self, output_dir: str, server_type: str, enable_sof...
    method build_generate_cmd (line 85) | def build_generate_cmd(self, output_dir: str, server_type: str, input_...
  class OutputManager (line 96) | class OutputManager:
    method setup_output_dir (line 100) | def setup_output_dir(model_type: str, test_name: str) -> str:
    method setup_io_files (line 107) | def setup_io_files(output_dir: str) -> tuple[str, str]:
  class MetricsValidator (line 116) | class MetricsValidator:
    method __init__ (line 119) | def __init__(self, config: TestConfig):
    method validate_eval_metrics (line 122) | def validate_eval_metrics(self, output_dir: str) -> Dict[str, Any]:
    method validate_eval_failure (line 138) | def validate_eval_failure(self, output_dir: str) -> bool:
    method validate_eval_completion_but_empty_generation (line 149) | def validate_eval_completion_but_empty_generation(self, output_dir: st...
    method validate_generation_output (line 163) | def validate_generation_output(self, output_file: str) -> bool:
  function _create_large_input_file (line 168) | def _create_large_input_file(input_file: str, num_samples: int):
  class ContextRetryTestSuite (line 187) | class ContextRetryTestSuite:
    method __init__ (line 190) | def __init__(self):
    method run_no_strategy_test (line 198) | def run_no_strategy_test(self, server_type: str, test_name: str, enabl...
    method run_reduce_generation_test (line 206) | def run_reduce_generation_test(
    method run_reduce_prompt_test (line 225) | def run_reduce_prompt_test(self, server_type: str, test_name: str, ret...
  function test_context_retry_no_strategy (line 244) | def test_context_retry_no_strategy(server_type):
  function test_context_retry_reduce_generation_enabled (line 256) | def test_context_retry_reduce_generation_enabled(server_type):
  function test_context_retry_disabled (line 268) | def test_context_retry_disabled(server_type):
  function test_context_retry_reduce_prompt_start (line 282) | def test_context_retry_reduce_prompt_start(server_type):
  function test_context_retry_reduce_prompt_end (line 299) | def test_context_retry_reduce_prompt_end(server_type):

FILE: tests/gpu-tests/test_eval.py
  function get_preparable_datasets (line 75) | def get_preparable_datasets():
  function test_aaa_prepare_and_eval_all_datasets (line 87) | def test_aaa_prepare_and_eval_all_datasets():
  function test_trtllm_eval (line 205) | def test_trtllm_eval():
  function test_trtllm_code_execution_eval (line 238) | def test_trtllm_code_execution_eval(server_type):
  function test_hf_eval (line 290) | def test_hf_eval(server_type, server_args):
  function test_megatron_eval (line 357) | def test_megatron_eval():

FILE: tests/gpu-tests/test_external_benchmark_eval.py
  function _wait_for_server (line 36) | def _wait_for_server(server_address, timeout=300, interval=5):
  function sglang_server (line 50) | def sglang_server():
  function test_external_benchmark_prepare_and_eval (line 82) | def test_external_benchmark_prepare_and_eval(run_location, use_data_dir,...

FILE: tests/gpu-tests/test_generate.py
  function test_vllm_generate_greedy (line 28) | def test_vllm_generate_greedy():
  function test_vllm_generate_greedy_chunked (line 64) | def test_vllm_generate_greedy_chunked():
  function test_vllm_generate_seeds (line 100) | def test_vllm_generate_seeds():

FILE: tests/gpu-tests/test_judge.py
  function test_trtllm_judge (line 27) | def test_trtllm_judge():

FILE: tests/gpu-tests/test_nemo_evaluator.py
  function test_nemo_evaluator_vllm (line 37) | def test_nemo_evaluator_vllm():

FILE: tests/gpu-tests/test_nemo_gym_rollouts.py
  function test_nemo_gym_rollouts_dry_run (line 24) | def test_nemo_gym_rollouts_dry_run():
  function test_nemo_gym_rollouts_dry_run_with_seeds (line 52) | def test_nemo_gym_rollouts_dry_run_with_seeds():

FILE: tests/gpu-tests/test_run_cmd_llm_infer.py
  function test_run_cmd_llm_infer (line 26) | def test_run_cmd_llm_infer():

FILE: tests/gpu-tests/test_sandbox_mounts.py
  function test_sandbox_mounts_read_only_functional (line 25) | def test_sandbox_mounts_read_only_functional():

FILE: tests/gpu-tests/test_tool_calling.py
  function _create_test_input_file (line 41) | def _create_test_input_file():
  function _run_tool_calling_test (line 50) | def _run_tool_calling_test(server_type: str, server_args: str, output_di...
  function test_vllm_tool_calling (line 111) | def test_vllm_tool_calling():
  function test_sglang_tool_calling (line 124) | def test_sglang_tool_calling():

FILE: tests/gpu-tests/test_train.py
  function test_sft_nemo_rl (line 27) | def test_sft_nemo_rl(backend):
  function test_sft_nemo_rl_messages_format (line 81) | def test_sft_nemo_rl_messages_format():
  function test_grpo_nemo_rl (line 137) | def test_grpo_nemo_rl(backend):

FILE: tests/gpu-tests/test_vllm_audio.py
  function test_vllm_audio_generation (line 28) | def test_vllm_audio_generation():

FILE: tests/gpu-tests/utils.py
  function require_env_var (line 18) | def require_env_var(var_name: str) -> str:

FILE: tests/slurm-tests/asr_nim/check_results.py
  function normalize_text (line 25) | def normalize_text(text):
  function check_server_startup (line 45) | def check_server_startup(workspace: str, server_timeout: int):
  function load_references (line 103) | def load_references(workspace: str):
  function check_generation_results (line 135) | def check_generation_results(workspace: str):
  function main (line 253) | def main():

FILE: tests/slurm-tests/asr_nim/run_test.py
  function load_nim_config (line 23) | def load_nim_config(config_file, config_key):
  function get_container_path (line 52) | def get_container_path(cluster, nim_config):
  function setup_workspace_and_mounts (line 71) | def setup_workspace_and_mounts(workspace, cluster):
  function start_server_only (line 105) | def start_server_only(workspace, cluster, expname_prefix, server_timeout...
  function run_generation_only (line 133) | def run_generation_only(workspace, cluster, expname_prefix, server_host,...
  function run_full_pipeline (line 175) | def run_full_pipeline(workspace, cluster, expname_prefix, nim_config):
  function main (line 212) | def main():

FILE: tests/slurm-tests/gpt_oss_python_aime25/check_results.py
  function parse_timeout_counts (line 34) | def parse_timeout_counts(eval_file: Path) -> int:
  function check_timeouts (line 48) | def check_timeouts(eval_dir: str):
  function check_results (line 78) | def check_results(eval_dir: str):
  function main (line 88) | def main():

FILE: tests/slurm-tests/gpt_oss_python_aime25/run_test.py
  function eval_gpt_oss_python (line 20) | def eval_gpt_oss_python(workspace, cluster, expname_prefix, wandb_project):
  function main (line 51) | def main():

FILE: tests/slurm-tests/nano_30b_tool_calling/check_results.py
  function check_tool_usage (line 74) | def check_tool_usage(eval_dir: str):
  function check_timeouts (line 106) | def check_timeouts(eval_dir: str):
  function check_math_tool_calling (line 144) | def check_math_tool_calling(eval_dir: str):
  function check_streaming (line 156) | def check_streaming(workspace: str):
  function main (line 208) | def main():

FILE: tests/slurm-tests/nano_30b_tool_calling/run_test.py
  function eval_math_tool_calling (line 48) | def eval_math_tool_calling(workspace, cluster, expname_prefix, wandb_pro...
  function eval_math_tool_calling_streaming (line 70) | def eval_math_tool_calling_streaming(workspace, cluster, expname_prefix,...
  function main (line 94) | def main():

FILE: tests/slurm-tests/omr_simple_recipe/check_results.py
  function check_results (line 35) | def check_results(benchmark: str, baseline_results: dict, after_training...
  function main (line 53) | def main():

FILE: tests/slurm-tests/omr_simple_recipe/run_test.py
  function main (line 20) | def main():

FILE: tests/slurm-tests/qwen3_4b_evals/check_results.py
  function check_results (line 44) | def check_results(eval_dir: str):
  function main (line 85) | def main():

FILE: tests/slurm-tests/qwen3_4b_evals/run_test.py
  function eval_qwen3_bfcl (line 20) | def eval_qwen3_bfcl(workspace, cluster, expname_prefix, wandb_project):
  function eval_qwen3_online_genselect (line 48) | def eval_qwen3_online_genselect(workspace, cluster, expname_prefix, wand...
  function eval_qwen3_offline_genselect (line 79) | def eval_qwen3_offline_genselect(workspace, cluster, expname_prefix, wan...
  function main (line 136) | def main():

FILE: tests/slurm-tests/qwen3_4b_ray_executor/check_results.py
  function check_results (line 35) | def check_results(workspace: str):
  function main (line 91) | def main():

FILE: tests/slurm-tests/qwen3_4b_ray_executor/run_test.py
  function submit_ray_smoke_task (line 49) | def submit_ray_smoke_task(workspace, cluster, expname_prefix):
  function main (line 82) | def main():

FILE: tests/slurm-tests/qwen3coder_30b_swebench/check_results.py
  function check_results (line 35) | def check_results(eval_dir: str, agent_framework: str):
  function main (line 44) | def main():

FILE: tests/slurm-tests/qwen3coder_30b_swebench/run_test.py
  function eval_qwen3coder (line 20) | def eval_qwen3coder(workspace, cluster, expname_prefix, wandb_project, a...
  function main (line 45) | def main():

FILE: tests/slurm-tests/stem_sdg_pipeline/run_test.py
  function repo_root (line 85) | def repo_root() -> Path:
  function pipeline_script_path (line 89) | def pipeline_script_path() -> Path:
  function settings_path (line 93) | def settings_path(name: str) -> Path:
  function build_overrides (line 100) | def build_overrides(
  function prepare_variant (line 122) | def prepare_variant(
  function launch_pipeline (line 146) | def launch_pipeline(config_path: Path, settings: list[str], overrides: l...
  function main (line 164) | def main():

FILE: tests/slurm-tests/super_120b_aime25/check_results.py
  function check_results (line 37) | def check_results(workspace: str, backend: str):
  function iter_output_rows (line 50) | def iter_output_rows(bench_dir: Path):
  function check_tir_results (line 61) | def check_tir_results(workspace: str, backend: str):
  function main (line 103) | def main():

FILE: tests/slurm-tests/super_120b_aime25/run_test.py
  function _get_trtllm_server_args (line 65) | def _get_trtllm_server_args() -> str:
  function eval_backend (line 78) | def eval_backend(
  function eval_backend_tir (line 108) | def eval_backend_tir(
  function main (line 139) | def main():

FILE: tests/slurm-tests/super_49b_evals/check_results.py
  function check_reasoning (line 155) | def check_reasoning(eval_dir: str, mode: str):
  function check_ruler (line 178) | def check_ruler(eval_dir: str, mode: str):
  function main (line 187) | def main():

FILE: tests/slurm-tests/super_49b_evals/run_test.py
  function setup (line 33) | def setup(workspace, cluster, expname_prefix):
  function eval_reasoning_on (line 48) | def eval_reasoning_on(workspace, cluster, expname_prefix, wandb_project):
  function eval_reasoning_off (line 152) | def eval_reasoning_off(workspace, cluster, expname_prefix, wandb_project):
  function main (line 272) | def main():

FILE: tests/slurm-tests/tts_nim/check_results.py
  function check_server_startup (line 25) | def check_server_startup(workspace: str, server_timeout: int):
  function check_generation_results (line 83) | def check_generation_results(workspace: str):
  function main (line 168) | def main():

FILE: tests/slurm-tests/tts_nim/run_test.py
  function load_nim_config (line 23) | def load_nim_config(config_file, config_key):
  function get_container_path (line 52) | def get_container_path(cluster, nim_config):
  function setup_workspace_and_mounts (line 71) | def setup_workspace_and_mounts(workspace, cluster):
  function start_server_only (line 106) | def start_server_only(workspace, cluster, expname_prefix, server_timeout...
  function run_generation_only (line 134) | def run_generation_only(workspace, cluster, expname_prefix, server_host,...
  function run_full_pipeline (line 176) | def run_full_pipeline(workspace, cluster, expname_prefix, nim_config):
  function main (line 212) | def main():

FILE: tests/slurm-tests/unified_asr/check_results.py
  function normalize_text (line 38) | def normalize_text(text: str) -> str:
  function load_references (line 50) | def load_references() -> dict[str, str]:
  function load_outputs (line 64) | def load_outputs(output_dir: Path) -> list[dict]:
  function check_asr_results (line 76) | def check_asr_results(workspace: str):
  function main (line 111) | def main():

FILE: tests/slurm-tests/unified_asr/run_test.py
  function ensure_workspace_exists (line 28) | def ensure_workspace_exists(workspace: str, cluster: str, config_dir: st...
  function run_unified_asr_test (line 33) | def run_unified_asr_test(
  function main (line 78) | def main():

FILE: tests/slurm-tests/unified_tts/check_results.py
  function load_outputs (line 26) | def load_outputs(output_dir: Path) -> list[dict]:
  function resolve_audio_path (line 38) | def resolve_audio_path(audio_path: str, workspace: str) -> Path:
  function check_tts_results (line 45) | def check_tts_results(workspace: str):
  function main (line 75) | def main():

FILE: tests/slurm-tests/unified_tts/run_test.py
  function ensure_workspace_exists (line 29) | def ensure_workspace_exists(workspace: str, cluster: str, config_dir: st...
  function run_unified_tts_test (line 34) | def run_unified_tts_test(
  function main (line 87) | def main():

FILE: tests/slurm-tests/utils.py
  function load_json (line 18) | def load_json(path):
  function get_nested_value (line 24) | def get_nested_value(nested_dict, nested_keys):
  function soft_assert (line 36) | def soft_assert(condition: bool, message: str):
  function assert_all (line 46) | def assert_all():

FILE: tests/slurm-tests/wmt24pp_gym_topology/check_results.py
  function _load_rows (line 52) | def _load_rows(path):
  function check_topology (line 57) | def check_topology(workspace: str, name: str) -> None:
  function main (line 105) | def main():

FILE: tests/slurm-tests/wmt24pp_gym_topology/run_test.py
  function _server_args (line 63) | def _server_args(tp_size: int, dp_size: int, reasoning_parser: bool) -> ...
  function _submit (line 79) | def _submit(
  function main (line 146) | def main():

FILE: tests/test_arena_metrics.py
  function _make_prediction (line 20) | def _make_prediction(gen_base_score, base_gen_score, category=None):
  function test_arena_metrics_per_category_scoring_v2 (line 31) | def test_arena_metrics_per_category_scoring_v2():
  function test_arena_metrics_single_category_v1 (line 69) | def test_arena_metrics_single_category_v1():
  function test_arena_metrics_legacy_data_no_category (line 96) | def test_arena_metrics_legacy_data_no_category():
  function test_arena_metrics_score_parsing (line 123) | def test_arena_metrics_score_parsing():
  function test_arena_metrics_invalid_score_handling (line 142) | def test_arena_metrics_invalid_score_handling():

FILE: tests/test_base_metrics.py
  class MockMetrics (line 20) | class MockMetrics(BaseMetrics):
    method _get_score_dict (line 21) | def _get_score_dict(self, prediction):
  function test_base_metrics_add_std_metrics (line 154) | def test_base_metrics_add_std_metrics(
  function test_base_metrics_update (line 190) | def test_base_metrics_update(predictions, expected_all_scores):

FILE: tests/test_code_execution.py
  function _get_sandbox (line 30) | def _get_sandbox():
  function test_triple_quotes (line 37) | async def test_triple_quotes(language):
  function test_no_output (line 51) | async def test_no_output(language):
  function test_execution_error (line 62) | async def test_execution_error(language):
  function test_syntax_error (line 85) | async def test_syntax_error(language):
  function test_timeout_error (line 107) | async def test_timeout_error(language):
  function test_std_input (line 121) | async def test_std_input(language):
  function test_multiple_prints_python (line 132) | async def test_multiple_prints_python(language):
  function test_multiple_code_blocks_ipython (line 149) | async def test_multiple_code_blocks_ipython():
  function test_multiple_code_blocks (line 169) | async def test_multiple_code_blocks():
  function test_real_generations (line 188) | async def test_real_generations():
  function test_few_shots (line 228) | async def test_few_shots(code_begin, code_end, code_output_begin, code_o...
  function test_lean4_basic_code_execution (line 285) | async def test_lean4_basic_code_execution():
  function test_lean4_mathlib_code_execution (line 309) | async def test_lean4_mathlib_code_execution():
  function test_shell_code_execution (line 330) | async def test_shell_code_execution():
  function test_lean4_code_execution_failure (line 360) | async def test_lean4_code_execution_failure():
  function test_state_restoration (line 384) | async def test_state_restoration():
  function test_minif2f_deepseek_fewshots (line 420) | async def test_minif2f_deepseek_fewshots():
  function test_ioi_eval_execution (line 461) | async def test_ioi_eval_execution():
  function test_ccc_eval_execution (line 477) | async def test_ccc_eval_execution():
  function test_math_to_lean4_fewshots (line 494) | async def test_math_to_lean4_fewshots():
  function test_code_exec_eval_execution (line 535) | async def test_code_exec_eval_execution():

FILE: tests/test_configs.py
  function test_error_on_extra_params (line 24) | def test_error_on_extra_params():
  function test_get_mounted_path (line 71) | def test_get_mounted_path(mount_source, mount_dest, input_path, expected):
  function test_get_benchmark_args_input_file_should_be_local_path_for_executor_none (line 85) | def test_get_benchmark_args_input_file_should_be_local_path_for_executor...

FILE: tests/test_data_preparation.py
  function compute_md5 (line 22) | def compute_md5(file_path):
  function test_multiple_files (line 30) | def test_multiple_files():
  function test_exclude_keys (line 62) | def test_exclude_keys():
  function test_code_sft_data (line 94) | def test_code_sft_data():
  function test_aggregate_answers_fill (line 121) | def test_aggregate_answers_fill():
  function test_aggregate_answers_extract (line 144) | def test_aggregate_answers_extract():

FILE: tests/test_declarative_pipeline.py
  class DummyScript (line 30) | class DummyScript:
    method __init__ (line 33) | def __init__(self, inline: str | Callable | None = "echo test"):
    method set_inline (line 39) | def set_inline(self, inline):
    method hostname_ref (line 42) | def hostname_ref(self) -> str:
  function make_command (line 48) | def make_command(*, inline: str | Callable | None = "echo test", name: s...
  class TestCommand (line 54) | class TestCommand:
    method test_command_basic_script (line 57) | def test_command_basic_script(self):
    method test_command_prepare_for_execution_string (line 63) | def test_command_prepare_for_execution_string(self):
    method test_command_prepare_for_execution_callable (line 73) | def test_command_prepare_for_execution_callable(self):
    method test_command_prepare_for_execution_callable_with_metadata (line 81) | def test_command_prepare_for_execution_callable_with_metadata(self):
    method test_command_hostname_ref_none (line 93) | def test_command_hostname_ref_none(self):
    method test_command_hostname_ref_heterogeneous (line 100) | def test_command_hostname_ref_heterogeneous(self):
  class TestCommandGroup (line 109) | class TestCommandGroup:
    method test_commandgroup_basic (line 112) | def test_commandgroup_basic(self):
    method test_commandgroup_with_hardware (line 123) | def test_commandgroup_with_hardware(self):
    method test_commandgroup_with_log_dir (line 134) | def test_commandgroup_with_log_dir(self):
  class TestPipeline (line 142) | class TestPipeline:
    method test_pipeline_with_single_job (line 145) | def test_pipeline_with_single_job(self):
    method test_pipeline_with_jobs (line 162) | def test_pipeline_with_jobs(self):
    method test_pipeline_requires_jobs (line 182) | def test_pipeline_requires_jobs(self):
    method test_pipeline_with_run_after (line 190) | def test_pipeline_with_run_after(self):
    method test_pipeline_with_run_after_list (line 206) | def test_pipeline_with_run_after_list(self):
    method test_pipeline_cluster_config_passed_directly (line 222) | def test_pipeline_cluster_config_passed_directly(self):
  class TestPipelineExecution (line 239) | class TestPipelineExecution:
    method test_pipeline_run_basic (line 245) | def test_pipeline_run_basic(self, mock_run_exp, mock_env_vars, mock_ge...
    method test_pipeline_run_with_dependencies (line 275) | def test_pipeline_run_with_dependencies(self, mock_run_exp, mock_env_v...
    method test_pipeline_hf_home_validation (line 320) | def test_pipeline_hf_home_validation(self, mock_get_executor, mock_is_...
    method test_pipeline_hf_home_missing (line 346) | def test_pipeline_hf_home_missing(self, mock_env_vars):
    method test_pipeline_hf_home_not_mounted (line 360) | def test_pipeline_hf_home_not_mounted(self, mock_is_mounted, mock_env_...
  class TestHetGroupIndices (line 374) | class TestHetGroupIndices:
    method test_het_group_index_non_heterogeneous (line 379) | def test_het_group_index_non_heterogeneous(self, mock_env_vars, mock_g...
    method test_het_group_index_heterogeneous (line 409) | def test_het_group_index_heterogeneous(self, mock_env_vars, mock_get_e...
    method test_het_group_index_per_job_not_global (line 440) | def test_het_group_index_per_job_not_global(self, mock_env_vars, mock_...
  class TestDependencyResolution (line 479) | class TestDependencyResolution:
    method test_dependency_none_handling (line 484) | def test_dependency_none_handling(self, mock_env_vars, mock_get_exp):
    method test_pipeline_run_after_applies_to_jobs (line 507) | def test_pipeline_run_after_applies_to_jobs(self, mock_env_vars, mock_...
  class TestErrorHandling (line 534) | class TestErrorHandling:
    method test_pipeline_job_missing_group_or_groups (line 537) | def test_pipeline_job_missing_group_or_groups(self):
    method test_commandgroup_missing_log_dir (line 546) | def test_commandgroup_missing_log_dir(self):
  class TestJobDependencies (line 558) | class TestJobDependencies:
    method test_multiple_internal_dependencies (line 561) | def test_multiple_internal_dependencies(self):
    method test_dependencies_separated_internal_vs_external (line 629) | def test_dependencies_separated_internal_vs_external(self):
    method test_run_after_dependencies_across_experiments (line 725) | def test_run_after_dependencies_across_experiments(self, tmp_path):
    method test_run_after_with_nonexistent_experiment (line 799) | def test_run_after_with_nonexistent_experiment(self):
    method test_run_after_with_experiment_object (line 807) | def test_run_after_with_experiment_object(self):
  class TestGenerateEnvironmentVariables (line 829) | class TestGenerateEnvironmentVariables:
    method test_generate_with_sandbox_passes_env_vars_correctly (line 834) | def test_generate_with_sandbox_passes_env_vars_correctly(self, mock_te...
  class TestMountsResolution (line 925) | class TestMountsResolution:
    method _make_script (line 935) | def _make_script(self, *, keep_mounts=None):
    method test_stage_a_resolved_mounts_and_keep_mounts (line 961) | def test_stage_a_resolved_mounts_and_keep_mounts(
    method _run_pipeline_and_capture_mounts (line 976) | def _run_pipeline_and_capture_mounts(self, command_mounts, keep_mounts...
    method test_mounts_none_no_keep_mounts_attr_inherits_cluster (line 1027) | def test_mounts_none_no_keep_mounts_attr_inherits_cluster(self):
    method test_mounts_none_keep_mounts_true_inherits_cluster (line 1033) | def test_mounts_none_keep_mounts_true_inherits_cluster(self):
    method test_mounts_empty_no_keep_mounts_attr_inherits_cluster (line 1038) | def test_mounts_empty_no_keep_mounts_attr_inherits_cluster(self):
    method test_mounts_empty_keep_mounts_true_inherits_cluster (line 1043) | def test_mounts_empty_keep_mounts_true_inherits_cluster(self):
    method test_mounts_extra_no_keep_mounts_attr_additive_merge (line 1048) | def test_mounts_extra_no_keep_mounts_attr_additive_merge(self):
    method test_mounts_extra_keep_mounts_true_additive_merge (line 1053) | def test_mounts_extra_keep_mounts_true_additive_merge(self):
    method test_sandbox_script_mounts_override_keep_mounts_true (line 1060) | def test_sandbox_script_mounts_override_keep_mounts_true(self, _mock_p...
    method test_bug_row_1_mounts_none_keep_mounts_false_no_cluster_leak (line 1099) | def test_bug_row_1_mounts_none_keep_mounts_false_no_cluster_leak(self):
    method test_bug_row_2_mounts_empty_keep_mounts_false_no_cluster_leak (line 1105) | def test_bug_row_2_mounts_empty_keep_mounts_false_no_cluster_leak(self):
    method test_bug_row_3_mounts_extra_keep_mounts_false_no_cluster_merge (line 1110) | def test_bug_row_3_mounts_extra_keep_mounts_false_no_cluster_merge(self):

FILE: tests/test_default_args.py
  function test_error_on_missing_default (line 20) | def test_error_on_missing_default():
  function test_wrap_arguments_empty_string (line 25) | def test_wrap_arguments_empty_string():
  function test_wrap_arguments_drops_extra_spaces (line 29) | def test_wrap_arguments_drops_extra_spaces():

FILE: tests/test_dependency_isolation.py
  function _discover_core_modules (line 32) | def _discover_core_modules():
  function test_core_module_imports_without_nemo_run (line 53) | def test_core_module_imports_without_nemo_run(module_name):
  function test_pipeline_can_import_core (line 77) | def test_pipeline_can_import_core():

FILE: tests/test_eval.py
  class FakeExp (line 30) | class FakeExp:
    method __enter__ (line 31) | def __enter__(self):
    method __exit__ (line 34) | def __exit__(self, *args):
  function test_eval_client_script_parallel_fails_if_any_unit_fails (line 38) | def test_eval_client_script_parallel_fails_if_any_unit_fails(monkeypatch...
  function test_prepare_eval_commands_propagates_cli_with_sandbox_to_generation_cmd (line 60) | def test_prepare_eval_commands_propagates_cli_with_sandbox_to_generation...
  function test_resolve_child_sbatch_kwargs_inherits_or_overrides (line 119) | def test_resolve_child_sbatch_kwargs_inherits_or_overrides():
  function _patch_eval_for_sbatch_tests (line 127) | def _patch_eval_for_sbatch_tests(monkeypatch, benchmark_args):
  function test_eval_summarize_sbatch_kwargs_and_account (line 155) | def test_eval_summarize_sbatch_kwargs_and_account(
  function test_eval_judge_sbatch_kwargs_override (line 201) | def test_eval_judge_sbatch_kwargs_override(monkeypatch, tmp_path):
  function test_eval_multi_model_generation_module_smoke (line 248) | def test_eval_multi_model_generation_module_smoke(tmp_path):

FILE: tests/test_external_benchmarks.py
  function clean_global_state (line 59) | def clean_global_state():
  function dummy_benchmark_git (line 92) | def dummy_benchmark_git(tmp_path):
  function dummy_benchmark_no_git (line 115) | def dummy_benchmark_no_git(tmp_path):
  function benchmark_map_path (line 123) | def benchmark_map_path(dummy_benchmark_git):
  function word_count_path (line 128) | def word_count_path(dummy_benchmark_git):
  function simple_bench_path (line 133) | def simple_bench_path(dummy_benchmark_git):
  class TestDatasetResolution (line 142) | class TestDatasetResolution:
    method test_get_dataset_name_short (line 143) | def test_get_dataset_name_short(self):
    method test_get_dataset_name_path (line 146) | def test_get_dataset_name_path(self):
    method test_get_dataset_path_builtin (line 149) | def test_get_dataset_path_builtin(self):
    method test_get_dataset_path_with_slash (line 154) | def test_get_dataset_path_with_slash(self, word_count_path):
    method test_get_dataset_path_from_map (line 158) | def test_get_dataset_path_from_map(self, benchmark_map_path, dummy_ben...
    method test_get_dataset_path_from_map_arg (line 164) | def test_get_dataset_path_from_map_arg(self, dummy_benchmark_git):
    method test_get_dataset_path_from_map_file_arg (line 172) | def test_get_dataset_path_from_map_file_arg(self, benchmark_map_path, ...
  class TestExtraBenchmarkMap (line 185) | class TestExtraBenchmarkMap:
    method test_empty_when_no_env_var (line 186) | def test_empty_when_no_env_var(self):
    method test_loads_map (line 190) | def test_loads_map(self, benchmark_map_path, dummy_benchmark_git):
    method test_relative_path_resolved (line 198) | def test_relative_path_resolved(self, benchmark_map_path):
    method test_absolute_path_kept (line 204) | def test_absolute_path_kept(self, tmp_path):
    method test_dict_arg_returned_as_is (line 212) | def test_dict_arg_returned_as_is(self):
    method test_file_path_arg (line 219) | def test_file_path_arg(self, benchmark_map_path, dummy_benchmark_git):
  class TestGetDatasetModule (line 233) | class TestGetDatasetModule:
    method test_full_path (line 234) | def test_full_path(self, word_count_path):
    method test_builtin (line 239) | def test_builtin(self):
    method test_from_map (line 244) | def test_from_map(self, benchmark_map_path):
    method test_from_map_arg (line 249) | def test_from_map_arg(self, benchmark_map_path):
    method test_from_map_dict_arg (line 255) | def test_from_map_dict_arg(self, dummy_benchmark_git):
    method test_simple_bench_from_map (line 263) | def test_simple_bench_from_map(self, benchmark_map_path):
    method test_collision_builtin_and_map (line 268) | def test_collision_builtin_and_map(self, tmp_path):
    method test_collision_builtin_and_map_dict_arg (line 276) | def test_collision_builtin_and_map_dict_arg(self, tmp_path):
    method test_not_found_no_map (line 283) | def test_not_found_no_map(self):
    method test_not_found_with_map (line 288) | def test_not_found_with_map(self, benchmark_map_path):
    method test_missing_init_py (line 293) | def test_missing_init_py(self, tmp_path):
  class TestBuildCommand (line 306) | class TestBuildCommand:
    method test_builtin_dataset_appended (line 307) | def test_builtin_dataset_appended(self):
    method test_external_dataset_local (line 319) | def test_external_dataset_local(self, benchmark_map_path, dummy_benchm...
    method test_data_dir_collision_raises (line 337) | def test_data_dir_collision_raises(self, benchmark_map_path, dummy_ben...
    method test_extra_args_appended (line 359) | def test_extra_args_appended(self):
  class TestParsePrepareCLIArguments (line 377) | class TestParsePrepareCLIArguments:
    method test_datasets_separated_from_unknown_args (line 378) | def test_datasets_separated_from_unknown_args(self):
    method test_parallelism_and_retries_passed_through (line 384) | def test_parallelism_and_retries_passed_through(self):
    method test_default_parallelism_and_retries_included (line 396) | def test_default_parallelism_and_retries_included(self):
  class TestEvaluatorResolution (line 408) | class TestEvaluatorResolution:
    method test_builtin_class (line 409) | def test_builtin_class(self):
    method test_builtin_function (line 414) | def test_builtin_function(self):
    method test_file_path_class_evaluator (line 419) | def test_file_path_class_evaluator(self, dummy_benchmark_git):
    method test_not_found (line 426) | def test_not_found(self):
  class TestMetricsResolution (line 437) | class TestMetricsResolution:
    method test_builtin_math (line 438) | def test_builtin_math(self):
    method test_file_path_with_colons (line 442) | def test_file_path_with_colons(self, dummy_benchmark_git):
    method test_not_found (line 447) | def test_not_found(self):
  class TestPackager (line 457) | class TestPackager:
    method test_register_external_repo (line 458) | def test_register_external_repo(self, dummy_benchmark_git):
    method test_resolve_external_data_path (line 464) | def test_resolve_external_data_path(self, dummy_benchmark_git):
    method test_resolve_external_data_path_no_match (line 472) | def test_resolve_external_data_path_no_match(self, tmp_path):
    method test_repo_metadata_invalid_path (line 476) | def test_repo_metadata_invalid_path(self):
    method test_get_packager_in_non_editable_git_repo_has_matching_include_paths (line 480) | def test_get_packager_in_non_editable_git_repo_has_matching_include_pa...
  class TestPrepareDatasets (line 494) | class TestPrepareDatasets:
    method test_prepare_word_count_via_full_path (line 495) | def test_prepare_word_count_via_full_path(self, word_count_path):
    method test_prepare_simple_bench_via_map (line 506) | def test_prepare_simple_bench_via_map(self, benchmark_map_path, dummy_...
  class TestExternalModuleAttributes (line 525) | class TestExternalModuleAttributes:
    method test_word_count_init_attributes (line 526) | def test_word_count_init_attributes(self, word_count_path):
    method test_simple_bench_init_attributes (line 532) | def test_simple_bench_init_attributes(self, simple_bench_path):
    method test_init_triggers_registration (line 538) | def test_init_triggers_registration(self, word_count_path):
    method test_module_data_path_is_parent (line 542) | def test_module_data_path_is_parent(self, word_count_path):
  class TestCustomGenerationModule (line 553) | class TestCustomGenerationModule:
    method test_generation_module_has_task_class (line 554) | def test_generation_module_has_task_class(self, dummy_benchmark_git):
    method test_custom_evaluator_class_resolution (line 560) | def test_custom_evaluator_class_resolution(self, dummy_benchmark_git):
    method test_custom_metrics_class_resolution (line 567) | def test_custom_metrics_class_resolution(self, dummy_benchmark_git):
  class TestPromptConfigResolution (line 579) | class TestPromptConfigResolution:
    method test_builtin_config (line 580) | def test_builtin_config(self):
    method test_absolute_yaml_path (line 586) | def test_absolute_yaml_path(self, dummy_benchmark_git):
    method test_relative_yaml_resolves_to_repo_root (line 593) | def test_relative_yaml_resolves_to_repo_root(self):
    method test_load_config_builtin (line 599) | def test_load_config_builtin(self):

FILE: tests/test_generation.py
  function test_eval_gsm8k_api (line 32) | def test_eval_gsm8k_api(tmp_path):
  function test_eval_judge_api (line 63) | def test_eval_judge_api(tmp_path):
  function test_fail_on_api_key_env_var (line 99) | def test_fail_on_api_key_env_var(tmp_path):
  function test_succeed_on_api_key_env_var (line 119) | def test_succeed_on_api_key_env_var(tmp_path):
  function test_generate_openai_format (line 154) | def test_generate_openai_format(tmp_path, format):
  function test_server_metadata_from_num_tasks (line 177) | def test_server_metadata_from_num_tasks(tmp_path):
  class TokenizerProbeModel (line 219) | class TokenizerProbeModel(BaseModel):
    method __init__ (line 220) | def __init__(self, **kwargs):
    method _get_tokenizer (line 224) | def _get_tokenizer(self, tokenizer):
    method _build_chat_request_params (line 228) | def _build_chat_request_params(self, **kwargs):
    method _build_completion_request_params (line 231) | def _build_completion_request_params(self, **kwargs):
  function test_base_model_initializes_tokenizer_only_when_needed (line 247) | def test_base_model_initializes_tokenizer_only_when_needed(
  function test_generation_task_sets_tokenizer_only_for_prompt_retry_strategies (line 278) | def test_generation_task_sets_tokenizer_only_for_prompt_retry_strategies(
  function test_generation_task_keeps_text_endpoint_tokenizer (line 300) | def test_generation_task_keeps_text_endpoint_tokenizer(monkeypatch):
  function test_configure_client_hosted_server_host_depends_on_num_nodes (line 326) | def test_configure_client_hosted_server_host_depends_on_num_nodes(server...
  function test_configure_client_preserves_explicit_server_type_override (line 348) | def test_configure_client_preserves_explicit_server_type_override():
  function test_judge_generations_with_structured_output (line 367) | def test_judge_generations_with_structured_output(tmp_path):
  function test_process_chat_chunk_never_yields_none_generation (line 399) | def test_process_chat_chunk_never_yields_none_generation():
  function test_parse_completion_response_token_counts (line 454) | def test_parse_completion_response_token_counts(usage_kwargs, expected_i...

FILE: tests/test_magpie_tts_backend.py
  function test_context_audio_path_is_disabled_without_allowlist (line 22) | def test_context_audio_path_is_disabled_without_allowlist(tmp_path: Path):
  function test_context_audio_path_must_be_under_allowed_roots (line 33) | def test_context_audio_path_must_be_under_allowed_roots(tmp_path: Path):

FILE: tests/test_math_equal.py
  function test_correct_examples (line 55) | def test_correct_examples(output_pair):
  function test_incorrect_examples (line 74) | def test_incorrect_examples(output_pair):

FILE: tests/test_mcp_clients.py
  class DummyClient (line 24) | class DummyClient(MCPClient):
    method __init__ (line 25) | def __init__(self):
    method list_tools (line 52) | async def list_tools(self):
    method call_tool (line 55) | async def call_tool(self, tool: str, args: dict):
  class MinimalClient (line 65) | class MinimalClient(MCPClient):
    method list_tools (line 67) | async def list_tools(self):
    method call_tool (line 70) | async def call_tool(self, tool: str, args: dict):
  function test_metaclass_list_tools_hides_and_filters (line 75) | async def test_metaclass_list_tools_hides_and_filters():
  function test_metaclass_enabled_tools_allowlist_and_missing_check (line 96) | async def test_metaclass_enabled_tools_allowlist_and_missing_check():
  function test_metaclass_call_tool_output_formatter_and_init_hook (line 108) | async def test_metaclass_call_tool_output_formatter_and_init_hook():
  function test_minimal_client_defaults_and_sanitize (line 129) | def test_minimal_client_defaults_and_sanitize():
  function test_stdio_env_inheritance_with_minimal_server (line 143) | async def test_stdio_env_inheritance_with_minimal_server(monkeypatch, tm...
  class DummyTool (line 182) | class DummyTool(Tool):
    method __init__ (line 183) | def __init__(self) -> None:
    method default_config (line 186) | def default_config(self):
    method configure (line 189) | def configure(self, overrides=None, context=None):
    method list_tools (line 192) | async def list_tools(self):
    method execute (line 214) | async def execute(self, tool_name: str, arguments: dict, extra_args: d...
  class CountingTool (line 224) | class CountingTool(DummyTool):
    method __init__ (line 229) | def __init__(self) -> None:
    method list_tools (line 232) | async def list_tools(self):
  class DupTool (line 239) | class DupTool(DummyTool):
    method list_tools (line 240) | async def list_tools(self):
  function test_tool_manager_list_and_execute_with_class_locator (line 246) | async def test_tool_manager_list_and_execute_with_class_locator():
  function test_tool_manager_cache_and_duplicate_detection (line 259) | async def test_tool_manager_cache_and_duplicate_detection():
  function test_stdio_client_list_tools_hide_and_call_tool_with_output_formatter (line 285) | async def test_stdio_client_list_tools_hide_and_call_tool_with_output_fo...
  function test_stdio_client_enabled_tools_enforcement (line 390) | async def test_stdio_client_enabled_tools_enforcement(monkeypatch):
  function test_streamable_http_client_list_and_call_tool (line 443) | async def test_streamable_http_client_list_and_call_tool(monkeypatch):
  function test_streamable_http_client_enforcement (line 514) | async def test_streamable_http_client_enforcement(monkeypatch):
  function test_tool_manager_with_schema_overrides (line 562) | async def test_tool_manager_with_schema_overrides():
  function test_schema_override_nonexistent_param_fails (line 591) | def test_schema_override_nonexistent_param_fails():
  function test_stdio_client_returns_list_for_multiple_content_items (line 611) | async def test_stdio_client_returns_list_for_multiple_content_items(tmp_...
  function _run_tool_sequence (line 644) | async def _run_tool_sequence(tool_impl, tool_calls):
  function test_direct_python_tool_basic_execution (line 662) | async def test_direct_python_tool_basic_execution():
  function test_direct_python_tool_session_persistence (line 687) | async def test_direct_python_tool_session_persistence():
  function test_direct_python_tool_sanitizes_hidden_args (line 709) | async def test_direct_python_tool_sanitizes_hidden_args():
  function test_direct_python_tool_separate_sessions (line 737) | async def test_direct_python_tool_separate_sessions():
  function test_direct_python_tool_cleanup_request_deletes_session (line 770) | async def test_direct_python_tool_cleanup_request_deletes_session():
  function test_mcp_vs_direct_python_tool_parity (line 818) | async def test_mcp_vs_direct_python_tool_parity():
  function test_mcp_vs_direct_error_parity (line 861) | async def test_mcp_vs_direct_error_parity():
  class _StubSandbox (line 894) | class _StubSandbox:
    method __init__ (line 897) | def __init__(self, execute_code=None, delete_session=None, close=None):
    method execute_code (line 904) | async def execute_code(self, code, language="ipython", timeout=10, ses...
    method delete_session (line 909) | async def delete_session(self, session_id):
    method close (line 914) | async def close(self):
  function _direct_tool_with_stub (line 920) | def _direct_tool_with_stub(stub):
  function test_direct_python_tool_missing_code_returns_error_not_raise (line 933) | async def test_direct_python_tool_missing_code_returns_error_not_raise():
  function test_direct_python_tool_sandbox_exception_returns_generic_error (line 950) | async def test_direct_python_tool_sandbox_exception_returns_generic_erro...
  function test_direct_python_tool_shutdown_tolerates_delete_failure (line 971) | async def test_direct_python_tool_shutdown_tolerates_delete_failure():
  function test_direct_python_tool_cleanup_request_tolerates_delete_failure (line 992) | async def test_direct_python_tool_cleanup_request_tolerates_delete_failu...
  class TestRadioactivedecayTool (line 1010) | class TestRadioactivedecayTool:
    method test_radioactivedecay_tool_config (line 1011) | def test_radioactivedecay_tool_config(self):
    method test_radioactivedecay_direct_list_tools (line 1018) | async def test_radioactivedecay_direct_list_tools(self):
    method test_radioactivedecay_rejects_non_finite_time (line 1031) | async def test_radioactivedecay_rejects_non_finite_time(self):
  class TestParticleTool (line 1043) | class TestParticleTool:
    method test_particle_tool_config (line 1044) | def test_particle_tool_config(self):
    method test_particle_direct_list_tools (line 1051) | async def test_particle_direct_list_tools(self):
    method test_particle_tool_rejects_extra_args (line 1061) | async def test_particle_tool_rejects_extra_args(self):
  class TestPeriodictableTool (line 1072) | class TestPeriodictableTool:
    method test_periodictable_tool_config (line 1073) | def test_periodictable_tool_config(self):
    method test_periodictable_direct_list_tools (line 1080) | async def test_periodictable_direct_list_tools(self):
  class TestCoolPropTool (line 1093) | class TestCoolPropTool:
    method test_coolprop_tool_config (line 1094) | def test_coolprop_tool_config(self):
    method test_coolprop_direct_list_tools (line 1101) | async def test_coolprop_direct_list_tools(self):
  class TestWikipediaTool (line 1114) | class TestWikipediaTool:
    method test_wikipedia_tool_config (line 1115) | def test_wikipedia_tool_config(self):
    method test_wikipedia_search_rejects_out_of_range_num_results (line 1122) | async def test_wikipedia_search_rejects_out_of_range_num_results(self):
    method test_wikipedia_direct_list_tools (line 1131) | async def test_wikipedia_direct_list_tools(self):
    method test_wikipedia_execute_dispatch_contracts (line 1156) | async def test_wikipedia_execute_dispatch_contracts(self, monkeypatch):
  class TestArxivTool (line 1188) | class TestArxivTool:
    method test_arxiv_tool_config (line 1189) | def test_arxiv_tool_config(self):
    method test_arxiv_search_rejects_non_positive_max_results (line 1196) | async def test_arxiv_search_rejects_non_positive_max_results(self):
    method test_arxiv_direct_list_tools (line 1205) | async def test_arxiv_direct_list_tools(self):

FILE: tests/test_metrics.py
  function test_metrics (line 24) | def test_metrics(tmp_path, max_seq_len):

FILE: tests/test_nemo_asr_backend.py
  class _FakeHypothesis (line 19) | class _FakeHypothesis:
    method __init__ (line 20) | def __init__(self, text: str):
  class _FakeTimestampHypothesis (line 25) | class _FakeTimestampHypothesis:
    method __init__ (line 26) | def __init__(self):
  class _FakeASRModel (line 37) | class _FakeASRModel:
    method __init__ (line 38) | def __init__(self):
    method transcribe (line 41) | def transcribe(self, audio=None, **kwargs):
  function test_nemo_asr_backend_validate_request_requires_audio (line 46) | def test_nemo_asr_backend_validate_request_requires_audio():
  function test_generation_params_preserve_explicit_zero_values (line 52) | def test_generation_params_preserve_explicit_zero_values():
  function test_nemo_asr_backend_generate_batched_with_words (line 65) | def test_nemo_asr_backend_generate_batched_with_words():
  function test_nemo_asr_backend_prefers_timestamp_words_when_words_are_strings (line 89) | def test_nemo_asr_backend_prefers_timestamp_words_when_words_are_strings():

FILE: tests/test_nemo_evaluator_pipeline.py
  function real_evaluator_config (line 31) | def real_evaluator_config(monkeypatch):
  class Ctx (line 38) | class Ctx:
    method __init__ (line 41) | def __init__(self, args=None):
  function _create_base_kwargs (line 45) | def _create_base_kwargs(tmp_path, real_evaluator_config, **overrides):
  function test_no_servers_external_urls (line 96) | def test_no_servers_external_urls(
  function test_main_server_hosted (line 150) | def test_main_server_hosted(
  function test_judge_server_hosted (line 207) | def test_judge_server_hosted(
  function test_both_servers_hosted_separate_groups (line 260) | def test_both_servers_hosted_separate_groups(
  function test_multiple_tasks (line 330) | def test_multiple_tasks(
  function test_output_dir_structure (line 361) | def test_output_dir_structure(

FILE: tests/test_nvidia_inference_api.py
  function test_nvidia_api_text_only (line 44) | def test_nvidia_api_text_only():
  function test_nvidia_api_audio_input (line 72) | def test_nvidia_api_audio_input():
  function test_nvidia_api_audio_with_transcription_prompt (line 105) | def test_nvidia_api_audio_with_transcription_prompt():

FILE: tests/test_pipeline_utils.py
  function create_done_files (line 30) | def create_done_files(output_dir, seed_chunk_pairs):
  function test_get_chunked_rs_filename (line 40) | def test_get_chunked_rs_filename():
  function test_get_expected_done_files (line 50) | def test_get_expected_done_files():
  function test_get_remaining_jobs_small (line 64) | def test_get_remaining_jobs_small():
  function test_get_remaining_jobs_large (line 81) | def test_get_remaining_jobs_large():
  function test_get_remaining_jobs_rerun_done (line 102) | def test_get_remaining_jobs_rerun_done():
  function test_get_remaining_jobs_no_chunks (line 122) | def test_get_remaining_jobs_no_chunks():
  function test_batch_processing_fallback (line 140) | def test_batch_processing_fallback():
  function test_slurm_execution (line 187) | def test_slurm_execution(mock_get_tunnel):
  function test_separate_hydra_args_empty (line 207) | def test_separate_hydra_args_empty():
  function test_separate_hydra_args_only_hydra (line 214) | def test_separate_hydra_args_only_hydra():
  function test_separate_hydra_args_only_overrides (line 221) | def test_separate_hydra_args_only_overrides():
  function test_separate_hydra_args_mixed (line 228) | def test_separate_hydra_args_mixed():
  function test_separate_hydra_args_equals_format (line 238) | def test_separate_hydra_args_equals_format():
  function test_separate_hydra_args_mixed_formats (line 245) | def test_separate_hydra_args_mixed_formats():
  function test_separate_hydra_args_with_special_chars (line 253) | def test_separate_hydra_args_with_special_chars():
  function test_separate_hydra_args_complex (line 261) | def test_separate_hydra_args_complex():
  function test_separate_hydra_args_hydra_no_value_flags (line 278) | def test_separate_hydra_args_hydra_no_value_flags():
  function test_separate_hydra_args_hydra_with_value_flags_space_and_equals (line 286) | def test_separate_hydra_args_hydra_with_value_flags_space_and_equals():
  function test_separate_hydra_args_hydra_help_and_version (line 299) | def test_separate_hydra_args_hydra_help_and_version():
  function test_separate_hydra_args_config_at_end (line 307) | def test_separate_hydra_args_config_at_end():
  function test_separate_hydra_args_config_in_middle (line 317) | def test_separate_hydra_args_config_in_middle():
  function test_separate_hydra_args_interspersed (line 325) | def test_separate_hydra_args_interspersed():
  function test_separate_hydra_args_only_config_name (line 339) | def test_separate_hydra_args_only_config_name():
  function test_separate_hydra_args_with_spaces_in_values (line 347) | def test_separate_hydra_args_with_spaces_in_values():
  function test_separate_hydra_args_with_quoted_special_chars (line 355) | def test_separate_hydra_args_with_quoted_special_chars():
  function test_sandbox_keep_mounts_false_produces_empty_mounts (line 368) | def test_sandbox_keep_mounts_false_produces_empty_mounts(mock_port, mock...
  function test_sandbox_keep_mounts_true_produces_none_mounts (line 388) | def test_sandbox_keep_mounts_true_produces_none_mounts(mock_port, mock_c...
  function test_non_sandbox_command_mounts_unchanged (line 401) | def test_non_sandbox_command_mounts_unchanged():
  function test_normalize_mounts_list_supports_sandbox_modes (line 411) | def test_normalize_mounts_list_supports_sandbox_modes(monkeypatch):
  function test_normalize_mounts_list_rejects_invalid_regular_mounts (line 423) | def test_normalize_mounts_list_rejects_invalid_regular_mounts(mount):
  function test_add_task_sandbox_mounts_override_keep_mounts_true (line 432) | def test_add_task_sandbox_mounts_override_keep_mounts_true(mock_port, mo...

FILE: tests/test_prompts.py
  function test_get_token_count (line 25) | def test_get_token_count():
  function test_generic_math_problem_augmentation_prompt (line 51) | def test_generic_math_problem_augmentation_prompt():
  function test_generic_codegen_prompt (line 140) | def test_generic_codegen_prompt():
  function test_generic_default_prompt (line 162) | def test_generic_default_prompt():
  function test_thinking_assistant_response (line 174) | def test_thinking_assistant_response():
  function test_generic_math_prompt (line 186) | def test_generic_math_prompt():
  function test_generic_math_prompt_code_examples (line 204) | def test_generic_math_prompt_code_examples():
  function test_llama_code_output_format_examples (line 381) | def test_llama_code_output_format_examples():
  function test_qwen_code_output_format_examples (line 577) | def test_qwen_code_output_format_examples():
  function test_judge_arena (line 759) | def test_judge_arena():
  function test_judge_math (line 815) | def test_judge_math():
  function test_judge_check_contamination (line 912) | def test_judge_check_contamination():
  function test_generic_general_boxed_prompt (line 941) | def test_generic_general_boxed_prompt():
  function test_llm_as_judge_hle_original_prompt (line 977) | def test_llm_as_judge_hle_original_prompt():
  function test_prompt_config_with_openai_format (line 1019) | def test_prompt_config_with_openai_format():
  function test_user_message_override_openai_no_prompt_config (line 1064) | def test_user_message_override_openai_no_prompt_config():
  function test_user_message_override_ns_path (line 1089) | def test_user_message_override_ns_path():
  function test_openai_prompt_suffix (line 1115) | def test_openai_prompt_suffix():
  function test_openai_system_message_override (line 1137) | def test_openai_system_message_override():
  function test_ns_path_template_fill (line 1170) | def test_ns_path_template_fill():
  function test_merge_audio_from_data_role_based (line 1194) | def test_merge_audio_from_data_role_based():
  function test_prompt_config_with_openai_and_suffix (line 1217) | def test_prompt_config_with_openai_and_suffix():
  function test_user_message_override_openai_multimodal_content (line 1240) | def test_user_message_override_openai_multimodal_content():
  function test_prompt_suffix_openai_multimodal_content (line 1269) | def test_prompt_suffix_openai_multimodal_content():
  function test_audio_field_in_prompt_config (line 1296) | def test_audio_field_in_prompt_config():
  function test_audio_list_field_in_prompt_config (line 1309) | def test_audio_list_field_in_prompt_config():
  function test_prompt_config_inline_dict (line 1320) | def test_prompt_config_inline_dict():
  function test_user_message_override_in_get_prompt (line 1333) | def test_user_message_override_in_get_prompt():

FILE: tests/test_prover.py
  class FakePrompt (line 29) | class FakePrompt:
    method __init__ (line 30) | def __init__(self, messages=None):
    method fill (line 33) | def fill(self, _):
  class FakeRefinePrompt (line 37) | class FakeRefinePrompt:
    method fill (line 38) | def fill(self, data):
  class RecordingTokenizer (line 42) | class RecordingTokenizer:
    method __init__ (line 43) | def __init__(self, token_count_fn=None):
    method apply_chat_template (line 47) | def apply_chat_template(self, conversation, tokenize=False, add_genera...
  class FakeSandbox (line 54) | class FakeSandbox:
    method __init__ (line 55) | def __init__(self, result):
    method execute_code (line 58) | async def execute_code(self, *args, **kwargs):  # noqa: ARG002
  function build_task (line 62) | def build_task(
  function test_parse_failure_retries_cleanly_from_base_prompt (line 92) | async def test_parse_failure_retries_cleanly_from_base_prompt():
  function test_parse_failure_does_not_use_nemotron_refinement_state (line 118) | async def test_parse_failure_does_not_use_nemotron_refinement_state():
  function test_compile_timeout_keeps_refinement_feedback_retries (line 144) | async def test_compile_timeout_keeps_refinement_feedback_retries():
  function test_remove_cot_delete_wrong_turns_keeps_only_latest_clean_code_on_success (line 177) | async def test_remove_cot_delete_wrong_turns_keeps_only_latest_clean_cod...
  function test_prefix_too_long_exits_before_generation (line 209) | async def test_prefix_too_long_exits_before_generation():
  function test_parse_gpt_oss_output_extracts_channels (line 231) | def test_parse_gpt_oss_output_extracts_channels():

FILE: tests/test_ray_executor.py
  function _ray_cluster (line 61) | def _ray_cluster():
  function client (line 79) | def client(_ray_cluster):
  function _wait_until_terminal (line 84) | def _wait_until_terminal(client: RayJobClient, job_id: str, timeout: flo...
  function _raiser (line 95) | def _raiser(exc: BaseException):
  function test_submit_job_returns_submission_id (line 109) | def test_submit_job_returns_submission_id(client, tmp_path):
  function test_submit_job_env_vars_reach_the_worker (line 134) | def test_submit_job_env_vars_reach_the_worker(client, tmp_path):
  function test_submit_job_preserves_runtime_env_overrides (line 156) | def test_submit_job_preserves_runtime_env_overrides(client, tmp_path):
  function test_submit_job_creates_log_dir (line 183) | def test_submit_job_creates_log_dir(client, tmp_path):
  function test_wait_for_dependencies_returns_on_succeeded (line 209) | def test_wait_for_dependencies_returns_on_succeeded(client, tmp_path):
  function test_wait_for_dependencies_raises_on_terminal_failure (line 228) | def test_wait_for_dependencies_raises_on_terminal_failure(client, tmp_pa...
  function test_wait_for_dependencies_raises_on_timeout (line 246) | def test_wait_for_dependencies_raises_on_timeout(client, tmp_path):
  function test_get_job_status_stringifies (line 269) | def test_get_job_status_stringifies(client, tmp_path):
  function test_get_job_logs_returns_underlying_logs_on_success (line 288) | def test_get_job_logs_returns_underlying_logs_on_success(client, tmp_path):
  function test_get_job_logs_returns_empty_string_on_error (line 307) | def test_get_job_logs_returns_empty_string_on_error(client, monkeypatch,...
  function test_get_job_logs_propagates_unexpected_errors (line 320) | def test_get_job_logs_propagates_unexpected_errors(client, monkeypatch):
  function test_cancel_job_swallows_error_and_logs_warning (line 333) | def test_cancel_job_swallows_error_and_logs_warning(client, monkeypatch,...
  function test_list_jobs_returns_list_against_real_cluster (line 344) | def test_list_jobs_returns_list_against_real_cluster(client):
  function test_list_jobs_returns_empty_list_on_error (line 350) | def test_list_jobs_returns_empty_list_on_error(client, monkeypatch):
  function test_get_ray_client_reads_address_and_namespace (line 362) | def test_get_ray_client_reads_address_and_namespace(monkeypatch):
  function test_get_ray_client_uses_defaults_when_ray_block_absent (line 386) | def test_get_ray_client_uses_defaults_when_ray_block_absent(monkeypatch):

FILE: tests/test_sandbox_fork_exc_leak.py
  function _test_shell_worker (line 37) | def _test_shell_worker(conn):
  function _get_server_module (line 86) | def _get_server_module():
  function test_error_after_shell_restart_has_no_exception_chain (line 104) | def test_error_after_shell_restart_has_no_exception_chain():

FILE: tests/test_sandbox_network_blocking.py
  function blocked_sandbox (line 38) | def blocked_sandbox():
  class TestNetworkBlocking (line 80) | class TestNetworkBlocking:
    method test_direct_socket_blocked (line 84) | async def test_direct_socket_blocked(self, blocked_sandbox):
    method test_underscore_socket_blocked (line 96) | async def test_underscore_socket_blocked(self, blocked_sandbox):
    method test_requests_library_blocked (line 108) | async def test_requests_library_blocked(self, blocked_sandbox):
    method test_urllib_blocked (line 121) | async def test_urllib_blocked(self, blocked_sandbox):
    method test_subprocess_curl_blocked (line 133) | async def test_subprocess_curl_blocked(self, blocked_sandbox):
    method test_subprocess_wget_blocked (line 149) | async def test_subprocess_wget_blocked(self, blocked_sandbox):
    method test_subprocess_env_clear_blocked (line 164) | async def test_subprocess_env_clear_blocked(self, blocked_sandbox):
    method test_subprocess_python_socket_blocked (line 181) | async def test_subprocess_python_socket_blocked(self, blocked_sandbox):
    method test_local_operations_still_work (line 197) | async def test_local_operations_still_work(self, blocked_sandbox):

FILE: tests/test_session_affinity.py
  class SessionAffinityTester (line 33) | class SessionAffinityTester:
    method __init__ (line 34) | def __init__(self, base_url=BASE_URL):
    method execute_code (line 38) | def execute_code(self, code, session_id, timeout=30, language="ipython"):
    method test_session_persistence (line 93) | def test_session_persistence(self, session_id, num_operations=5):
    method get_worker_info (line 175) | def get_worker_info(self, session_id=None, language="ipython"):
  function tester (line 229) | def tester():
  class TestSessionAffinity (line 234) | class TestSessionAffinity:
    method test_server_health (line 237) | def test_server_health(self):
    method test_basic_session_persistence (line 243) | def test_basic_session_persistence(self, tester):
    method test_session_persistence_various_lengths (line 258) | def test_session_persistence_various_lengths(self, tester, num_operati...
    method test_multiple_concurrent_sessions (line 266) | def test_multiple_concurrent_sessions(self, tester):
    method test_session_affinity_routing (line 294) | def test_session_affinity_routing(self, tester):
    method test_session_persistence_large_payload (line 312) | def test_session_persistence_large_payload(self, tester):
    method test_multiple_large_payloads_concurrent (line 344) | def test_multiple_large_payloads_concurrent(self, tester):
    method test_different_sessions_can_hit_different_workers (line 370) | def test_different_sessions_can_hit_different_workers(self, tester):
    method test_load_balancing_without_session_id (line 385) | def test_load_balancing_without_session_id(self, tester):
    method test_session_affinity_under_load (line 408) | def test_session_affinity_under_load(self, tester, session_config):
    method test_session_cleanup_endpoint (line 446) | def test_session_cleanup_endpoint(self, tester):
    method test_session_list_endpoint (line 468) | def test_session_list_endpoint(self):
    method test_infinite_loop_timeout_then_simple_job (line 504) | def test_infinite_loop_timeout_then_simple_job(self, tester):
    method test_multiple_timeouts_different_sessions (line 556) | def test_multiple_timeouts_different_sessions(self, tester):
    method test_timeout_with_resource_intensive_code (line 619) | def test_timeout_with_resource_intensive_code(self, tester):
    method test_sandbox_session_history_after_timeout (line 677) | async def test_sandbox_session_history_after_timeout(self):

FILE: tests/test_streaming_tool_calling.py
  class FakeTokenizer (line 26) | class FakeTokenizer:
    method encode (line 29) | def encode(self, text):
  function _make_wrapper (line 35) | def _make_wrapper(tokenizer=None):
  function _collect (line 52) | def _collect(agen):
  function test_stream_no_tool_calls (line 67) | def test_stream_no_tool_calls():
  function test_generate_async_duplicates_reasoning_key_in_conversation (line 92) | def test_generate_async_duplicates_reasoning_key_in_conversation():
  function test_stream_final_conversation_duplicates_reasoning_key (line 123) | def test_stream_final_conversation_duplicates_reasoning_key():
  function test_stream_with_tool_call (line 147) | def test_stream_with_tool_call():
  function test_stream_tool_response_tokens_counted (line 202) | def test_stream_tool_response_tokens_counted():
  function test_stream_max_tool_calls_stops_loop (line 258) | def test_stream_max_tool_calls_stops_loop():
  function test_execute_tool_calls_preserves_order (line 307) | async def test_execute_tool_calls_preserves_order():
  function test_stream_no_tokenizer_raises (line 332) | def test_stream_no_tokenizer_raises():

FILE: tests/test_unified_server_audio_parser.py
  function _b64 (line 33) | def _b64(data: bytes) -> str:
  function test_extract_audio_from_messages_audio_url_only (line 37) | def test_extract_audio_from_messages_audio_url_only():
  function test_extract_audio_from_messages_input_audio_only (line 54) | def test_extract_audio_from_messages_input_audio_only():
  function test_extract_audio_from_messages_mixed_order_is_preserved (line 71) | def test_extract_audio_from_messages_mixed_order_is_preserved():
  function test_extract_audio_from_messages_skips_non_audio_or_malformed_blocks (line 93) | def test_extract_audio_from_messages_skips_non_audio_or_malformed_blocks():
  function test_extract_text_from_messages_ignores_system_role (line 113) | def test_extract_text_from_messages_ignores_system_role():

FILE: tests/test_unified_server_batcher.py
  class _MismatchedBackend (line 27) | class _MismatchedBackend:
    method generate (line 28) | def generate(self, requests):
  function test_request_batcher_fails_on_batch_result_length_mismatch (line 33) | def test_request_batcher_fails_on_batch_result_length_mismatch():

FILE: tests/test_unified_server_error_handling.py
  class _ErrorBackend (line 35) | class _ErrorBackend(InferenceBackend):
    method get_config_class (line 37) | def get_config_class(cls) -> type:
    method name (line 41) | def name(self) -> str:
    method supported_modalities (line 45) | def supported_modalities(self):
    method load_model (line 48) | def load_model(self) -> None:
    method generate (line 51) | def generate(self, requests: list[GenerationRequest]) -> list[Generati...
  class _OkBackend (line 58) | class _OkBackend(InferenceBackend):
    method get_config_class (line 60) | def get_config_class(cls) -> type:
    method name (line 64) | def name(self) -> str:
    method supported_modalities (line 68) | def supported_modalities(self):
    method load_model (line 71) | def load_model(self) -> None:
    method generate (line 74) | def generate(self, requests: list[GenerationRequest]) -> list[Generati...
  function test_chat_completion_does_not_leak_raw_backend_error (line 79) | def test_chat_completion_does_not_leak_raw_backend_error(monkeypatch):
  function test_chat_completion_returns_500_if_audio_save_dir_cannot_be_prepared (line 100) | def test_chat_completion_returns_500_if_audio_save_dir_cannot_be_prepare...

FILE: tests/test_vllm_audio.py
  function test_audio_file_to_base64 (line 28) | def test_audio_file_to_base64():
  function _is_valid_audio_content (line 45) | def _is_valid_audio_content(content_item: dict) -> bool:
  function mock_vllm_multimodal_model (line 55) | def mock_vllm_multimodal_model(tmp_path):
  function mock_vllm_multimodal_model_input_audio (line 71) | def mock_vllm_multimodal_model_input_audio(tmp_path):
  function test_content_text_to_list_with_audio (line 86) | def test_content_text_to_list_with_audio(mock_vllm_multimodal_model, tmp...
  function test_content_text_to_list_with_input_audio_format (line 105) | def test_content_text_to_list_with_input_audio_format(mock_vllm_multimod...
  function test_content_text_to_list_with_multiple_audios (line 123) | def test_content_text_to_list_with_multiple_audios(mock_vllm_multimodal_...
  function test_content_text_to_list_no_audio (line 151) | def test_content_text_to_list_no_audio(mock_vllm_multimodal_model):
  function test_preprocess_messages_preserves_no_think (line 160) | def test_preprocess_messages_preserves_no_think(mock_vllm_multimodal_mod...
  function test_needs_audio_chunking_disabled (line 174) | def test_needs_audio_chunking_disabled(mock_vllm_multimodal_model):
  function test_needs_audio_chunking_task_type_filter (line 186) | def test_needs_audio_chunking_task_type_filter(mock_vllm_multimodal_model):

FILE: tests/test_vlm.py
  function test_encode_image_to_base64 (line 26) | def test_encode_image_to_base64():
  function test_encode_image_file_not_found (line 47) | def test_encode_image_file_not_found():
  function test_process_image_content_text_only (line 52) | def test_process_image_content_text_only():
  function test_process_image_content_preserves_text_items (line 58) | def test_process_image_content_preserves_text_items():
  function test_process_image_content_none (line 64) | def test_process_image_content_none():
  function test_process_image_content_http_url_passthrough (line 69) | def test_process_image_content_http_url_passthrough():
  function test_process_image_content_data_url_passthrough (line 79) | def test_process_image_content_data_url_passthrough():
  function test_vllm_model_registered (line 86) | def test_vllm_model_registered():
  function test_get_model_vllm (line 91) | def test_get_model_vllm():
  function test_prompt_with_image_field (line 96) | def test_prompt_with_image_field():
  function test_prompt_without_image_field (line 114) | def test_prompt_without_image_field():
  function test_prompt_image_position_after (line 124) | def test_prompt_image_position_after():
  function test_prompt_image_position_invalid (line 136) | def test_prompt_image_position_invalid():