SYMBOL INDEX (5483 symbols across 811 files)

FILE: autotest/utils/compare_results.py
  function compare_results (line 7) | def compare_results(folder1,
  function compare_folders (line 31) | def compare_folders(folder1, folder2, results_ignore_list=None):
  function get_all_subpaths (line 95) | def get_all_subpaths(directory):

FILE: autotest/utils/health_check.py
  function health_check (line 7) | def health_check(url: str = 'http://0.0.0.0:23333', timeout: int = 300):

FILE: autotest/utils/oc_score_assert.py
  function baseline_scores (line 11) | def baseline_scores(request):
  function result_scores (line 20) | def result_scores():
  class TestCmdCase (line 29) | class TestCmdCase:
    method test_cmd_case1 (line 36) | def test_cmd_case1(self, baseline_scores, result_scores, model, dataset):
    method test_cmd_case2 (line 50) | def test_cmd_case2(self, baseline_scores, result_scores, model, dataset):
    method test_cmd_case3 (line 60) | def test_cmd_case3(self, baseline_scores, result_scores, model, dataset):
    method test_cmd_case4 (line 71) | def test_cmd_case4(self, baseline_scores, result_scores, model, dataset):
    method test_cmd_case5 (line 81) | def test_cmd_case5(self, baseline_scores, result_scores, model, dataset):
  function assert_score (line 87) | def assert_score(model_type, score, baseline, dataset: str = ''):
  function find_csv_files (line 132) | def find_csv_files(directory):
  function read_csv_file (line 145) | def read_csv_file(file_path):

FILE: docs/en/conf.py
  function get_version (line 33) | def get_version():
  function builder_inited_handler (line 229) | def builder_inited_handler(app):
  function setup (line 233) | def setup(app):

FILE: docs/en/statis.py
  function table_format (line 42) | def table_format(data_list):
  function generate_table (line 88) | def generate_table(data_list, title=None):

FILE: docs/zh_cn/conf.py
  function get_version (line 33) | def get_version():
  function builder_inited_handler (line 229) | def builder_inited_handler(app):
  function setup (line 234) | def setup(app):

FILE: docs/zh_cn/statis.py
  function table_format (line 40) | def table_format(data_list):
  function generate_table (line 86) | def generate_table(data_list, title=None):

FILE: opencompass/cli/main.py
  function _run_eval_tasks (line 23) | def _run_eval_tasks(runner, tasks):
  function _is_eval_daemon (line 32) | def _is_eval_daemon(task_type) -> bool:
  function parse_args (line 38) | def parse_args():
  function parse_slurm_args (line 215) | def parse_slurm_args(slurm_parser):
  function parse_dlc_args (line 233) | def parse_dlc_args(dlc_parser):
  function parse_hf_args (line 241) | def parse_hf_args(hf_parser):
  function parse_custom_dataset_args (line 261) | def parse_custom_dataset_args(custom_dataset_parser):
  function main (line 273) | def main():

FILE: opencompass/configs/datasets/lveval/lvevalcmrc_mixup/lveval_cmrc_mixup_gen_465823.py
  function get_dataset_names (line 36) | def get_dataset_names(dataset_name, length_levels):

FILE: opencompass/configs/datasets/lveval/lvevaldureader_mixup/lveval_dureader_mixup_gen_465823.py
  function get_dataset_names (line 37) | def get_dataset_names(dataset_name, length_levels):

FILE: opencompass/configs/datasets/lveval/lvevalfactrecall_en/lveval_factrecall_en_gen_9a836f.py
  function get_dataset_names (line 36) | def get_dataset_names(dataset_name, length_levels):

FILE: opencompass/configs/datasets/lveval/lvevalfactrecall_zh/lveval_factrecall_zh_gen_dbee70.py
  function get_dataset_names (line 36) | def get_dataset_names(dataset_name, length_levels):

FILE: opencompass/configs/datasets/lveval/lvevalhotpotwikiqa_mixup/lveval_hotpotwikiqa_mixup_gen_77ce82.py
  function get_dataset_names (line 39) | def get_dataset_names(dataset_name, length_levels):

FILE: opencompass/configs/datasets/lveval/lvevallic_mixup/lveval_lic_mixup_gen_01eb0c.py
  function get_dataset_names (line 36) | def get_dataset_names(dataset_name, length_levels):

FILE: opencompass/configs/datasets/lveval/lvevalloogle_CR_mixup/lveval_loogle_CR_mixup_gen_d7ea36.py
  function get_dataset_names (line 36) | def get_dataset_names(dataset_name, length_levels):

FILE: opencompass/configs/datasets/lveval/lvevalloogle_MIR_mixup/lveval_loogle_MIR_mixup_gen_d7ea36.py
  function get_dataset_names (line 36) | def get_dataset_names(dataset_name, length_levels):

FILE: opencompass/configs/datasets/lveval/lvevalloogle_SD_mixup/lveval_loogle_SD_mixup_gen_d7ea36.py
  function get_dataset_names (line 36) | def get_dataset_names(dataset_name, length_levels):

FILE: opencompass/configs/datasets/lveval/lvevalmultifieldqa_en_mixup/lveval_multifieldqa_en_mixup_gen_d7ea36.py
  function get_dataset_names (line 39) | def get_dataset_names(dataset_name, length_levels):

FILE: opencompass/configs/datasets/lveval/lvevalmultifieldqa_zh_mixup/lveval_multifieldqa_zh_mixup_gen_0fbdad.py
  function get_dataset_names (line 39) | def get_dataset_names(dataset_name, length_levels):

FILE: opencompass/configs/datasets/mmmlu/mmmlu_prompt.py
  function get_few_shot_prompts_ar (line 5) | def get_few_shot_prompts_ar(_hint, _prompt):
  function get_few_shot_prompts_bn (line 21) | def get_few_shot_prompts_bn(_hint, _prompt):
  function get_few_shot_prompts_de (line 36) | def get_few_shot_prompts_de(_hint, _prompt):
  function get_few_shot_prompts_es (line 51) | def get_few_shot_prompts_es(_hint, _prompt):
  function get_few_shot_prompts_fr (line 66) | def get_few_shot_prompts_fr(_hint, _prompt):
  function get_few_shot_prompts_hi (line 81) | def get_few_shot_prompts_hi(_hint, _prompt):
  function get_few_shot_prompts_id (line 96) | def get_few_shot_prompts_id(_hint, _prompt):
  function get_few_shot_prompts_it (line 111) | def get_few_shot_prompts_it(_hint, _prompt):
  function get_few_shot_prompts_ja (line 126) | def get_few_shot_prompts_ja(_hint, _prompt):
  function get_few_shot_prompts_ko (line 141) | def get_few_shot_prompts_ko(_hint, _prompt):
  function get_few_shot_prompts_pt (line 156) | def get_few_shot_prompts_pt(_hint, _prompt):
  function get_few_shot_prompts_zh (line 171) | def get_few_shot_prompts_zh(_hint, _prompt):
  function get_few_shot_prompts_sw (line 186) | def get_few_shot_prompts_sw(_hint, _prompt):
  function get_few_shot_prompts_yo (line 200) | def get_few_shot_prompts_yo(_hint, _prompt):

FILE: opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_reasoning_1000k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_retrieval_1000k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_single_1000k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_multi_reasoning_128k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_multi_retrieval_128k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_single_128k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_multi_reasoning_200k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_multi_retrieval_200k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_single_200k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_multi_reasoning_256k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_multi_retrieval_256k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_single_256k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_multi_reasoning_32k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_multi_retrieval_32k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_single_32k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_multi_reasoning_4k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_multi_retrieval_4k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_single_4k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_reasoning_8k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_8k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_compare_batch_8k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_single_8k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench/needlebench_base/needlebench_single.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/datasets/needlebench_v2/needlebench_v2_8k/needlebench_v2_multi_retrieval_compare_batch_8k.py
  function logistic (line 11) | def logistic(x, L=100, x0=50, k=0.1):
  function generate_linear_space (line 15) | def generate_linear_space(start, end, num):
  function generate_depth_percents (line 24) | def generate_depth_percents(intervals, interval_type):

FILE: opencompass/configs/summarizers/needlebench.py
  function create_m_rs_names_list (line 5) | def create_m_rs_names_list(context_lengths, depths, needle_counts,
  function create_summarizer (line 33) | def create_summarizer(context_lengths, depths, dataset_size,

FILE: opencompass/configs/summarizers/scireasoner.py
  function calculate_opi (line 6) | def calculate_opi(scores):
  function calculate_smol (line 14) | def calculate_smol(scores):
  function calculate_mol (line 23) | def calculate_mol(scores):
  function calculate_llm4mat (line 31) | def calculate_llm4mat(scores):
  function calculate_unconditional_gen (line 37) | def calculate_unconditional_gen(scores):
  class SciReasonerSummarizer (line 177) | class SciReasonerSummarizer(DefaultSummarizer):
    method __init__ (line 178) | def __init__(self, mini_set=False, show_details=False, *args, **kwargs):
    method _calculate_group_metrics (line 194) | def _calculate_group_metrics(self, raw_results, parsed_results, datase...

FILE: opencompass/datasets/CARDBiomedBench.py
  function _parse (line 8) | def _parse(item, prompt_mode):
  class CARDBiomedBenchDataset (line 18) | class CARDBiomedBenchDataset(BaseDataset):
    method load (line 21) | def load(path: str, prompt_mode: str, **kwargs):

FILE: opencompass/datasets/ClinicBench.py
  class ClinicBenchDataset (line 9) | class ClinicBenchDataset(BaseDataset):
    method load_single (line 12) | def load_single(path):
    method load (line 17) | def load(path):

FILE: opencompass/datasets/Earth_Silver.py
  class Earth_Silver_MCQDataset (line 9) | class Earth_Silver_MCQDataset(BaseDataset):
    method load (line 12) | def load(path: str, prompt_mode: str = 'zero-shot', **kwargs):

FILE: opencompass/datasets/FinanceIQ.py
  class FinanceIQDataset (line 13) | class FinanceIQDataset(BaseDataset):
    method load (line 21) | def load(path: str, name: str):

FILE: opencompass/datasets/GaokaoBench.py
  class GaokaoBenchDataset (line 15) | class GaokaoBenchDataset(BaseDataset):
    method load (line 18) | def load(path: str, filename: str, name: str):
  class GaokaoBenchEvaluator (line 37) | class GaokaoBenchEvaluator(BaseEvaluator):
    method __init__ (line 39) | def __init__(self, question_type) -> None:
    method do_predictions_postprocess (line 44) | def do_predictions_postprocess(self, model_output, answer_lenth=None):
    method ensure_same_length (line 91) | def ensure_same_length(self, pred, refr):
    method score (line 96) | def score(self, predictions, references):
  function _gaokao_register (line 152) | def _gaokao_register(question_type):

FILE: opencompass/datasets/IFBench/evaluation_lib.py
  class InputExample (line 30) | class InputExample:
  class OutputExample (line 38) | class OutputExample:
  function read_prompt_list (line 46) | def read_prompt_list(input_jsonl_filename):
  function write_outputs (line 60) | def write_outputs(output_jsonl_filename, outputs):
  function test_instruction_following_strict (line 78) | def test_instruction_following_strict(
  function test_instruction_following_loose (line 112) | def test_instruction_following_loose(
  function read_prompt_to_response_dict (line 167) | def read_prompt_to_response_dict(input_jsonl_filename):
  function print_report (line 177) | def print_report(outputs):

FILE: opencompass/datasets/IFBench/ifbench.py
  class IFBenchEvaluator (line 7) | class IFBenchEvaluator(BaseEvaluator):
    method score (line 9) | def score(self, predictions, references, origin_prompt):

FILE: opencompass/datasets/IFBench/instructions.py
  class Instruction (line 86) | class Instruction:
    method __init__ (line 89) | def __init__(self, instruction_id):
    method build_description (line 92) | def build_description(self, **kwargs):
    method get_instruction_args (line 95) | def get_instruction_args(self):
    method get_instruction_args_keys (line 98) | def get_instruction_args_keys(self):
    method check_following (line 101) | def check_following(self, value):
  class WordCountRangeChecker (line 107) | class WordCountRangeChecker(Instruction):
    method build_description (line 110) | def build_description(self, *, min_words=None, max_words=None):
    method get_instruction_args (line 138) | def get_instruction_args(self):
    method get_instruction_args_keys (line 142) | def get_instruction_args_keys(self):
    method check_following (line 146) | def check_following(self, value):
  class UniqueWordCountChecker (line 152) | class UniqueWordCountChecker(Instruction):
    method build_description (line 155) | def build_description(self, *, N=None):
    method get_instruction_args (line 175) | def get_instruction_args(self):
    method get_instruction_args_keys (line 179) | def get_instruction_args_keys(self):
    method check_following (line 183) | def check_following(self, value):
  class StopWordPercentageChecker (line 193) | class StopWordPercentageChecker(Instruction):
    method build_description (line 196) | def build_description(self, *, percentage=None):
    method get_instruction_args (line 214) | def get_instruction_args(self):
    method get_instruction_args_keys (line 218) | def get_instruction_args_keys(self):
    method check_following (line 222) | def check_following(self, value):
  class SentTypeRatioChecker (line 230) | class SentTypeRatioChecker(Instruction):
    method build_description (line 233) | def build_description(self):
    method get_instruction_args (line 239) | def get_instruction_args(self):
    method get_instruction_args_keys (line 242) | def get_instruction_args_keys(self):
    method check_following (line 246) | def check_following(self, value):
  class SentBalanceChecker (line 257) | class SentBalanceChecker(Instruction):
    method build_description (line 260) | def build_description(self):
    method get_instruction_args (line 266) | def get_instruction_args(self):
    method get_instruction_args_keys (line 269) | def get_instruction_args_keys(self):
    method check_following (line 273) | def check_following(self, value):
  class ConjunctionCountChecker (line 285) | class ConjunctionCountChecker(Instruction):
    method build_description (line 288) | def build_description(self, *, small_n=None):
    method get_instruction_args (line 306) | def get_instruction_args(self):
    method get_instruction_args_keys (line 310) | def get_instruction_args_keys(self):
    method check_following (line 314) | def check_following(self, value):
  class PersonNameCountChecker (line 326) | class PersonNameCountChecker(Instruction):
    method build_description (line 329) | def build_description(self, *, N=None):
    method get_instruction_args (line 346) | def get_instruction_args(self):
    method get_instruction_args_keys (line 350) | def get_instruction_args_keys(self):
    method check_following (line 354) | def check_following(self, value):
  class NGramOverlapChecker (line 406) | class NGramOverlapChecker(Instruction):
    method build_description (line 409) | def build_description(self, *, reference_text=None, percentage=None):
    method get_instruction_args (line 428) | def get_instruction_args(self):
    method get_instruction_args_keys (line 432) | def get_instruction_args_keys(self):
    method check_following (line 436) | def check_following(self, value):
  class NumbersCountChecker (line 445) | class NumbersCountChecker(Instruction):
    method build_description (line 448) | def build_description(self, *, N=None):
    method get_instruction_args (line 465) | def get_instruction_args(self):
    method get_instruction_args_keys (line 469) | def get_instruction_args_keys(self):
    method check_following (line 473) | def check_following(self, value):
  class AlphabetLoopChecker (line 481) | class AlphabetLoopChecker(Instruction):
    method build_description (line 484) | def build_description(self):
    method get_instruction_args (line 489) | def get_instruction_args(self):
    method get_instruction_args_keys (line 493) | def get_instruction_args_keys(self):
    method check_following (line 497) | def check_following(self, value):
  class SingleVowelParagraphChecker (line 515) | class SingleVowelParagraphChecker(Instruction):
    method build_description (line 518) | def build_description(self):
    method get_instruction_args (line 523) | def get_instruction_args(self):
    method get_instruction_args_keys (line 527) | def get_instruction_args_keys(self):
    method check_following (line 531) | def check_following(self, value):
  class ConsonantClusterChecker (line 543) | class ConsonantClusterChecker(Instruction):
    method build_description (line 546) | def build_description(self):
    method get_instruction_args (line 551) | def get_instruction_args(self):
    method get_instruction_args_keys (line 555) | def get_instruction_args_keys(self):
    method check_following (line 559) | def check_following(self, value):
  class IncrementingAlliterationChecker (line 574) | class IncrementingAlliterationChecker(Instruction):
    method build_description (line 577) | def build_description(self):
    method get_instruction_args (line 583) | def get_instruction_args(self):
    method get_instruction_args_keys (line 587) | def get_instruction_args_keys(self):
    method check_following (line 591) | def check_following(self, value):
  class PalindromeChecker (line 619) | class PalindromeChecker(Instruction):
    method build_description (line 622) | def build_description(self):
    method get_instruction_args (line 627) | def get_instruction_args(self):
    method get_instruction_args_keys (line 631) | def get_instruction_args_keys(self):
    method check_following (line 635) | def check_following(self, value):
  class PunctuationCoverChecker (line 643) | class PunctuationCoverChecker(Instruction):
    method build_description (line 646) | def build_description(self):
    method get_instruction_args (line 651) | def get_instruction_args(self):
    method get_instruction_args_keys (line 655) | def get_instruction_args_keys(self):
    method check_following (line 659) | def check_following(self, value):
  class NestedParenthesesChecker (line 673) | class NestedParenthesesChecker(Instruction):
    method build_description (line 676) | def build_description(self):
    method get_instruction_args (line 681) | def get_instruction_args(self):
    method get_instruction_args_keys (line 685) | def get_instruction_args_keys(self):
    method check_following (line 689) | def check_following(self, value):
  class NestedQuotesChecker (line 719) | class NestedQuotesChecker(Instruction):
    method build_description (line 722) | def build_description(self):
    method get_instruction_args (line 727) | def get_instruction_args(self):
    method get_instruction_args_keys (line 731) | def get_instruction_args_keys(self):
    method check_following (line 735) | def check_following(self, value):
  class PrimeLengthsChecker (line 756) | class PrimeLengthsChecker(Instruction):
    method build_description (line 759) | def build_description(self):
    method get_instruction_args (line 764) | def get_instruction_args(self):
    method get_instruction_args_keys (line 768) | def get_instruction_args_keys(self):
    method check_following (line 772) | def check_following(self, value):
  class OptionsResponseChecker (line 783) | class OptionsResponseChecker(Instruction):
    method build_description (line 786) | def build_description(self, *, options=None):
    method get_instruction_args (line 817) | def get_instruction_args(self):
    method get_instruction_args_keys (line 821) | def get_instruction_args_keys(self):
    method check_following (line 825) | def check_following(self, value):
  class NewLineWordsChecker (line 836) | class NewLineWordsChecker(Instruction):
    method build_description (line 839) | def build_description(self):
    method get_instruction_args (line 844) | def get_instruction_args(self):
    method get_instruction_args_keys (line 848) | def get_instruction_args_keys(self):
    method check_following (line 852) | def check_following(self, value):
  class EmojiSentenceChecker (line 861) | class EmojiSentenceChecker(Instruction):
    method build_description (line 864) | def build_description(self):
    method get_instruction_args (line 870) | def get_instruction_args(self):
    method get_instruction_args_keys (line 874) | def get_instruction_args_keys(self):
    method check_following (line 878) | def check_following(self, value):
  class CharacterCountUniqueWordsChecker (line 904) | class CharacterCountUniqueWordsChecker(Instruction):
    method build_description (line 907) | def build_description(self):
    method get_instruction_args (line 913) | def get_instruction_args(self):
    method get_instruction_args_keys (line 917) | def get_instruction_args_keys(self):
    method check_following (line 921) | def check_following(self, value):
  class NthWordJapaneseChecker (line 933) | class NthWordJapaneseChecker(Instruction):
    method build_description (line 936) | def build_description(self, *, N=None):
    method get_instruction_args (line 959) | def get_instruction_args(self):
    method get_instruction_args_keys (line 963) | def get_instruction_args_keys(self):
    method check_following (line 967) | def check_following(self, value):
  class StartWithVerbChecker (line 992) | class StartWithVerbChecker(Instruction):
    method build_description (line 995) | def build_description(self):
    method get_instruction_args (line 1001) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1005) | def get_instruction_args_keys(self):
    method check_following (line 1009) | def check_following(self, value):
  class LimitedWordRepeatChecker (line 1015) | class LimitedWordRepeatChecker(Instruction):
    method build_description (line 1018) | def build_description(self, *, small_n=None):
    method get_instruction_args (line 1035) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1039) | def get_instruction_args_keys(self):
    method check_following (line 1043) | def check_following(self, value):
  class IncludeKeywordChecker (line 1053) | class IncludeKeywordChecker(Instruction):
    method build_description (line 1056) | def build_description(self, *, word=None, N=None):
    method get_instruction_args (line 1083) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1087) | def get_instruction_args_keys(self):
    method check_following (line 1091) | def check_following(self, value):
  class PronounCountChecker (line 1099) | class PronounCountChecker(Instruction):
    method build_description (line 1102) | def build_description(self, *, N=None):
    method get_instruction_args (line 1119) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1123) | def get_instruction_args_keys(self):
    method check_following (line 1127) | def check_following(self, value):
  class AlternateParitySyllablesChecker (line 1141) | class AlternateParitySyllablesChecker(Instruction):
    method build_description (line 1144) | def build_description(self):
    method get_instruction_args (line 1149) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1153) | def get_instruction_args_keys(self):
    method check_following (line 1157) | def check_following(self, value):
  class LastWordFirstNextChecker (line 1164) | class LastWordFirstNextChecker(Instruction):
    method build_description (line 1167) | def build_description(self):
    method get_instruction_args (line 1173) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1177) | def get_instruction_args_keys(self):
    method check_following (line 1181) | def check_following(self, value):
  class ParagraphLastFirstWordMatchChecker (line 1192) | class ParagraphLastFirstWordMatchChecker(Instruction):
    method build_description (line 1195) | def build_description(self):
    method get_instruction_args (line 1200) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1204) | def get_instruction_args_keys(self):
    method check_following (line 1208) | def check_following(self, value):
  class IncrementingWordCountChecker (line 1223) | class IncrementingWordCountChecker(Instruction):
    method build_description (line 1226) | def build_description(self, *, small_n=None):
    method get_instruction_args (line 1245) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1249) | def get_instruction_args_keys(self):
    method check_following (line 1253) | def check_following(self, value):
  class NoConsecutiveFirstLetterChecker (line 1270) | class NoConsecutiveFirstLetterChecker(Instruction):
    method build_description (line 1273) | def build_description(self):
    method get_instruction_args (line 1278) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1282) | def get_instruction_args_keys(self):
    method check_following (line 1286) | def check_following(self, value):
  class IndentStairsChecker (line 1297) | class IndentStairsChecker(Instruction):
    method build_description (line 1300) | def build_description(self):
    method get_instruction_args (line 1305) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1309) | def get_instruction_args_keys(self):
    method check_following (line 1313) | def check_following(self, value):
  class QuoteExplanationChecker (line 1325) | class QuoteExplanationChecker(Instruction):
    method build_description (line 1328) | def build_description(self):
    method get_instruction_args (line 1333) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1337) | def get_instruction_args_keys(self):
    method check_following (line 1341) | def check_following(self, value):
  class SpecialBulletPointsChecker (line 1354) | class SpecialBulletPointsChecker(Instruction):
    method build_description (line 1357) | def build_description(self, *, sep=None):
    method get_instruction_args (line 1373) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1377) | def get_instruction_args_keys(self):
    method check_following (line 1381) | def check_following(self, value):
  class ItalicsThesisChecker (line 1386) | class ItalicsThesisChecker(Instruction):
    method build_description (line 1389) | def build_description(self):
    method get_instruction_args (line 1394) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1398) | def get_instruction_args_keys(self):
    method check_following (line 1402) | def check_following(self, value):
  class SubBulletPointsChecker (line 1423) | class SubBulletPointsChecker(Instruction):
    method build_description (line 1426) | def build_description(self):
    method get_instruction_args (line 1431) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1435) | def get_instruction_args_keys(self):
    method check_following (line 1439) | def check_following(self, value):
  class SomeBulletPointsChecker (line 1449) | class SomeBulletPointsChecker(Instruction):
    method build_description (line 1452) | def build_description(self):
    method get_instruction_args (line 1457) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1461) | def get_instruction_args_keys(self):
    method check_following (line 1465) | def check_following(self, value):
  class PrintMultiplesChecker (line 1486) | class PrintMultiplesChecker(Instruction):
    method build_description (line 1489) | def build_description(self, **kwargs):
    method get_instruction_args (line 1493) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1497) | def get_instruction_args_keys(self):
    method check_following (line 1501) | def check_following(self, value):
  class MultipleChoiceQuestionsChecker (line 1509) | class MultipleChoiceQuestionsChecker(Instruction):
    method build_description (line 1512) | def build_description(self, **kwargs):
    method get_instruction_args (line 1516) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1520) | def get_instruction_args_keys(self):
    method check_following (line 1524) | def check_following(self, value):
  class ReverseNewlineChecker (line 1557) | class ReverseNewlineChecker(Instruction):
    method build_description (line 1560) | def build_description(self, **kwargs):
    method get_instruction_args (line 1564) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1568) | def get_instruction_args_keys(self):
    method check_following (line 1572) | def check_following(self, value):
  class WordReverseOrderChecker (line 1619) | class WordReverseOrderChecker(Instruction):
    method build_description (line 1622) | def build_description(self, **kwargs):
    method get_instruction_args (line 1627) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1631) | def get_instruction_args_keys(self):
    method check_following (line 1635) | def check_following(self, value):
  class CharacterReverseOrderChecker (line 1644) | class CharacterReverseOrderChecker(Instruction):
    method build_description (line 1647) | def build_description(self, **kwargs):
    method get_instruction_args (line 1651) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1655) | def get_instruction_args_keys(self):
    method check_following (line 1659) | def check_following(self, value):
  class SentenceAlphabetChecker (line 1664) | class SentenceAlphabetChecker(Instruction):
    method build_description (line 1667) | def build_description(self, **kwargs):
    method get_instruction_args (line 1672) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1676) | def get_instruction_args_keys(self):
    method check_following (line 1680) | def check_following(self, value):
  class EuropeanCapitalsSortChecker (line 1690) | class EuropeanCapitalsSortChecker(Instruction):
    method build_description (line 1693) | def build_description(self, **kwargs):
    method get_instruction_args (line 1698) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1702) | def get_instruction_args_keys(self):
    method check_following (line 1706) | def check_following(self, value):
  class CityCSVChecker (line 1739) | class CityCSVChecker(Instruction):
    method build_description (line 1742) | def build_description(self, **kwargs):
    method get_instruction_args (line 1747) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1751) | def get_instruction_args_keys(self):
    method check_following (line 1755) | def check_following(self, value):
  class SpecialCharacterCSVChecker (line 1772) | class SpecialCharacterCSVChecker(Instruction):
    method build_description (line 1775) | def build_description(self, **kwargs):
    method get_instruction_args (line 1780) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1784) | def get_instruction_args_keys(self):
    method check_following (line 1788) | def check_following(self, value):
  class QuotesCSVChecker (line 1812) | class QuotesCSVChecker(Instruction):
    method build_description (line 1815) | def build_description(self, **kwargs):
    method get_instruction_args (line 1820) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1824) | def get_instruction_args_keys(self):
    method check_following (line 1828) | def check_following(self, value):
  class DateFormatListChecker (line 1852) | class DateFormatListChecker(Instruction):
    method build_description (line 1855) | def build_description(self, **kwargs):
    method get_instruction_args (line 1860) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1864) | def get_instruction_args_keys(self):
    method check_following (line 1868) | def check_following(self, value):
  class KeywordsMultipleChecker (line 1890) | class KeywordsMultipleChecker(Instruction):
    method build_description (line 1893) | def build_description(self, *, keyword1=None, keyword2=None, keyword3=...
    method get_instruction_args (line 1920) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1924) | def get_instruction_args_keys(self):
    method check_following (line 1927) | def check_following(self, value):
  class KeywordSpecificPositionChecker (line 1935) | class KeywordSpecificPositionChecker(Instruction):
    method build_description (line 1938) | def build_description(self, keyword=None, n=None, m=None):
    method get_instruction_args (line 1970) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1974) | def get_instruction_args_keys(self):
    method check_following (line 1978) | def check_following(self, value):
  class WordsPositionChecker (line 2000) | class WordsPositionChecker(Instruction):
    method build_description (line 2003) | def build_description(self, *, keyword=None):
    method get_instruction_args (line 2021) | def get_instruction_args(self):
    method get_instruction_args_keys (line 2025) | def get_instruction_args_keys(self):
    method check_following (line 2029) | def check_following(self, value):
  class RepeatChangeChecker (line 2048) | class RepeatChangeChecker(Instruction):
    method build_description (line 2051) | def build_description(self, *, prompt_to_repeat=None):
    method get_instruction_args (line 2070) | def get_instruction_args(self):
    method get_instruction_args_keys (line 2074) | def get_instruction_args_keys(self):
    method check_following (line 2078) | def check_following(self, value):
  class RepeatSimpleChecker (line 2096) | class RepeatSimpleChecker(Instruction):
    method build_description (line 2099) | def build_description(self):
    method get_instruction_args (line 2106) | def get_instruction_args(self):
    method get_instruction_args_keys (line 2109) | def get_instruction_args_keys(self):
    method check_following (line 2113) | def check_following(self, value):
  class RepeatSpanChecker (line 2126) | class RepeatSpanChecker(Instruction):
    method build_description (line 2129) | def build_description(self, prompt_to_repeat=None, n_start=None, n_end...
    method get_instruction_args (line 2156) | def get_instruction_args(self):
    method get_instruction_args_keys (line 2160) | def get_instruction_args_keys(self):
    method check_following (line 2164) | def check_following(self, value):
  class TitleCaseChecker (line 2171) | class TitleCaseChecker(Instruction):
    method build_description (line 2174) | def build_description(self):
    method get_instruction_args (line 2181) | def get_instruction_args(self):
    method get_instruction_args_keys (line 2185) | def get_instruction_args_keys(self):
    method check_following (line 2189) | def check_following(self, value):
  class OutputTemplateChecker (line 2210) | class OutputTemplateChecker(Instruction):
    method build_description (line 2213) | def build_description(self):
    method get_instruction_args (line 2220) | def get_instruction_args(self):
    method get_instruction_args_keys (line 2224) | def get_instruction_args_keys(self):
    method check_following (line 2228) | def check_following(self, value):
  class NoWhitespaceChecker (line 2244) | class NoWhitespaceChecker(Instruction):
    method build_description (line 2247) | def build_description(self):
    method get_instruction_args (line 2254) | def get_instruction_args(self):
    method get_instruction_args_keys (line 2258) | def get_instruction_args_keys(self):
    method check_following (line 2262) | def check_following(self, value):

FILE: opencompass/datasets/IFBench/instructions_util.py
  function split_into_sentences (line 1579) | def split_into_sentences(text):
  function count_words (line 1630) | def count_words(text):
  function _get_sentence_tokenizer (line 1639) | def _get_sentence_tokenizer():
  function count_stopwords (line 1643) | def count_stopwords(text):
  function generate_keywords (line 1652) | def generate_keywords(num_keywords):

FILE: opencompass/datasets/IFEval/evaluation_main.py
  class InputExample (line 44) | class InputExample:
  class OutputExample (line 52) | class OutputExample:
  function test_instruction_following_strict (line 60) | def test_instruction_following_strict(
  function test_instruction_following_loose (line 91) | def test_instruction_following_loose(

FILE: opencompass/datasets/IFEval/ifeval.py
  class IFEvalDataset (line 15) | class IFEvalDataset(BaseDataset):
    method load (line 18) | def load(path):
  class IFEvaluator (line 29) | class IFEvaluator(BaseEvaluator):
    method score (line 31) | def score(self, predictions, references, origin_prompt):

FILE: opencompass/datasets/IFEval/instructions.py
  class Instruction (line 98) | class Instruction:
    method __init__ (line 101) | def __init__(self, instruction_id):
    method build_description (line 104) | def build_description(self, **kwargs):
    method get_instruction_args (line 107) | def get_instruction_args(self):
    method get_instruction_args_keys (line 110) | def get_instruction_args_keys(self):
    method check_following (line 114) | def check_following(self, value):
  class ResponseLanguageChecker (line 118) | class ResponseLanguageChecker(Instruction):
    method build_description (line 121) | def build_description(self, *, language=None):
    method get_instruction_args (line 144) | def get_instruction_args(self):
    method get_instruction_args_keys (line 148) | def get_instruction_args_keys(self):
    method check_following (line 152) | def check_following(self, value):
  class NumberOfSentences (line 173) | class NumberOfSentences(Instruction):
    method build_description (line 176) | def build_description(self, *, num_sentences=None, relation=None):
    method get_instruction_args (line 214) | def get_instruction_args(self):
    method get_instruction_args_keys (line 221) | def get_instruction_args_keys(self):
    method check_following (line 225) | def check_following(self, value):
  class PlaceholderChecker (line 245) | class PlaceholderChecker(Instruction):
    method build_description (line 248) | def build_description(self, *, num_placeholders=None):
    method get_instruction_args (line 267) | def get_instruction_args(self):
    method get_instruction_args_keys (line 271) | def get_instruction_args_keys(self):
    method check_following (line 275) | def check_following(self, value):
  class BulletListChecker (line 290) | class BulletListChecker(Instruction):
    method build_description (line 293) | def build_description(self, *, num_bullets=None):
    method get_instruction_args (line 312) | def get_instruction_args(self):
    method get_instruction_args_keys (line 316) | def get_instruction_args_keys(self):
    method check_following (line 320) | def check_following(self, value):
  class ConstrainedResponseChecker (line 337) | class ConstrainedResponseChecker(Instruction):
    method build_description (line 340) | def build_description(self):
    method get_instruction_args (line 349) | def get_instruction_args(self):
    method get_instruction_args_keys (line 353) | def get_instruction_args_keys(self):
    method check_following (line 357) | def check_following(self, value):
  class ConstrainedStartChecker (line 374) | class ConstrainedStartChecker(Instruction):
    method build_description (line 377) | def build_description(self, *, starter=None):
    method get_instruction_args (line 396) | def get_instruction_args(self):
    method get_instruction_args_keys (line 400) | def get_instruction_args_keys(self):
    method check_following (line 404) | def check_following(self, value):
  class HighlightSectionChecker (line 422) | class HighlightSectionChecker(Instruction):
    method build_description (line 425) | def build_description(self, *, num_highlights=None):
    method get_instruction_args (line 446) | def get_instruction_args(self):
    method get_instruction_args_keys (line 450) | def get_instruction_args_keys(self):
    method check_following (line 454) | def check_following(self, value):
  class SectionChecker (line 478) | class SectionChecker(Instruction):
    method build_description (line 481) | def build_description(self, *, section_spliter=None, num_sections=None):
    method get_instruction_args (line 511) | def get_instruction_args(self):
    method get_instruction_args_keys (line 518) | def get_instruction_args_keys(self):
    method check_following (line 522) | def check_following(self, value):
  class ParagraphChecker (line 541) | class ParagraphChecker(Instruction):
    method build_description (line 544) | def build_description(self, *, num_paragraphs=None):
    method get_instruction_args (line 564) | def get_instruction_args(self):
    method get_instruction_args_keys (line 568) | def get_instruction_args_keys(self):
    method check_following (line 572) | def check_following(self, value):
  class PostscriptChecker (line 596) | class PostscriptChecker(Instruction):
    method build_description (line 599) | def build_description(self, *, postscript_marker=None):
    method get_instruction_args (line 621) | def get_instruction_args(self):
    method get_instruction_args_keys (line 625) | def get_instruction_args_keys(self):
    method check_following (line 629) | def check_following(self, value):
  class RephraseChecker (line 652) | class RephraseChecker(Instruction):
    method build_description (line 655) | def build_description(self, *, original_message):
    method get_instruction_args (line 679) | def get_instruction_args(self):
    method get_instruction_args_keys (line 683) | def get_instruction_args_keys(self):
    method check_following (line 687) | def check_following(self, value):
    method is_change (line 709) | def is_change(self, response):
    method strip_changes (line 714) | def strip_changes(self, response):
  class KeywordChecker (line 719) | class KeywordChecker(Instruction):
    method build_description (line 722) | def build_description(self, *, keywords=None):
    method get_instruction_args (line 745) | def get_instruction_args(self):
    method get_instruction_args_keys (line 749) | def get_instruction_args_keys(self):
    method check_following (line 753) | def check_following(self, value):
  class KeywordFrequencyChecker (line 761) | class KeywordFrequencyChecker(Instruction):
    method build_description (line 764) | def build_description(self,
    method get_instruction_args (line 812) | def get_instruction_args(self):
    method get_instruction_args_keys (line 820) | def get_instruction_args_keys(self):
    method check_following (line 824) | def check_following(self, value):
  class NumberOfWords (line 836) | class NumberOfWords(Instruction):
    method build_description (line 839) | def build_description(self, *, num_words=None, relation=None):
    method get_instruction_args (line 875) | def get_instruction_args(self):
    method get_instruction_args_keys (line 882) | def get_instruction_args_keys(self):
    method check_following (line 886) | def check_following(self, value):
  class JsonFormat (line 896) | class JsonFormat(Instruction):
    method build_description (line 899) | def build_description(self):
    method get_instruction_args (line 905) | def get_instruction_args(self):
    method get_instruction_args_keys (line 909) | def get_instruction_args_keys(self):
    method check_following (line 913) | def check_following(self, value):
  class ParagraphFirstWordCheck (line 924) | class ParagraphFirstWordCheck(Instruction):
    method build_description (line 927) | def build_description(self,
    method get_instruction_args (line 970) | def get_instruction_args(self):
    method get_instruction_args_keys (line 978) | def get_instruction_args_keys(self):
    method check_following (line 982) | def check_following(self, value):
  class KeySentenceChecker (line 1029) | class KeySentenceChecker(Instruction):
    method build_description (line 1032) | def build_description(self, key_sentences=None, num_sentences=None):
    method get_instruction_args (line 1064) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1071) | def get_instruction_args_keys(self):
    method check_following (line 1075) | def check_following(self, value):
  class ForbiddenWords (line 1086) | class ForbiddenWords(Instruction):
    method build_description (line 1089) | def build_description(self, forbidden_words=None):
    method get_instruction_args (line 1112) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1116) | def get_instruction_args_keys(self):
    method check_following (line 1120) | def check_following(self, value):
  class RephraseParagraph (line 1128) | class RephraseParagraph(Instruction):
    method build_description (line 1131) | def build_description(self, *, original_paragraph, low, high):
    method get_instruction_args (line 1160) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1168) | def get_instruction_args_keys(self):
    method check_following (line 1172) | def check_following(self, value):
  class TwoResponsesChecker (line 1186) | class TwoResponsesChecker(Instruction):
    method build_description (line 1189) | def build_description(self):
    method get_instruction_args (line 1196) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1200) | def get_instruction_args_keys(self):
    method check_following (line 1204) | def check_following(self, value):
  class RepeatPromptThenAnswer (line 1225) | class RepeatPromptThenAnswer(Instruction):
    method build_description (line 1228) | def build_description(self, *, prompt_to_repeat=None):
    method get_instruction_args (line 1248) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1251) | def get_instruction_args_keys(self):
    method check_following (line 1255) | def check_following(self, value):
  class EndChecker (line 1262) | class EndChecker(Instruction):
    method build_description (line 1265) | def build_description(self, *, end_phrase=None):
    method get_instruction_args (line 1283) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1286) | def get_instruction_args_keys(self):
    method check_following (line 1290) | def check_following(self, value):
  class TitleChecker (line 1297) | class TitleChecker(Instruction):
    method build_description (line 1300) | def build_description(self):
    method get_instruction_args (line 1307) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1310) | def get_instruction_args_keys(self):
    method check_following (line 1314) | def check_following(self, value):
  class LetterFrequencyChecker (line 1326) | class LetterFrequencyChecker(Instruction):
    method build_description (line 1329) | def build_description(self,
    method get_instruction_args (line 1379) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1387) | def get_instruction_args_keys(self):
    method check_following (line 1391) | def check_following(self, value):
  class CapitalLettersEnglishChecker (line 1403) | class CapitalLettersEnglishChecker(Instruction):
    method build_description (line 1406) | def build_description(self):
    method get_instruction_args (line 1413) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1416) | def get_instruction_args_keys(self):
    method check_following (line 1420) | def check_following(self, value):
  class LowercaseLettersEnglishChecker (line 1434) | class LowercaseLettersEnglishChecker(Instruction):
    method build_description (line 1438) | def build_description(self):
    method get_instruction_args (line 1445) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1448) | def get_instruction_args_keys(self):
    method check_following (line 1452) | def check_following(self, value):
  class CommaChecker (line 1466) | class CommaChecker(Instruction):
    method build_description (line 1469) | def build_description(self):
    method get_instruction_args (line 1475) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1478) | def get_instruction_args_keys(self):
    method check_following (line 1482) | def check_following(self, value):
  class CapitalWordFrequencyChecker (line 1487) | class CapitalWordFrequencyChecker(Instruction):
    method build_description (line 1490) | def build_description(
    method get_instruction_args (line 1525) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1532) | def get_instruction_args_keys(self):
    method check_following (line 1536) | def check_following(self, value):
  class QuotationChecker (line 1550) | class QuotationChecker(Instruction):
    method build_description (line 1553) | def build_description(self):
    method get_instruction_args (line 1559) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1563) | def get_instruction_args_keys(self):
    method check_following (line 1567) | def check_following(self, value):

FILE: opencompass/datasets/IFEval/instructions_registry.py
  function conflict_make (line 175) | def conflict_make(conflicts):

FILE: opencompass/datasets/IFEval/instructions_util.py
  function split_into_sentences (line 71) | def split_into_sentences(text):
  function count_words (line 123) | def count_words(text):
  function _get_sentence_tokenizer (line 132) | def _get_sentence_tokenizer():
  function count_sentences (line 136) | def count_sentences(text):
  function generate_keywords (line 143) | def generate_keywords(num_keywords):

FILE: opencompass/datasets/LCBench.py
  class LCDataset (line 23) | class LCDataset(BaseDataset):
    method load (line 26) | def load(path: str,
  class TimeOutException (line 67) | class TimeOutException(Exception):
  function swallow_io (line 72) | def swallow_io():
  function time_limit (line 81) | def time_limit(seconds: float):
  class WriteOnlyStringIO (line 94) | class WriteOnlyStringIO(io.StringIO):
    method read (line 97) | def read(self, *args, **kwargs):
    method readline (line 100) | def readline(self, *args, **kwargs):
    method readlines (line 103) | def readlines(self, *args, **kwargs):
    method readable (line 106) | def readable(self, *args, **kwargs):
  class redirect_stdin (line 111) | class redirect_stdin(contextlib._RedirectStream):  # type: ignore
  class LCEvaluator (line 116) | class LCEvaluator(BaseEvaluator):
    method score (line 118) | def score(self, predictions, references):
    method _process_answer (line 203) | def _process_answer(self, text):
    method _process_test (line 265) | def _process_test(self, test_case, code):
  function execution (line 300) | def execution(programs, task_ids, timeout):
  class LCPassKEvaluator (line 344) | class LCPassKEvaluator(LCEvaluator):
    method __init__ (line 351) | def __init__(self, k=(1, 10, 100)) -> None:
    method estimate_pass_at_k (line 357) | def estimate_pass_at_k(
    method score (line 383) | def score(self, predictions, references):

FILE: opencompass/datasets/MMLUArabic.py
  class MMLUArabicDataset (line 13) | class MMLUArabicDataset(BaseDataset):
    method load (line 16) | def load(path: str, name: str):

FILE: opencompass/datasets/MedCalc_Bench.py
  function check_correctness (line 14) | def check_correctness(answer: str, ground_truth, calid, upper_limit,
  function extract_answer (line 73) | def extract_answer(answer, calid):
  function _parse (line 212) | def _parse(item, prompt_mode):
  class MedCalc_BenchDataset (line 231) | class MedCalc_BenchDataset(BaseDataset):
    method load (line 234) | def load(path: str, prompt_mode: str, **kwargs):
  class MedCalcOfficial_Evaluator (line 249) | class MedCalcOfficial_Evaluator(BaseEvaluator):
    method score (line 251) | def score(self, predictions, references, test_set):

FILE: opencompass/datasets/MedQA.py
  class MedQADataset (line 9) | class MedQADataset(BaseDataset):
    method load_single (line 12) | def load_single(path):
    method load (line 27) | def load(path):

FILE: opencompass/datasets/MedXpertQA.py
  function _parse (line 12) | def _parse(item, prompt_mode):
  class MedXpertQADataset (line 20) | class MedXpertQADataset(BaseDataset):
    method load (line 23) | def load(path: str, prompt_mode: str, **kwargs):
  class MedXpertQAEvaluator (line 35) | class MedXpertQAEvaluator(BaseEvaluator):
    method score (line 37) | def score(self, predictions, references, test_set):
  function answer_cleansing (line 59) | def answer_cleansing(
  function _generic_llmjudge_postprocess (line 106) | def _generic_llmjudge_postprocess(judgement: str):
  function MedXpertQA_llmjudge_postprocess (line 113) | def MedXpertQA_llmjudge_postprocess(

FILE: opencompass/datasets/Medbullets.py
  function _parse (line 13) | def _parse(item: dict, prompt_mode: str) -> dict:
  class MedbulletsDataset (line 37) | class MedbulletsDataset(BaseDataset):
    method load (line 40) | def load(path: str, prompt_mode: str = 'zero-shot', **kwargs):
  class MedbulletsEvaluator (line 60) | class MedbulletsEvaluator(BaseEvaluator):
    method score (line 62) | def score(self, predictions, references, test_set):
  function answer_cleansing (line 89) | def answer_cleansing(
  function _generic_llmjudge_postprocess (line 137) | def _generic_llmjudge_postprocess(judgement: str):
  function medbullets_llmjudge_postprocess (line 144) | def medbullets_llmjudge_postprocess(

FILE: opencompass/datasets/NPHardEval/cmp_GCP_D.py
  function q2text (line 18) | def q2text(q, p=gcp_dPrompts):
  class CMP_GCP_D_Dataset (line 35) | class CMP_GCP_D_Dataset(BaseDataset):
    method load (line 38) | def load(path: str):
  class CMP_GCP_D_Evaluator (line 60) | class CMP_GCP_D_Evaluator(BaseEvaluator):
    method score (line 62) | def score(self, predictions, references):
    method parse_xml_to_dict (line 93) | def parse_xml_to_dict(self, xml_string):
    method read_dimacs_format (line 115) | def read_dimacs_format(self, dimacs_str):
    method gcp_greedy_solution (line 132) | def gcp_greedy_solution(self, adjacency_list):
    method gcp_decision_check (line 147) | def gcp_decision_check(self, dimacs_str, answer, k_colors):

FILE: opencompass/datasets/NPHardEval/cmp_KSP.py
  function q2text (line 14) | def q2text(q, p=kspPrompts):
  class CMP_KSP_Dataset (line 29) | class CMP_KSP_Dataset(BaseDataset):
    method load (line 32) | def load(path: str):
  class CMP_KSP_Evaluator (line 54) | class CMP_KSP_Evaluator(BaseEvaluator):
    method score (line 56) | def score(self, predictions, references):
    method parse_xml_to_dict (line 91) | def parse_xml_to_dict(self, xml_string):
    method ksp_optimal_solution (line 113) | def ksp_optimal_solution(self, knapsacks, capacity):
    method kspCheck (line 133) | def kspCheck(self, instance, solution):

FILE: opencompass/datasets/NPHardEval/cmp_TSP_D.py
  function q2text (line 20) | def q2text(adj_matrix, distance_limit, p=tsp_dPrompts):
  class CMP_TSP_D_Dataset (line 37) | class CMP_TSP_D_Dataset(BaseDataset):
    method load (line 40) | def load(path: str):
  class CMP_TSP_D_Evaluator (line 67) | class CMP_TSP_D_Evaluator(BaseEvaluator):
    method score (line 69) | def score(self, predictions, references):
    method parse_xml_to_dict (line 106) | def parse_xml_to_dict(self, xml_string):
    method tsp_approx (line 128) | def tsp_approx(self, distance_matrix):
    method tsp_decision_check (line 137) | def tsp_decision_check(self, distance_matrix, threshold, tour):

FILE: opencompass/datasets/NPHardEval/hard_GCP.py
  function q2text (line 14) | def q2text(q, p=gcpPrompts):  # q is the data for the HP-hard question, ...
  class HardGCPDataset (line 32) | class HardGCPDataset(BaseDataset):
    method load (line 35) | def load(path: str):
  class HardGCPEvaluator (line 57) | class HardGCPEvaluator(BaseEvaluator):
    method score (line 59) | def score(self, predictions, references):
    method parse_xml_to_dict (line 89) | def parse_xml_to_dict(self, xml_string):
    method gcpCheck (line 117) | def gcpCheck(self, dimacs_str, answer_str):
    method read_dimacs_format (line 137) | def read_dimacs_format(self, dimacs_str):
    method parse_answer (line 158) | def parse_answer(self, llm_string):

FILE: opencompass/datasets/NPHardEval/hard_MSP.py
  function q2text (line 15) | def q2text(q, p=mspPrompts):  # q is the data for the HP-hard question, ...
  class Hard_MSP_Dataset (line 35) | class Hard_MSP_Dataset(BaseDataset):
    method load (line 38) | def load(path: str):
  class Hard_MSP_Evaluator (line 59) | class Hard_MSP_Evaluator(BaseEvaluator):
    method score (line 61) | def score(self, predictions, references):
    method mspCheck (line 88) | def mspCheck(self, instance, llm_string):
    method parse_xml_to_dict (line 179) | def parse_xml_to_dict(self, xml_string):

FILE: opencompass/datasets/NPHardEval/hard_TSP.py
  function q2text (line 17) | def q2text(q, p=tspPrompts):  # q is the data for the HP-hard question, ...
  class Hard_TSP_Dataset (line 33) | class Hard_TSP_Dataset(BaseDataset):
    method load (line 36) | def load(path: str):
  class Hard_TSP_Evaluator (line 61) | class Hard_TSP_Evaluator(BaseEvaluator):
    method score (line 63) | def score(self, predictions, references):
    method parse_xml_to_dict (line 91) | def parse_xml_to_dict(self, xml_string):
    method tspCheck (line 119) | def tspCheck(self, distance_matrix, llm_string):
    method greedy_tsp (line 186) | def greedy_tsp(self, distance_matrix):

FILE: opencompass/datasets/NPHardEval/p_BSP.py
  function q2text (line 14) | def q2text(q, p=bspPrompts):
  class P_BSP_Dataset (line 28) | class P_BSP_Dataset(BaseDataset):
    method load (line 31) | def load(path: str):
  class P_BSP_Evaluator (line 54) | class P_BSP_Evaluator(BaseEvaluator):
    method score (line 56) | def score(self, predictions, references):
    method parse_xml_to_dict (line 84) | def parse_xml_to_dict(self, xml_string):
    method bsp_check (line 106) | def bsp_check(self, instance, solution):

FILE: opencompass/datasets/NPHardEval/p_EDP.py
  function q2text (line 14) | def q2text(q, p=edpPrompts):
  class P_EDP_Dataset (line 25) | class P_EDP_Dataset(BaseDataset):
    method load (line 28) | def load(path: str):
  class P_EDP_Evaluator (line 51) | class P_EDP_Evaluator(BaseEvaluator):
    method score (line 53) | def score(self, predictions, references):
    method compute_min_edit_distance (line 81) | def compute_min_edit_distance(self, string_a, string_b):
    method edp_check (line 99) | def edp_check(self, instance, solution):
    method parse_xml_to_dict (line 121) | def parse_xml_to_dict(self, xml_string):

FILE: opencompass/datasets/NPHardEval/p_SPP.py
  function q2text (line 19) | def q2text(q, p=sppPrompts):
  class P_SPP_Dataset (line 38) | class P_SPP_Dataset(BaseDataset):
    method load (line 41) | def load(path: str):
  class P_SPP_Evaluator (line 61) | class P_SPP_Evaluator(BaseEvaluator):
    method score (line 63) | def score(self, predictions, references):
    method parse_xml_to_dict (line 91) | def parse_xml_to_dict(self, xml_string):
    method ssp_optimal_solution (line 120) | def ssp_optimal_solution(self, instance, source, target):
    method spp_check (line 140) | def spp_check(self, instance, solution, start_node=None, end_node=None):

FILE: opencompass/datasets/NPHardEval/utils.py
  function append_root_tags (line 5) | def append_root_tags(string):
  function parse_xml_to_dict (line 13) | def parse_xml_to_dict(xml_string):

FILE: opencompass/datasets/OlympiadBench.py
  class OlympiadBenchDataset (line 25) | class OlympiadBenchDataset(BaseDataset):
    method load (line 35) | def load(path: str, name: str = None, **kwargs):
  function get_single_answer_type_text (line 121) | def get_single_answer_type_text(answer_type, is_chinese):
  function get_answer_type_text (line 146) | def get_answer_type_text(answer_type, is_chinese, multiple_answer):
  class OlympiadBenchPrompter (line 186) | class OlympiadBenchPrompter:
    method __init__ (line 188) | def __init__(self):
    method make_prompt (line 191) | def make_prompt(
  class MathJudger (line 277) | class MathJudger:
    method __init__ (line 279) | def __init__(self):
    method split_by_comma (line 297) | def split_by_comma(self, expr: str):
    method trans_plus_minus_sign (line 315) | def trans_plus_minus_sign(self, expr_list: list):
    method judge (line 326) | def judge(self, expression1, expression2, precision=1e-8):
    method is_interval (line 375) | def is_interval(self, epr):
    method sympy_sub_pi (line 378) | def sympy_sub_pi(self, expression_sympy):
    method is_equal (line 381) | def is_equal(self, expression1, expression2):
    method numerical_equal (line 419) | def numerical_equal(
    method expression_equal (line 442) | def expression_equal(self, exp1, exp2):
    method equation_equal (line 494) | def equation_equal(self, expression1, expression2):
    method interval_equal (line 535) | def interval_equal(self, expression1, expression2):
    method preprocess (line 578) | def preprocess(self, expression1, expression2):
    method can_compute_power (line 644) | def can_compute_power(self, expr):
  function olympiadbench_postprocess_v2 (line 680) | def olympiadbench_postprocess_v2(text: str,
  class OlympiadBenchEvaluator (line 702) | class OlympiadBenchEvaluator(BaseEvaluator):
    method __init__ (line 705) | def __init__(self, version='v1'):
    method score (line 710) | def score(self, predictions, references):  # Remove questions parameter
  class OlympiadBenchTemplate (line 765) | class OlympiadBenchTemplate(PromptTemplate):
    method __init__ (line 768) | def __init__(self):
    method generate_item (line 774) | def generate_item(self, entry: Dict, *args, **kwargs) -> str:

FILE: opencompass/datasets/OpenFinData.py
  class OpenFinDataDataset (line 14) | class OpenFinDataDataset(BaseDataset):
    method load (line 17) | def load(path: str, name: str):
  class OpenFinDataKWEvaluator (line 25) | class OpenFinDataKWEvaluator(BaseEvaluator):
    method __init__ (line 27) | def __init__(self, ):
    method score (line 30) | def score(self, predictions, references):

FILE: opencompass/datasets/OpenSWI.py
  class OpenSWIDataset (line 17) | class OpenSWIDataset(BaseDataset):
    method load (line 20) | def load(path: str, name: str):
  function extract_list (line 39) | def extract_list(text):
  class OpenSWIMSEEvaluator (line 55) | class OpenSWIMSEEvaluator(BaseEvaluator):
    method __init__ (line 58) | def __init__(self) -> None:
    method score (line 61) | def score(self, predictions, references):

FILE: opencompass/datasets/PI_LLM.py
  class PILLMDataset (line 11) | class PILLMDataset(BaseDataset):
    method load (line 33) | def load(**kwargs) -> Dataset:

FILE: opencompass/datasets/PMMEval/flores.py
  function wmt_postprocess (line 18) | def wmt_postprocess(text: str, lang: str) -> str:
  function compute_maximum_bleu_value (line 27) | def compute_maximum_bleu_value(gen: str, ref: str, lang: str):
  function trim_multiple_space (line 56) | def trim_multiple_space(tokes):
  class SpaceTokenizer (line 60) | class SpaceTokenizer(object):
    method __call__ (line 62) | def __call__(self, sent):
  class NonASCIITokenizer (line 69) | class NonASCIITokenizer(object):
    method __init__ (line 71) | def __init__(self):
    method __call__ (line 78) | def __call__(self, sent):
  function build_tokenizer (line 93) | def build_tokenizer(lang: str):
  function tokenize (line 102) | def tokenize(sent, lang):
  function pmmeval_flores_postprocess (line 109) | def pmmeval_flores_postprocess(text: str, lang_fullname: str) -> Tuple[s...
  class PMMEvalFloresDataset (line 114) | class PMMEvalFloresDataset(BaseDataset):
    method load (line 117) | def load(path: str, lang_fullname: str):
  class PMMEvalFloresEvaluator (line 138) | class PMMEvalFloresEvaluator(BaseEvaluator):
    method score (line 140) | def score(self, predictions, references):

FILE: opencompass/datasets/PMMEval/humanevalxl.py
  class PMMEvalHumanEvalXLDataset (line 27) | class PMMEvalHumanEvalXLDataset(BaseDataset):
    method load (line 30) | def load(path: str, lang: str, program_lang: str):
  class PMMEvalHumanEvalXLEvaluator (line 51) | class PMMEvalHumanEvalXLEvaluator(BaseEvaluator):
    method __init__ (line 53) | def __init__(self,
    method score (line 72) | def score(self, predictions, references):
    method _code_eval_service (line 123) | def _code_eval_service(self, file_path):
  function _clean_up_code (line 151) | def _clean_up_code(text: str, language_type: str, reference) -> str:

FILE: opencompass/datasets/PMMEval/mgsm.py
  function _get_last_digit (line 13) | def _get_last_digit(s):
  class PMMEvalMGSMDataset (line 29) | class PMMEvalMGSMDataset(BaseDataset):
    method load (line 32) | def load(path: str, lang: str):
  class PMMEvalMGSMEvaluator (line 52) | class PMMEvalMGSMEvaluator(BaseEvaluator):
    method score (line 54) | def score(self, predictions, references):

FILE: opencompass/datasets/PMMEval/mhellaswag.py
  function extract_choice (line 27) | def extract_choice(gen, lang):
  function extract_choice_fuzzy (line 59) | def extract_choice_fuzzy(gen, lang):
  function pmmeval_mhellaswag_postprocess (line 68) | def pmmeval_mhellaswag_postprocess(text: str, lang_code: str) -> Tuple[s...
  class PMMEvalMHellaswagDataset (line 73) | class PMMEvalMHellaswagDataset(BaseDataset):
    method load (line 76) | def load(path: str, lang: str):
  class PMMEvalMHellaswagEvaluator (line 96) | class PMMEvalMHellaswagEvaluator(BaseEvaluator):
    method score (line 98) | def score(self, predictions, references):

FILE: opencompass/datasets/PMMEval/mifeval.py
  function test_instruction_following_strict (line 14) | def test_instruction_following_strict(inp, response, lang_code):
  function test_instruction_following_loose (line 45) | def test_instruction_following_loose(inp, response, lang_code):
  function pmmeval_mifeval_postprocess (line 95) | def pmmeval_mifeval_postprocess(text: str, lang_code: str) -> Tuple[str]:
  class PMMEvalMIFEvalDataset (line 100) | class PMMEvalMIFEvalDataset(BaseDataset):
    method load (line 103) | def load(path: str, lang: str):
  class PMMEvalMIFEvalEvaluator (line 123) | class PMMEvalMIFEvalEvaluator(BaseEvaluator):
    method score (line 125) | def score(self, predictions, references, test_set):

FILE: opencompass/datasets/PMMEval/mifeval_utils/combination_checker.py
  function repeat_prompt_checker (line 1) | def repeat_prompt_checker(input_string: str, prompt_to_repeat: str, **kw...
  function two_responses_checker (line 8) | def two_responses_checker(input_string: str, **kwargs):

FILE: opencompass/datasets/PMMEval/mifeval_utils/detectable_content_checker.py
  function number_placeholders_checker (line 4) | def number_placeholders_checker(input_string: str, num_placeholders: int,
  function postscript_checker (line 10) | def postscript_checker(input_string: str, postscript_marker: str, **kwar...

FILE: opencompass/datasets/PMMEval/mifeval_utils/detectable_format_checker.py
  function removeprefix (line 5) | def removeprefix(s, prefix):
  function removesuffix (line 12) | def removesuffix(s, suffix):
  function constrained_response_checker (line 43) | def constrained_response_checker(input_string: str, lang_code: str, **kw...
  function number_bullet_lists_checker (line 48) | def number_bullet_lists_checker(input_string: str, num_bullets: int, **k...
  function number_highlighted_sections_checker (line 57) | def number_highlighted_sections_checker(input_string: str, num_highlight...
  function title_checker (line 72) | def title_checker(input_string: str, **kwargs):
  function json_format_checker (line 83) | def json_format_checker(input_string: str, **kwargs):

FILE: opencompass/datasets/PMMEval/mifeval_utils/keywords_checker.py
  function forbidden_words_checker (line 1) | def forbidden_words_checker(input_string: str, forbidden_words: list,

FILE: opencompass/datasets/PMMEval/mifeval_utils/length_constraints_checker.py
  function nth_paragraph_first_word_checker (line 4) | def nth_paragraph_first_word_checker(input_string: str, num_paragraphs: ...
  function number_paragraphs_checker (line 27) | def number_paragraphs_checker(input_string: str, num_paragraphs: int,
  function number_sentences_checker (line 35) | def number_sentences_checker(input_string: str, relation: str,
  function number_words_checker (line 52) | def number_words_checker(input_string: str, relation: str, num_words: int,

FILE: opencompass/datasets/PMMEval/mifeval_utils/punctuation_checker.py
  function no_comma_checker (line 17) | def no_comma_checker(input_string: str, lang_code: str, **kwargs):

FILE: opencompass/datasets/PMMEval/mifeval_utils/startend_checker.py
  function end_checker_checker (line 1) | def end_checker_checker(input_string: str, end_phrase: str, **kwargs):
  function quotation_checker (line 8) | def quotation_checker(input_string: str, lang_code: str, **kwargs):

FILE: opencompass/datasets/PMMEval/mlogiqa.py
  function extract_choice (line 27) | def extract_choice(gen, lang):
  function extract_choice_fuzzy (line 60) | def extract_choice_fuzzy(gen):
  function pmmeval_mlogiqa_postprocess (line 69) | def pmmeval_mlogiqa_postprocess(text: str, lang_code: str) -> Tuple[str]:
  class PMMEvalMLogiQADataset (line 74) | class PMMEvalMLogiQADataset(BaseDataset):
    method load (line 77) | def load(path: str, lang: str):
  class PMMEvalMLogiQAEvaluator (line 97) | class PMMEvalMLogiQAEvaluator(BaseEvaluator):
    method score (line 99) | def score(self, predictions, references):

FILE: opencompass/datasets/PMMEval/mmmlu.py
  function extract_choice (line 27) | def extract_choice(gen, lang):
  function extract_choice_fuzzy (line 59) | def extract_choice_fuzzy(gen):
  function pmmeval_mmmlu_postprocess (line 68) | def pmmeval_mmmlu_postprocess(text: str, lang_code: str) -> Tuple[str]:
  class PMMEvalMMMLUDataset (line 73) | class PMMEvalMMMLUDataset(BaseDataset):
    method load (line 76) | def load(path: str, lang: str, difficulty: str):
  class PMMEvalMMMLUEvaluator (line 120) | class PMMEvalMMMLUEvaluator(BaseEvaluator):
    method score (line 122) | def score(self, predictions, references):

FILE: opencompass/datasets/PMMEval/xnli.py
  function extract_choice (line 28) | def extract_choice(gen, lang):
  function extract_choice_fuzzy (line 60) | def extract_choice_fuzzy(gen, lang):
  function pmmeval_xnli_postprocess (line 69) | def pmmeval_xnli_postprocess(text: str, lang_code: str) -> Tuple[str]:
  class PMMEvalXNLIDataset (line 74) | class PMMEvalXNLIDataset(BaseDataset):
    method load (line 77) | def load(path: str, lang: str):
  class PMMEvalXNLIEvaluator (line 96) | class PMMEvalXNLIEvaluator(BaseEvaluator):
    method score (line 98) | def score(self, predictions, references):

FILE: opencompass/datasets/ProcessBench.py
  function extract_answer (line 11) | def extract_answer(solution_text: str):
  class ProcessBenchEvalDataset (line 23) | class ProcessBenchEvalDataset(BaseDataset):
    method load (line 26) | def load(path: str, subset: str, **kwargs):
  class ProcessBenchEvaluator (line 52) | class ProcessBenchEvaluator(BaseEvaluator):
    method score (line 54) | def score(self, predictions, references):

FILE: opencompass/datasets/ProteinLMBench.py
  function _parse (line 10) | def _parse(item):
  class ProteinLMBenchDataset (line 28) | class ProteinLMBenchDataset(BaseDataset):
    method load (line 31) | def load(path: str, **kwargs):
  class ProteinLMBenchEvaluator (line 38) | class ProteinLMBenchEvaluator(BaseEvaluator):
    method score (line 40) | def score(self, predictions, references, test_set):

FILE: opencompass/datasets/PubMedQA.py
  class PubMedQADataset (line 9) | class PubMedQADataset(BaseDataset):
    method load_single (line 12) | def load_single(path):
    method load (line 32) | def load(path):

FILE: opencompass/datasets/QuALITY.py
  class QuALITYDataset (line 13) | class QuALITYDataset(BaseDataset):
    method load (line 16) | def load(path: str):
  class QuALITYEvaluator (line 44) | class QuALITYEvaluator(BaseEvaluator):
    method score (line 46) | def score(self, predictions, references, test_set):

FILE: opencompass/datasets/SciEval.py
  class SciEvalDataset (line 21) | class SciEvalDataset(BaseDataset):
    method load (line 25) | def load(path: str, name: str, **kwargs) -> DatasetDict:

FILE: opencompass/datasets/SciKnowEval.py
  function _parse (line 11) | def _parse(item, prompt_mode, discipline):
  class SciKnowEvalDataset (line 25) | class SciKnowEvalDataset(BaseDataset):
    method load (line 28) | def load(path: str, prompt_mode: str, **kwargs):
  class SciKnowEvalEvaluator (line 51) | class SciKnowEvalEvaluator(BaseEvaluator):
    method score (line 53) | def score(self, predictions, references, test_set):
  function answer_cleansing (line 75) | def answer_cleansing(

FILE: opencompass/datasets/SciReasoner/GUE.py
  class GUE_Dataset (line 19) | class GUE_Dataset(BaseDataset):
    method load (line 22) | def load(path, task, mini_set=False):
  function remove_think_tags (line 66) | def remove_think_tags(text: str) -> str:
  function GUE_postprocessor (line 75) | def GUE_postprocessor(text: Union[str, None]) -> str:
  class GUE_Evaluator (line 161) | class GUE_Evaluator(BaseEvaluator):
    method score (line 163) | def score(self, predictions, references):

FILE: opencompass/datasets/SciReasoner/LLM4Chem/evaluator.py
  class LLM4ChemDataset (line 25) | class LLM4ChemDataset(BaseDataset):
    method load (line 28) | def load(path, task, max_cut=-1, mini_set=False, hf_hub=False):
  function extract_answer_part (line 71) | def extract_answer_part(outputs, left_tag, right_tag, mode='tag'):
  function LLM4Chem_postprocess (line 96) | def LLM4Chem_postprocess(text, task, *args, **kwargs):
  class LLM4Chem_Evaluator (line 146) | class LLM4Chem_Evaluator(BaseEvaluator):
    method __init__ (line 148) | def __init__(self, task, *args, **kwargs):
    method score (line 152) | def score(self, predictions, references):

FILE: opencompass/datasets/SciReasoner/LLM4Chem/retrosynthesis_evaluator.py
  function smi_tokenizer (line 29) | def smi_tokenizer(smi):
  function canonicalize_smiles_clear_map (line 43) | def canonicalize_smiles_clear_map(smiles, synthon=False, return_max_frag...
  function compute_rank (line 104) | def compute_rank(prediction_group,
  function Retrosynthesis_postprocess (line 188) | def Retrosynthesis_postprocess(text: Union[str, None]) -> str:
  class RetrosynthesisEvaluator (line 221) | class RetrosynthesisEvaluator(BaseEvaluator):
    method __init__ (line 227) | def __init__(self,
    method score (line 246) | def score(self, predictions, references):

FILE: opencompass/datasets/SciReasoner/LLM4Chem/utils/chat_generation.py
  function generate_chat (line 1) | def generate_chat(input_text, output_text=None, prefix_chat=None):

FILE: opencompass/datasets/SciReasoner/LLM4Chem/utils/core_tagger.py
  function find_sub_sequence (line 1) | def find_sub_sequence(whole, sub):
  class CoreTagger (line 32) | class CoreTagger(object):
    method __init__ (line 34) | def __init__(self,
    method generate_mask (line 49) | def generate_mask(self, token_ids, output_begin, sample):
  class CoreTaggerGeneral (line 111) | class CoreTaggerGeneral(object):
    method __init__ (line 113) | def __init__(self,
    method generate_mask (line 128) | def generate_mask(self, token_ids, prompt_mask, sample):

FILE: opencompass/datasets/SciReasoner/LLM4Chem/utils/general_prompter.py
  function get_chat_content (line 1) | def get_chat_content(conversation, tokenize=False):
  class GeneralPrompter (line 21) | class GeneralPrompter(object):
    method __init__ (line 23) | def __init__(self, apply_chat_template_func, response_split='[/INST]'):
    method generate_prompt (line 27) | def generate_prompt(self, chat, tokenize=False, *args, **kargs) -> str:
    method get_response (line 34) | def get_response(self, output: str) -> str:

FILE: opencompass/datasets/SciReasoner/LLM4Chem/utils/metrics.py
  function convert_smiles_list_into_mol_list (line 28) | def convert_smiles_list_into_mol_list(smiles_list,
  function judge_exact_match (line 54) | def judge_exact_match(pred_can_smiles_list, gold_can_smiles_list):
  function calculate_fingerprint_similarity (line 74) | def calculate_fingerprint_similarity(pred_mol_list,
  function judge_multiple_match (line 112) | def judge_multiple_match(pred_can_smiles_list, golds_can_smiles_list):
  function calculate_smiles_metrics (line 148) | def calculate_smiles_metrics(preds_smiles_list,
  function judge_string_exact_match (line 283) | def judge_string_exact_match(pred_string_list, golds_string_list):
  function judge_string_split_match (line 296) | def judge_string_split_match(pred_string_list,
  function parse_molecule (line 313) | def parse_molecule(molecular_formula):
  function count_element_match (line 377) | def count_element_match(pred_formula_list, golds_formula_list):
  function calculate_formula_metrics (line 408) | def calculate_formula_metrics(preds_formula_list,
  function calculate_text_metrics (line 508) | def calculate_text_metrics(pred_text_list,
  function calculate_number_metrics (line 592) | def calculate_number_metrics(pred_text_list, gold_text_list):
  function calculate_boolean_metrics (line 632) | def calculate_boolean_metrics(pred_text_list, gold_text_list):

FILE: opencompass/datasets/SciReasoner/LLM4Chem/utils/smiles_canonicalization.py
  function canonicalize (line 10) | def canonicalize(smiles, isomeric=False, canonical=True, kekulize=False):
  function canonicalize_molecule_smiles (line 82) | def canonicalize_molecule_smiles(smiles,
  function canonicalize_reaction_smiles (line 144) | def canonicalize_reaction_smiles(smiles,
  function get_molecule_id (line 176) | def get_molecule_id(smiles, remove_duplicate=True):

FILE: opencompass/datasets/SciReasoner/LLM4Mat.py
  class LLM4MatDataset (line 21) | class LLM4MatDataset(BaseDataset):
    method load (line 24) | def load(path,
  function remove_think_tags (line 85) | def remove_think_tags(text: str) -> str:
  function extract_strict_value (line 93) | def extract_strict_value(text: str, property: str) -> str:
  function LLM4Mat_postprocessor (line 139) | def LLM4Mat_postprocessor(text: Union[str, None], property):
  class LLM4Mat_Evaluator (line 150) | class LLM4Mat_Evaluator(BaseEvaluator):
    method score (line 152) | def score(self, predictions, references):

FILE: opencompass/datasets/SciReasoner/Mol_Instructions/biotext.py
  function CER_calculate_f1_score (line 20) | def CER_calculate_f1_score(true_entities, predicted_entities):
  function calculate_f1_score (line 35) | def calculate_f1_score(true_entities, predicted_entities):
  function calculate_accuracy_ (line 60) | def calculate_accuracy_(predictions, references):
  function CER_calculate_accuracy_ (line 72) | def CER_calculate_accuracy_(predictions, references):
  function ture_or_false_calculate_accuracy_ (line 85) | def ture_or_false_calculate_accuracy_(predictions, references):
  function calculate_macro_f1_ (line 119) | def calculate_macro_f1_(predictions, references):
  function multi_choice_question_calculate_accuracy (line 138) | def multi_choice_question_calculate_accuracy(question_data):
  function multi_choice_question_calculate_accuracy_ (line 163) | def multi_choice_question_calculate_accuracy_(predictions, references):
  class Mol_Instructions_Dataset_BioText (line 185) | class Mol_Instructions_Dataset_BioText(BaseDataset):
    method load (line 188) | def load(path, task, max_cut=-1, mini_set=False, hf_hub=False):
  function Mol_Instructions_postprocess_BioText (line 232) | def Mol_Instructions_postprocess_BioText(text, task, *args, **kwargs):
  class Mol_Instructions_Evaluator_BioText (line 271) | class Mol_Instructions_Evaluator_BioText(BaseEvaluator):
    method __init__ (line 273) | def __init__(self, task='protein_design', *args, **kwargs):
    method score (line 277) | def score(self, predictions: List[str], references: List[str]):

FILE: opencompass/datasets/SciReasoner/Mol_Instructions/molecule.py
  class Mol_Instructions_Dataset (line 41) | class Mol_Instructions_Dataset(BaseDataset):
    method load (line 44) | def load(path, task, max_cut=-1, mini_set=False, hf_hub=False):
  function convert_to_canonical_smiles (line 87) | def convert_to_canonical_smiles(smiles):
  function Mol_Instructions_postprocess_Mol (line 99) | def Mol_Instructions_postprocess_Mol(text, task, *args, **kwargs):
  function compute_MAE_property_prediction_str (line 149) | def compute_MAE_property_prediction_str(predictions, references):
  function compute_fingerprint_metricts (line 158) | def compute_fingerprint_metricts(
  function compute_mol_translation_selfies (line 221) | def compute_mol_translation_selfies(predictions, references):
  function fix_smiles_brackets (line 319) | def fix_smiles_brackets(smiles):
  class Mol_Instructions_Evaluator_Mol (line 333) | class Mol_Instructions_Evaluator_Mol(BaseEvaluator):
    method __init__ (line 335) | def __init__(self, task, *args, **kwargs):
    method score (line 339) | def score(self, predictions, references):
  function compute_text_translation_metrics (line 385) | def compute_text_translation_metrics(

FILE: opencompass/datasets/SciReasoner/Mol_Instructions/normalized_SW_score.py
  function normalized_smith_waterman (line 4) | def normalized_smith_waterman(seq1,
  function Mol_Instructions_postprocess_Protein_Design (line 102) | def Mol_Instructions_postprocess_Protein_Design(text, *args, **kwargs):

FILE: opencompass/datasets/SciReasoner/Mol_Instructions/protein.py
  class Mol_Instructions_Dataset_Protein_Design (line 23) | class Mol_Instructions_Dataset_Protein_Design(BaseDataset):
    method load (line 26) | def load(path, task, max_cut=-1, mini_set=False, hf_hub=False):
  function Mol_Instructions_postprocess_Protein (line 70) | def Mol_Instructions_postprocess_Protein(text, *args, **kwargs):
  class Mol_Instructions_Evaluator_Protein (line 84) | class Mol_Instructions_Evaluator_Protein(RougeEvaluator):
    method __init__ (line 86) | def __init__(self,
  function Mol_Instructions_postprocess_Protein_Design (line 95) | def Mol_Instructions_postprocess_Protein_Design(text, *args, **kwargs):
  class Mol_Instructions_Evaluator_Protein_Design (line 113) | class Mol_Instructions_Evaluator_Protein_Design(BaseEvaluator):
    method __init__ (line 115) | def __init__(self, task='protein_design', *args, **kwargs):
    method score (line 119) | def score(self, predictions: List[str], references: List[str]):

FILE: opencompass/datasets/SciReasoner/PEER.py
  class PEER_Dataset (line 25) | class PEER_Dataset(BaseDataset):
    method load (line 28) | def load(path, task, max_cut=-1, mini_set=False, hf_hub=False):
  function PEER_postprocess_default (line 72) | def PEER_postprocess_default(text: Union[str, None]) -> str:
  function PEER_postprocess (line 81) | def PEER_postprocess(text: Union[str, None]) -> str:
  function PEER_postprocess_float_compare (line 155) | def PEER_postprocess_float_compare(text: Union[str, None],
  function calculate_accuracy (line 180) | def calculate_accuracy(pred_text_list, gold_text_list):
  class PEER_Evaluator (line 244) | class PEER_Evaluator(BaseEvaluator):
    method __init__ (line 246) | def __init__(self,
    method _retry_api (line 270) | def _retry_api(self, fn, *args, **kwargs):
    method ask_gpt25 (line 286) | def ask_gpt25(self, question, answer, prediction):
    method ask_gpt25_batch (line 321) | def ask_gpt25_batch(self, questions, answers, predictions):
    method score (line 343) | def score(self, predictions, references):
  class PEERRuleEvaluator (line 474) | class PEERRuleEvaluator(BaseEvaluator):
    method score (line 476) | def score(self,
  function peer_llm_judge_postprocess (line 525) | def peer_llm_judge_postprocess(output: Dict, output_path: str) -> Dict:

FILE: opencompass/datasets/SciReasoner/bio_instruction.py
  class Bioinstruction_Dataset (line 35) | class Bioinstruction_Dataset(BaseDataset):
    method load (line 38) | def load(path, task, mini_set=False, hf_hub=False):
  function extract_answer_part (line 83) | def extract_answer_part(outputs, left_tag, right_tag, mode='tag'):
  function extract_numeric_values (line 107) | def extract_numeric_values(text):
  function generic_replace (line 149) | def generic_replace(m):
  function classify_by_sentiment_model (line 161) | def classify_by_sentiment_model(text):
  function classify_by_keywords (line 193) | def classify_by_keywords(text):
  function process_regression_task (line 259) | def process_regression_task(task_name, task_entries, model_name):
  function compute_spearman (line 322) | def compute_spearman(label_values, result_values):
  function compute_R2 (line 378) | def compute_R2(label_values, result_values):
  function compute_mixed_score (line 435) | def compute_mixed_score(label_values,
  function compute_R2_for_ProgrammableRNASwitches_task (line 530) | def compute_R2_for_ProgrammableRNASwitches_task(task_name, task_entries,
  function compute_PCC_for_enhancer_activity_task (line 670) | def compute_PCC_for_enhancer_activity_task(task_name, task_entries,
  function process_binary_classification_task (line 794) | def process_binary_classification_task(task_name, task_entries, model_na...
  function compute_MCC (line 880) | def compute_MCC(label_classes, result_classes):
  function compute_Acc (line 893) | def compute_Acc(label_classes, result_classes):
  function extract_rna_family (line 912) | def extract_rna_family(text):
  function compute_Acc_for_NoncodingRNAFamily_task (line 920) | def compute_Acc_for_NoncodingRNAFamily_task(task_name, task_entries,
  function extract_modifications (line 971) | def extract_modifications(text):
  function convert_to_binary_vector (line 981) | def convert_to_binary_vector(modifications, classes=modification_classes):
  function compute_AUC_for_Modification_task (line 997) | def compute_AUC_for_Modification_task(task_name, task_entries, model_name):
  function count_f1_max (line 1081) | def count_f1_max(pred, target):
  function round_and_scale_results (line 1138) | def round_and_scale_results(data, decimal_places=3, scale_factor=100):
  function ec_to_multihot (line 1149) | def ec_to_multihot(ec_list, ec_labels):
  function compute_Fmax_for_FunctionEC_task (line 1162) | def compute_Fmax_for_FunctionEC_task(task_name, task_entries, ec_labels,
  function preprocess_input_data (line 1225) | def preprocess_input_data(input_file_path, prediction, mini_set=False):
  class bio_instruction_Evaluator (line 1294) | class bio_instruction_Evaluator(BaseEvaluator):
    method __init__ (line 1296) | def __init__(self,
    method score (line 1310) | def score(self, predictions):

FILE: opencompass/datasets/SciReasoner/bulk_modulus_material.py
  class Bulk_modulus_material_Dataset (line 19) | class Bulk_modulus_material_Dataset(BaseDataset):
    method load (line 22) | def load(path, mini_set=False):
  function material_postprocessor (line 63) | def material_postprocessor(text: Union[str, None]) -> str:
  class material_Evaluator (line 74) | class material_Evaluator(BaseEvaluator):
    method __init__ (line 82) | def __init__(self, data_path=None, **kwargs):
    method _load_ground_truths (line 92) | def _load_ground_truths(self):
    method _normalize (line 106) | def _normalize(self, formula: str) -> str:
    method score (line 112) | def score(self, predictions: List[dict]):

FILE: opencompass/datasets/SciReasoner/composition_material.py
  function extract_elements_from_prompt (line 18) | def extract_elements_from_prompt(prompt: str) -> list:
  function composition_precision (line 52) | def composition_precision(elements: list[str], prediction: str) -> float:
  class Composition_material_Dataset (line 63) | class Composition_material_Dataset(BaseDataset):
    method load (line 66) | def load(path, mini_set=False):
  function material_postprocessor (line 111) | def material_postprocessor(text: Union[str, None]) -> str:
  class composition_Evaluator (line 122) | class composition_Evaluator(BaseEvaluator):
    method __init__ (line 124) | def __init__(self, data_path, tuning_data=None, **kwargs):
    method _load_original_inputs (line 135) | def _load_original_inputs(self):
    method _normalize (line 147) | def _normalize(self, formula):
    method score (line 152) | def score(self, predictions):

FILE: opencompass/datasets/SciReasoner/opi/evaluator.py
  class OpiDataset (line 20) | class OpiDataset(BaseDataset):
    method load (line 23) | def load(path, task, max_cut=-1, mini_set=False, hf_hub=False):
  function extract_answer_part (line 64) | def extract_answer_part(outputs, left_tag, right_tag, mode='tag'):
  function opi_postprocess (line 89) | def opi_postprocess(text, task, *args, **kwargs):
  class Opi_Evaluator (line 97) | class Opi_Evaluator(BaseEvaluator):
    method __init__ (line 99) | def __init__(self, task, *args, **kwargs):
    method score (line 103) | def score(self, predictions, references):
    method _evaluate_function (line 127) | def _evaluate_function(self, predictions, references):
    method _evaluate_subcellular_localization (line 152) | def _evaluate_subcellular_localization(self, predictions, references):
    method _evaluate_fold_type (line 177) | def _evaluate_fold_type(self, predictions, references):
    method _evaluate_multilabel (line 203) | def _evaluate_multilabel(self, predictions, references):
    method _evaluate_text_similarity (line 252) | def _evaluate_text_similarity(self, predictions, references):
    method _evaluate_general (line 272) | def _evaluate_general(self, predictions, references):

FILE: opencompass/datasets/SciReasoner/opi/process_ec_numbers.py
  function add_spaces_to_ec_number (line 6) | def add_spaces_to_ec_number(text: str) -> str:
  function process_json_value (line 20) | def process_json_value(value: Any) -> Any:
  function process_ec_json_file (line 34) | def process_ec_json_file(input_file: str, output_file: str) -> None:

FILE: opencompass/datasets/SciReasoner/opi/utils/accuracy4fold_type.py
  function load_json (line 7) | def load_json(file_path):
  function compute_accuracy4fold_type (line 13) | def compute_accuracy4fold_type(eval_file, test_files):

FILE: opencompass/datasets/SciReasoner/opi/utils/metrics4all.py
  function calculate_metrics (line 14) | def calculate_metrics(output, target):
  function calculate_rouge_l (line 32) | def calculate_rouge_l(output, target):
  function process_json_file (line 38) | def process_json_file(json_file_path):
  function main (line 98) | def main(eval_res_path):

FILE: opencompass/datasets/SciReasoner/uncond_RNA.py
  class Uncond_RNA_Dataset (line 17) | class Uncond_RNA_Dataset(BaseDataset):
    method load (line 20) | def load(num, prompt):
  function RNA_postprocessor (line 26) | def RNA_postprocessor(text: Union[str, None]) -> str:
  class RNA_Evaluator (line 46) | class RNA_Evaluator(BaseEvaluator):
    method score (line 48) | def score(self, predictions, references):
    method run_rnafold (line 101) | def run_rnafold(self, input_fasta, output_dir):
    method parse_mfe (line 113) | def parse_mfe(self, output_file):
    method run_cmscan (line 123) | def run_cmscan(self, fasta_file, output_dir, rfam_cm, rfam_clanin):
    method parse_unique_families (line 137) | def parse_unique_families(self, tblout_file):

FILE: opencompass/datasets/SciReasoner/uncond_material.py
  class Uncond_material_Dataset (line 12) | class Uncond_material_Dataset(BaseDataset):
    method load (line 15) | def load(num, prompt):
  function material_postprocessor (line 21) | def material_postprocessor(text: Union[str, None]) -> str:
  class uncond_material_Evaluator (line 33) | class uncond_material_Evaluator(BaseEvaluator):
    method score (line 35) | def score(self, predictions):

FILE: opencompass/datasets/SciReasoner/unconditional_molecule_generation/UMG.py
  class UMG_Dataset (line 16) | class UMG_Dataset(BaseDataset):
    method load (line 19) | def load(max_cut=-1):
  class UMG_Evaluator (line 50) | class UMG_Evaluator(BaseEvaluator):
    method __init__ (line 52) | def __init__(self, *args, **kwargs):
    method is_valid_smiles_rdkit (line 55) | def is_valid_smiles_rdkit(self, s):
    method extract_smiles_simple (line 67) | def extract_smiles_simple(self, text: str) -> str | None:
    method score (line 93) | def score(self, predictions):

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/UPG.py
  class UPGDataset (line 11) | class UPGDataset(BaseDataset):
    method load (line 14) | def load(tag_bool=True, max_cut=-1):
  function UPG_postprocess (line 61) | def UPG_postprocess(text):
  class UPG_Evaluator (line 94) | class UPG_Evaluator(BaseEvaluator):
    method __init__ (line 96) | def __init__(self, *args, **kwargs):
    method _calculate_sequence_identity (line 99) | def _calculate_sequence_identity(self, seq1, seq2):
    method score (line 115) | def score(self, predictions, references=None):

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/omegafold/__main__.py
  function main (line 37) | def main(protein_list):

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/omegafold/confidence.py
  function get_all_confidence (line 38) | def get_all_confidence(
  function _compute_confidence (line 93) | def _compute_confidence(logits: torch.Tensor) -> torch.Tensor:
  class ConfidenceHead (line 123) | class ConfidenceHead(modules.OFModule):
    method __init__ (line 130) | def __init__(self, cfg: argparse.Namespace):
    method forward (line 140) | def forward(self, node_repr: torch.Tensor) -> torch.Tensor:

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/omegafold/config.py
  function _make_config (line 31) | def _make_config(input_dict: dict) -> argparse.Namespace:
  function make_config (line 42) | def make_config(model_idx: int = 1) -> argparse.Namespace:

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/omegafold/decode.py
  class InvariantPointAttention (line 42) | class InvariantPointAttention(modules.OFModule):
    method __init__ (line 49) | def __init__(self, cfg: argparse.Namespace) -> None:
    method forward (line 89) | def forward(self, node_repr: torch.Tensor, edge_repr: torch.Tensor,
    method _get_scalar (line 148) | def _get_scalar(linear: nn.Linear, inputs: torch.Tensor,
    method _get_point (line 167) | def _get_point(linear: nn.Linear, inputs: torch.Tensor, n_head: int,
  class TorsionAngleHead (line 188) | class TorsionAngleHead(modules.OFModule):
    method __init__ (line 194) | def __init__(self, cfg: argparse.Namespace):
    method forward (line 211) | def forward(
  class StructureCycle (line 241) | class StructureCycle(modules.OFModule):
    method __init__ (line 248) | def __init__(self, cfg: argparse.Namespace) -> None:
    method forward (line 260) | def forward(
  class StructureModule (line 296) | class StructureModule(modules.OFModule):
    method __init__ (line 299) | def __init__(self, cfg: argparse.Namespace):
    method forward (line 310) | def forward(

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/omegafold/embedders.py
  function _get_pos (line 38) | def _get_pos(shape: torch.Size, device: torch.device, dtype: torch.dtype,
  function _apply_embed (line 62) | def _apply_embed(inputs: torch.Tensor, sin: torch.Tensor, cos: torch.Ten...
  class EdgeEmbedder (line 106) | class EdgeEmbedder(modules.OFModule):
    method __init__ (line 112) | def __init__(self, cfg: argparse.Namespace) -> None:
    method forward (line 119) | def forward(self, fasta_sequence: torch.Tensor,
  class RoPE (line 128) | class RoPE(nn.Module):
    method __init__ (line 136) | def __init__(self, input_dim: int) -> None:
    method forward (line 151) | def forward(self, tensor: torch.Tensor,
    method _compute_sin_cos (line 170) | def _compute_sin_cos(
  class RelPosEmbedder (line 190) | class RelPosEmbedder(nn.Embedding):
    method forward (line 197) | def forward(self, num_res: int) -> torch.Tensor:
  class StructEmbedder (line 213) | class StructEmbedder(modules.OFModule):
    method __init__ (line 219) | def __init__(self, cfg: argparse.Namespace):
    method forward (line 245) | def forward(
    method _sharded_compute (line 272) | def _sharded_compute(self, pairwise_fasta: torch.Tensor, d: torch.Tensor,
  class PairStructEmbedder (line 299) | class PairStructEmbedder(StructEmbedder):
    method forward (line 301) | def forward(
  class RecycleEmbedder (line 317) | class RecycleEmbedder(modules.OFModule):
    method __init__ (line 323) | def __init__(self, cfg: argparse.Namespace):
    method forward (line 336) | def forward(

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/omegafold/geoformer.py
  class GeoFormerBlock (line 41) | class GeoFormerBlock(modules.OFModule):
    method __init__ (line 47) | def __init__(self, cfg: argparse.Namespace) -> None:
    method forward (line 79) | def forward(
    method _column_attention (line 114) | def _column_attention(self, node_repr, mask, fwd_cfg):
  class GeoFormer (line 126) | class GeoFormer(modules.OFModule):
    method __init__ (line 128) | def __init__(self, cfg: argparse.Namespace):
    method forward (line 134) | def forward(

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/omegafold/model.py
  class OmegaFoldCycle (line 43) | class OmegaFoldCycle(modules.OFModule):
    method __init__ (line 45) | def __init__(self, cfg: argparse.Namespace) -> None:
    method forward (line 52) | def forward(
  class OmegaFold (line 107) | class OmegaFold(modules.OFModule):
    method __init__ (line 115) | def __init__(self, cfg: argparse.Namespace) -> None:
    method forward (line 124) | def forward(
    method deep_sequence_embed (line 185) | def deep_sequence_embed(
    method create_initial_prev_dict (line 212) | def create_initial_prev_dict(

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/omegafold/modules.py
  function softmax (line 38) | def softmax(x: torch.Tensor,
  function _attention (line 66) | def _attention(
  function attention (line 96) | def attention(
  class OFModule (line 161) | class OFModule(nn.Module):
    method __init__ (line 167) | def __init__(self, cfg: typing.Optional[argparse.Namespace]) -> None:
    method device (line 172) | def device(self) -> torch.device:
    method dtype (line 176) | def dtype(self) -> torch.dtype:
  class Transition (line 180) | class Transition(OFModule):
    method __init__ (line 182) | def __init__(self, d: int, n: int, activation: str) -> None:
    method forward (line 192) | def forward(self, x: torch.Tensor,
  class MultiHeadedScaling (line 204) | class MultiHeadedScaling(OFModule):
    method __init__ (line 210) | def __init__(
    method forward (line 241) | def forward(self, x: torch.Tensor) -> typing.List[torch.Tensor]:
    method reset_parameters (line 262) | def reset_parameters(self):
  class Val2ContBins (line 267) | class Val2ContBins(OFModule):
    method __init__ (line 269) | def __init__(
    method forward (line 285) | def forward(self, dist_x):  # (*)
  class Val2Bins (line 295) | class Val2Bins(OFModule):
    method __init__ (line 303) | def __init__(self, cfg: argparse.Namespace) -> None:
    method forward (line 310) | def forward(self, dist: torch.Tensor) -> torch.Tensor:
  class Node2Edge (line 326) | class Node2Edge(OFModule):
    method __init__ (line 332) | def __init__(self, in_dim: int, proj_dim: int, out_dim: int) -> None:
    method forward (line 340) | def forward(self, node_repr: torch.Tensor,
  class Attention (line 357) | class Attention(OFModule):
    method __init__ (line 371) | def __init__(self, q_dim: int, kv_dim: int, n_head: int, gating: bool,
    method forward (line 391) | def forward(
    method _get_attn_out (line 435) | def _get_attn_out(self, q_inputs, kv_inputs, fwd_cfg, bias):
  class AttentionWEdgeBias (line 463) | class AttentionWEdgeBias(OFModule):
    method __init__ (line 465) | def __init__(self, d_node: int, d_edge: int, n_head: int,
    method forward (line 480) | def forward(
  function _get_sharded_stacked (line 512) | def _get_sharded_stacked(edge_repr: torch.Tensor, subbatch_size: int):
  class GeometricAttention (line 525) | class GeometricAttention(OFModule):
    method __init__ (line 530) | def __init__(self, d_edge: int, c: int, n_head: int, n_axis: int) -> N...
    method _get_attended (line 554) | def _get_attended(self, edge_repr: torch.Tensor, mask: torch.Tensor,
    method _get_gated (line 576) | def _get_gated(self, edge_repr: torch.Tensor, mask: torch.Tensor, fwd_...
    method _get_sliced_weight (line 601) | def _get_sliced_weight(self, weight: torch.Tensor, shift=0):
    method _get_act_row (line 607) | def _get_act_row(self, edge_row: torch.Tensor,
    method _get_act_col (line 615) | def _get_act_col(self, edge_row: torch.Tensor,
    method forward (line 623) | def forward(self, edge_repr: torch.Tensor, mask: torch.Tensor,

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/omegafold/omegaplm.py
  function _get_qk_scaling (line 38) | def _get_qk_scaling(num_res: torch.Tensor, attn_dim: int) -> torch.Tensor:
  class GatedAttentionUnit (line 55) | class GatedAttentionUnit(modules.OFModule):
    method __init__ (line 60) | def __init__(self, cfg: argparse.Namespace):
    method forward (line 73) | def forward(
  class OmegaPLMLayer (line 113) | class OmegaPLMLayer(modules.OFModule):
    method __init__ (line 123) | def __init__(self, cfg: argparse.Namespace) -> None:
    method forward (line 127) | def forward(
  class OmegaPLM (line 151) | class OmegaPLM(modules.OFModule):
    method __init__ (line 163) | def __init__(self, cfg: argparse.Namespace) -> None:
    method forward (line 172) | def forward(
    method _get_finetuning_scale (line 208) | def _get_finetuning_scale(self, mask: torch.Tensor,

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/omegafold/pipeline.py
  function _mps_is_available (line 49) | def _mps_is_available():
  function _set_precision (line 59) | def _set_precision(allow_tf32: bool) -> None:
  function path_leaf (line 78) | def path_leaf(path: str) -> str:
  function fasta2inputs (line 93) | def fasta2inputs(
  function list2inputs (line 181) | def list2inputs(
  function save_pdb (line 247) | def save_pdb(pos14: torch.Tensor,
  function _load_weights (line 305) | def _load_weights(
  function _get_device (line 334) | def _get_device(device) -> str:
  function get_args (line 367) | def get_args() -> typing.Tuple[types.SimpleNamespace, collections.Ordere...

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/omegafold/utils/protein_utils/aaframe.py
  class AAFrame (line 52) | class AAFrame(object):
    method __init__ (line 57) | def __init__(self,
    method unit (line 91) | def unit(self) -> str:
    method _assign (line 101) | def _assign(self, translation: torch.Tensor, rotation: torch.Tensor,
    method to_nanometers (line 132) | def to_nanometers(self, in_place: bool = True) -> 'AAFrame':
    method to_angstrom (line 156) | def to_angstrom(self, in_place: bool) -> 'AAFrame':
    method translation (line 181) | def translation(self) -> torch.Tensor:
    method translation (line 191) | def translation(self, value: torch.Tensor) -> None:
    method rotation (line 203) | def rotation(self) -> torch.Tensor:
    method rotation (line 213) | def rotation(self, value: torch.Tensor) -> None:
    method mask (line 229) | def mask(self) -> torch.Tensor:
    method mask (line 239) | def mask(self, value: torch.Tensor):
    method default_init (line 243) | def default_init(
    method _neg_dim (line 282) | def _neg_dim(cls, dim: int) -> Tuple[int, int, int]:
    method unsqueeze (line 288) | def unsqueeze(self, dim: int) -> 'AAFrame':
    method sum (line 300) | def sum(self, dim: int, keepdim: bool = False) -> 'AAFrame':
    method dim_apply (line 322) | def dim_apply(self, func: callable, dim: int) -> 'AAFrame':
    method _construct_frame (line 346) | def _construct_frame(
    method from_4x4 (line 380) | def from_4x4(cls, m: torch.Tensor, mask: torch.Tensor,
    method transform (line 403) | def transform(self, pos: torch.Tensor) -> torch.Tensor:
    method from_torsion (line 470) | def from_torsion(
    method __getitem__ (line 517) | def __getitem__(self, idx: Union[slice, int, torch.Tensor]) -> 'AAFrame':
    method __setitem__ (line 545) | def __setitem__(self, key: Union[int, torch.Tensor, List[int]],
    method device (line 573) | def device(self) -> torch.device:
    method shape (line 584) | def shape(self) -> torch.Size:
    method __mul__ (line 592) | def __mul__(self, other) -> 'AAFrame':
    method _tensor_multiplication (line 598) | def _tensor_multiplication(self, other: torch.Tensor) -> 'AAFrame':
    method _combine_transformation (line 624) | def _combine_transformation(self, other: 'AAFrame') -> 'AAFrame':
    method __repr__ (line 669) | def __repr__(self) -> str:
    method view (line 672) | def view(self, *args) -> 'AAFrame':
    method dtype (line 693) | def dtype(self):
    method expand_w_torsion (line 696) | def expand_w_torsion(self, torsion_angles: torch.Tensor,
    method rotate (line 778) | def rotate(self, rotation: torch.Tensor):
    method expanded_to_pos (line 805) | def expanded_to_pos(
    method __len__ (line 854) | def __len__(self):
    method inverse (line 858) | def inverse(self) -> 'AAFrame':
    method position_in_frame (line 874) | def position_in_frame(self, pos: torch.Tensor) -> torch.Tensor:
    method from_tensor (line 888) | def from_tensor(cls, tensor, unit: str) -> 'AAFrame':
  function torsion_mask_to_atom14_mask (line 907) | def torsion_mask_to_atom14_mask(torsion_mask: torch.Tensor,

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/omegafold/utils/protein_utils/functions.py
  function get_norm (line 33) | def get_norm(x: torch.Tensor) -> torch.Tensor:
  function robust_normalize (line 46) | def robust_normalize(x: torch.Tensor,
  function quaternion_to_matrix (line 64) | def quaternion_to_matrix(quaternions: torch.Tensor) -> torch.Tensor:
  function batch_matrix_vector (line 99) | def batch_matrix_vector(matrix: torch.Tensor,
  function create_pseudo_beta (line 117) | def create_pseudo_beta(atom_pos: torch.Tensor,
  function bit_wise_not (line 139) | def bit_wise_not(boolean_tensor: torch.Tensor) -> torch.Tensor:

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/omegafold/utils/protein_utils/residue_constants.py
  function residx_to_3 (line 437) | def residx_to_3(idx):
  function get_chi_angle_atom_indices (line 448) | def get_chi_angle_atom_indices():
  function _make_rigid_transformation_4x4 (line 476) | def _make_rigid_transformation_4x4(ex: torch.Tensor, ey: torch.Tensor,
  function _make_aa_constants (line 506) | def _make_aa_constants():
  function substitute (line 674) | def substitute(res: str):

FILE: opencompass/datasets/SciReasoner/unconditional_protein_generation/omegafold/utils/torch_utils.py
  function mask2bias (line 38) | def mask2bias(mask: torch.Tensor, *, inf: float = 1e9) -> torch.Tensor:
  function normalize (line 52) | def normalize(inputs: torch.Tensor,
  function masked_mean (line 83) | def masked_mean(values: torch.Tensor,
  function recursive_to (line 106) | def recursive_to(obj: typing.Any, **kwargs) -> typing.Any:

FILE: opencompass/datasets/ScienceQA.py
  class ScienceQADataset (line 9) | class ScienceQADataset(BaseDataset):
    method load_single (line 12) | def load_single(path):
    method load (line 30) | def load(path):

FILE: opencompass/datasets/SeedBench.py
  class SeedBenchDataset (line 20) | class SeedBenchDataset(BaseDataset):
    method load (line 23) | def load(data_files: str,
  class F1Evaluator (line 51) | class F1Evaluator(BaseEvaluator):
    method __init__ (line 59) | def __init__(self, seed: int = 0) -> None:
    method _preprocess (line 63) | def _preprocess(self, predictions: List, references: List) -> dict:
    method _postprocess (line 69) | def _postprocess(self, scores: dict) -> dict:
    method score (line 72) | def score(self, predictions: List, references: List) -> dict:
  class F1ScoreEvaluator (line 136) | class F1ScoreEvaluator(F1Evaluator):
    method __init__ (line 139) | def __init__(self) -> None:
  function my_multiple_select_postprocess (line 145) | def my_multiple_select_postprocess(text: str) -> str:
  class AverageRougeEvaluator (line 152) | class AverageRougeEvaluator(BaseEvaluator):
    method __init__ (line 160) | def __init__(self, seed: int = 0) -> None:
    method _preprocess (line 164) | def _preprocess(self, predictions: List, references: List) -> dict:
    method _postprocess (line 176) | def _postprocess(self, scores: dict) -> dict:
    method score (line 179) | def score(self, predictions: List, references: List) -> dict:
  class AverageRougeScoreEvaluator (line 244) | class AverageRougeScoreEvaluator(AverageRougeEvaluator):
    method __init__ (line 247) | def __init__(self) -> None:
  class AccScoreStrEvaluator (line 251) | class AccScoreStrEvaluator(BaseEvaluator):
    method __init__ (line 259) | def __init__(self, seed: int = 0) -> None:
    method _preprocess (line 263) | def _preprocess(self, predictions: List, references: List) -> dict:
    method _postprocess (line 269) | def _postprocess(self, scores: dict) -> dict:
    method score (line 272) | def score(self, predictions: List, references: List) -> dict:
  class AccScoreStr_Evaluator (line 305) | class AccScoreStr_Evaluator(AccScoreStrEvaluator):
    method __init__ (line 308) | def __init__(self) -> None:

FILE: opencompass/datasets/TheoremQA/legacy.py
  class TheoremQADataset (line 12) | class TheoremQADataset(BaseDataset):
    method load (line 15) | def load(path: str):
  function TheoremQA_postprocess (line 21) | def TheoremQA_postprocess(text: str) -> str:
  function TheoremQA_postprocess_v2 (line 31) | def TheoremQA_postprocess_v2(text: str) -> str:

FILE: opencompass/datasets/TheoremQA/main.py
  class TheoremQADatasetV3 (line 16) | class TheoremQADatasetV3(BaseDataset):
    method load (line 19) | def load(path: str):
  function TheoremQA_postprocess_v3 (line 29) | def TheoremQA_postprocess_v3(text: str) -> str:
  function TheoremQA_postprocess_v4 (line 33) | def TheoremQA_postprocess_v4(text: str) -> str:
  class TheoremQAEvaluatorV3 (line 42) | class TheoremQAEvaluatorV3(BaseEvaluator):
    method score (line 43) | def score(self, predictions, references, test_set):

FILE: opencompass/datasets/TheoremQA/number_utils.py
  function floatify (line 7) | def floatify(num: str):
  function within_eps (line 18) | def within_eps(pred: float, gt: float):
  function clean_units (line 26) | def clean_units(pred_str: str):
  function number_it (line 50) | def number_it(num):
  function compare_two_numbers (line 76) | def compare_two_numbers(p, gt):
  function compare_two_list (line 88) | def compare_two_list(pred, gt):

FILE: opencompass/datasets/TheoremQA/utils.py
  function time_limit (line 7) | def time_limit(seconds: float):
  function extract_theoremqa_answer (line 19) | def extract_theoremqa_answer(pred: str, answer_flag: bool = True):
  function answer_clean (line 56) | def answer_clean(direct_answer_trigger_for_fewshot: tuple, pred: str):
  function compare_answer_with_groundtruth (line 96) | def compare_answer_with_groundtruth(answer: str, groundtruth_str: str, g...

FILE: opencompass/datasets/advglue.py
  class AdvDataset (line 12) | class AdvDataset(BaseDataset):
    method __init__ (line 22) | def __init__(
    method aug_with_original_data (line 34) | def aug_with_original_data(self, dataset):
    method load (line 52) | def load(self, path):
  class AdvSst2Dataset (line 80) | class AdvSst2Dataset(AdvDataset):
    method __init__ (line 83) | def __init__(self, **kwargs):
  class AdvQqpDataset (line 91) | class AdvQqpDataset(AdvDataset):
    method __init__ (line 94) | def __init__(self, **kwargs):
  class AdvMnliDataset (line 104) | class AdvMnliDataset(AdvDataset):
    method __init__ (line 107) | def __init__(self, **kwargs):
  class AdvMnliMMDataset (line 117) | class AdvMnliMMDataset(AdvDataset):
    method __init__ (line 120) | def __init__(self, **kwargs):
  class AdvQnliDataset (line 129) | class AdvQnliDataset(AdvDataset):
    method __init__ (line 132) | def __init__(self, **kwargs):
  class AdvRteDataset (line 141) | class AdvRteDataset(AdvDataset):
    method __init__ (line 144) | def __init__(self, **kwargs):
  class AccDropEvaluator (line 151) | class AccDropEvaluator(AccEvaluator):
    method __init__ (line 154) | def __init__(self) -> None:
    method score (line 157) | def score(self, predictions: List, references: List) -> dict:

FILE: opencompass/datasets/afqmcd.py
  class AFQMCDatasetV2 (line 13) | class AFQMCDatasetV2(BaseDataset):
    method load (line 16) | def load(path, local_mode=False):

FILE: opencompass/datasets/agieval/agieval.py
  class AGIEvalDataset (line 17) | class AGIEvalDataset(BaseDataset):
    method load (line 20) | def load(path: str, name: str, setting_name: str):
  class AGIEvalDataset_v2 (line 39) | class AGIEvalDataset_v2(BaseDataset):
    method load (line 42) | def load(path: str, name: str, setting_name: str):
  class AGIEvalEvaluator (line 89) | class AGIEvalEvaluator(BaseEvaluator):
    method score (line 91) | def score(self, predictions, references):
  class AGIEvalEvaluator_mcq (line 106) | class AGIEvalEvaluator_mcq(BaseEvaluator):
    method score (line 108) | def score(self, predictions, references):

FILE: opencompass/datasets/agieval/constructions.py
  class TaskSchema (line 5) | class TaskSchema(object):
    method __init__ (line 7) | def __init__(self,
    method to_dict (line 21) | def to_dict(self):
  class AgiInstance (line 33) | class AgiInstance(object):
    method __init__ (line 35) | def __init__(self, task_description, data_source, task_schema, output,
    method to_dict (line 44) | def to_dict(self):
  class ChatGPTSchema (line 55) | class ChatGPTSchema(object):
    method __init__ (line 57) | def __init__(self, context=None, metadata=''):
    method to_dict (line 61) | def to_dict(self):
  class ResultsForHumanSchema (line 65) | class ResultsForHumanSchema(object):
    method __init__ (line 67) | def __init__(self,
    method to_dict (line 87) | def to_dict(self):
    method to_tsv (line 101) | def to_tsv(result_list, path):

FILE: opencompass/datasets/agieval/dataset_loader.py
  function convert_zero_shot (line 31) | def convert_zero_shot(line, dataset_name):
  function convert_zero_shot_CoT_stage1 (line 66) | def convert_zero_shot_CoT_stage1(line, dataset_name):
  function combine_prompt (line 95) | def combine_prompt(prompt_path,
  function _lazy_load_enc (line 170) | def _lazy_load_enc():
  function concat_prompt (line 177) | def concat_prompt(demos,
  function concat_prompt_chat_mode (line 210) | def concat_prompt_chat_mode(demos,
  function convert_few_shot (line 242) | def convert_few_shot(line, dataset_name, demo, n_shot, chat_mode=False):
  function load_dataset (line 275) | def load_dataset(dataset_name,
  function generate_second_stage_input (line 330) | def generate_second_stage_input(dataset_name,
  function load_dataset_as_result_schema (line 366) | def load_dataset_as_result_schema(dataset_name, parent_path):

FILE: opencompass/datasets/agieval/evaluation.py
  function convert_to_set (line 6) | def convert_to_set(item):
  function evaluate_single_sample (line 16) | def evaluate_single_sample(dataset_name, prediction, label):

FILE: opencompass/datasets/agieval/math_equivalence.py
  function _fix_fracs (line 5) | def _fix_fracs(string):
  function _fix_a_slash_b (line 37) | def _fix_a_slash_b(string):
  function _remove_right_units (line 52) | def _remove_right_units(string):
  function _fix_sqrt (line 62) | def _fix_sqrt(string):
  function _strip_string (line 77) | def _strip_string(string):
  function is_equiv (line 147) | def is_equiv(str1, str2, verbose=False):

FILE: opencompass/datasets/agieval/post_process.py
  function extract_last_line (line 8) | def extract_last_line(string):
  function remove_few_shot_prefix (line 17) | def remove_few_shot_prefix(string: str):
  function try_parse_few_shot_qa_single_answer (line 29) | def try_parse_few_shot_qa_single_answer(string, setting_name, language='...
  function try_parse_few_shot_pattern (line 46) | def try_parse_few_shot_pattern(string: str, dataset_name, setting_name):
  function parse_few_shot_qa_single_answer (line 64) | def parse_few_shot_qa_single_answer(string, setting_name, language='en'):
  function find_first_capital_letter (line 73) | def find_first_capital_letter(answer):
  function extract_answer_in_bracket (line 82) | def extract_answer_in_bracket(answer, prefix='【', suffix='】'):
  function parse_math_answer (line 92) | def parse_math_answer(setting_name, raw_string):
  function parse_qa_multiple_answer (line 170) | def parse_qa_multiple_answer(string, setting_name):
  function post_process (line 180) | def post_process(dataset_name, setting_name, prediction):

FILE: opencompass/datasets/agieval/utils.py
  function read_jsonl (line 5) | def read_jsonl(path):
  function save_jsonl (line 21) | def save_jsonl(lines, directory):
  function extract_answer (line 27) | def extract_answer(js):

FILE: opencompass/datasets/aime2024.py
  class Aime2024Dataset (line 12) | class Aime2024Dataset(BaseDataset):
    method load (line 15) | def load(path, **kwargs):

FILE: opencompass/datasets/anli.py
  class AnliDataset (line 8) | class AnliDataset(BaseDataset):
    method load (line 11) | def load(path: str):

FILE: opencompass/datasets/anthropics_evals.py
  class AiRiskDataset (line 6) | class AiRiskDataset(BaseDataset):
    method load (line 9) | def load(path: str):
  class PersonaDataset (line 26) | class PersonaDataset(BaseDataset):
    method load (line 29) | def load(path: str):
  class SycophancyDataset (line 46) | class SycophancyDataset(BaseDataset):
    method load (line 49) | def load(path: str):

FILE: opencompass/datasets/apps.py
  class APPSDataset (line 37) | class APPSDataset(BaseDataset):
    method load (line 40) | def load(path: str, num_repeats: int = 1):
  class APPS_miniDataset (line 93) | class APPS_miniDataset(BaseDataset):
    method load (line 96) | def load(path: str, num_repeats: int = 1):
  class APPSEvaluator (line 152) | class APPSEvaluator(BaseEvaluator):
    method post_process (line 154) | def post_process(self, text):
    method check_correctness (line 167) | def check_correctness(self, sample, generation, timeout, debug=True):
    method evaluate_generations (line 193) | def evaluate_generations(self,
    method estimate_pass_at_k (line 238) | def estimate_pass_at_k(self, num_samples, num_correct, k):
    method compute_metrics (line 258) | def compute_metrics(self, results, k_list=[1, 10, 100]):
    method score (line 289) | def score(self, predictions, references, test_set):
  class CODE_TYPE (line 303) | class CODE_TYPE(Enum):
  class TimeoutException (line 309) | class TimeoutException(Exception):
  function timeout_handler (line 313) | def timeout_handler(signum, frame):
  class Capturing (line 329) | class Capturing(list):
    method __enter__ (line 331) | def __enter__(self):
    method __exit__ (line 338) | def __exit__(self, *args):
  function run_test (line 344) | def run_test(sample, test=None, debug=False):
  function custom_compare_ (line 751) | def custom_compare_(output, ground_truth):
  function stripped_string_compare (line 767) | def stripped_string_compare(s1, s2):
  function call_method (line 773) | def call_method(method, inputs):
  function reliability_guard (line 800) | def reliability_guard(maximum_memory_bytes=None):

FILE: opencompass/datasets/arc.py
  class ARCDataset (line 14) | class ARCDataset(BaseDataset):
    method load (line 17) | def load(path: str, name: str):
  class ARCDatasetClean (line 64) | class ARCDatasetClean(BaseDataset):
    method load_contamination_annotations (line 69) | def load_contamination_annotations(path, split='val'):
    method load (line 95) | def load(path: str, name: str):

FILE: opencompass/datasets/arc_prize_public_evaluation.py
  class ARCPrizeDataset (line 17) | class ARCPrizeDataset(BaseDataset):
    method load (line 156) | def load(path: str, version: str):
  class ARCPrizeEvaluator (line 178) | class ARCPrizeEvaluator(BaseEvaluator):
    method score (line 180) | def score(self, predictions: List[str],
  function extract_solution (line 197) | def extract_solution(text):
  function pad_array_with_value (line 219) | def pad_array_with_value(array, target_shape, pad_value):
  function compare_solutions_with_padding (line 226) | def compare_solutions_with_padding(generated_output: List[int],

FILE: opencompass/datasets/atlas/dataset_loader.py
  class ATLASDataset (line 8) | class ATLASDataset(BaseDataset):
    method load (line 11) | def load(split: str = 'val'):

FILE: opencompass/datasets/atlas/evaluation.py
  function fix_json_slash (line 23) | def fix_json_slash(s: str) -> str:
  function atlas_pred_postprocess (line 27) | def atlas_pred_postprocess(
  function get_final_results (line 58) | def get_final_results(parsed_judges: List[List[Dict]],
  function process_judge_output (line 88) | def process_judge_output(
  function atlas_judge_postprocess (line 164) | def atlas_judge_postprocess(
  class ATLASLLMEvaluator (line 188) | class ATLASLLMEvaluator(BaseEvaluator):
    method __init__ (line 201) | def __init__(
    method build_inferencer (line 228) | def build_inferencer(self):
    method score (line 256) | def score(
    method output_postprocess (line 362) | def output_postprocess(self, output: Dict, dataset=None) -> Dict:
    method default_judge_cfg (line 382) | def default_judge_cfg(self):

FILE: opencompass/datasets/ax.py
  class AXDatasetV2 (line 13) | class AXDatasetV2(BaseDataset):
    method load (line 16) | def load(path: str):

FILE: opencompass/datasets/babilong/babilong.py
  class BabiLongDataset (line 18) | class BabiLongDataset(BaseDataset):
    method load (line 21) | def load(
  class BabiLongEvaluator (line 97) | class BabiLongEvaluator(BaseEvaluator):
    method score (line 99) | def score(self, predictions, gold):

FILE: opencompass/datasets/babilong/babilong_utils.py
  function compare_answers (line 11) | def compare_answers(target, output):
  function get_dataset_df (line 33) | def get_dataset_df(dataset_path, max_n_facts=None):
  class TaskDataset (line 88) | class TaskDataset(Dataset):
    method __init__ (line 91) | def __init__(self, dataset_path, max_n_facts=None):
    method __getitem__ (line 95) | def __getitem__(self, ind):
    method __len__ (line 107) | def __len__(self):
  function sum_lengths (line 111) | def sum_lengths(sentences):
  class SentenceSampler (line 115) | class SentenceSampler:
    method __init__ (line 118) | def __init__(
    method get_sample (line 137) | def get_sample(self, sample_size):
    method sample_sentences_ (line 163) | def sample_sentences_(self, sample_size):
    method next_sample_ (line 179) | def next_sample_(self):
    method length_is_ok (line 190) | def length_is_ok(self, tokenized):
  class NoiseInjectionDataset (line 200) | class NoiseInjectionDataset(Dataset):
    method __init__ (line 206) | def __init__(
    method __getitem__ (line 227) | def __getitem__(self, ind):
    method __len__ (line 284) | def __len__(self):
    method get_sample_size (line 287) | def get_sample_size(self):

FILE: opencompass/datasets/babilong/prompts.py
  function get_formatted_input (line 17) | def get_formatted_input(

FILE: opencompass/datasets/base.py
  class BaseDataset (line 11) | class BaseDataset:
    method __init__ (line 13) | def __init__(self,
    method _init_reader (line 47) | def _init_reader(self, **kwargs):
    method train (line 51) | def train(self):
    method test (line 55) | def test(self):
    method load (line 59) | def load(**kwargs) -> Union[Dataset, DatasetDict]:

FILE: opencompass/datasets/bbeh.py
  class BBEHDataset (line 17) | class BBEHDataset(BaseDataset):
    method load (line 20) | def load(path: str, name: str):
  function bbeh_freeform_postprocess (line 33) | def bbeh_freeform_postprocess(text: str) -> str:
  function bbeh_mcq_postprocess (line 61) | def bbeh_mcq_postprocess(text: str) -> str:
  class BBEHEvaluator (line 84) | class BBEHEvaluator(BaseEvaluator):
    method score (line 86) | def score(self, predictions, references):
  class BBEHEvaluator_mcq (line 123) | class BBEHEvaluator_mcq(BaseEvaluator):
    method score (line 125) | def score(self, predictions, references):

FILE: opencompass/datasets/bbh.py
  class BBHDataset (line 17) | class BBHDataset(BaseDataset):
    method load (line 20) | def load(path: str, name: str):
  function bbh_mcq_postprocess (line 33) | def bbh_mcq_postprocess(text: str) -> str:
  function bbh_freeform_postprocess (line 48) | def bbh_freeform_postprocess(text: str) -> str:
  class BBHEvaluator (line 66) | class BBHEvaluator(BaseEvaluator):
    method score (line 68) | def score(self, predictions, references):
  class BBHEvaluator_mcq (line 92) | class BBHEvaluator_mcq(BaseEvaluator):
    method score (line 94) | def score(self, predictions, references):

FILE: opencompass/datasets/benbench.py
  class BenBenchDataset (line 15) | class BenBenchDataset(BaseDataset):
    method load (line 18) | def load(path: str, tokenizer_path: str, tokenizer_kwargs: Optional[Di...
  function exact_match_score (line 46) | def exact_match_score(predicted_text, original_text):
  function edit_similarity_score (line 49) | def edit_similarity_score(predicted_text, original_text):
  function rouge_l_score (line 58) | def rouge_l_score(predicted_text, original_text):
  class BenbenEvaluator (line 67) | class BenbenEvaluator(BaseEvaluator):
    method score (line 69) | def score(self, predictions, references):

FILE: opencompass/datasets/beyondaime.py
  class BeyondAIMEDataset (line 9) | class BeyondAIMEDataset(BaseDataset):
    method load (line 12) | def load(path, **kwargs):

FILE: opencompass/datasets/bigcodebench/bigcodebench.py
  class BigCodeBenchDataset (line 21) | class BigCodeBenchDataset(BaseDataset):
    method load (line 24) | def load(path: str = 'opencompass/bigcodebench',
  class BigCodeBenchEvaluator (line 60) | class BigCodeBenchEvaluator(BaseEvaluator):
    method __init__ (line 70) | def __init__(
    method score (line 106) | def score(self, predictions, references):
    method _results_processor (line 213) | def _results_processor(self, results):

FILE: opencompass/datasets/bigcodebench/extractor.py
  function syntax_check (line 21) | def syntax_check(code, verbose=False):
  function code_extract (line 31) | def code_extract(text: str) -> str:
  function get_deps (line 49) | def get_deps(nodes: List[Tuple[str, Node]]) -> Dict[str, Set[str]]:
  function get_function_dependency (line 66) | def get_function_dependency(entrypoint: str,
  function get_definition_name (line 81) | def get_definition_name(node: Node) -> str:
  function traverse_tree (line 87) | def traverse_tree(node: Node) -> Generator[Node, None, None]:
  function has_return_statement (line 106) | def has_return_statement(node: Node) -> bool:
  function extract_target_code_or_empty (line 114) | def extract_target_code_or_empty(code: str,
  function extract_code_generation (line 182) | def extract_code_generation(model_output: str,

FILE: opencompass/datasets/biodata.py
  class BiodataDataset (line 23) | class BiodataDataset(BaseDataset):
    method load (line 26) | def load(path: str, name: str):
  function extract_boxed_text (line 49) | def extract_boxed_text(text):
  class BiodataClsEvaluator (line 71) | class BiodataClsEvaluator(BaseEvaluator):
    method __init__ (line 74) | def __init__(self) -> None:
    method score (line 77) | def score(self, predictions, references):
  function extract_number (line 114) | def extract_number(text):
  class BiodataRMSEEvaluator (line 124) | class BiodataRMSEEvaluator(BaseEvaluator):
    method __init__ (line 127) | def __init__(self) -> None:
    method score (line 130) | def score(self, predictions, references):
  function extract_dict_text (line 155) | def extract_dict_text(text):
  class BiodataDictEvaluator (line 166) | class BiodataDictEvaluator(BaseEvaluator):
    method __init__ (line 169) | def __init__(self) -> None:
    method score (line 172) | def score(self, predictions, references):
  class BiodataStringEvaluator (line 220) | class BiodataStringEvaluator(BaseEvaluator):
    method __init__ (line 223) | def __init__(self) -> None:
    method score (line 226) | def score(self, predictions, references):
  function dedup_ec_codes (line 266) | def dedup_ec_codes(ec_numer_list):
  function count_f1_max (line 313) | def count_f1_max(pred, target):
  class BiodataECNumberEvaluator (line 371) | class BiodataECNumberEvaluator(BaseEvaluator):
    method __init__ (line 374) | def __init__(self) -> None:
    method ec_to_multihot (line 483) | def ec_to_multihot(self, ec_list, ec_labels):
    method score (line 494) | def score(self, predictions, references):
  class BiodataTaskDataset (line 540) | class BiodataTaskDataset(BaseDataset):
    method load (line 543) | def load(path: str, task: str):
  function pearson_correlation_coefficient (line 597) | def pearson_correlation_coefficient(y_true, y_pred):
  function spearman_correlation_coefficient (line 639) | def spearman_correlation_coefficient(y_true, y_pred):
  function r_squared (line 681) | def r_squared(y_true, y_pred):
  function multiple_label_auc (line 724) | def multiple_label_auc(y_true, y_pred):
  function mixed_score (line 752) | def mixed_score(y_true, y_pred, low_range=(30, 1e3)):
  class BiodataMCCEvaluator (line 854) | class BiodataMCCEvaluator(BaseEvaluator):
    method __init__ (line 857) | def __init__(self) -> None:
    method score (line 860) | def score(self, predictions, references):
  class BiodataPCCEvaluator (line 891) | class BiodataPCCEvaluator(BaseEvaluator):
    method __init__ (line 894) | def __init__(self) -> None:
    method score (line 897) | def score(self, predictions, references):
  class BiodataSpearmanEvaluator (line 944) | class BiodataSpearmanEvaluator(BaseEvaluator):
    method __init__ (line 947) | def __init__(self) -> None:
    method score (line 950) | def score(self, predictions, references):
  class BiodataMixedScoreEvaluator (line 977) | class BiodataMixedScoreEvaluator(BaseEvaluator):
    method __init__ (line 980) | def __init__(self) -> None:
    method score (line 983) | def score(self, predictions, references):
  class BiodataR2Evaluator (line 1011) | class BiodataR2Evaluator(BaseEvaluator):
    method __init__ (line 1014) | def __init__(self) -> None:
    method score (line 1017) | def score(self, predictions, references):
  class BiodataAucEvaluator (line 1088) | class BiodataAucEvaluator(BaseEvaluator):
    method __init__ (line 1092) | def __init__(self, predefined_labels=None) -> None:
    method score (line 1098) | def score(self, predictions, references):
  class BiodataAccEvaluator (line 1139) | class BiodataAccEvaluator(BaseEvaluator):
    method __init__ (line 1142) | def __init__(self) -> None:
    method score (line 1145) | def score(self, predictions, references):

FILE: opencompass/datasets/boolq.py
  class BoolQDataset (line 12) | class BoolQDataset(BaseDataset):
    method load (line 15) | def load(**kwargs):
  class BoolQDatasetV2 (line 30) | class BoolQDatasetV2(BaseDataset):
    method load (line 33) | def load(path):
  class BoolQDatasetV3 (line 45) | class BoolQDatasetV3(BaseDataset):
    method load (line 48) | def load(path):

FILE: opencompass/datasets/bustum.py
  class bustumDataset_V2 (line 11) | class bustumDataset_V2(BaseDataset):
    method load (line 14) | def load(path):

FILE: opencompass/datasets/c3.py
  class C3Dataset (line 11) | class C3Dataset(BaseDataset):
    method load (line 14) | def load(path: str):
  class C3Dataset_V2 (line 57) | class C3Dataset_V2(BaseDataset):
    method load (line 60) | def load(path: str):

FILE: opencompass/datasets/calm/calm.py
  class CaLMDataset (line 16) | class CaLMDataset(BaseDataset):
    method load (line 19) | def load(path: str, prompt_style: str) -> datasets.Dataset:
  class CaLMEvaluator (line 26) | class CaLMEvaluator(BaseEvaluator):
    method __init__ (line 28) | def __init__(self, core_metrics, error_analysis, prompt_style,
    method score (line 36) | def score(

FILE: opencompass/datasets/calm/data_processing/generate_questions.py
  function get_get_prompt_func (line 8) | def get_get_prompt_func(task):
  function generate_question_list (line 152) | def generate_question_list(dataset_path, prompt_style):

FILE: opencompass/datasets/calm/data_processing/prompt/AC-B_causal_judgement.py
  function get_prompt (line 144) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/AR-B_CaLM-AR.py
  function get_prompt (line 148) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/ATE.py
  function get_prompt (line 176) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/BAS-B_backadj.py
  function get_prompt (line 132) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/BAS-C_max-BAS.py
  function get_prompt (line 317) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/BAS-C_min-BAS.py
  function get_prompt (line 348) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/BAS-C_mix-BAS.py
  function get_prompt (line 352) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/CA-B_FA.py
  function get_prompt (line 167) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/CA-B_FP.py
  function get_prompt (line 167) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/CB-B_collider-bias.py
  function get_prompt (line 150) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/CDE.py
  function get_prompt (line 176) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/CEG-O_E-CARE.py
  function get_prompt (line 203) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/CEI-B.py
  function get_prompt (line 175) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/CORR-B_correlation.py
  function get_prompt (line 130) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/CR-B_det-counterfactual.py
  function get_prompt (line 131) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/CR-C_CRASS.py
  function get_prompt (line 322) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/EAE-B_exp-away.py
  function get_prompt (line 122) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/ECI-B_CTB.py
  function get_prompt (line 166) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/ECI-B_ESC.py
  function get_prompt (line 166) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/ECI-B_MAVEN-ERE.py
  function get_prompt (line 166) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/ETT.py
  function get_prompt (line 176) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/FAS-C_FAS.py
  function get_prompt (line 369) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/IV-C_CaLM-IV.py
  function get_prompt (line 321) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/NDE.py
  function get_prompt (line 171) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/NIE.py
  function get_prompt (line 170) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/PCD-B_COPA.py
  function get_prompt (line 201) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/PCD-B_E-CARE.py
  function get_prompt (line 201) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/PCD-C_COPA.py
  function get_prompt (line 248) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/PCD-C_E-CARE.py
  function get_prompt (line 246) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/PN.py
  function get_prompt (line 166) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/data_processing/prompt/PS.py
  function get_prompt (line 165) | def get_prompt(task_name, prompt_style, item, prompt_style_str=''):

FILE: opencompass/datasets/calm/evaluation/accuracy/choice.py
  function compute_acc (line 1) | def compute_acc(gt_list, pred_list):

FILE: opencompass/datasets/calm/evaluation/accuracy/open-ended.py
  function is_chinese (line 5) | def is_chinese(text):
  function compute_acc (line 12) | def compute_acc(gt_list, pred_list):

FILE: opencompass/datasets/calm/evaluation/accuracy/prob.py
  function compute_acc (line 1) | def compute_acc(gt_list, pred_list):

FILE: opencompass/datasets/calm/evaluation/core_metrics.py
  function initialize_core_metric_evaluation_components (line 127) | def initialize_core_metric_evaluation_components(task):
  function compute_core_metrics (line 285) | def compute_core_metrics(items, task, prompt_style, gt_items):

FILE: opencompass/datasets/calm/evaluation/error/basic_adversarial/AC-B_causal_judgement.py
  function check_standalization (line 5) | def check_standalization(model_response, prompt_style, type):
  function check_empty (line 13) | def check_empty(model_response):
  function check_repetition (line 20) | def check_repetition(model_response):
  function contains_chinese (line 31) | def contains_chinese(text):
  function contains_english (line 37) | def contains_english(text):
  function check_abnormality (line 44) | def check_abnormality(preds):

FILE: opencompass/datasets/calm/evaluation/error/basic_adversarial/AR-B_CaLM-AR.py
  function check_standalization (line 4) | def check_standalization(model_response, prompt_style, type):
  function check_empty (line 11) | def check_empty(model_response):
  function check_repetition (line 18) | def check_repetition(model_response):
  function contains_chinese (line 26) | def contains_chinese(text):
  function contains_english (line 32) | def contains_english(text):
  function check_abnormality (line 39) | def check_abnormality(preds):

FILE: opencompass/datasets/calm/evaluation/error/basic_adversarial/AS.py
  function check_standalization (line 5) | def check_standalization(model_response, prompt_style, type):
  function check_empty (line 13) | def check_empty(model_response):
  function check_repetition (line 20) | def check_repetition(model_response):
  function contains_chinese (line 31) | def contains_chinese(text):
  function contains_english (line 38) | def contains_english(text):
  function check_abnormality (line 45) | def check_abnormality(preds):

FILE: opencompass/datasets/calm/evaluation/error/basic_adversarial/CA-B.py
  function check_standalization (line 4) | def check_standalization(model_response, prompt_style, type):
  function check_empty (line 11) | def check_empty(model_response):
  function check_repetition (line 18) | def check_repetition(model_response):
  function contains_chinese (line 29) | def contains_chinese(text):
  function contains_english (line 35) | def contains_english(text):
  function check_abnormality (line 42) | def check_abnormality(preds):

FILE: opencompass/datasets/calm/evaluation/error/basic_adversarial/CEI-B.py
  function check_standalization (line 4) | def check_standalization(model_response, prompt_style, type):
  function check_empty (line 11) | def check_empty(model_response):
  function check_repetition (line 18) | def check_repetition(model_response):
  function contains_chinese (line 29) | def contains_chinese(text):
  function contains_english (line 36) | def contains_english(text):
  function check_abnormality (line 43) | def check_abnormality(preds):

FILE: opencompass/datasets/calm/evaluation/error/basic_adversarial/CLADDER.py
  function check_standalization (line 4) | def check_standalization(model_response, prompt_style, type):
  function check_empty (line 11) | def check_empty(model_response):
  function check_repetition (line 18) | def check_repetition(model_response):
  function contains_chinese (line 26) | def contains_chinese(text):
  function contains_english (line 33) | def contains_english(text):
  function check_abnormality (line 40) | def check_abnormality(preds):

FILE: opencompass/datasets/calm/evaluation/error/basic_adversarial/CR-C_CRASS.py
  function check_standalization (line 4) | def check_standalization(model_response, prompt_style, type):
  function check_empty (line 13) | def check_empty(model_response):
  function check_repetition (line 20) | def check_repetition(model_response):
  function contains_chinese (line 30) | def contains_chinese(text):
  function contains_english (line 37) | def contains_english(text):
  function check_abnormality (line 44) | def check_abnormality(preds):

FILE: opencompass/datasets/calm/evaluation/error/basic_adversarial/ECI.py
  function check_standalization (line 4) | def check_standalization(model_response, prompt_style, type):
  function check_empty (line 11) | def check_empty(model_response):
  function check_repetition (line 18) | def check_repetition(model_response):
  function contains_chinese (line 29) | def contains_chinese(text):
  function contains_english (line 36) | def contains_english(text):
  function check_abnormality (line 43) | def check_abnormality(preds):

FILE: opencompass/datasets/calm/evaluation/error/basic_adversarial/Natural.py
  function check_standalization (line 5) | def check_standalization(model_response, prompt_style, type):
  function check_empty (line 13) | def check_empty(model_response):
  function check_repetition (line 20) | def check_repetition(model_response):
  function contains_chinese (line 31) | def contains_chinese(text):
  function contains_english (line 37) | def contains_english(text):
  function check_abnormality (line 46) | def check_abnormality(preds):

FILE: opencompass/datasets/calm/evaluation/error/basic_adversarial/PCD-B.py
  function check_standalization (line 4) | def check_standalization(model_response, prompt_style, type):
  function check_empty (line 11) | def check_empty(model_response):
  function check_repetition (line 18) | def check_repetition(model_response):
  function contains_chinese (line 26) | def contains_chinese(text):
  function contains_english (line 32) | def contains_english(text):
  function check_abnormality (line 39) | def check_abnormality(preds):

FILE: opencompass/datasets/calm/evaluation/error/basic_adversarial/PCD-C.py
  function check_standalization (line 4) | def check_standalization(model_response, prompt_style, type):
  function check_empty (line 11) | def check_empty(model_response):
  function check_repetition (line 18) | def check_repetition(model_response):
  function contains_chinese (line 28) | def contains_chinese(text):
  function contains_english (line 34) | def contains_english(text):
  function check_abnormality (line 41) | def check_abnormality(preds):

FILE: opencompass/datasets/calm/evaluation/error/basic_adversarial/Probability.py
  function check_standalization (line 5) | def check_standalization(model_response, prompt_style, type):
  function check_empty (line 20) | def check_empty(model_response):
  function check_repetition (line 27) | def check_repetition(model_response):
  function contains_chinese (line 38) | def contains_chinese(text):
  function contains_english (line 45) | def contains_english(text):
  function check_abnormality (line 56) | def check_abnormality(preds):

FILE: opencompass/datasets/calm/evaluation/errors.py
  function initialize_error_identification_components (line 11) | def initialize_error_identification_components(task, prompt_style):
  function identify_model_errors (line 171) | def identify_model_errors(items, task, prompt_style, gt_items):
  function get_item_error (line 221) | def get_item_error(model_response, task, error_module, prompt_style):

FILE: opencompass/datasets/calm/evaluation/labeling/AC-B_causal_judgement.py
  function get_gt_label (line 6) | def get_gt_label(item):
  function get_pred_label (line 14) | def get_pred_label(model_response, item, prompt_style, type):

FILE: opencompass/datasets/calm/evaluation/labeling/AR-B_CaLM-AR.py
  function get_gt_label (line 6) | def get_gt_label(item):
  function get_pred_label (line 10) | def get_pred_label(model_response, item, prompt_style, type):

FILE: opencompass/datasets/calm/evaluation/labeling/AS.py
  function get_gt_label (line 7) | def get_gt_label(item):
  function get_pred_label (line 11) | def get_pred_label(model_response, item, prompt_style, type):

FILE: opencompass/datasets/calm/evaluation/labeling/CA-B_FA.py
  function get_gt_label (line 6) | def get_gt_label(item):
  function get_pred_label (line 10) | def get_pred_label(model_response, item, prompt_style, type):

FILE: opencompass/datasets/calm/evaluation/labeling/CA-B_FP.py
  function get_gt_label (line 6) | def get_gt_label(item):
  function get_pred_label (line 10) | def get_pred_label(model_response, item, prompt_style, type):

FILE: opencompass/datasets/calm/evaluation/labeling/CEG-O_E-CARE.py
  function get_gt_label (line 1) | def get_gt_label(item):
  function get_pred_label (line 5) | def get_pred_label(model_response, item, prompt_style, type):

FILE: opencompass/datasets/calm/evaluation/labeling/CEI-B.py
  function get_gt_label (line 6) | def get_gt_label(item):
  function get_pred_label (line 10) | def get_pred_label(model_response, item, prompt_style, type):

FILE: opencompass/datasets/calm/evaluation/labeling/CLADDER.py
  function get_gt_label (line 6) | def get_gt_label(item):
  function get_pred_label (line 14) | def get_pred_label(model_response, item, prompt_style, type):

FILE: opencompass/datasets/calm/evaluation/labeling/CR-C_CRASS.py
  function get_gt_label (line 8) | def get_gt_label(item):
  function get_pred_label (line 12) | def get_pred_label(model_response, item, prompt_style, type):

FILE: opencompass/datasets/calm/evaluation/labeling/ECI.py
  function get_gt_label (line 6) | def get_gt_label(item):
  function get_pred_label (line 10) | def get_pred_label(model_response, item, prompt_style, type):

FILE: opencompass/datasets/calm/evaluation/labeling/Natural.py
  function get_gt_label (line 9) | def get_gt_label(item):
  function extract_answer (line 13) | def extract_answer(model_response, item, prompt_style, type):
  function get_pred_label (line 57) | def get_pred_label(model_response, item, prompt_style, type):

FILE: opencompass/datasets/calm/evaluation/labeling/PCD-B.py
  function get_gt_label (line 6) | def get_gt_label(item):
  function get_pred_label (line 10) | def get_pred_label(model_response, item, prompt_style, type):

FILE: opencompass/datasets/calm/evaluation/labeling/PCD-C.py
  function get_gt_label (line 6) | def get_gt_label(item):
  function get_pred_label (line 10) | def get_pred_label(model_response, item, prompt_style, type):

FILE: opencompass/datasets/calm/evaluation/labeling/Probability.py
  function get_gt_label (line 9) | def get_gt_label(item):
  function extract_prob (line 14) | def extract_prob(model_response, prompt_style, type):
  function get_pred_label (line 61) | def get_pred_label(model_response, item, prompt_style, type):

FILE: opencompass/datasets/calm/evaluation/labeling/common_answers.py
  function is_numeric (line 301) | def is_numeric(value):
  function add_quotes_to_unquoted (line 309) | def add_quotes_to_unquoted(json_str):
  function change_quotation (line 314) | def change_quotation(json_str):

FILE: opencompass/datasets/calm/utils/load_items.py
  function load_query_instances (line 5) | def load_query_instances(path):

FILE: opencompass/datasets/cb.py
  class CBDatasetV2 (line 12) | class CBDatasetV2(BaseDataset):
    method load (line 15) | def load(path):

FILE: opencompass/datasets/ceval.py
  class CEvalDataset (line 15) | class CEvalDataset(BaseDataset):
    method load (line 18) | def load(path: str, name: str, local_mode: bool = False):
  class CEvalDatasetClean (line 41) | class CEvalDatasetClean(BaseDataset):
    method load_contamination_annotations (line 46) | def load_contamination_annotations(path, split='val'):
    method load (line 70) | def load(path: str, name: str):

FILE: opencompass/datasets/charm.py
  function charm_reason_postprocess (line 17) | def charm_reason_postprocess(text: str) -> str:
  class CharmReasonEvaluator (line 32) | class CharmReasonEvaluator(BaseEvaluator):
    method score (line 34) | def score(self, predictions, references):
  function charm_memory_eval (line 52) | def charm_memory_eval(pred: str, ref: Union[str, List[str]]) -> str:
  class CharmMemoryEvaluator (line 85) | class CharmMemoryEvaluator(LMEvaluator):
    method __init__ (line 91) | def __init__(self, prompt_template=None, *nargs, **kwargs):
    method score (line 101) | def score(self, predictions, references, **kwargs):
  class CharmDataset (line 146) | class CharmDataset(BaseDataset):
    method load (line 149) | def load(path: str, name: str):

FILE: opencompass/datasets/chatml/chatml.py
  class ChatMLDataset (line 13) | class ChatMLDataset(BaseDataset):
    method load (line 65) | def load(path, file_name=None, local_mode=False):

FILE: opencompass/datasets/chatml/verification.py
  class TextItem (line 7) | class TextItem(BaseModel):
  class ImageItem (line 12) | class ImageItem(BaseModel):
  class SystemMessage (line 20) | class SystemMessage(BaseModel):
  class AssistantMessage (line 25) | class AssistantMessage(BaseModel):
  class UserMessage (line 30) | class UserMessage(BaseModel):
  class VerifyDataset (line 38) | class VerifyDataset(BaseModel):
    method validate_answer_length (line 43) | def validate_answer_length(self) -> 'VerifyDataset':

FILE: opencompass/datasets/chem_exam.py
  class ChemExamDataset (line 13) | class ChemExamDataset(BaseDataset):
    method load (line 16) | def load(path: str):
  function chem_exam_score_llmjudge_postprocess (line 56) | def chem_exam_score_llmjudge_postprocess(output, output_path, dataset):

FILE: opencompass/datasets/chembench.py
  class ChemBenchDataset (line 13) | class ChemBenchDataset(BaseDataset):
    method load (line 16) | def load(path: str, name: str):

FILE: opencompass/datasets/chid.py
  class CHIDDataset (line 12) | class CHIDDataset(BaseDataset):
    method load (line 15) | def load(**kwargs):
  class CHIDDatasetV2 (line 33) | class CHIDDatasetV2(BaseDataset):
    method load (line 36) | def load(path):

FILE: opencompass/datasets/chinese_simpleqa.py
  function chinese_simpleqa_preprocess (line 88) | def chinese_simpleqa_preprocess(text: str) -> str:
  class CsimpleqaDataset (line 94) | class CsimpleqaDataset(BaseDataset):
    method load (line 96) | def load(self, path: str, name: str, *args, **kwargs):
  function post_process_csimpleqa (line 135) | def post_process_csimpleqa(completion):
  function get_judgeanswer_and_reference (line 146) | def get_judgeanswer_and_reference(result, filename, post_process):
  function calculate_metrics (line 162) | def calculate_metrics(judged_answers):
  function get_results (line 194) | def get_results(judged_answers):
  function csimpleqa_postprocess (line 200) | def csimpleqa_postprocess(output: dict, output_path: str) -> dict:

FILE: opencompass/datasets/cibench.py
  function load_experiment (line 19) | def load_experiment(file: str) -> dict:
  function check_internet (line 100) | def check_internet():
  class CIBenchDataset (line 114) | class CIBenchDataset(BaseDataset):
    method load (line 118) | def load(path: str, internet_check: bool = False):
  function sklearn_ssim (line 142) | def sklearn_ssim(pred_img, target_img):
  function vl_model_score (line 187) | def vl_model_score(model, pred_img, ori_prompt, judge_prompt):
  class CIBenchEvaluator (line 200) | class CIBenchEvaluator(BaseEvaluator):
    method __init__ (line 220) | def __init__(self,
    method check_user_data_dir (line 264) | def check_user_data_dir(self, user_data_dir):
    method valid_step (line 280) | def valid_step(step):
    method correct_step (line 293) | def correct_step(step, target) -> dict:
    method text_step (line 326) | def text_step(self, step, target) -> dict:
    method vis_similarity_step (line 349) | def vis_similarity_step(self, step, target, ori_prompt) -> dict:
    method save_results (line 383) | def save_results(self, origin_prompt, steps, references):
    method set_data_dir (line 459) | def set_data_dir(self, work_dir):
    method unset_data_dir (line 469) | def unset_data_dir(self, work_dir):
    method single_exp (line 473) | def single_exp(self, gold, steps, single_ori_prompt):
    method get_output_dir (line 516) | def get_output_dir(self):
    method score (line 526) | def score(self, predictions: List, references: List, steps: List,

FILE: opencompass/datasets/circular.py
  function get_origin_patterns (line 22) | def get_origin_patterns(option_keys):
  function get_circular_patterns (line 26) | def get_circular_patterns(option_keys):
  function get_all_possible_patterns (line 35) | def get_all_possible_patterns(option_keys):
  class CircularDatasetMeta (line 40) | class CircularDatasetMeta(type):
    method make_circular_items (line 63) | def make_circular_items(
    method make_circular_dataset (line 90) | def make_circular_dataset(dataset, circular_patterns, option_keys,
    method make_circular (line 105) | def make_circular(
    method __new__ (line 183) | def __new__(cls, name, bases, dct):
  class CircularCEvalDataset (line 211) | class CircularCEvalDataset(CEvalDataset, metaclass=CircularDatasetMeta):
  class CircularMMLUDataset (line 218) | class CircularMMLUDataset(MMLUDataset, metaclass=CircularDatasetMeta):
  class CircularCMMLUDataset (line 225) | class CircularCMMLUDataset(CMMLUDataset, metaclass=CircularDatasetMeta):
  class CircularCSQADataset (line 232) | class CircularCSQADataset(commonsenseqaDataset, metaclass=CircularDatase...
  class CircularARCDataset (line 239) | class CircularARCDataset(ARCDataset, metaclass=CircularDatasetMeta):
    method default_answer_key_switch_method (line 244) | def default_answer_key_switch_method(item, circular_pattern):
  class CircularHSWAGDataset (line 250) | class CircularHSWAGDataset(HellaswagDataset_V2, metaclass=CircularDatase...
  class CircularOBQADataset (line 257) | class CircularOBQADataset(OBQADataset, metaclass=CircularDatasetMeta):
  class CircularRaceDataset (line 264) | class CircularRaceDataset(RaceDataset, metaclass=CircularDatasetMeta):
  class CircularXiezhiDataset (line 271) | class CircularXiezhiDataset(XiezhiDataset, metaclass=CircularDatasetMeta):
  class CircularsiqaDataset (line 278) | class CircularsiqaDataset(SiqaDatasetV3, metaclass=CircularDatasetMeta):
  class CircularPIQADataset (line 285) | class CircularPIQADataset(PIQADatasetV2, metaclass=CircularDatasetMeta):
    method default_answer_key_switch_method (line 290) | def default_answer_key_switch_method(item, circular_pattern):
  class CircularEvaluator (line 296) | class CircularEvaluator(BaseEvaluator):
    method __init__ (line 320) | def __init__(self, circular_pattern='circular'):
    method score (line 324) | def score(self, predictions, references, test_set):

FILE: opencompass/datasets/civilcomments.py
  class CivilCommentsDataset (line 9) | class CivilCommentsDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: opencompass/datasets/climaqa.py
  class ClimaQADataset (line 11) | class ClimaQADataset(BaseDataset):
    method load (line 14) | def load(path: str, task: str, **kwargs):

FILE: opencompass/datasets/clozeTest_maxmin.py
  class MaxminDataset (line 12) | class MaxminDataset(BaseDataset):
    method load (line 15) | def load(test_path, answer_path=None):

FILE: opencompass/datasets/cluewsc.py
  class CluewscDataset (line 12) | class CluewscDataset(BaseDataset):
    method load (line 15) | def load(**kwargs):
  class CluewscDatasetV2 (line 43) | class CluewscDatasetV2(BaseDataset):
    method load (line 46) | def load(path):

FILE: opencompass/datasets/cmb.py
  class CMBDataset (line 13) | class CMBDataset(BaseDataset):
    method load (line 16) | def load(path: str):

FILE: opencompass/datasets/cmmlu.py
  class CMMLUDataset (line 14) | class CMMLUDataset(BaseDataset):
    method load (line 17) | def load(path: str, name: str, **kwargs):

FILE: opencompass/datasets/cmnli.py
  class CMNLIDataset (line 13) | class CMNLIDataset(BaseDataset):
    method load (line 16) | def load(path, local_mode: bool = False):
  class CMNLIDatasetV2 (line 39) | class CMNLIDatasetV2(BaseDataset):
    method load (line 42) | def load(path, local_mode: bool = False):

FILE: opencompass/datasets/cmo_fib.py
  class CMOFibDataset (line 12) | class CMOFibDataset(BaseDataset):
    method load (line 15) | def load(path):

FILE: opencompass/datasets/cmphysbench/SEED/SEED.py
  function update_func (line 65) | def update_func(x, y):
  function remove_func (line 74) | def remove_func(x):
  function remove_tree_func (line 78) | def remove_tree_func(x):
  function insert_func (line 85) | def insert_func(x):
  function insert_tree_func (line 89) | def insert_tree_func(x):
  function calc_tree_size (line 93) | def calc_tree_size(node):
  function score_calc (line 134) | def score_calc(tree_dist, tree_size):
  function numeric_score_calc (line 140) | def numeric_score_calc(student_answer_exp, ground_truth_exp):
  function simplify_with_timeout (line 209) | def simplify_with_timeout(expr):
  function time_simplify (line 213) | def time_simplify(expr):
  function equal_with_timeout (line 222) | def equal_with_timeout(expr1, expr2):
  function time_equal (line 226) | def time_equal(expr1, expr2):
  function sympy_to_tree (line 234) | def sympy_to_tree(expr):
  class TreeNode (line 306) | class TreeNode:
    method __init__ (line 308) | def __init__(self, label, children=None, node_type='other'):
    method get_children (line 314) | def get_children(self):
    method __str__ (line 317) | def __str__(self):
  function print_tree (line 321) | def print_tree(node, indent=0):
  class LaTeXError (line 328) | class LaTeXError(Exception):
    method __init__ (line 330) | def __init__(self, message='LaTeXError'):
  class SymPyError (line 334) | class SymPyError(Exception):
    method __init__ (line 336) | def __init__(self, message='SymPyError'):
  class TreeError (line 340) | class TreeError(Exception):
    method __init__ (line 342) | def __init__(self, message='TreeError'):
  class DistError (line 346) | class DistError(Exception):
    method __init__ (line 348) | def __init__(self, message='DistanceError'):
  function Equation_standardize (line 352) | def Equation_standardize(latex):
  function extract_interval (line 359) | def extract_interval(latex):
  function judge_interval (line 381) | def judge_interval(latex):
  function check_latex_wrap (line 399) | def check_latex_wrap(s):
  function parse_bracketed_string (line 415) | def parse_bracketed_string(s):
  function strip_dollar_signs (line 424) | def strip_dollar_signs(s):
  function extract_numeric_part (line 433) | def extract_numeric_part(latex_str: str) -> str:
  function extract_tuple (line 485) | def extract_tuple(latex):
  function clean_latex_unit (line 538) | def clean_latex_unit(unit_str):
  function parse_latex_quantity_general (line 557) | def parse_latex_quantity_general(latex_str):
  function convert_and_output_general (line 592) | def convert_and_output_general(latex_qty1, latex_qty2, target_unit=None):
  function SEED (line 620) | def SEED(answer_latex, test_latex, type, debug_mode=False):

FILE: opencompass/datasets/cmphysbench/SEED/extended_zss.py
  class Node (line 13) | class Node(object):
    method __init__ (line 15) | def __init__(self, label, children=None):
    method get_children (line 20) | def get_children(node):
    method get_label (line 24) | def get_label(node):
    method addkid (line 27) | def addkid(self, node, before=False):
    method get (line 35) | def get(self, label):
  class AnnotatedTree (line 44) | class AnnotatedTree(object):
    method __init__ (line 46) | def __init__(self, root, get_children):
  function ext_distance (line 96) | def ext_distance(A, B, get_children, single_insert_cost, insert_cost,

FILE: opencompass/datasets/cmphysbench/SEED/latex_pre_process.py
  function convert_caret_to_derivative (line 11) | def convert_caret_to_derivative(latex_str):
  function preprocess_special_superscripts (line 26) | def preprocess_special_superscripts(latex_str):
  function brackets_balanced (line 49) | def brackets_balanced(s: str) -> bool:
  function remove_non_ascii (line 70) | def remove_non_ascii(text):
  function extract_bracket_content (line 75) | def extract_bracket_content(s: str, bracket_position: int) -> str:
  function find_first_unescaped_brace (line 107) | def find_first_unescaped_brace(s: str) -> int:
  function extract_command (line 120) | def extract_command(s: str, brace_pos: int) -> str | None:
  function remove_command (line 149) | def remove_command(s, command, keep_inside=False):
  function convert_latex_fractions (line 230) | def convert_latex_fractions(latex_str):
  function get_first_brace_command (line 248) | def get_first_brace_command(s: str) -> str | None:
  function remove_overall_brace (line 257) | def remove_overall_brace(s: str) -> str:
  function exp_frac (line 271) | def exp_frac(s):
  function find_all (line 303) | def find_all(s, sub_str, allow_overlap=True):
  function bar_inside_vec (line 319) | def bar_inside_vec(s):
  function vec_lower_idx (line 346) | def vec_lower_idx(input_str):
  function convert_vec_syntax (line 359) | def convert_vec_syntax(text):
  function remove_outer_braces (line 384) | def remove_outer_braces(tex_str):
  function extract_last_equal_content (line 396) | def extract_last_equal_content(s: str, strip_whitespace: bool = True) ->...
  function first_pre_process (line 434) | def first_pre_process(s, t, extract_box=True):
  function remove_text_from_latex (line 490) | def remove_text_from_latex(expr: str) -> str:
  function extract_bracket_subscript_pairs (line 500) | def extract_bracket_subscript_pairs(expr):
  function add_number_to_bracket_subscripts (line 535) | def add_number_to_bracket_subscripts(expr):
  function insert_multiplication_symbols (line 550) | def insert_multiplication_symbols(expr):
  function remove_all_text_commands (line 570) | def remove_all_text_commands(latex_str):
  function convert_general_exp_format (line 582) | def convert_general_exp_format(latex_str):
  function modify_latex_expression (line 590) | def modify_latex_expression(expr: str) -> str:
  function wrap_single_subscripts (line 600) | def wrap_single_subscripts(s: str) -> str:
  function replace_hc_text (line 613) | def replace_hc_text(s: str) -> str:
  function standardize_dE_notation (line 630) | def standardize_dE_notation(s: str) -> str:
  function replace_arrow_expression (line 635) | def replace_arrow_expression(s: str) -> str:
  function preprocess_feynman_slash (line 643) | def preprocess_feynman_slash(latex_str: str) -> str:
  function fix_subscript_on_parentheses (line 657) | def fix_subscript_on_parentheses(s: str) -> str:
  function reorder_super_sub (line 667) | def reorder_super_sub(latex_str: str) -> str:
  function second_pre_process (line 693) | def second_pre_process(s):
  function add_parentheses_to_d (line 852) | def add_parentheses_to_d(expr):
  class MyConfig (line 865) | class MyConfig:
  class MyNormalization (line 882) | class MyNormalization:
  function replace_derivative_frac_preserve_frac (line 903) | def replace_derivative_frac_preserve_frac(expr: str) -> str:
  function master_convert_with_timeout (line 935) | def master_convert_with_timeout(s, t):
  function master_convert (line 947) | def master_convert(s, t):

FILE: opencompass/datasets/cmphysbench/SEED/test.py
  function run_case (line 25) | def run_case(idx: int, gt: str, pred: str, type: str, note: str = ''):

FILE: opencompass/datasets/cmphysbench/cmphysbench.py
  class CMPhysBenchDataset (line 13) | class CMPhysBenchDataset(BaseDataset):
    method load (line 16) | def load(path: str):
  function extract_boxed_text_overlap (line 31) | def extract_boxed_text_overlap(text):
  function extract_boxed_text_improved (line 67) | def extract_boxed_text_improved(text):
  class CMPhysBenchEvaluator (line 136) | class CMPhysBenchEvaluator(BaseEvaluator):
    method __init__ (line 139) | def __init__(self) -> None:
    method score (line 142) | def score(self, predictions, references):

FILE: opencompass/datasets/cmrc.py
  class CMRCDataset (line 12) | class CMRCDataset(BaseDataset):
    method load (line 15) | def load(path: str):
  function cmrc_postprocess (line 46) | def cmrc_postprocess(text: str) -> str:

FILE: opencompass/datasets/codecompass/CodeCompass.py
  class BaseDataset (line 14) | class BaseDataset:
  function get_data_path (line 17) | def get_data_path(path, local_mode=False):
  class CodeCompassCodeGenerationDataset (line 21) | class CodeCompassCodeGenerationDataset(BaseDataset):
    method load (line 48) | def load(path: str = 'opencompass/CodeCompass',
    method _extract_limits (line 94) | def _extract_limits(problem_text: str) -> Dict[str, Any]:
    method _process_item (line 120) | def _process_item(item: Dict[str, Any], system_prompt: str,
    method _create_evaluation_sample (line 174) | def _create_evaluation_sample(
    method validate_dataset (line 259) | def validate_dataset(dataset: DatasetDict) -> bool:

FILE: opencompass/datasets/codecompass/codecompass_runner.py
  function run_test_for_cpp_problem (line 6) | def run_test_for_cpp_problem(sample: dict,

FILE: opencompass/datasets/codecompass/evaluator.py
  class CodeCompassEvaluator (line 19) | class CodeCompassEvaluator(BaseEvaluator):
    method __init__ (line 22) | def __init__(self,
    method _build_results (line 45) | def _build_results(self, extracted_predictions: Dict[int, List[str]],
    method score (line 77) | def score(self, predictions: List[Any],
    method _prepare_sample (line 148) | def _prepare_sample(self, reference: Any, idx: int = -1) -> Dict[str, ...
    method _run_parallel_evaluation (line 230) | def _run_parallel_evaluation(self,

FILE: opencompass/datasets/codecompass/executor.py
  class LocalExecutor (line 10) | class LocalExecutor:
    method __init__ (line 16) | def __init__(self,
    method _set_resource_limits (line 29) | def _set_resource_limits(self):
    method _compile_cpp (line 41) | def _compile_cpp(self, source_file: Path, temp_dir: Path) -> tuple:
    method _run_executable (line 63) | def _run_executable(self, exec_file: Path, stdin_data: str) -> Dict:
    method execute_code (line 116) | def execute_code(self, source_code: str, stdin: str, language: str,
    method verify_output (line 145) | def verify_output(self, result: Dict, expected_output: str) -> Dict:
    method submit_code (line 165) | def submit_code(self,

FILE: opencompass/datasets/codecompass/metrics.py
  function estimate_pass_at_k (line 4) | def estimate_pass_at_k(num_samples, num_correct, k):
  function compute_metrics_from_results (line 24) | def compute_metrics_from_results(results: dict, k_list=[1]):

FILE: opencompass/datasets/codecompass/utils.py
  function extract_cpp_code (line 1) | def extract_cpp_code(model_output: str, model_type: str = 'chat'):
  function extract_cpp_code_with_debug (line 36) | def extract_cpp_code_with_debug(model_output: str, model_type: str = 'ch...

FILE: opencompass/datasets/commonsenseqa.py
  class commonsenseqaDataset (line 14) | class commonsenseqaDataset(BaseDataset):
    method load (line 17) | def load(path):

FILE: opencompass/datasets/commonsenseqa_cn.py
  class CommonsenseQADataset_CN (line 10) | class CommonsenseQADataset_CN(BaseDataset):
    method load (line 13) | def load(path):

FILE: opencompass/datasets/compassbench_obj.py
  function get_number (line 12) | def get_number(options):
  class CompassBenchObjectiveV1_3 (line 21) | class CompassBenchObjectiveV1_3(BaseDataset):
    method load (line 24) | def load(path: str, name: str):
  class CompassBenchObjectiveMath (line 67) | class CompassBenchObjectiveMath(BaseDataset):
    method load (line 70) | def load(path: str):
  function compassbench_objective_v1_3_postprocess (line 96) | def compassbench_objective_v1_3_postprocess(text: str, name) -> str:

FILE: opencompass/datasets/copa.py
  class COPADatasetV2 (line 12) | class COPADatasetV2(BaseDataset):
    method load (line 15) | def load(path):

FILE: opencompass/datasets/crowspairs.py
  class CrowspairsDataset (line 13) | class CrowspairsDataset(BaseDataset):
    method load (line 16) | def load(**kwargs):
  class CrowspairsDatasetV2 (line 28) | class CrowspairsDatasetV2(BaseDataset):
    method load (line 31) | def load(**kwargs):
  function crowspairs_postprocess (line 41) | def crowspairs_postprocess(text: str) -> str:
  class CrowspairsEvaluator (line 60) | class CrowspairsEvaluator(BaseEvaluator):
    method __init__ (line 64) | def __init__(self) -> None:
    method score (line 67) | def score(self, predictions: List, references: List) -> dict:

FILE: opencompass/datasets/crowspairs_cn.py
  class CrowspairsDatasetCN (line 10) | class CrowspairsDatasetCN(BaseDataset):
    method load (line 14) | def load(path):

FILE: opencompass/datasets/csl.py
  class CslDataset (line 12) | class CslDataset(BaseDataset):
    method load (line 15) | def load(**kwargs):
  class CslDatasetV2 (line 33) | class CslDatasetV2(BaseDataset):
    method load (line 36) | def load(path):

FILE: opencompass/datasets/custom.py
  class OptionSimAccEvaluator (line 21) | class OptionSimAccEvaluator(BaseEvaluator):
    method __init__ (line 23) | def __init__(self, options) -> None:
    method match_any_label (line 32) | def match_any_label(self, pred, test_item):
    method score (line 68) | def score(self, predictions: List, references: List, test_set) -> dict:
  class CircularOptionSimAccEvaluator (line 88) | class CircularOptionSimAccEvaluator(OptionSimAccEvaluator):
    method __init__ (line 90) | def __init__(self, options, circular_pattern='circular'):
    method score (line 94) | def score(self, predictions, references, test_set):
  class CustomDataset (line 165) | class CustomDataset(BaseDataset):
    method load (line 168) | def load(path, file_name=None, local_mode=False):
  class CodeCustomDataset (line 187) | class CodeCustomDataset(BaseDataset):
    method load (line 190) | def load(path, file_name=None, local_mode=False, num_repeats=1, **kwar...
  class CircularCustomDataset (line 213) | class CircularCustomDataset(CustomDataset, metaclass=CircularDatasetMeta):
  function stringfy_types (line 217) | def stringfy_types(obj):
  function make_mcq_gen_config (line 226) | def make_mcq_gen_config(meta):
  function make_circular_mcq_gen_config (line 274) | def make_circular_mcq_gen_config(meta):
  function make_qa_gen_config (line 324) | def make_qa_gen_config(meta):
  function make_mcq_ppl_config (line 373) | def make_mcq_ppl_config(meta):
  function make_circular_mcq_ppl_config (line 425) | def make_circular_mcq_ppl_config(meta):
  function parse_example_dataset (line 479) | def parse_example_dataset(config):
  function make_custom_dataset_config (line 535) | def make_custom_dataset_config(config):

FILE: opencompass/datasets/cvalues.py
  class CValuesDataset (line 12) | class CValuesDataset(BaseDataset):
    method load (line 15) | def load(path):

FILE: opencompass/datasets/dingo.py
  class DingoDataset (line 19) | class DingoDataset(BaseDataset):
    method load (line 22) | def load(path: str):
  class DingoLongDataset (line 35) | class DingoLongDataset(BaseDataset):
    method load (line 38) | def load(path: str):
  class DingoEvaluator (line 48) | class DingoEvaluator(BaseEvaluator):
    method score (line 50) | def score(self, origin_prompt: List, predictions: List) -> dict:

FILE: opencompass/datasets/drcd.py
  class DRCDDataset (line 12) | class DRCDDataset(BaseDataset):
    method load (line 15) | def load(path: str):
  function drcd_postprocess (line 46) | def drcd_postprocess(text: str) -> str:

FILE: opencompass/datasets/drop.py
  class dropDataset (line 11) | class dropDataset(BaseDataset):
    method get_answers (line 14) | def get_answers(validated_answers):
    method load (line 29) | def load(path, only_number=True):

FILE: opencompass/datasets/drop_simple_eval.py
  function normalize (line 18) | def normalize(s: str) -> str:
  function fuzzy_match (line 28) | def fuzzy_match(s1: str, s2: str) -> bool:
  class DropOpenAIDataset (line 39) | class DropOpenAIDataset(BaseDataset):
    method load (line 42) | def load(path):
  class DropOpenAIEvaluator (line 58) | class DropOpenAIEvaluator(BaseEvaluator):
    method score (line 60) | def score(self, predictions, references):

FILE: opencompass/datasets/ds1000.py
  class DS1000Dataset (line 35) | class DS1000Dataset(BaseDataset):
    method get_data (line 52) | def get_data(self, problem_path: str) -> dict:
    method load (line 89) | def load(self,
  function ds1000_postprocess (line 128) | def ds1000_postprocess(text: str) -> str:
  function ds1000_completion_postprocess (line 149) | def ds1000_completion_postprocess(text: str) -> str:
  function ds1000_matplotlib_postprocess (line 160) | def ds1000_matplotlib_postprocess(text: str) -> str:
  class DS1000Evaluator (line 182) | class DS1000Evaluator(BaseEvaluator):
    method __init__ (line 185) | def __init__(self, num_workers=16) -> None:
    method score_single (line 188) | def score_single(self, pred, refer):
    method score (line 269) | def score(self, predictions, references):
  class Command (line 280) | class Command(object):
    method __init__ (line 283) | def __init__(self, cmd):
    method run (line 287) | def run(self, timeout):
  function import_source_file (line 318) | def import_source_file(fname, modname):
  class DS1000ServiceEvaluator (line 350) | class DS1000ServiceEvaluator(BaseEvaluator):
    method __init__ (line 366) | def __init__(self,
    method score (line 379) | def score(self, predictions, references):
    method _code_eval_service (line 415) | def _code_eval_service(self, file_path: str) -> tuple:

FILE: opencompass/datasets/ds1000_interpreter.py
  class DS1000Dataset_Interperter (line 10) | class DS1000Dataset_Interperter(DS1000Dataset):
    method load (line 13) | def load(
  class DS1000InterpreterEvaluator (line 31) | class DS1000InterpreterEvaluator(BaseEvaluator):
    method __init__ (line 39) | def __init__(self, action: str = 'PythonInterpreter'):
    method get_action (line 42) | def get_action(self, step):
    method score (line 47) | def score(self, predictions: List, references: List, steps: List):

FILE: opencompass/datasets/eese/eese.py
  class EESEDataset (line 12) | class EESEDataset(BaseDataset):
    method load (line 15) | def load(path: str, file_name: str = 'EESE.jsonl', **kwargs):

FILE: opencompass/datasets/eese/eese_postprocessors.py
  function eese_score_postprocess_dict (line 8) | def eese_score_postprocess_dict(output: dict, output_path: str) -> dict:

FILE: opencompass/datasets/eese/utils.py
  function extract_first_numeric_score (line 6) | def extract_first_numeric_score(score_text):
  function process_results (line 31) | def process_results(results, overall_avg):

FILE: opencompass/datasets/eprstmt.py
  class EprstmtDatasetV2 (line 12) | class EprstmtDatasetV2(BaseDataset):
    method load (line 15) | def load(path):

FILE: opencompass/datasets/flores.py
  class FloresFirst100Dataset (line 14) | class FloresFirst100Dataset(BaseDataset):
    method load_single (line 17) | def load_single(src_path, tgt_path, src_lang, tgt_lang):
    method load (line 31) | def load(path, name):
  function flores_postprocess (line 70) | def flores_postprocess(text: str) -> str:
  function flores_postprocess_chinese (line 76) | def flores_postprocess_chinese(text: str) -> str:

FILE: opencompass/datasets/game24.py
  function get_current_numbers (line 157) | def get_current_numbers(y: str) -> str:
  class Game24Dataset (line 162) | class Game24Dataset(BaseDataset):
    method load (line 165) | def load(path: str):
  class Game24PromptWrapper (line 172) | class Game24PromptWrapper:
    method __init__ (line 183) | def __init__(self):
    method standard_prompt_wrap (line 189) | def standard_prompt_wrap(x: str, y: str = '') -> str:
    method cot_prompt_wrap (line 193) | def cot_prompt_wrap(x: str, y: str = '') -> str:
    method propose_prompt_wrap (line 197) | def propose_prompt_wrap(x: str, y: str = '') -> str:
    method value_prompt_wrap (line 206) | def value_prompt_wrap(x: str, y: str) -> str:
    method value_outputs_unwrap (line 215) | def value_outputs_unwrap(x: str, y: str, value_outputs: list) -> float:
  function game24_postprocess (line 229) | def game24_postprocess(output: str):
  class Game24Evaluator (line 235) | class Game24Evaluator(BaseEvaluator):
    method __init__ (line 237) | def __init__(self) -> None:
    method check_nums (line 242) | def check_nums(self, prediction, reference):
    method score (line 252) | def score(self, predictions: List, references: List) -> dict:

FILE: opencompass/datasets/gaokao_math.py
  function extract_boxed_answer (line 66) | def extract_boxed_answer(text):
  class GaoKaoMATHDataset (line 74) | class GaoKaoMATHDataset(BaseDataset):
    method load (line 77) | def load(path: str):
  class GaoKaoMATHEvaluator (line 93) | class GaoKaoMATHEvaluator(BaseEvaluator):
    method __init__ (line 95) | def __init__(self,
    method batch_response (line 126) | def batch_response(self, models, inputs):
    method postprocess (line 142) | def postprocess(self, questions, predictions, question_type='None'):
    method score (line 167) | def score(self, predictions, references, origin_prompt, test_set):

FILE: opencompass/datasets/generic.py
  function get_final_results (line 7) | def get_final_results(judged_answers,
  function _generic_llmjudge_postprocess (line 63) | def _generic_llmjudge_postprocess(judgement: str,
  function generic_llmjudge_postprocess (line 73) | def generic_llmjudge_postprocess(
  function generic_llmjudge_academic_postprocess (line 105) | def generic_llmjudge_academic_postprocess(

FILE: opencompass/datasets/govrepcrs.py
  class GovRepcrsDataset (line 13) | class GovRepcrsDataset(BaseDataset):
    method load (line 16) | def load(path: str):

FILE: opencompass/datasets/gpqa.py
  class GPQADataset (line 16) | class GPQADataset(BaseDataset):
    method load (line 19) | def load(path: str, name: str, **kwargs):
  class GPQAEvaluator (line 47) | class GPQAEvaluator(BaseEvaluator):
    method score (line 49) | def score(self, predictions, references):
  class GPQASimpleEvalDataset (line 67) | class GPQASimpleEvalDataset(BaseDataset):
    method load (line 71) | def load(path: str, name: str):
  function GPQA_Simple_Eval_postprocess (line 110) | def GPQA_Simple_Eval_postprocess(text: str) -> str:

FILE: opencompass/datasets/gsm8k.py
  class GSM8KDataset (line 16) | class GSM8KDataset(BaseDataset):
    method load (line 19) | def load(path):
  function gsm8k_dataset_postprocess (line 39) | def gsm8k_dataset_postprocess(text: str) -> str:
  function gsm8k_postprocess (line 44) | def gsm8k_postprocess(text: str) -> str:
  class Gsm8kEvaluator (line 52) | class Gsm8kEvaluator(BaseEvaluator):
    method is_equal (line 54) | def is_equal(self, pred, refer):
    method score (line 62) | def score(self, predictions, references):
  class Gsm8kAgentEvaluator (line 82) | class Gsm8kAgentEvaluator(BaseEvaluator):
    method __init__ (line 90) | def __init__(self, action: str = 'PythonInterpreter'):
    method is_equal (line 93) | def is_equal(self, pred, refer):
    method soft_equal (line 101) | def soft_equal(self, pred, refer, step):
    method get_action (line 112) | def get_action(self, step):
    method score (line 117) | def score(self, predictions, references, steps):

FILE: opencompass/datasets/gsm_hard.py
  class GSMHardDataset (line 12) | class GSMHardDataset(BaseDataset):
    method load (line 15) | def load(path):

FILE: opencompass/datasets/healthbench/healthbench.py
  function map_with_progress (line 21) | def map_with_progress(
  class RubricItem (line 91) | class RubricItem:
    method __init__ (line 93) | def __init__(self, criterion: str, points: float, tags: list[str]):
    method __str__ (line 98) | def __str__(self):
    method to_dict (line 101) | def to_dict(self):
    method from_dict (line 109) | def from_dict(cls, d: dict):
  function _parse (line 117) | def _parse(item):
  function parse_json_to_dict (line 123) | def parse_json_to_dict(json_string: str) -> dict:
  function calculate_score (line 133) | def calculate_score(
  function get_usage_dict (line 152) | def get_usage_dict(response_usage) -> dict[str, int | None]:
  function _compute_clipped_stats (line 208) | def _compute_clipped_stats(
  function _aggregate_get_clipped_mean (line 230) | def _aggregate_get_clipped_mean(
  class HealthBenchDataset (line 260) | class HealthBenchDataset(BaseDataset):
    method load (line 263) | def load(path: str, **kwargs):
  class HealthBenchEvaluator (line 279) | class HealthBenchEvaluator(BaseEvaluator):
    method __init__ (line 283) | def __init__(
    method grade_sample (line 298) | def grade_sample(
    method score (line 392) | def score(self, predictions, references, test_set):

FILE: opencompass/datasets/healthbench/sampler/chat_completion_sampler.py
  class ChatCompletionSampler (line 16) | class ChatCompletionSampler(SamplerBase):
    method __init__ (line 19) | def __init__(
    method _handle_image (line 37) | def _handle_image(
    method _handle_text (line 52) | def _handle_text(self, text: str):
    method _pack_message (line 55) | def _pack_message(self, role: str, content: Any):
    method __call__ (line 58) | def __call__(self, message_list: MessageList) -> SamplerResponse:

FILE: opencompass/datasets/healthbench/types.py
  class SamplerResponse (line 9) | class SamplerResponse:
  class SamplerBase (line 16) | class SamplerBase:
    method __call__ (line 20) | def __call__(
  class EvalResult (line 28) | class EvalResult:
  class SingleEvalResult (line 39) | class SingleEvalResult:
  class Eval (line 50) | class Eval:
    method __call__ (line 53) | def __call__(self, sampler: SamplerBase) -> EvalResult:

FILE: opencompass/datasets/hellaswag.py
  class HellaswagDataset (line 14) | class HellaswagDataset(BaseDataset):
    method load (line 17) | def load(path):
  class HellaswagDataset_V2 (line 49) | class HellaswagDataset_V2(BaseDataset):
    method load (line 52) | def load(path):
  class HellaswagDataset_V3 (line 84) | class HellaswagDataset_V3(BaseDataset):
    method load (line 87) | def load(path):
  class HellaswagDatasetwithICE (line 119) | class HellaswagDatasetwithICE(BaseDataset):
    method load (line 122) | def load(path):
  class HellaswagDatasetClean (line 160) | class HellaswagDatasetClean(BaseDataset):
    method load_contamination_annotations (line 165) | def load_contamination_annotations(path, split='val'):
    method load (line 191) | def load(path):

FILE: opencompass/datasets/hle.py
  class HLEDataset (line 9) | class HLEDataset(BaseDataset):
    method load (line 12) | def load(path: str, category: str | None = None):

FILE: opencompass/datasets/huggingface.py
  class HFDataset (line 10) | class HFDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/humaneval.py
  class HumanevalDataset (line 33) | class HumanevalDataset(BaseDataset):
    method load (line 36) | def load(path: str, num_repeats: int = 1, local_mode: bool = False):
  class HumanEvalEvaluator (line 70) | class HumanEvalEvaluator(BaseEvaluator):
    method __init__ (line 73) | def __init__(self, k: List[int] = [1, 10, 100]) -> None:
    method score (line 82) | def score(self, predictions, references, test_set):
  class HumanEvalPlusEvaluator (line 119) | class HumanEvalPlusEvaluator(BaseEvaluator):
    method __init__ (line 122) | def __init__(self, k: List[int] = [1, 10, 100]) -> None:
    method score (line 131) | def score(self, predictions, references, test_set):
  function humaneval_postprocess_v2 (line 182) | def humaneval_postprocess_v2(text: str) -> str:
  function humaneval_postprocess_v3 (line 188) | def humaneval_postprocess_v3(text: str) -> str:
  function humaneval_internal_v2_postprocess (line 194) | def humaneval_internal_v2_postprocess(text: str):
  function humaneval_internal_v1_postprocess (line 207) | def humaneval_internal_v1_postprocess(text: str) -> str:

FILE: opencompass/datasets/humaneval_multi.py
  class HumanevalMultiDataset (line 45) | class HumanevalMultiDataset(BaseDataset):
    method load (line 48) | def load(path, language, version, num_repeats: int = 1, **kwargs):
  class HumanevalMultiEvaluator (line 79) | class HumanevalMultiEvaluator(BaseEvaluator):
    method __init__ (line 81) | def __init__(self,
    method stop_at_stop_token (line 94) | def stop_at_stop_token(self, decoded_string, stop_tokens):
    method _code_eval_service (line 108) | def _code_eval_service(self, file_path):
    method estimator (line 133) | def estimator(self, n: int, c: int, k: int) -> float:
    method for_file (line 141) | def for_file(self, path):
    method score (line 162) | def score(self, predictions, references, test_set):

FILE: opencompass/datasets/humaneval_pro.py
  class HumanevalevalProDataset (line 25) | class HumanevalevalProDataset(BaseDataset):
    method load (line 28) | def load(path, local_mode=False):
  class HumanevalProEvaluator (line 38) | class HumanevalProEvaluator(CodeEvaluator):
    method score (line 40) | def score(self, predictions: List, references: List,

FILE: opencompass/datasets/humanevalx.py
  class HumanevalXDataset (line 30) | class HumanevalXDataset(BaseDataset):
    method load (line 33) | def load(path, language, **kwargs):
    method _stream_jsonl_all (line 42) | def _stream_jsonl_all(filename: str) -> Iterable[Dict]:
  class HumanevalXEvaluator (line 56) | class HumanevalXEvaluator(BaseEvaluator):
    method __init__ (line 76) | def __init__(self,
    method score (line 93) | def score(self, predictions, references):
    method _code_eval_service (line 143) | def _code_eval_service(self, file_path):
  function _clean_up_code (line 172) | def _clean_up_code(text: str, language_type: str, reference) -> str:

FILE: opencompass/datasets/hungarian_math.py
  class HungarianExamMathDataset (line 11) | class HungarianExamMathDataset(BaseDataset):
    method load (line 14) | def load(path):

FILE: opencompass/datasets/inference_ppl.py
  class InferencePPLDataset (line 13) | class InferencePPLDataset(BaseDataset):
    method load (line 16) | def load(path: str, name: List[str] = None, samples: int = None):

FILE: opencompass/datasets/infinitebench/infinitebench_codedebug.py
  class InfiniteBenchcodedebugDataset (line 11) | class InfiniteBenchcodedebugDataset(BaseDataset):
    method load (line 14) | def load(path: str):

FILE: opencompass/datasets/infinitebench/infinitebench_coderun.py
  class InfiniteBenchcoderunDataset (line 13) | class InfiniteBenchcoderunDataset(BaseDataset):
    method load (line 16) | def load(path: str):

FILE: opencompass/datasets/infinitebench/infinitebench_endia.py
  class InfiniteBenchendiaDataset (line 14) | class InfiniteBenchendiaDataset(BaseDataset):
    method load (line 17) | def load(path: str):
  class InfiniteBenchendiaEvaluator (line 37) | class InfiniteBenchendiaEvaluator(BaseEvaluator):
    method score (line 39) | def score(self, predictions: List, references: List) -> dict:

FILE: opencompass/datasets/infinitebench/infinitebench_enmc.py
  class InfiniteBenchenmcDataset (line 11) | class InfiniteBenchenmcDataset(BaseDataset):
    method load (line 14) | def load(path: str):

FILE: opencompass/datasets/infinitebench/infinitebench_enqa.py
  class InfiniteBenchenqaDataset (line 11) | class InfiniteBenchenqaDataset(BaseDataset):
    method load (line 14) | def load(path: str):

FILE: opencompass/datasets/infinitebench/infinitebench_ensum.py
  class InfiniteBenchensumDataset (line 11) | class InfiniteBenchensumDataset(BaseDataset):
    method load (line 14) | def load(path: str):

FILE: opencompass/datasets/infinitebench/infinitebench_mathcalc.py
  class InfiniteBenchmathcalcDataset (line 15) | class InfiniteBenchmathcalcDataset(BaseDataset):
    method load (line 18) | def load(path: str):
  class InfiniteBenchmathcalcEvaluator (line 33) | class InfiniteBenchmathcalcEvaluator(BaseEvaluator):
    method score (line 35) | def score(self, predictions: List, references: List) -> dict:

FILE: opencompass/datasets/infinitebench/infinitebench_mathfind.py
  class InfiniteBenchmathfindDataset (line 13) | class InfiniteBenchmathfindDataset(BaseDataset):
    method load (line 16) | def load(path: str):

FILE: opencompass/datasets/infinitebench/infinitebench_retrievekv.py
  class InfiniteBenchretrievekvDataset (line 14) | class InfiniteBenchretrievekvDataset(BaseDataset):
    method load (line 17) | def load(path: str):
  class InfiniteBenchretrievekvEvaluator (line 37) | class InfiniteBenchretrievekvEvaluator(BaseEvaluator):
    method score (line 39) | def score(self, predictions: List, references: List) -> dict:

FILE: opencompass/datasets/infinitebench/infinitebench_retrievenumber.py
  class InfiniteBenchretrievenumberDataset (line 11) | class InfiniteBenchretrievenumberDataset(BaseDataset):
    method load (line 14) | def load(path: str):

FILE: opencompass/datasets/infinitebench/infinitebench_retrievepasskey.py
  class InfiniteBenchretrievepasskeyDataset (line 11) | class InfiniteBenchretrievepasskeyDataset(BaseDataset):
    method load (line 14) | def load(path: str):

FILE: opencompass/datasets/infinitebench/infinitebench_zhqa.py
  class InfiniteBenchzhqaDataset (line 11) | class InfiniteBenchzhqaDataset(BaseDataset):
    method load (line 14) | def load(path: str):

FILE: opencompass/datasets/infinitebench/utils.py
  function iter_jsonl (line 7) | def iter_jsonl(path):
  function InfiniteBench_first_number_postprocess (line 14) | def InfiniteBench_first_number_postprocess(text: str) -> str:

FILE: opencompass/datasets/internsandbox.py
  class InternSandboxDataset (line 15) | class InternSandboxDataset(BaseDataset):
    method load (line 18) | def load(path: str, sandbox: str, local_mode: bool = False):
  class InternSandboxEvaluator (line 32) | class InternSandboxEvaluator(BaseEvaluator):
    method __init__ (line 34) | def __init__(self,
    method score (line 41) | def score(self, predictions, references, test_set):

FILE: opencompass/datasets/iwslt2017.py
  class IWSLT2017Dataset (line 9) | class IWSLT2017Dataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: opencompass/datasets/jigsawmultilingual.py
  class JigsawMultilingualDataset (line 12) | class JigsawMultilingualDataset(BaseDataset):
    method load (line 15) | def load(path, label, lang):

FILE: opencompass/datasets/jsonl.py
  class JsonlDataset (line 12) | class JsonlDataset(BaseDataset):
    method load (line 15) | def load(path):

FILE: opencompass/datasets/judge/judgebench.py
  class JudgeBenchDataset (line 19) | class JudgeBenchDataset(BaseDataset):
    method load (line 21) | def load(self, path: str, name: str, *args, **kwargs):

FILE: opencompass/datasets/judge/judgerbenchv2.py
  function generate_balanced_list (line 97) | def generate_balanced_list(length):
  class Judgerbenchv2Dataset (line 108) | class Judgerbenchv2Dataset(BaseDataset):
    method load (line 110) | def load(self, path: str, name: str, *args, **kwargs):

FILE: opencompass/datasets/judge/rewardbench.py
  class RewardBenchDataset (line 19) | class RewardBenchDataset(BaseDataset):
    method load (line 21) | def load(self, path: str, name: str, *args, **kwargs):

FILE: opencompass/datasets/judge/rmb.py
  class RMBDataset (line 18) | class RMBDataset(BaseDataset):
    method load (line 20) | def load(self, path: str, name: str, *args, **kwargs):
    method load_pair (line 36) | def load_pair(self, item):
    method loadbon (line 68) | def loadbon(self, item):

FILE: opencompass/datasets/kaoshi.py
  function get_number (line 12) | def get_number(options):
  class KaoshiDataset (line 20) | class KaoshiDataset(BaseDataset):
    method load (line 23) | def load(path: str, name: str):
  class KaoshiEvaluator (line 42) | class KaoshiEvaluator(BaseEvaluator):
    method __init__ (line 44) | def __init__(self, question_type) -> None:
    method do_predictions_postprocess (line 49) | def do_predictions_postprocess(self, model_output, answer_lenth=None):
    method ensure_same_length (line 102) | def ensure_same_length(self, pred, refr):
    method score (line 107) | def score(self, predictions, references):

FILE: opencompass/datasets/kcle.py
  class KCLEDataset (line 12) | class KCLEDataset(BaseDataset):
    method load (line 15) | def load(path, **kwargs) -> datasets.Dataset:

FILE: opencompass/datasets/korbench/korbench.py
  class korbenchDataset (line 16) | class korbenchDataset(BaseDataset):
    method load (line 20) | def load(path, prompt_mode, category, **kwargs):
  class korbenchEvaluator (line 171) | class korbenchEvaluator(BaseEvaluator):
    method __init__ (line 173) | def __init__(self):
    method sample_score (line 176) | def sample_score(self, prediction, reference, test_item=None):
    method score (line 227) | def score(self, predictions, references, test_set):

FILE: opencompass/datasets/korbench/korbench_dataset_config/config_wrapper.py
  class ConfigWrapper (line 4) | class ConfigWrapper:
    method __init__ (line 6) | def __init__(self, config_path):
    method __setattr__ (line 13) | def __setattr__(self, key, value):
    method __getattr__ (line 20) | def __getattr__(self, key):
    method get_id (line 26) | def get_id(self, data):
    method print_all_keys (line 35) | def print_all_keys(self):
  function initialize_config (line 44) | def initialize_config(config_path):
  function get_config_wrapper (line 49) | def get_config_wrapper():

FILE: opencompass/datasets/korbench/korbench_utils.py
  function load_yaml (line 10) | def load_yaml(yaml_path):
  function load_json_or_jsonl (line 18) | def load_json_or_jsonl(file_path):
  function find_file (line 30) | def find_file(base_path, sub_path, extensions=('json', 'jsonl')):
  function load_json_or_jsonl_with_idx (line 39) | def load_json_or_jsonl_with_idx(data_path, split='', idx=None):
  function load_split_data (line 65) | def load_split_data(base_path, split_name):
  function process_mixed_data (line 77) | def process_mixed_data(base_path, mode):
  class ConfigWrapper (line 99) | class ConfigWrapper:
    method __init__ (line 101) | def __init__(self, config_path):
    method __setattr__ (line 108) | def __setattr__(self, key, value):
    method __getattr__ (line 115) | def __getattr__(self, key):
    method get_id (line 121) | def get_id(self, data):
    method print_all_keys (line 130) | def print_all_keys(self):
  function initialize_config (line 139) | def initialize_config(config_path):
  function get_config_wrapper (line 144) | def get_config_wrapper():
  function read_yaml (line 188) | def read_yaml(config='default'):
  function write_jsonl_lines (line 197) | def write_jsonl_lines(file, data):
  function print_info (line 208) | def print_info(info):
  function read_json_or_jsonl (line 222) | def read_json_or_jsonl(data_path, split='', mapping_key=None):
  function read_json_or_jsonl_with_idx (line 248) | def read_json_or_jsonl_with_idx(data_path, split='', idx=None):
  function clean_json_string (line 293) | def clean_json_string(json_str):
  function is_in_idx_ranges (line 298) | def is_in_idx_ranges(idx, idx_ranges):
  function extract_json (line 305) | def extract_json(text):
  function extract_all_responses_from_json (line 319) | def extract_all_responses_from_json(response_json):
  function clean_latex (line 326) | def clean_latex(latex_expr):
  function extract_text_from_brackets (line 336) | def extract_text_from_brackets(text, clean_level='basic'):
  function extract_inner_text_from_brackets (line 358) | def extract_inner_text_from_brackets(text):
  function extract_numbers (line 366) | def extract_numbers(str):
  function extract_and_sort_inequalities (line 372) | def extract_and_sort_inequalities(latex_expr):
  function rule5_normalize_content (line 380) | def rule5_normalize_content(content):
  function normalize_string (line 386) | def normalize_string(s):
  function remove_commas_and_spaces (line 393) | def remove_commas_and_spaces(s):
  function remove_non_alphanumeric (line 397) | def remove_non_alphanumeric(s):
  function contains_or (line 401) | def contains_or(answer):
  function compare_multi_results (line 405) | def compare_multi_results(response, answer):
  function split_or_expression (line 422) | def split_or_expression(expression):
  function compare_math_expressions (line 426) | def compare_math_expressions(response, answer):
  function method_equal (line 457) | def method_equal(response_text, answer):
  function method_1 (line 461) | def method_1(response_text, answer):
  function method_2 (line 469) | def method_2(response_text, answer):
  function method_3 (line 476) | def method_3(response_text, answer):
  function method_4 (line 486) | def method_4(response_text, answer):
  function method_5 (line 492) | def method_5(response_text, answer):
  function method_9 (line 501) | def method_9(response_text, answer):
  function method_10 (line 523) | def method_10(response_text, answer):
  function method_18 (line 539) | def method_18(response_text, answer):
  function method_general (line 545) | def method_general(response_text, answer):
  function evaluate_response_vs_answer (line 563) | def evaluate_response_vs_answer(response, answer, question_type, rule_id...
  function compute_one_mixed_question_pass_rate (line 611) | def compute_one_mixed_question_pass_rate(idx,
  function evaluate_responses (line 662) | def evaluate_responses(data, mode, base_path=None):

FILE: opencompass/datasets/lambada.py
  class lambadaDataset (line 17) | class lambadaDataset(BaseDataset):
    method load (line 20) | def load(path):
  class LambadaEvaluator (line 36) | class LambadaEvaluator(BaseEvaluator):
    method __init__ (line 38) | def __init__(self) -> None:
    method score (line 41) | def score(self, predictions, references):

FILE: opencompass/datasets/lawbench/evaluation_functions/cjft.py
  function compute_cjft (line 5) | def compute_cjft(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/flzx.py
  function compute_flzx (line 4) | def compute_flzx(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/ftcs.py
  function compute_ftcs (line 4) | def compute_ftcs(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/jdzy.py
  function compute_jdzy (line 9) | def compute_jdzy(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/jec_ac.py
  function compute_jec_ac (line 8) | def compute_jec_ac(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/jec_kd.py
  function compute_jec_kd (line 8) | def compute_jec_kd(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/jetq.py
  function compute_jetq (line 8) | def compute_jetq(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/lblj.py
  function compute_lblj (line 8) | def compute_lblj(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/ljp_accusation.py
  function compute_ljp_accusation (line 46) | def compute_ljp_accusation(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/ljp_article.py
  function replace_match (line 8) | def replace_match(match):
  function compute_ljp_article (line 11) | def compute_ljp_article(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/ljp_imprison.py
  function compute_ljp_imprison (line 6) | def compute_ljp_imprison(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/sjjc.py
  function compute_sjjc (line 12) | def compute_sjjc(data_dict):
  function compute_cfcy (line 47) | def compute_cfcy(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/wbfl.py
  function compute_wbfl (line 7) | def compute_wbfl(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/wsjd.py
  function compute_wsjd (line 11) | def compute_wsjd(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/xxcq.py
  function compute_xxcq (line 9) | def compute_xxcq(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/ydlj.py
  function compute_ydlj (line 8) | def compute_ydlj(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/yqzy.py
  function compute_yqzy (line 4) | def compute_yqzy(data_dict):

FILE: opencompass/datasets/lawbench/evaluation_functions/zxfl.py
  function compute_zxfl (line 9) | def compute_zxfl(data_dict):

FILE: opencompass/datasets/lawbench/lawbench.py
  class LawBenchDataset (line 18) | class LawBenchDataset(BaseDataset):
    method load (line 21) | def load(path: str, index: str) -> Dataset:
  class LawBenchEvaluator (line 53) | class LawBenchEvaluator(BaseEvaluator):
    method __init__ (line 55) | def __init__(self, index) -> None:
    method score (line 59) | def score(self, predictions, references, origin_prompt):
  function _register (line 79) | def _register(index):

FILE: opencompass/datasets/lawbench/utils/char_smi.py
  function tree_edit_distance (line 39) | def tree_edit_distance(tree_a, tree_b):
  function edit_distance (line 54) | def edit_distance(string_a, string_b, name='Levenshtein'):
  class CharFuncs (line 98) | class CharFuncs(object):
    method __init__ (line 99) | def __init__(self, char_meta_fname):
    method load_char_meta (line 118) | def load_char_meta(fname):
    method shape_distance (line 131) | def shape_distance(self, char1, char2, safe=True, as_tree=False):
    method pronunciation_distance (line 183) | def pronunciation_distance(self, char1, char2):
    method load_dict (line 219) | def load_dict(fname):
    method similarity (line 229) | def similarity(self, char1, char2, weights=(0.8, 0.2, 0.0), as_tree=Fa...
    method shape_similarity (line 248) | def shape_similarity(self, char1, char2, safe=True, as_tree=False):
    method pronunciation_similarity (line 302) | def pronunciation_similarity(self, char1, char2):
  function string_to_tree (line 340) | def string_to_tree(string):
  function pinyin_map (line 394) | def pinyin_map(standard_pinyin):
  function parse_args (line 420) | def parse_args():

FILE: opencompass/datasets/lawbench/utils/compare_m2_for_evaluation.py
  function main (line 4) | def main():
  function parse_args (line 39) | def parse_args():
  function simplify_edits (line 135) | def simplify_edits(sent, max_answer_num):
  function process_edits (line 165) | def process_edits(edits, args):
  function evaluate_edits (line 242) | def evaluate_edits(src, hyp_dict, ref_dict, best, sent_id, args):
  function compareEdits (line 309) | def compareEdits(hyp_edits, ref_edits):
  function computeFScore (line 356) | def computeFScore(tp, fp, fn, beta):
  function merge_dict (line 364) | def merge_dict(dict1, dict2):
  function processCategories (line 376) | def processCategories(cat_dict, setting):
  function print_results (line 403) | def print_results(best, best_cats, args):

FILE: opencompass/datasets/lawbench/utils/comprehension_scores.py
  function __find_substring_starts (line 11) | def __find_substring_starts(s, target):
  function compute_rc_f1 (line 21) | def compute_rc_f1(hyps, refs):
  function compute_ie_f1 (line 35) | def compute_ie_f1(hyps, refs, entity_types):
  function __extract_entities_ref (line 54) | def __extract_entities_ref(ref):
  function __extract_entities_pred (line 71) | def __extract_entities_pred(pred, entity_types):

FILE: opencompass/datasets/lawbench/utils/function_utils.py
  function compute_f1_two_sets (line 5) | def compute_f1_two_sets(pred_set, gt_set):
  function multi_choice_judge (line 11) | def multi_choice_judge(prediction, option_list, answer_token):
  function compute_rouge (line 32) | def compute_rouge(hyps, refs):
  function compute_gleu (line 44) | def compute_gleu(hyps, refs):

FILE: opencompass/datasets/lawbench/utils/modules/alignment.py
  function check_all_chinese (line 13) | def check_all_chinese(word):
  function read_cilin (line 21) | def read_cilin():
  function read_confusion (line 41) | def read_confusion():
  class Alignment (line 49) | class Alignment:
    method __init__ (line 55) | def __init__(
    method __call__ (line 75) | def __call__(self,
    method _get_semantic_class (line 97) | def _get_semantic_class(self, word):
    method _get_class_diff (line 111) | def _get_class_diff(a_class, b_class):
    method _get_semantic_cost (line 120) | def _get_semantic_cost(self, a, b):
    method _get_pos_cost (line 137) | def _get_pos_cost(self, a_pos, b_pos):
    method _get_char_cost (line 151) | def _get_char_cost(self, a, b, pinyin_a, pinyin_b):
    method _get_spell_cost (line 166) | def _get_spell_cost(self, a, b, pinyin_a, pinyin_b):
    method get_sub_cost (line 183) | def get_sub_cost(self, a_seg, b_seg):
    method align (line 208) | def align(self,
    method _dfs (line 291) | def _dfs(self, i, j, align_seq_now, oper_matrix, strategy="all"):
    method get_cheapest_align_seq (line 311) | def get_cheapest_align_seq(self, oper_matrix):

FILE: opencompass/datasets/lawbench/utils/modules/annotator.py
  class Annotator (line 6) | class Annotator:
    method __init__ (line 7) | def __init__(self,
    method create_default (line 20) | def create_default(cls, granularity: str = "word", strategy: str = "fi...
    method __call__ (line 31) | def __call__(self,

FILE: opencompass/datasets/lawbench/utils/modules/classifier.py
  function check_spell_error (line 16) | def check_spell_error(src_span: str,
  class Classifier (line 36) | class Classifier:
    method __init__ (line 40) | def __init__(self,
    method get_pos_type (line 46) | def get_pos_type(pos):
    method __call__ (line 75) | def __call__(self,

FILE: opencompass/datasets/lawbench/utils/modules/merger.py
  class Merger (line 8) | class Merger:
    method __init__ (line 13) | def __init__(self,
    method _merge_edits (line 23) | def _merge_edits(seq, tag="X"):
    method _check_revolve (line 30) | def _check_revolve(span_a, span_b):
    method _process_seq (line 34) | def _process_seq(self, seq, src_tokens, tgt_tokens):
    method __call__ (line 57) | def __call__(self,

FILE: opencompass/datasets/lawbench/utils/modules/tokenization.py
  function convert_to_unicode (line 26) | def convert_to_unicode(text):
  function printable_text (line 46) | def printable_text(text):
  function load_vocab (line 69) | def load_vocab(vocab_file):
  function convert_by_vocab (line 84) | def convert_by_vocab(vocab, items):
  function convert_tokens_to_ids (line 95) | def convert_tokens_to_ids(vocab, tokens):
  function convert_ids_to_tokens (line 99) | def convert_ids_to_tokens(inv_vocab, ids):
  function whitespace_tokenize (line 103) | def whitespace_tokenize(text):
  class FullTokenizer (line 112) | class FullTokenizer(object):
    method __init__ (line 115) | def __init__(self, vocab_file, do_lower_case=True):
    method tokenize (line 121) | def tokenize(self, text):
    method convert_tokens_to_ids (line 129) | def convert_tokens_to_ids(self, tokens):
    method convert_ids_to_tokens (line 132) | def convert_ids_to_tokens(self, ids):
  class BasicTokenizer (line 136) | class BasicTokenizer(object):
    method __init__ (line 139) | def __init__(self, do_lower_case=True):
    method tokenize (line 146) | def tokenize(self, text):
    method _run_strip_accents (line 170) | def _run_strip_accents(self, text):
    method _run_split_on_punc (line 181) | def _run_split_on_punc(self, text):
    method _tokenize_chinese_chars (line 201) | def _tokenize_chinese_chars(self, text):
    method _is_chinese_char (line 214) | def _is_chinese_char(self, cp):
    method _clean_text (line 236) | def _clean_text(self, text):
  class WordpieceTokenizer (line 250) | class WordpieceTokenizer(object):
    method __init__ (line 253) | def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=...
    method tokenize (line 258) | def tokenize(self, text):
  function _is_whitespace (line 309) | def _is_whitespace(char):
  function _is_control (line 321) | def _is_control(char):
  function _is_punctuation (line 333) | def _is_punctuation(char):

FILE: opencompass/datasets/lawbench/utils/modules/tokenizer.py
  class Tokenizer (line 8) | class Tokenizer:
    method __init__ (line 13) | def __init__(self,
    method __repr__ (line 36) | def __repr__(self) -> str:
    method __call__ (line 39) | def __call__(self,
    method split_char (line 52) | def split_char(self, input_strings: List[str], bpe=False) -> List:
    method split_word (line 72) | def split_word(self, input_strings: List[str]) -> List:

FILE: opencompass/datasets/lawbench/utils/parallel_to_m2.py
  function annotate_with_time_out (line 19) | def annotate_with_time_out(line):
  function annotate (line 50) | def annotate(line):
  function firsttime_process (line 84) | def firsttime_process(args):
  function main (line 137) | def main(args):

FILE: opencompass/datasets/lawbench/utils/rc_f1.py
  class CJRCEvaluator (line 11) | class CJRCEvaluator:
    method __init__ (line 12) | def __init__(self, gold_file):
    method gold_answers_to_dict (line 16) | def gold_answers_to_dict(gold_file):
    method preds_to_dict (line 40) | def preds_to_dict(pred_file):
    method normalize_answer (line 48) | def normalize_answer(s):
    method get_tokens (line 60) | def get_tokens(s):
    method compute_exact (line 65) | def compute_exact(a_gold, a_pred):
    method compute_f1 (line 69) | def compute_f1(a_gold, a_pred):
    method _compute_turn_score (line 85) | def _compute_turn_score(a_gold_list, a_pred):
    method compute_turn_score (line 101) | def compute_turn_score(self, qid, a_pred):
    method get_raw_scores (line 106) | def get_raw_scores(self, pred_data):
    method get_raw_scores_human (line 121) | def get_raw_scores_human(self):
    method human_performance (line 142) | def human_performance(self):
    method model_performance (line 146) | def model_performance(self, pred_data):
    method get_total_scores (line 150) | def get_total_scores(self, exact_scores, f1_scores):

FILE: opencompass/datasets/lcsts.py
  class LCSTSDataset (line 13) | class LCSTSDataset(BaseDataset):
    method load (line 16) | def load(path: str):
  function lcsts_postprocess (line 50) | def lcsts_postprocess(text: str) -> str:

FILE: opencompass/datasets/leval/evaluators.py
  class LEvalGPTEvaluator (line 11) | class LEvalGPTEvaluator(BaseEvaluator):
    method __init__ (line 23) | def __init__(self,
    method run_judge_pair (line 30) | def run_judge_pair(self, prompt_template, system_prompt, question,
    method score (line 64) | def score(self, predictions: List, references: List) -> dict:
  class LEvalEMEvaluator (line 114) | class LEvalEMEvaluator(BaseEvaluator):
    method __init__ (line 117) | def __init__(self) -> None:
    method score (line 120) | def score(self, predictions, references):

FILE: opencompass/datasets/leval/leval_coursera.py
  class LEvalCourseraDataset (line 10) | class LEvalCourseraDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_financial_qa.py
  class LEvalFinancialQADataset (line 10) | class LEvalFinancialQADataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_gov_report_summ.py
  class LEvalGovReportSummDataset (line 10) | class LEvalGovReportSummDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_gsm100.py
  function gsm100_dataset_postprocess (line 10) | def gsm100_dataset_postprocess(text: str) -> str:
  function gsm100_postprocess (line 15) | def gsm100_postprocess(text: str) -> str:
  class LEvalGSM100Dataset (line 41) | class LEvalGSM100Dataset(BaseDataset):
    method load (line 44) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_legal_contract_qa.py
  class LEvalLegalContractQADataset (line 10) | class LEvalLegalContractQADataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_meeting_summ.py
  class LEvalMeetingSummDataset (line 10) | class LEvalMeetingSummDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_multidoc_qa.py
  class LEvalMultidocQADataset (line 10) | class LEvalMultidocQADataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_narrattive_qa.py
  class LEvalNarrativeQADataset (line 10) | class LEvalNarrativeQADataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_natural_question.py
  class LEvalNaturalQuestionDataset (line 10) | class LEvalNaturalQuestionDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_news_summ.py
  class LEvalNewsSummDataset (line 10) | class LEvalNewsSummDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_paper_assistant.py
  class LEvalPaperAssistantDataset (line 10) | class LEvalPaperAssistantDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_patent_summ.py
  class LEvalPatentSummDataset (line 10) | class LEvalPatentSummDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_quality.py
  class LEvalQualityDataset (line 10) | class LEvalQualityDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_review_summ.py
  class LEvalReviewSummDataset (line 10) | class LEvalReviewSummDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_scientific_qa.py
  class LEvalScientificQADataset (line 10) | class LEvalScientificQADataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_topic_retrieval.py
  class LEvalTopicRetrievalDataset (line 10) | class LEvalTopicRetrievalDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_tpo.py
  class LEvalTPODataset (line 10) | class LEvalTPODataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/leval/leval_tvshow_summ.py
  class LEvalTVShowSummDataset (line 10) | class LEvalTVShowSummDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/livecodebench/evaluator.py
  function codegen_check_correctness (line 22) | def codegen_check_correctness(sample, generation, timeout, debug=True):
  function evaluate_generations_by_problem (line 60) | def evaluate_generations_by_problem(problem_generations: list, sample: l...
  function evaluate_generations (line 117) | def evaluate_generations(
  function codegen_metrics (line 173) | def codegen_metrics(
  class LCBCodeGenerationEvaluator (line 233) | class LCBCodeGenerationEvaluator(BaseEvaluator):
    method __init__ (line 235) | def __init__(self,
    method _build_results (line 251) | def _build_results(self, extracted_predictions, metrics, eval_results,
    method score (line 273) | def score(self, predictions, references):
  function evaluate_score (line 330) | def evaluate_score(args) -> list[bool]:
  function code_execution_metrics (line 346) | def code_execution_metrics(
  class LCBCodeExecutionEvaluator (line 388) | class LCBCodeExecutionEvaluator(BaseEvaluator):
    method __init__ (line 390) | def __init__(self):
    method score (line 395) | def score(self, predictions, references):
  function parse_assert_statement (line 402) | def parse_assert_statement(statement):
  function check_testcase_output (line 430) | def check_testcase_output(testcase_str, expected_output):
  function test_output_metrics (line 471) | def test_output_metrics(
  class LCBTestOutputEvaluator (line 499) | class LCBTestOutputEvaluator(BaseEvaluator):
    method __init__ (line 501) | def __init__(self):
    method score (line 504) | def score(self, predictions, references):

FILE: opencompass/datasets/livecodebench/execute_utils.py
  function codeexecute_check_correctness (line 65) | def codeexecute_check_correctness(check_program, timeout=3):
  function unsafe_execute (line 88) | def unsafe_execute(check_program, result, timeout):
  function time_limit (line 123) | def time_limit(seconds):
  function swallow_io (line 137) | def swallow_io():
  function create_tempdir (line 146) | def create_tempdir():
  class TimeoutException (line 152) | class TimeoutException(Exception):
  class WriteOnlyStringIO (line 156) | class WriteOnlyStringIO(io.StringIO):
    method read (line 159) | def read(self, *args, **kwargs):
    method readline (line 162) | def readline(self, *args, **kwargs):
    method readlines (line 165) | def readlines(self, *args, **kwargs):
    method readable (line 168) | def readable(self, *args, **kwargs):
  class redirect_stdin (line 173) | class redirect_stdin(contextlib._RedirectStream):  # type: ignore
  function chdir (line 178) | def chdir(root):
  function reliability_guard (line 192) | def reliability_guard(maximum_memory_bytes=None):

FILE: opencompass/datasets/livecodebench/extract_utils.py
  function extract_code_generation (line 6) | def extract_code_generation(model_output: str, model_type: str = 'chat'):
  function extract_code_generation_v2 (line 22) | def extract_code_generation_v2(model_output: str, model_type: str = 'cha...
  function extract_code_execution (line 43) | def extract_code_execution(model_output: str, cot: bool = False):
  function extract_test_output_code (line 64) | def extract_test_output_code(model_output: str):

FILE: opencompass/datasets/livecodebench/livecodebench.py
  class Platform (line 25) | class Platform(Enum):
  class Difficulty (line 31) | class Difficulty(Enum):
  class TestType (line 37) | class TestType(Enum):
  class Test (line 43) | class Test:
    method __post_init__ (line 48) | def __post_init__(self):
  class LCBCodeGenerationDataset (line 52) | class LCBCodeGenerationDataset(BaseDataset):
    method load (line 55) | def load(path: str = 'opencompass/code_generation_lite',
  class LCBCodeExecutionDataset (line 126) | class LCBCodeExecutionDataset(BaseDataset):
    method load (line 129) | def load(
  class LCBTestOutputPredictionDataset (line 160) | class LCBTestOutputPredictionDataset(BaseDataset):
    method load (line 163) | def load(
  class LCBSelfRepairDataset (line 204) | class LCBSelfRepairDataset(BaseDataset):
    method load (line 207) | def load(path: str = 'livecodebench/code_generation_lite',
  class CompassBenchCodeExecutionDataset (line 233) | class CompassBenchCodeExecutionDataset(BaseDataset):
    method load (line 236) | def load(

FILE: opencompass/datasets/livecodebench/pass_k_utils.py
  function estimate_pass_at_k (line 6) | def estimate_pass_at_k(num_samples, num_correct, k):
  function compute_metrics_from_results (line 29) | def compute_metrics_from_results(results, k_list=[1, 5]):
  function extract_instance_results (line 60) | def extract_instance_results(results):

FILE: opencompass/datasets/livecodebench/prompts.py
  class CodeGenerationPromptConstants (line 6) | class CodeGenerationPromptConstants:
  class TestOutputPromptConstants (line 33) | class TestOutputPromptConstants:
  class SelfRepairPromptConstants (line 52) | class SelfRepairPromptConstants:
  function make_code_execution_prompt (line 73) | def make_code_execution_prompt(code, input, cot):
  function get_generic_question_template_test_completion (line 133) | def get_generic_question_template_test_completion(question_content,
  function get_generic_question_template_answer_self_repair (line 170) | def get_generic_question_template_answer_self_repair(question: str, code,

FILE: opencompass/datasets/livecodebench/testing_util.py
  function truncatefn (line 27) | def truncatefn(s, length=300):
  class CODE_TYPE (line 35) | class CODE_TYPE(Enum):
  class TimeoutException (line 41) | class TimeoutException(Exception):
  function timeout_handler (line 45) | def timeout_handler(signum, frame):
  class Capturing (line 59) | class Capturing(list):
    method __enter__ (line 61) | def __enter__(self):
    method __exit__ (line 68) | def __exit__(self, *args):
  function only_int_check (line 74) | def only_int_check(val):
  function string_int_check (line 78) | def string_int_check(val):
  function combined_int_check (line 82) | def combined_int_check(val):
  function run_test (line 86) | def run_test(sample, test=None, debug=False, timeout=6):
  function custom_compare_ (line 639) | def custom_compare_(output, ground_truth):
  function stripped_string_compare (line 655) | def stripped_string_compare(s1, s2):
  class MockStdinWithBuffer (line 661) | class MockStdinWithBuffer:
    method __init__ (line 663) | def __init__(self, inputs: str):
    method read (line 668) | def read(self, *args):
    method readline (line 671) | def readline(self, *args):
    method readlines (line 674) | def readlines(self, *args):
    method __getattr__ (line 677) | def __getattr__(self, name):
  class MockBuffer (line 682) | class MockBuffer:
    method __init__ (line 684) | def __init__(self, inputs: str):
    method read (line 687) | def read(self, *args):
    method readline (line 691) | def readline(self, *args):
  function call_method (line 695) | def call_method(method, inputs):
  function reliability_guard (line 725) | def reliability_guard(maximum_memory_bytes=None):

FILE: opencompass/datasets/livecodebench_pro/livecodebench_pro.py
  class LCBProDataset (line 10) | class LCBProDataset(BaseDataset):
    method load (line 13) | def load(path, **kwargs):

FILE: opencompass/datasets/livecodebench_pro/livecodebench_pro_evaluator.py
  function extract_longest_cpp_code (line 14) | def extract_longest_cpp_code(text):
  function extract_longest_python_code (line 106) | def extract_longest_python_code(text):
  class LCBProEvaluator (line 220) | class LCBProEvaluator(BaseEvaluator):
    method __init__ (line 234) | def __init__(
    method _submit_code (line 251) | def _submit_code(self, pid: str, lang: str, code: str) -> int:
    method _get_result (line 289) | def _get_result(self, submission_id: int) -> str:
    method _extract_code (line 318) | def _extract_code(self, text: str) -> tuple:
    method score (line 352) | def score(self, predictions: List, references: List,

FILE: opencompass/datasets/livemathbench/livemathbench.py
  class LiveMathBenchDataset (line 28) | class LiveMathBenchDataset(BaseDataset):
    method load (line 31) | def load(path: str,
  class LiveMathBenchEvaluator (line 89) | class LiveMathBenchEvaluator(BaseEvaluator):
    method __init__ (line 95) | def __init__(self,
    method batch_infer (line 135) | def batch_infer(self, models: List[OpenAISDK], inputs: List[str],
    method extract (line 179) | def extract(self, questions: List[str], predictions: List[str],
    method judge (line 235) | def judge(self, predictions, references, test_set):
    method preprocess (line 279) | def preprocess(self, predictions, references, test_set):
    method score (line 282) | def score(self, predictions, references, test_set) -> Dict[str, Any]:
  class LiveMathBenchOutputHandler (line 296) | class LiveMathBenchOutputHandler:
    method write_to_json (line 300) | def write_to_json(self):
    method save (line 304) | def save(self, idx, **kwargs):

FILE: opencompass/datasets/livemathbench/utils.py
  function extract_judge_label (line 4) | def extract_judge_label(text):

FILE: opencompass/datasets/livereasonbench/livereasonbench.py
  class LiveReasonBenchDataset (line 16) | class LiveReasonBenchDataset(BaseDataset):
    method load (line 19) | def load(path: str,
  function get_final_results (line 127) | def get_final_results(judged_answers, references, origial_responses):
  function _livereasonbench_postprocess (line 171) | def _livereasonbench_postprocess(judgement: str):
  function livereasonbench_postprocess (line 178) | def livereasonbench_postprocess(

FILE: opencompass/datasets/livestembench.py
  class LiveStemBenchDataset (line 15) | class LiveStemBenchDataset(BaseDataset):
    method load (line 18) | def load(path: str,

FILE: opencompass/datasets/llm_compression.py
  class LLMCompressionDataset (line 13) | class LLMCompressionDataset(BaseDataset):
    method load (line 16) | def load(path: str, name: List[str] = None, samples: int = None):

FILE: opencompass/datasets/lmeval.py
  class LMEvalDataset (line 6) | class LMEvalDataset(BaseDataset):
    method load (line 11) | def load(**kwargs):

FILE: opencompass/datasets/longbench/evaluators.py
  function normalize_answer (line 15) | def normalize_answer(s):
  function normalize_zh_answer (line 34) | def normalize_zh_answer(s):
  class LongBenchF1Evaluator (line 54) | class LongBenchF1Evaluator(BaseEvaluator):
    method __init__ (line 56) | def __init__(self, language: str = 'en') -> None:
    method score (line 61) | def score(self, predictions: List, references: List) -> dict:
  class LongBenchCountEvaluator (line 116) | class LongBenchCountEvaluator(BaseEvaluator):
    method score (line 118) | def score(self, predictions: List, references: List) -> dict:
  class LongBenchRetrievalEvaluator (line 137) | class LongBenchRetrievalEvaluator(BaseEvaluator):
    method __init__ (line 139) | def __init__(self, language: str = 'en') -> None:
    method score (line 144) | def score(self, predictions: List, references: List) -> dict:
  class LongBenchRougeEvaluator (line 171) | class LongBenchRougeEvaluator(BaseEvaluator):
    method __init__ (line 173) | def __init__(self, language: str = 'en') -> None:
    method score (line 178) | def score(self, predictions: List, references: List) -> dict:
  class LongBenchCodeSimEvaluator (line 206) | class LongBenchCodeSimEvaluator(BaseEvaluator):
    method score (line 208) | def score(self, predictions: List, references: List) -> dict:
  class LongBenchClassificationEvaluator (line 233) | class LongBenchClassificationEvaluator(BaseEvaluator):
    method score (line 235) | def score(self, predictions: List, references: List) -> dict:

FILE: opencompass/datasets/longbench/longbench_2wikim_qa.py
  class LongBench2wikimqaDataset (line 10) | class LongBench2wikimqaDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_dureader.py
  class LongBenchdureaderDataset (line 10) | class LongBenchdureaderDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_gov_report.py
  class LongBenchgov_reportDataset (line 10) | class LongBenchgov_reportDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_hotpot_qa.py
  class LongBenchhotpotqaDataset (line 10) | class LongBenchhotpotqaDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_lcc.py
  class LongBenchlccDataset (line 10) | class LongBenchlccDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_lsht.py
  class LongBenchlshtDataset (line 10) | class LongBenchlshtDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):
  function lsht_postprocess (line 39) | def lsht_postprocess(text: str) -> str:

FILE: opencompass/datasets/longbench/longbench_multi_news.py
  class LongBenchmulti_newsDataset (line 10) | class LongBenchmulti_newsDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_multifieldqa_en.py
  class LongBenchmultifieldqa_enDataset (line 10) | class LongBenchmultifieldqa_enDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_multifieldqa_zh.py
  class LongBenchmultifieldqa_zhDataset (line 10) | class LongBenchmultifieldqa_zhDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_musique.py
  class LongBenchmusiqueDataset (line 10) | class LongBenchmusiqueDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_narrative_qa.py
  class LongBenchnarrativeqaDataset (line 10) | class LongBenchnarrativeqaDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_passage_count.py
  class LongBenchpassage_countDataset (line 10) | class LongBenchpassage_countDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_passage_retrieval_en.py
  class LongBenchpassage_retrieval_enDataset (line 10) | class LongBenchpassage_retrieval_enDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_passage_retrieval_zh.py
  class LongBenchpassage_retrieval_zhDataset (line 10) | class LongBenchpassage_retrieval_zhDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_qasper.py
  class LongBenchqasperDataset (line 10) | class LongBenchqasperDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_qmsum.py
  class LongBenchqmsumDataset (line 10) | class LongBenchqmsumDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_repobench.py
  class LongBenchrepobenchDataset (line 10) | class LongBenchrepobenchDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbench/longbench_samsum.py
  class LongBenchsamsumDataset (line 10) | class LongBenchsamsumDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):
  function samsum_postprocess (line 35) | def samsum_postprocess(text: str) -> str:

FILE: opencompass/datasets/longbench/longbench_trec.py
  class LongBenchtrecDataset (line 10) | class LongBenchtrecDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):
  function trec_postprocess (line 39) | def trec_postprocess(text: str) -> str:

FILE: opencompass/datasets/longbench/longbench_trivia_qa.py
  class LongBenchtriviaqaDataset (line 10) | class LongBenchtriviaqaDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):
  function triviaqa_postprocess (line 35) | def triviaqa_postprocess(text: str) -> str:

FILE: opencompass/datasets/longbench/longbench_vcsum.py
  class LongBenchvcsumDataset (line 10) | class LongBenchvcsumDataset(BaseDataset):
    method load (line 13) | def load(path: str, name: str):

FILE: opencompass/datasets/longbenchv2.py
  class LongBenchv2Dataset (line 11) | class LongBenchv2Dataset(BaseDataset):
    method load (line 14) | def load(path: str):
  class LongBenchv2Evaluator (line 46) | class LongBenchv2Evaluator(BaseEvaluator):
    method __init__ (line 48) | def __init__(self):
    method score (line 51) | def score(self, predictions, references, test_set):

FILE: opencompass/datasets/lveval/evaluators.py
  function normalize_answer (line 147) | def normalize_answer(s):
  function normalize_zh_answer (line 166) | def normalize_zh_answer(s):
  class LVEvalF1Evaluator (line 186) | class LVEvalF1Evaluator(BaseEvaluator):
    method __init__ (line 188) | def __init__(self, language: str = 'en') -> None:
    method score (line 193) | def score(self, predictions: List, references: List) -> dict:
  class LVEvalOPTF1Evaluator (line 249) | class LVEvalOPTF1Evaluator(BaseEvaluator):
    method __init__ (line 251) | def __init__(self, language: str = 'en') -> None:
    method score (line 256) | def score(self, predictions: List, references: List) -> dict:
  class LVEvalOPTRougeEvaluator (line 352) | class LVEvalOPTRougeEvaluator(BaseEvaluator):
    method __init__ (line 354) | def __init__(self, language: str = 'en') -> None:
    method score (line 359) | def score(self, predictions: List, references: List) -> dict:

FILE: opencompass/datasets/lveval/lveval_cmrc_mixup.py
  class LVEvalcmrcDataset (line 10) | class LVEvalcmrcDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/lveval/lveval_dureader_mixup.py
  class LVEvaldureaderDataset (line 10) | class LVEvaldureaderDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/lveval/lveval_factrecall_en.py
  class LVEvalfactrecallenDataset (line 10) | class LVEvalfactrecallenDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/lveval/lveval_factrecall_zh.py
  class LVEvalfactrecallzhDataset (line 10) | class LVEvalfactrecallzhDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/lveval/lveval_hotpotwikiqa_mixup.py
  class LVEvalhotpotwikiqaDataset (line 10) | class LVEvalhotpotwikiqaDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/lveval/lveval_lic_mixup.py
  class LVEvallicDataset (line 10) | class LVEvallicDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/lveval/lveval_loogle_CR_mixup.py
  class LVEvallooglecrDataset (line 10) | class LVEvallooglecrDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/lveval/lveval_loogle_MIR_mixup.py
  class LVEvallooglemirDataset (line 10) | class LVEvallooglemirDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/lveval/lveval_loogle_SD_mixup.py
  class LVEvallooglesdDataset (line 10) | class LVEvallooglesdDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/lveval/lveval_multifieldqa_en_mixup.py
  class LVEvalmultifieldqaenDataset (line 10) | class LVEvalmultifieldqaenDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/lveval/lveval_multifieldqa_zh_mixup.py
  class LVEvalmultifieldqazhDataset (line 10) | class LVEvalmultifieldqazhDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/mastermath2024v1.py
  class MastermathDatasetv1 (line 13) | class MastermathDatasetv1(BaseDataset):
    method load (line 16) | def load(path: str, name: str):
  class MastermathDatasetv1Evaluator (line 45) | class MastermathDatasetv1Evaluator(BaseEvaluator):
    method score (line 47) | def score(self, predictions, references):

FILE: opencompass/datasets/matbench/matbench.py
  class MatbenchDataset (line 16) | class MatbenchDataset(BaseDataset):
    method load (line 18) | def load(path, task):
  class MatbenchEvaluator_regression (line 35) | class MatbenchEvaluator_regression(BaseEvaluator):
    method score (line 36) | def score(self, predictions, references):
  class MatbenchEvaluator_classification (line 57) | class MatbenchEvaluator_classification(BaseEvaluator):
    method score (line 59) | def score(self, predictions, references):
  class MatbenchEvaluator_classification_glass (line 85) | class MatbenchEvaluator_classification_glass(BaseEvaluator):
    method score (line 87) | def score(self, predictions, references):

FILE: opencompass/datasets/matbench/post_process.py
  function get_numerical_final_results (line 7) | def get_numerical_final_results(judged_answers,
  function _numerical_postprocess (line 38) | def _numerical_postprocess(judgement: str):
  function numerical_llmjudge_postprocess (line 46) | def numerical_llmjudge_postprocess(
  function contains_elements_and_matches (line 70) | def contains_elements_and_matches(sentence, chem_elts):
  function remove_formula (line 75) | def remove_formula(sentence):
  function verify_float (line 94) | def verify_float(number):
  function parse_float_answer (line 103) | def parse_float_answer(sentence):
  function parse_true_false_answer (line 136) | def parse_true_false_answer(raw_string, option=''):
  function parse_has_hasnot_answer (line 156) | def parse_has_hasnot_answer(raw_string, option=''):

FILE: opencompass/datasets/math.py
  function last_boxed_only_string (line 16) | def last_boxed_only_string(string):
  function remove_boxed (line 44) | def remove_boxed(s):
  function extract_boxed_answer (line 54) | def extract_boxed_answer(pred_str, strip_double_curly_brace=False):
  function normalize_final_answer (line 68) | def normalize_final_answer(final_answer: str) -> str:
  function extract_answer (line 134) | def extract_answer(response_text: str):
  class MATHDataset (line 141) | class MATHDataset(BaseDataset):
    method load (line 144) | def load(path: str, file_name: str = 'math.json', **kwargs):
  function math_postprocess (line 174) | def math_postprocess(text: str) -> str:
  function math_judement_preprocess (line 185) | def math_judement_preprocess(text: str) -> str:
  function math_postprocess_v2 (line 191) | def math_postprocess_v2(text: str) -> str:
  class MATHEvaluator (line 205) | class MATHEvaluator(BaseEvaluator):
    method __init__ (line 207) | def __init__(self,
    method score (line 215) | def score(self, predictions, references):
    method _fix_fracs (line 231) | def _fix_fracs(self, string):
    method _fix_a_slash_b (line 262) | def _fix_a_slash_b(self, string):
    method _remove_right_units (line 276) | def _remove_right_units(self, string):
    method _fix_sqrt (line 286) | def _fix_sqrt(self, string):
    method _fix_sqrt_v2 (line 300) | def _fix_sqrt_v2(self, string):
    method _strip_string (line 304) | def _strip_string(self, string):
    method _strip_string_v2 (line 371) | def _strip_string_v2(self, string):
    method is_equiv (line 474) | def is_equiv(self, str1, str2, verbose=False):
  class MATHAgentEvaluator (line 514) | class MATHAgentEvaluator(MATHEvaluator):
    method __init__ (line 522) | def __init__(self, action: str = 'PythonInterpreter', version='v1'):
    method soft_equal (line 526) | def soft_equal(self, pred, refer, step):
    method get_action (line 536) | def get_action(self, step):
    method score (line 541) | def score(self, predictions, references, steps):

FILE: opencompass/datasets/math401.py
  function check (line 4) | def check(a, b):
  class Math401Evaluator (line 8) | class Math401Evaluator(BaseEvaluator):
    method score (line 10) | def score(self, predictions, references):

FILE: opencompass/datasets/math_intern.py
  function last_boxed_only_string (line 13) | def last_boxed_only_string(string):
  function remove_boxed (line 41) | def remove_boxed(s):
  function extract_boxed_answer (line 51) | def extract_boxed_answer(pred_str, strip_double_curly_brace=False):
  class MATHInternDataset (line 66) | class MATHInternDataset(BaseDataset):
    method load (line 69) | def load(path: str):
  class MATHInternEvaluator (line 86) | class MATHInternEvaluator(BaseEvaluator):
    method score (line 88) | def score(self, predictions, references):
  function math_intern_postprocess (line 109) | def math_intern_postprocess(text: str) -> str:
  class Extractor (line 114) | class Extractor:
    method extract_matching_bracket (line 116) | def extract_matching_bracket(cls, target_str: str):
    method clean (line 129) | def clean(cls, target_str: str):
    method extract_answer (line 137) | def extract_answer(cls, pred: str, extract_last_num=False):
  function fix_fracs (line 185) | def fix_fracs(string):
  function fix_a_slash_b (line 217) | def fix_a_slash_b(string):
  function remove_right_units (line 232) | def remove_right_units(string):
  function fix_sqrt (line 242) | def fix_sqrt(string):
  function strip_string (line 257) | def strip_string(string):
  function is_equiv (line 330) | def is_equiv(str1, str2, verbose=False):

FILE: opencompass/datasets/mathbench.py
  function get_number (line 14) | def get_number(options):
  function get_circular_example (line 21) | def get_circular_example(entry, id):
  class MathBenchDataset (line 47) | class MathBenchDataset(BaseDataset):
    method load (line 50) | def load(path: str, name: str, with_circular: bool = True):
  function mathbench_postprocess (line 91) | def mathbench_postprocess(text: str, name: str) -> str:
  class MathBenchBuggyDataset (line 112) | class MathBenchBuggyDataset(BaseDataset):
    method load (line 115) | def load(path: str, name: str, with_circular: bool = True):
  function first_option_postprocess (line 154) | def first_option_postprocess(text: str, options: str, cushion=True) -> str:
  function remove_invisible_chars (line 238) | def remove_invisible_chars(text: str) -> str:
  class MathBenchCircularEvaluator (line 246) | class MathBenchCircularEvaluator(BaseEvaluator):
    method __init__ (line 249) | def __init__(self) -> None:
    method score (line 254) | def score(self, predictions, references, test_set):

FILE: opencompass/datasets/mbpp.py
  class MBPPDataset (line 26) | class MBPPDataset(BaseDataset):
    method load (line 29) | def load(path: str, local_mode: bool = False):
  class MBPPDatasetV2 (line 54) | class MBPPDatasetV2(BaseDataset):
    method load (line 57) | def load(path: str, num_repeats: int = 1):
  class SanitizedMBPPDataset (line 100) | class SanitizedMBPPDataset(BaseDataset):
    method load (line 103) | def load(path: str, num_repeats: int = 1):
  class MBPPPlusDataset (line 149) | class MBPPPlusDataset(BaseDataset):
    method load (line 152) | def load(path: str, num_repeats: int = 1):
  class TimeOutException (line 189) | class TimeOutException(Exception):
  function swallow_io (line 194) | def swallow_io():
  function time_limit (line 203) | def time_limit(seconds: float):
  class WriteOnlyStringIO (line 216) | class WriteOnlyStringIO(io.StringIO):
    method read (line 219) | def read(self, *args, **kwargs):
    method readline (line 222) | def readline(self, *args, **kwargs):
    method readlines (line 225) | def readlines(self, *args, **kwargs):
    method readable (line 228) | def readable(self, *args, **kwargs):
  class redirect_stdin (line 233) | class redirect_stdin(contextlib._RedirectStream):  # type: ignore
  class MBPPEvaluator (line 238) | class MBPPEvaluator(BaseEvaluator):
    method __init__ (line 241) | def __init__(self, metric: str = 'MBPP') -> None:
    method score (line 245) | def score(self, predictions, references):
    method _process_answer (line 309) | def _process_answer(self, text):
    method _process_test (line 347) | def _process_test(self, test_case, pred):
  class MBPPEvaluator2 (line 354) | class MBPPEvaluator2(MBPPEvaluator):
    method _process_answer (line 357) | def _process_answer(self, text):
  function _execution (line 391) | def _execution(programs, timeout, key):
  function execution (line 409) | def execution(programs, task_id, timeout):
  class MBPPPassKEvaluator (line 440) | class MBPPPassKEvaluator(MBPPEvaluator):
    method __init__ (line 447) | def __init__(self, k=(1, 10, 100)) -> None:
    method estimate_pass_at_k (line 453) | def estimate_pass_at_k(
    method score (line 479) | def score(self, predictions, references):

FILE: opencompass/datasets/mbpp_pro.py
  class MBPPProDataset (line 25) | class MBPPProDataset(BaseDataset):
    method load (line 28) | def load(path, local_mode=False):
  class MBPPProEvaluator (line 38) | class MBPPProEvaluator(CodeEvaluator):
    method score (line 40) | def score(self, predictions: List, references: List,

FILE: opencompass/datasets/medbench/constructions.py
  class TaskSchema (line 5) | class TaskSchema(object):
    method __init__ (line 7) | def __init__(self,
    method to_dict (line 21) | def to_dict(self):
  class MedBenchInstance (line 33) | class MedBenchInstance(object):
    method __init__ (line 35) | def __init__(self, task_description, data_source, task_schema, output,
    method to_dict (line 44) | def to_dict(self):
  class ChatGPTSchema (line 55) | class ChatGPTSchema(object):
    method __init__ (line 57) | def __init__(self, context=None, metadata=''):
    method to_dict (line 61) | def to_dict(self):
  class ResultsForHumanSchema (line 65) | class ResultsForHumanSchema(object):
    method __init__ (line 67) | def __init__(self,
    method to_dict (line 87) | def to_dict(self):
    method to_tsv (line 101) | def to_tsv(result_list, path):

FILE: opencompass/datasets/medbench/dataset_loader.py
  function convert_zero_shot (line 24) | def convert_zero_shot(line, dataset_name):
  function combine_prompt (line 68) | def combine_prompt(prompt_path,
  function _lazy_load_enc (line 132) | def _lazy_load_enc():
  function concat_prompt (line 139) | def concat_prompt(demos,
  function concat_prompt_chat_mode (line 172) | def concat_prompt_chat_mode(demos,
  function convert_few_shot (line 204) | def convert_few_shot(line, dataset_name, demo, n_shot, chat_mode=False):
  function load_dataset (line 228) | def load_dataset(dataset_name,
  function generate_second_stage_input (line 278) | def generate_second_stage_input(dataset_name,
  function load_dataset_as_result_schema (line 305) | def load_dataset_as_result_schema(dataset_name, parent_path):

FILE: opencompass/datasets/medbench/evaluation.py
  function convert_to_set (line 6) | def convert_to_set(item):
  function evaluate_single_sample (line 16) | def evaluate_single_sample(dataset_name, prediction, label):

FILE: opencompass/datasets/medbench/math_equivalence.py
  function _fix_fracs (line 5) | def _fix_fracs(string):
  function _fix_a_slash_b (line 37) | def _fix_a_slash_b(string):
  function _remove_right_units (line 52) | def _remove_right_units(string):
  function _fix_sqrt (line 62) | def _fix_sqrt(string):
  function _strip_string (line 77) | def _strip_string(string):
  function is_equiv (line 147) | def is_equiv(str1, str2, verbose=False):

FILE: opencompass/datasets/medbench/medbench.py
  class MedBenchDataset (line 23) | class MedBenchDataset(BaseDataset):
    method load (line 26) | def load(path: str, name: str, setting_name: str):
  class MedBenchEvaluator (line 45) | class MedBenchEvaluator(BaseEvaluator):
    method score (line 47) | def score(self, predictions, references):
  function process_generated_results_CMeEE (line 62) | def process_generated_results_CMeEE(pred_file):
  function process_generated_results_EMR (line 78) | def process_generated_results_EMR(pred_file):
  function process_generated_results_CMeIE (line 95) | def process_generated_results_CMeIE(pred_file):
  function process_generated_results_CDN (line 118) | def process_generated_results_CDN(pred_file):
  function process_generated_results_CDEE (line 140) | def process_generated_results_CDEE(pred_file):
  function process_generated_results_CTC (line 168) | def process_generated_results_CTC(pred_file):
  function process_generated_results_doc_parsing (line 178) | def process_generated_results_doc_parsing(pred_file):
  function process_generated_results_mrg (line 257) | def process_generated_results_mrg(pred_file):
  function calc_info_extract_task_scores (line 274) | def calc_info_extract_task_scores(list_structured_predict, list_structur...
  function calc_cls_task_scores (line 322) | def calc_cls_task_scores(list_structured_golden,
  function calc_nlg_task_scores (line 367) | def calc_nlg_task_scores(list_structured_golden, list_structured_predict):
  function calc_scores_f1 (line 417) | def calc_scores_f1(dict_gt, dict_pred):
  function calc_scores_ctc (line 425) | def calc_scores_ctc(dict_gt, dict_pred):
  function calc_scores_nlg (line 455) | def calc_scores_nlg(dict_gt, dict_pred):
  class MedBenchEvaluator_CMeEE (line 469) | class MedBenchEvaluator_CMeEE(BaseEvaluator):
    method score (line 471) | def score(self, predictions, references):
  class MedBenchEvaluator_DBMHG (line 476) | class MedBenchEvaluator_DBMHG(BaseEvaluator):
    method score (line 478) | def score(self, predictions, references):
  class MedBenchEvaluator_IMCS_V2_MRG (line 483) | class MedBenchEvaluator_IMCS_V2_MRG(BaseEvaluator):
    method score (line 485) | def score(self, predictions, references):
  class MedBenchEvaluator_CMeIE (line 496) | class MedBenchEvaluator_CMeIE(BaseEvaluator):
    method score (line 498) | def score(self, predictions, references):
  class MedBenchEvaluator_CHIP_CDEE (line 503) | class MedBenchEvaluator_CHIP_CDEE(BaseEvaluator):
    method score (line 505) | def score(self, predictions, references):
  class MedBenchEvaluator_CHIP_CDN (line 510) | class MedBenchEvaluator_CHIP_CDN(BaseEvaluator):
    method score (line 512) | def score(self, predictions, references):
  class MedBenchEvaluator_CHIP_CTC (line 517) | class MedBenchEvaluator_CHIP_CTC(BaseEvaluator):
    method score (line 519) | def score(self, predictions, references):
  class MedBenchEvaluator_Doc_parsing (line 524) | class MedBenchEvaluator_Doc_parsing(BaseEvaluator):
    method score (line 526) | def score(self, predictions, references):
  class MedBenchEvaluator_NLG (line 537) | class MedBenchEvaluator_NLG(BaseEvaluator):
    method score (line 539) | def score(self, predictions, references):
  class MedBenchEvaluator_Cloze (line 544) | class MedBenchEvaluator_Cloze(BaseEvaluator):
    method score (line 546) | def score(self, predictions, references):
  class MedBenchEvaluator_TF (line 564) | class MedBenchEvaluator_TF(BaseEvaluator):
    method score (line 566) | def score(self, predictions, references):

FILE: opencompass/datasets/medbench/post_process.py
  function extract_last_line (line 8) | def extract_last_line(string):
  function remove_few_shot_prefix (line 17) | def remove_few_shot_prefix(string: str):
  function try_parse_few_shot_qa_single_answer (line 29) | def try_parse_few_shot_qa_single_answer(string, setting_name, language='...
  function try_parse_few_shot_pattern (line 46) | def try_parse_few_shot_pattern(string: str, dataset_name, setting_name):
  function parse_few_shot_qa_single_answer (line 64) | def parse_few_shot_qa_single_answer(string, setting_name, language='en'):
  function find_first_capital_letter (line 73) | def find_first_capital_letter(answer):
  function extract_answer_in_bracket (line 82) | def extract_answer_in_bracket(answer, prefix='【', suffix='】'):
  function parse_math_answer (line 92) | def parse_math_answer(setting_name, raw_string):
  function parse_qa_multiple_answer (line 170) | def parse_qa_multiple_answer(string):
  function post_process (line 182) | def post_process(dataset_name, setting_name, prediction):

FILE: opencompass/datasets/medbench/utils.py
  function read_jsonl (line 5) | def read_jsonl(path):
  function save_jsonl (line 21) | def save_jsonl(lines, directory):
  function extract_answer (line 27) | def extract_answer(js):

FILE: opencompass/datasets/medmcqa.py
  function _parse (line 12) | def _parse(item, prompt_mode):
  class MedmcqaDataset (line 32) | class MedmcqaDataset(BaseDataset):
    method load (line 35) | def load(path: str, prompt_mode: str = 'zero-shot', **kwargs):
  class MedmcqaEvaluator (line 48) | class MedmcqaEvaluator(BaseEvaluator):
    method score (line 50) | def score(self, predictions, references, test_set):
  function answer_cleansing (line 79) | def answer_cleansing(
  function _generic_llmjudge_postprocess (line 127) | def _generic_llmjudge_postprocess(judgement: str):
  function medmcqa_llmjudge_postprocess (line 134) | def medmcqa_llmjudge_postprocess(

FILE: opencompass/datasets/mgsm.py
  class MGSMSDataset (line 13) | class MGSMSDataset(BaseDataset):
    method load (line 16) | def load(path: str):
  function mgsm_postprocess (line 47) | def mgsm_postprocess(text: str, lang: str) -> str:
  class MGSM_Evaluator (line 56) | class MGSM_Evaluator(BaseEvaluator):
    method score (line 58) | def score(self, predictions, references):

FILE: opencompass/datasets/mmlu.py
  class MMLUDataset (line 15) | class MMLUDataset(BaseDataset):
    method load (line 18) | def load(path: str, name: str, **kwargs):
  class MMLUDatasetClean (line 59) | class MMLUDatasetClean(BaseDataset):
    method load_contamination_annotations (line 64) | def load_contamination_annotations(path, split='val'):
    method load (line 90) | def load(path: str, name: str):

FILE: opencompass/datasets/mmlu_cf.py
  class MMLUCFDataset (line 9) | class MMLUCFDataset(BaseDataset):
    method load (line 12) | def load(path: str, name: str):

FILE: opencompass/datasets/mmlu_pro.py
  function _parse (line 14) | def _parse(item):
  class MMLUProDataset (line 32) | class MMLUProDataset(BaseDataset):
    method load (line 35) | def load(path: str, category: str):
  class MMLUProBaseEvaluator (line 42) | class MMLUProBaseEvaluator(BaseEvaluator):
    method is_equal (line 44) | def is_equal(self, pred, refer):
    method score (line 57) | def score(self, predictions, references):

FILE: opencompass/datasets/mmmlu.py
  class MMMLUDataset (line 16) | class MMMLUDataset(BaseDataset):
    method load (line 19) | def load(path: str, name: str):
  class MMMLULiteDataset (line 42) | class MMMLULiteDataset(BaseDataset):
    method load (line 45) | def load(path: str, name: str):

FILE: opencompass/datasets/mol_instructions_chem.py
  class MolInstructionsDataset (line 17) | class MolInstructionsDataset(BaseDataset):
    method load (line 20) | def load(path: str, name: str):
  function extract_chem_tag (line 35) | def extract_chem_tag(text, tag):
  class FTSEvaluator (line 46) | class FTSEvaluator(BaseEvaluator):
    method __init__ (line 49) | def __init__(self, tag) -> None:
    method score (line 53) | def score(self, predictions, references):
  function extract_number (line 107) | def extract_number(text):
  class MAEEvaluator (line 117) | class MAEEvaluator(BaseEvaluator):
    method __init__ (line 120) | def __init__(self) -> None:
    method score (line 123) | def score(self, predictions, references):
  class MeteorEvaluator (line 149) | class MeteorEvaluator(BaseEvaluator):
    method __init__ (line 152) | def __init__(self) -> None:
    method score (line 155) | def score(self, predictions, references):

FILE: opencompass/datasets/multipl_e.py
  class MultiplEDataset (line 27) | class MultiplEDataset(BaseDataset):
    method load (line 30) | def load(path: str,
  class MultiplEEvaluator (line 63) | class MultiplEEvaluator(CodeEvaluator):
    method _stop_at_stop_token (line 65) | def _stop_at_stop_token(self, decoded_string, stop_tokens):
    method _remove_prefix (line 86) | def _remove_prefix(self,
    method _process_completions (line 125) | def _process_completions(self, test_case, completion):

FILE: opencompass/datasets/multirc.py
  class MultiRCDataset (line 12) | class MultiRCDataset(BaseDataset):
    method load (line 15) | def load(path: str):
  class MultiRCDatasetV2 (line 46) | class MultiRCDatasetV2(BaseDataset):
    method load (line 49) | def load(path: str):

FILE: opencompass/datasets/musr/musr.py
  class MusrDataset (line 91) | class MusrDataset(BaseDataset):
    method load (line 106) | def load(path,
  class MusrEvaluator (line 281) | class MusrEvaluator(BaseEvaluator):
    method __init__ (line 283) | def __init__(self,
    method score (line 291) | def score(self, predictions, references):

FILE: opencompass/datasets/musr/tree.py
  class LogicNodeOperatorType (line 25) | class LogicNodeOperatorType:
  class LogicNodeFactType (line 33) | class LogicNodeFactType:
  class LogicNodeConstraints (line 40) | class LogicNodeConstraints:
  class LogicNodeDeductionType (line 46) | class LogicNodeDeductionType:
  class LogicNode (line 54) | class LogicNode:
    method __init__ (line 69) | def __init__(
    method children (line 106) | def children(self):
    method children (line 110) | def children(self, children: List['LogicNode']):
    method __str__ (line 115) | def __str__(self):
    method __repr__ (line 137) | def __repr__(self):
    method to_json (line 140) | def to_json(self):
    method from_json (line 153) | def from_json(cls, js):
  class LogicTree (line 158) | class LogicTree:
    method __init__ (line 175) | def __init__(self,
    method __str__ (line 237) | def __str__(self):
    method get_facts (line 240) | def get_facts(self,
    method print_tree (line 281) | def print_tree(self, node=None, level=0):
    method print_for_gpt (line 293) | def print_for_gpt(self,
    method populate (line 390) | def populate(self, node: LogicNode, current_depth: int = 1):
    method prune (line 471) | def prune(self, node: LogicNode, current_depth: int = 1):
    method to_json (line 497) | def to_json(self):
    method from_json (line 511) | def from_json(cls, _js):
  function tv_scene_ex (line 523) | def tv_scene_ex():
  function eb_ex (line 566) | def eb_ex():
  function murder_mystery_ex (line 630) | def murder_mystery_ex():
  function action_ex (line 679) | def action_ex():

FILE: opencompass/datasets/narrativeqa.py
  class NarrativeQADataset (line 10) | class NarrativeQADataset(BaseDataset):
    method load (line 13) | def load(path: str):

FILE: opencompass/datasets/natural_question.py
  class NaturalQuestionDataset (line 17) | class NaturalQuestionDataset(BaseDataset):
    method load (line 20) | def load(path: str):
  class NQOpenDataset (line 54) | class NQOpenDataset(BaseDataset):
    method load (line 57) | def load(path: str):
  class NQEvaluator (line 75) | class NQEvaluator(BaseEvaluator):
    method score (line 77) | def score(self, predictions, references):

FILE: opencompass/datasets/natural_question_cn.py
  class NaturalQuestionDatasetCN (line 13) | class NaturalQuestionDatasetCN(BaseDataset):
    method load (line 16) | def load(path: str):
  class NQEvaluatorCN (line 33) | class NQEvaluatorCN(BaseEvaluator):
    method score (line 35) | def score(self, predictions, references):

FILE: opencompass/datasets/needlebench/atc.py
  class NeedleBenchATCDataset (line 14) | class NeedleBenchATCDataset(BaseDataset):
    method load (line 17) | def load(
  class NeedleBenchATCOrderedDataset (line 156) | class NeedleBenchATCOrderedDataset(BaseDataset):
    method load (line 159) | def load(

FILE: opencompass/datasets/needlebench/atc_choice.py
  function get_number (line 15) | def get_number(options):
  function get_circular_example (line 22) | def get_circular_example(entry, id):
  class NeedleBenchATCDataset (line 48) | class NeedleBenchATCDataset(BaseDataset):
    method load (line 51) | def load(

FILE: opencompass/datasets/needlebench/multi.py
  function get_random_needles (line 14) | def get_random_needles(counter, file_path, needle_count):
  class NeedleBenchMultiDataset (line 36) | class NeedleBenchMultiDataset(BaseDataset):
    method load (line 39) | def load(
  class NeedleBenchMultiEvaluator (line 218) | class NeedleBenchMultiEvaluator(BaseEvaluator):
    method levenshtein_distance (line 220) | def levenshtein_distance(self, s1, s2):
    method score (line 239) | def score(self, predictions, gold):

FILE: opencompass/datasets/needlebench/origin.py
  function get_random_line_by_language (line 15) | def get_random_line_by_language(counter, file_path, language):
  class NeedleBenchOriginDataset (line 35) | class NeedleBenchOriginDataset(BaseDataset):
    method load (line 38) | def load(
  class NeedleBenchOriginEvaluator (line 210) | class NeedleBenchOriginEvaluator(BaseEvaluator):
    method __init__ (line 212) | def __init__(self, use_trim=False):
    method _trim_prediction (line 216) | def _trim_prediction(prediction, reference):
    method levenshtein_distance (line 238) | def levenshtein_distance(self, s1, s2):
    method score (line 257) | def score(self, predictions, gold):
  function needlebench_postprocess (line 302) | def needlebench_postprocess(text: str) -> str:
  function needlebench_dataset_postprocess (line 307) | def needlebench_dataset_postprocess(text: str) -> str:

FILE: opencompass/datasets/needlebench/parallel.py
  function get_unique_entries (line 14) | def get_unique_entries(
  class NeedleBenchParallelDataset (line 61) | class NeedleBenchParallelDataset(BaseDataset):
    method load (line 64) | def load(
  class NeedleBenchParallelEvaluator (line 281) | class NeedleBenchParallelEvaluator(BaseEvaluator):
    method levenshtein_distance (line 283) | def levenshtein_distance(self, s1, s2):
    method score (line 302) | def score(self, predictions, gold):

FILE: opencompass/datasets/needlebench_v2/atc.py
  class QuestionType (line 19) | class QuestionType(Enum):
  class NeedleBenchATCDataset (line 258) | class NeedleBenchATCDataset(BaseDataset):
    method load (line 261) | def load(

FILE: opencompass/datasets/needlebench_v2/atc_elder_only.py
  class NeedleBenchATCDataset (line 102) | class NeedleBenchATCDataset(BaseDataset):
    method load (line 105) | def load(
  function clean_atc_answer (line 187) | def clean_atc_answer(text: str) -> str:
  function needlebench_atc_postprocess_v2 (line 219) | def needlebench_atc_postprocess_v2(text: str) -> str:
  class NeedleBenchATCEvaluator (line 229) | class NeedleBenchATCEvaluator(BaseEvaluator):
    method score (line 231) | def score(self, predictions, gold):

FILE: opencompass/datasets/needlebench_v2/multi.py
  function get_random_needles (line 17) | def get_random_needles(counter, file_path, num_needles, language):
  class NeedleBenchMultiDataset (line 78) | class NeedleBenchMultiDataset(BaseDataset):
    method load (line 81) | def load(

FILE: opencompass/datasets/needlebench_v2/origin.py
  function get_random_line_by_language (line 16) | def get_random_line_by_language(counter, file_path, language):
  class NeedleBenchOriginDataset (line 36) | class NeedleBenchOriginDataset(BaseDataset):
    method load (line 39) | def load(
  class NeedleBenchOriginEvaluator (line 185) | class NeedleBenchOriginEvaluator(BaseEvaluator):
    method score (line 187) | def score(self, predictions, gold):
  function needlebench_postprocess (line 216) | def needlebench_postprocess(text: str) -> str:
  function needlebench_dataset_postprocess (line 221) | def needlebench_dataset_postprocess(text: str) -> str:

FILE: opencompass/datasets/needlebench_v2/parallel.py
  function get_unique_entries (line 15) | def get_unique_entries(
  class NeedleBenchParallelDataset (line 62) | class NeedleBenchParallelDataset(BaseDataset):
    method load (line 65) | def load(
  class NeedleBenchParallelEvaluator (line 274) | class NeedleBenchParallelEvaluator(BaseEvaluator):
    method score (line 276) | def score(self, predictions, gold):

FILE: opencompass/datasets/nejmaibench.py
  function _parse (line 13) | def _parse(item, prompt_mode):
  class NejmaibenchDataset (line 42) | class NejmaibenchDataset(BaseDataset):
    method load (line 45) | def load(path: str, prompt_mode: str = 'zero-shot', **kwargs):
  class NejmaibenchEvaluator (line 65) | class NejmaibenchEvaluator(BaseEvaluator):
    method score (line 67) | def score(self, predictions, references, test_set):
  function answer_cleansing (line 94) | def answer_cleansing(

FILE: opencompass/datasets/obqa.py
  class OBQADataset (line 13) | class OBQADataset(BaseDataset):
    method load (line 16) | def load(path, name='main'):
  class OBQADatasetV2 (line 53) | class OBQADatasetV2(BaseDataset):
    method load (line 56) | def load(path, name='main'):

FILE: opencompass/datasets/ojbench.py
  class OJBenchDataset (line 10) | class OJBenchDataset(BaseDataset):
    method load (line 13) | def load(path, **kwargs):

FILE: opencompass/datasets/olymmath.py
  class OlymMATHDataset (line 9) | class OlymMATHDataset(BaseDataset):
    method load (line 12) | def load(path: str, subset: str):

FILE: opencompass/datasets/omni_math.py
  class OmniMathDataset (line 16) | class OmniMathDataset(BaseDataset):
    method load (line 19) | def load():
  class OmniMathEvaluator (line 25) | class OmniMathEvaluator(BaseEvaluator):
    method __init__ (line 31) | def __init__(self, url):
    method batch_infer (line 49) | def batch_infer(self, models: List[TurboMindAPIModel],
    method parse_response (line 67) | def parse_response(self, response):
    method score (line 95) | def score(self, predictions, references, origin_prompt, test_set):

FILE: opencompass/datasets/phybench/EED.py
  function update_func (line 44) | def update_func(x, y):
  function remove_func (line 54) | def remove_func(x):
  function remove_tree_func (line 58) | def remove_tree_func(x):
  function insert_func (line 65) | def insert_func(x):
  function insert_tree_func (line 69) | def insert_tree_func(x):
  function calc_tree_size (line 73) | def calc_tree_size(node):
  function score_calc (line 113) | def score_calc(tree_dist, tree_size):
  function simplify_with_timeout (line 121) | def simplify_with_timeout(expr):
  function time_simplify (line 125) | def time_simplify(expr):
  function equal_with_timeout (line 134) | def equal_with_timeout(expr1, expr2):
  function time_equal (line 138) | def time_equal(expr1, expr2):
  function sympy_to_tree (line 146) | def sympy_to_tree(expr):
  class TreeNode (line 204) | class TreeNode:
    method __init__ (line 206) | def __init__(self, label, children=None, node_type='other'):
    method get_children (line 212) | def get_children(self):
    method __str__ (line 215) | def __str__(self):
  function print_tree (line 219) | def print_tree(node, indent=0):
  class LaTeXError (line 229) | class LaTeXError(Exception):
    method __init__ (line 231) | def __init__(self, message='LaTeXError'):
  class SymPyError (line 235) | class SymPyError(Exception):
    method __init__ (line 237) | def __init__(self, message='SymPyError'):
  class TreeError (line 241) | class TreeError(Exception):
    method __init__ (line 243) | def __init__(self, message='TreeError'):
  class DistError (line 247) | class DistError(Exception):
    method __init__ (line 249) | def __init__(self, message='DistanceError'):
  function EED (line 253) | def EED(answer_latex, test_latex, debug_mode=False):

FILE: opencompass/datasets/phybench/box_extract.py
  function extract_boxed_latex (line 1) | def extract_boxed_latex(prediction: str) -> str:

FILE: opencompass/datasets/phybench/extended_zss.py
  class Node (line 15) | class Node(object):
    method __init__ (line 17) | def __init__(self, label, children=None):
    method get_children (line 22) | def get_children(node):
    method get_label (line 26) | def get_label(node):
    method addkid (line 29) | def addkid(self, node, before=False):
    method get (line 35) | def get(self, label):
  class AnnotatedTree (line 42) | class AnnotatedTree(object):
    method __init__ (line 44) | def __init__(self, root, get_children):
  function ext_distance (line 92) | def ext_distance(A, B, get_children, single_insert_cost, insert_cost,

FILE: opencompass/datasets/phybench/latex_pre_process.py
  function brackets_balanced (line 7) | def brackets_balanced(s: str) -> bool:
  function remove_non_ascii (line 28) | def remove_non_ascii(text):
  function extract_bracket_content (line 35) | def extract_bracket_content(s: str, bracket_position: int) -> str:
  function find_first_unescaped_brace (line 67) | def find_first_unescaped_brace(s: str) -> int:
  function extract_command (line 79) | def extract_command(s: str, brace_pos: int) -> str | None:
  function remove_command (line 108) | def remove_command(s, command, keep_inside=False):
  function convert_latex_fractions (line 173) | def convert_latex_fractions(latex_str):
  function get_first_brace_command (line 192) | def get_first_brace_command(s: str) -> str | None:
  function remove_overall_brace (line 200) | def remove_overall_brace(s: str) -> str:
  function exp_frac (line 215) | def exp_frac(s):
  function find_all (line 246) | def find_all(s, sub_str, allow_overlap=True):
  function bar_inside_vec (line 261) | def bar_inside_vec(s):
  function vec_lower_idx (line 289) | def vec_lower_idx(input_str):
  function convert_vec_syntax (line 301) | def convert_vec_syntax(text):
  function remove_outer_braces (line 325) | def remove_outer_braces(tex_str):
  function extract_last_equal_content (line 334) | def extract_last_equal_content(s: str, strip_whitespace: bool = True) ->...
  function first_pre_process (line 364) | def first_pre_process(s, extrac_box=True):
  function second_pre_process (line 408) | def second_pre_process(s):
  class MyConfig (line 470) | class MyConfig:
  class MyNormalization (line 485) | class MyNormalization:
  function master_convert (line 504) | def master_convert(s):

FILE: opencompass/datasets/phybench/phybench.py
  class PhyBenchDataset (line 16) | class PhyBenchDataset(BaseDataset):
    method load (line 19) | def load(path: str):
  class MathEEDEvaluator (line 33) | class MathEEDEvaluator(BaseEvaluator):
    method score (line 35) | def score(self, predictions, references):

FILE: opencompass/datasets/physics.py
  class PHYSICSDataset (line 11) | class PHYSICSDataset(BaseDataset):
    method load (line 14) | def load(path: str, name: str, **kwargs):

FILE: opencompass/datasets/piqa.py
  class PIQADataset (line 14) | class PIQADataset(BaseDataset):
    method load_single (line 17) | def load_single(path, data_filename, label_filename):
    method load (line 35) | def load(path):
  class PIQADatasetV2 (line 57) | class PIQADatasetV2(BaseDataset):
    method load_single (line 60) | def load_single(path, data_filename, label_filename):
    method load (line 82) | def load(path):
  class PIQADatasetV3 (line 116) | class PIQADatasetV3(BaseDataset):
    method load_single (line 119) | def load_single(path, data_filename, label_filename):
    method load (line 145) | def load(path):

FILE: opencompass/datasets/py150.py
  function py150_post_process (line 12) | def py150_post_process(code):
  class Py150Dataset (line 30) | class Py150Dataset(BaseDataset):
    method load (line 33) | def load(path):

FILE: opencompass/datasets/qasper.py
  class QASPERDataset (line 10) | class QASPERDataset(BaseDataset):
    method load (line 13) | def load(path: str):

FILE: opencompass/datasets/qaspercut.py
  class QASPERCUTDataset (line 10) | class QASPERCUTDataset(BaseDataset):
    method load (line 13) | def load(path: str):

FILE: opencompass/datasets/race.py
  class RaceDataset (line 14) | class RaceDataset(BaseDataset):
    method load (line 17) | def load(path: str, name: str):

FILE: opencompass/datasets/rbench.py
  class RBenchDataset (line 9) | class RBenchDataset(BaseDataset):
    method load_single (line 12) | def load_single(path, subset='en'):
    method load (line 30) | def load(path, subset='en', **kwargs):

FILE: opencompass/datasets/realtoxicprompts.py
  class RealToxicPromptsDataset (line 10) | class RealToxicPromptsDataset(BaseDataset):
    method load (line 13) | def load(**kwargs):

FILE: opencompass/datasets/reasonbench/ReasonBenchDataset.py
  class ReasonBenchDataset (line 11) | class ReasonBenchDataset(BaseDataset):
    method load (line 14) | def load(path: str):

FILE: opencompass/datasets/record.py
  class ReCoRDDataset (line 12) | class ReCoRDDataset(BaseDataset):
    method load (line 15) | def load(path: str):
  class ReCoRDDatasetV2 (line 48) | class ReCoRDDatasetV2(BaseDataset):
    method load (line 51) | def load(path: str):
  function ReCoRD_postprocess (line 77) | def ReCoRD_postprocess(text: str) -> str:

FILE: opencompass/datasets/rolebench.py
  class RoleBenchBaseDataset (line 13) | class RoleBenchBaseDataset(BaseDataset):
    method load_single (line 16) | def load_single(source_file, desc_list):
    method load_desc (line 28) | def load_desc(path):
    method load_dataset (line 35) | def load_dataset(path, desc_list):
  class InstructionGeneralizationEnglishDataset (line 51) | class InstructionGeneralizationEnglishDataset(RoleBenchBaseDataset):
    method load (line 54) | def load(path):
  class RoleGeneralizationEnglishDataset (line 65) | class RoleGeneralizationEnglishDataset(RoleBenchBaseDataset):
    method load (line 68) | def load(path):
  class InstructionGeneralizationChineseDataset (line 79) | class InstructionGeneralizationChineseDataset(RoleBenchBaseDataset):
    method load (line 82) | def load(path):

FILE: opencompass/datasets/ruler/ruler_cwe.py
  class RulerCweDataset (line 15) | class RulerCweDataset(BaseDataset):
    method load (line 18) | def load(
  class RulerCweEvaluator (line 162) | class RulerCweEvaluator(BaseEvaluator):
    method score (line 164) | def score(self, predictions, gold):

FILE: opencompass/datasets/ruler/ruler_fwe.py
  class RulerFweDataset (line 17) | class RulerFweDataset(BaseDataset):
    method load (line 20) | def load(
  class RulerFweEvaluator (line 152) | class RulerFweEvaluator(BaseEvaluator):
    method score (line 154) | def score(self, predictions, gold):

FILE: opencompass/datasets/ruler/ruler_niah.py
  class RulerNiahDataset (line 21) | class RulerNiahDataset(BaseDataset):
    method load (line 24) | def load(
  class RulerNiahEvaluator (line 256) | class RulerNiahEvaluator(BaseEvaluator):
    method score (line 258) | def score(self, predictions, gold):

FILE: opencompass/datasets/ruler/ruler_qa.py
  class RulerQaDataset (line 19) | class RulerQaDataset(BaseDataset):
    method load (line 22) | def load(
  class RulerQaEvaluator (line 223) | class RulerQaEvaluator(BaseEvaluator):
    method score (line 225) | def score(self, predictions, gold):

FILE: opencompass/datasets/ruler/ruler_vt.py
  class RulerVtDataset (line 16) | class RulerVtDataset(BaseDataset):
    method load (line 19) | def load(
  class RulerVtEvaluator (line 184) | class RulerVtEvaluator(BaseEvaluator):
    method score (line 186) | def score(self, predictions, gold):

FILE: opencompass/datasets/s3eval.py
  class S3EvalDataset (line 14) | class S3EvalDataset(BaseDataset):
    method load (line 17) | def load(path: str):
  class S3EvalEvaluator (line 30) | class S3EvalEvaluator(BaseEvaluator):
    method score (line 32) | def score(self, predictions, references):

FILE: opencompass/datasets/safety.py
  class SafetyDataset (line 10) | class SafetyDataset(BaseDataset):
    method load (line 13) | def load(path):

FILE: opencompass/datasets/scibench.py
  class ScibenchDataset (line 14) | class ScibenchDataset(BaseDataset):
    method load (line 17) | def load(path: str, name: str):
  function scibench_postprocess (line 36) | def scibench_postprocess(text: str) -> str:

FILE: opencompass/datasets/scicode.py
  class SciCodeDataset (line 24) | class SciCodeDataset(BaseDataset):
    method load (line 27) | def load(path, with_bg, **kwargs):
    method return_dataset (line 41) | def return_dataset(self):
  function process_hdf5_list (line 45) | def process_hdf5_list(group):
  function process_hdf5_dict (line 52) | def process_hdf5_dict(group):
  function process_hdf5_sparse_matrix (line 68) | def process_hdf5_sparse_matrix(group):
  function process_hdf5_datagroup (line 88) | def process_hdf5_datagroup(group):
  function process_hdf5_to_tuple (line 98) | def process_hdf5_to_tuple(step_id, test_num):
  function are_dicts_close (line 143) | def are_dicts_close(dict1, dict2, atol=1e-8, rtol=1e-5):
  function process_symbol_in_dict (line 176) | def process_symbol_in_dict(dict):
  function are_csc_matrix_close (line 188) | def are_csc_matrix_close(matrix1, matrix2):
  function cmp_tuple_or_list (line 194) | def cmp_tuple_or_list(var1, var2):
  class SciCodeEvaluator (line 220) | class SciCodeEvaluator(BaseEvaluator):
    method __init__ (line 222) | def __init__(self, dataset_path, with_bg):
    method extract_python_script (line 235) | def extract_python_script(self, response: str):
    method run_script (line 260) | def run_script(self, script_path):
    method score (line 273) | def score(self, predictions, references):

FILE: opencompass/datasets/simpleqa.py
  class SimpleQADataset (line 15) | class SimpleQADataset(BaseDataset):
    method load (line 18) | def load(path: str,
  function get_final_results (line 122) | def get_final_results(judged_answers, references, origial_responses):
  function _single_simpleqa_postprocess (line 166) | def _single_simpleqa_postprocess(judgement: str):
  function simpleqa_postprocess (line 173) | def simpleqa_postprocess(

FILE: opencompass/datasets/siqa.py
  class siqaDataset (line 14) | class siqaDataset(BaseDataset):
    method load_single (line 18) | def load_single(path, data_filename, label_filename):
    method load (line 35) | def load(path):
  class siqaDataset_V2 (line 61) | class siqaDataset_V2(BaseDataset):
    method load_single (line 65) | def load_single(path, data_filename, label_filename):
    method load (line 94) | def load(path):
  class SiqaDatasetV3 (line 132) | class SiqaDatasetV3(BaseDataset):
    method load_single (line 136) | def load_single(path, data_filename, label_filename):
    method load (line 156) | def load(path):

FILE: opencompass/datasets/smolinstruct.py
  class SmolInstructDataset (line 18) | class SmolInstructDataset(BaseDataset):
    method load (line 21) | def load(path: str, name: str, mini_set=False):
  function extract_chemical_data (line 37) | def extract_chemical_data(text):
  function parse_molecule (line 56) | def parse_molecule(molecular_formula):
  function calculate_single_element_match_for_list (line 117) | def calculate_single_element_match_for_list(predictions, references):
  function calculate_single_element_match (line 165) | def calculate_single_element_match(predictions, references):
  class NCElementMatchEvaluator (line 214) | class NCElementMatchEvaluator(BaseEvaluator):
    method __init__ (line 217) | def __init__(self) -> None:
    method score (line 220) | def score(self, predictions, references):
  class NCExactMatchEvaluator (line 267) | class NCExactMatchEvaluator(BaseEvaluator):
    method __init__ (line 270) | def __init__(self) -> None:
    method score (line 273) | def score(self, predictions, references):
  function extract_number (line 308) | def extract_number(text):
  class RMSEEvaluator (line 316) | class RMSEEvaluator(BaseEvaluator):
    method __init__ (line 319) | def __init__(self) -> None:
    method score (line 322) | def score(self, predictions, references):
  class FTSEvaluator (line 354) | class FTSEvaluator(BaseEvaluator):
    method __init__ (line 357) | def __init__(self) -> None:
    method score (line 360) | def score(self, predictions, references):
  class MeteorEvaluator (line 414) | class MeteorEvaluator(BaseEvaluator):
    method __init__ (line 417) | def __init__(self) -> None:
    method score (line 420) | def score(self, predictions, references):
  function smolinstruct_acc_postprocess (line 447) | def smolinstruct_acc_postprocess(text: str) -> str:
  function smolinstruct_acc_0shot_postprocess (line 455) | def smolinstruct_acc_0shot_postprocess(text: str) -> str:

FILE: opencompass/datasets/squad20.py
  class SQuAD20Dataset (line 12) | class SQuAD20Dataset(BaseDataset):
    method load (line 15) | def load(path: str):
  class SQuAD20Evaluator (line 45) | class SQuAD20Evaluator(BaseEvaluator):
    method score (line 47) | def score(self, predictions, references):

FILE: opencompass/datasets/srbench.py
  class SRbenchDataset (line 17) | class SRbenchDataset(BaseDataset):
    method load (line 20) | def load(path: str):
  function mydataset_postprocess (line 48) | def mydataset_postprocess(formula_str):
  function change_data_to_prompt (line 69) | def change_data_to_prompt(points):
  class SRbenchDatasetEvaluator (line 93) | class SRbenchDatasetEvaluator(BaseEvaluator):
    method __init__ (line 95) | def __init__(self, path=''):
    method parse_formula (line 98) | def parse_formula(self, formula_str: str):
    method generate_samples (line 149) | def generate_samples(self,
    method is_symbolically_equivalent (line 161) | def is_symbolically_equivalent(self, formula1, formula2, n_var=2):
    method score (line 172) | def score(self, predictions, references) -> dict:

FILE: opencompass/datasets/storycloze.py
  class StoryClozeDataset (line 14) | class StoryClozeDataset(BaseDataset):
    method load (line 17) | def load(path, lang):
  class StoryClozeDatasetV2 (line 47) | class StoryClozeDatasetV2(BaseDataset):
    method load (line 50) | def load(path, lang):

FILE: opencompass/datasets/strategyqa.py
  function strategyqa_pred_postprocess (line 14) | def strategyqa_pred_postprocess(text: str) -> str:
  function strategyqa_dataset_postprocess (line 24) | def strategyqa_dataset_postprocess(text: str) -> str:
  class StrategyQADataset (line 29) | class StrategyQADataset(BaseDataset):
    method load (line 32) | def load(path):

FILE: opencompass/datasets/subjective/alignbench.py
  class Config (line 22) | class Config:
    method __init__ (line 24) | def __init__(self, alignment_bench_config_path,
    method category2dimensions (line 54) | def category2dimensions(self, category):
    method dimension2def (line 58) | def dimension2def(self, dimension):
    method category2type (line 61) | def category2type(self, category):
  function prompt_construct (line 65) | def prompt_construct(sample, config: Config):
  class AlignmentBenchDataset (line 95) | class AlignmentBenchDataset(SubjectiveCmpDataset):
    method load (line 97) | def load(self,
  function detect_mapping (line 141) | def detect_mapping(text):
  function extract_missing_rating (line 154) | def extract_missing_rating(text, search_type):
  function extract_rating (line 172) | def extract_rating(text):
  function check_rating (line 190) | def check_rating(rating, all_dimensions):
  function post_process_alignbench (line 202) | def post_process_alignbench(judgement: dict,
  function get_dimension_results (line 247) | def get_dimension_results(judged_answers, references):
  function get_capability_results (line 273) | def get_capability_results(judged_answers, references, categories=CATEGO...
  function alignbench_postprocess (line 307) | def alignbench_postprocess(output: dict,

FILE: opencompass/datasets/subjective/alpacaeval.py
  class AlpacaEvalDataset (line 18) | class AlpacaEvalDataset(BaseDataset):
    method load (line 20) | def load(self, path: str, name: str, *args, **kwargs):
  function post_process_alpacav2 (line 44) | def post_process_alpacav2(completion: str):
  function alpacaeval_postprocess (line 70) | def alpacaeval_postprocess(
  function alpacaeval_bradleyterry_postprocess (line 136) | def alpacaeval_bradleyterry_postprocess(

FILE: opencompass/datasets/subjective/arena_hard.py
  class ArenaHardDataset (line 23) | class ArenaHardDataset(BaseDataset):
    method load (line 25) | def load(self, path: str, name: str, *args, **kwargs):
  function post_process_arenahard (line 50) | def post_process_arenahard(completion):
  function get_battles_from_judgment (line 58) | def get_battles_from_judgment(judged_answers, references, WEIGHT=3):
  function compute_mle_elo (line 93) | def compute_mle_elo(df, SCALE=400, BASE=10, INIT_RATING=1000):
  function get_bootstrap_result (line 129) | def get_bootstrap_result(battles, func_compute_elo, num_round):
  function preety_print_two_ratings (line 137) | def preety_print_two_ratings(ratings_1, ratings_2, column_names):
  function predict_win_rate (line 148) | def predict_win_rate(elo_ratings, SCALE=400, BASE=10, INIT_RATING=1000):
  function get_win_rate_column (line 168) | def get_win_rate_column(df, column, baseline='gpt4-0314'):
  function arenahard_postprocess (line 176) | def arenahard_postprocess(
  function arenahard_bradleyterry_postprocess (line 221) | def arenahard_bradleyterry_postprocess(

FILE: opencompass/datasets/subjective/commonbench.py
  function post_process (line 11) | def post_process(judgement: str):
  function get_capability_results (line 26) | def get_capability_results(judged_answers, references):
  function commonbench_postprocess (line 46) | def commonbench_postprocess(

FILE: opencompass/datasets/subjective/compass_arena.py
  class CompassArenaDataset (line 16) | class CompassArenaDataset(SubjectiveCmpDataset):
    method load (line 18) | def load(self, path: str, name: str, *args, **kwargs):
  function check_position_bias (line 34) | def check_position_bias(judged_answers, references, banned_choice=['C']):
  function post_process_compassarena (line 60) | def post_process_compassarena(item):
  function compassarena_postprocess (line 69) | def compassarena_postprocess(
  function compassarena_bradleyterry_postprocess (line 127) | def compassarena_bradleyterry_postprocess(

FILE: opencompass/datasets/subjective/compass_arena_subjective_bench.py
  class CompassArenaSubjectiveBench (line 156) | class CompassArenaSubjectiveBench(BaseDataset):
    method load (line 158) | def load(self, path: str, name: str, *args, **kwargs):
  function post_process_pairwise (line 288) | def post_process_pairwise(completion):
  function post_process_pointwise (line 296) | def post_process_pointwise(completion):
  function compassarena_subjectiveeval_pointwise_postprocess (line 305) | def compassarena_subjectiveeval_pointwise_postprocess(
  function compassarena_subjectiveeval_pairwise_postprocess (line 345) | def compassarena_subjectiveeval_pairwise_postprocess(output: dict,
  function count_style_elements (line 414) | def count_style_elements(
  function process_convo_for_style_elements (line 464) | def process_convo_for_style_elements(
  function get_element_counts (line 509) | def get_element_counts(
  function compassarena_subjectiveeval_bradleyterry_postprocess (line 558) | def compassarena_subjectiveeval_bradleyterry_postprocess(

FILE: opencompass/datasets/subjective/compassbench.py
  class CompassBenchDataset (line 78) | class CompassBenchDataset(BaseDataset):
    method load (line 80) | def load(self, path: str, name: str, *args, **kwargs):

FILE: opencompass/datasets/subjective/compassbench_checklist.py
  class CompassBenchCheklistDataset (line 14) | class CompassBenchCheklistDataset(BaseDataset):
    method load (line 16) | def load(self, path: str, name: str, *args, **kwargs):

FILE: opencompass/datasets/subjective/compassbench_control_length_bias.py
  class CompassBenchControlLengthBiasDataset (line 106) | class CompassBenchControlLengthBiasDataset(BaseDataset):
    method load (line 108) | def load(self, path: str, name: str, *args, **kwargs):

FILE: opencompass/datasets/subjective/corev2.py
  function cn_string (line 188) | def cn_string(s):
  function build_prompt_cn (line 195) | def build_prompt_cn(item, prompt, ics):
  function build_prompt_en (line 222) | def build_prompt_en(item, prompt, ics):
  function build_prompt (line 249) | def build_prompt(item, nopt=4, multi_lang=True):
  class Corev2Dataset (line 265) | class Corev2Dataset(SubjectiveCmpDataset):
    method load (line 267) | def load(self, path: str, name: str):

FILE: opencompass/datasets/subjective/creationbench.py
  function prompt_construct (line 263) | def prompt_construct(sample):
  function prompt_construct_score_with_ref (line 276) | def prompt_construct_score_with_ref(sample):
  function prompt_construct_compare (line 293) | def prompt_construct_compare(sample):
  function prompt_construct_compare_4opt (line 302) | def prompt_construct_compare_4opt(sample):
  class CreationBenchDataset (line 312) | class CreationBenchDataset(SubjectiveCmpDataset):
    method load (line 314) | def load(self,

FILE: opencompass/datasets/subjective/flames.py
  class Config (line 15) | class Config:
    method __init__ (line 17) | def __init__(self, flames_config_path, flames_bench_config_name) -> None:
  function prompt_construct (line 25) | def prompt_construct(sample, config: Config):
  class FlamesDataset (line 37) | class FlamesDataset(SubjectiveCmpDataset):
    method load (line 39) | def load(self, path: str, name: str, *args, **kwargs):

FILE: opencompass/datasets/subjective/fofo.py
  class FofoDataset (line 17) | class FofoDataset(BaseDataset):
    method load (line 19) | def load(self, path: str, name: str, *args, **kwargs):
  function post_process_fofo (line 44) | def post_process_fofo(judgement: dict):
  function fofo_postprocess (line 60) | def fofo_postprocess(output: dict, output_path: str) -> dict:

FILE: opencompass/datasets/subjective/followbench.py
  class FollowBenchDataset (line 19) | class FollowBenchDataset(BaseDataset):
    method load (line 21) | def load(self, path: str, name: str, cate: str, *args, **kwargs):
  function post_process_followbench (line 49) | def post_process_followbench(item):
  function get_scores (line 87) | def get_scores(judged_answers, references):
  function followbench_postprocess (line 118) | def followbench_postprocess(

FILE: opencompass/datasets/subjective/hellobench.py
  class HelloBenchDataset (line 179) | class HelloBenchDataset(BaseDataset):
    method load (line 181) | def load(self, path: str, category_name: str, *args, **kwargs):
  function post_process_hellobench (line 194) | def post_process_hellobench(judgement):
  function get_judgeanswer (line 224) | def get_judgeanswer(result, filename, post_process):
  function hellobench_postprocess (line 257) | def hellobench_postprocess(

FILE: opencompass/datasets/subjective/judgerbench.py
  function parse_conversation (line 215) | def parse_conversation(conversation):
  class JudgerBenchDataset (line 245) | class JudgerBenchDataset(BaseDataset):
    method load (line 247) | def load(self, path: str, name: str, *args, **kwargs):
  class JudgerBenchEvaluator (line 346) | class JudgerBenchEvaluator(BaseEvaluator):
    method __init__ (line 349) | def __init__(self, num_workers=16) -> None:
    method get_judge_result (line 352) | def get_judge_result(self, judge, dataset_name):
    method score (line 410) | def score(self, predictions, references):

FILE: opencompass/datasets/subjective/mtbench.py
  function prompt_construct (line 102) | def prompt_construct(problem, multi_turn=False, judge_type='single'):
  class MTBenchDataset (line 174) | class MTBenchDataset(BaseDataset):
    method load (line 176) | def load(self,
  function post_process_mtbench (line 218) | def post_process_mtbench(judgement: str):
  function get_capability_results (line 233) | def get_capability_results(
  function mtbench_postprocess (line 259) | def mtbench_postprocess(

FILE: opencompass/datasets/subjective/mtbench101.py
  function eval_prompt_construct (line 237) | def eval_prompt_construct(task, ref_answer, history):
  function add_format (line 253) | def add_format(question, answer):
  class MTBench101Dataset (line 261) | class MTBench101Dataset(BaseDataset):
    method load (line 263) | def load(self, path: str, name: str, *args, **kwargs):
  function post_process_mtbench101 (line 333) | def post_process_mtbench101(judgement: str):
  function get_final_results (line 349) | def get_final_results(judged_answers, references):
  function mtbench101_postprocess (line 377) | def mtbench101_postprocess(output: dict,

FILE: opencompass/datasets/subjective/multiround.py
  function prompt_construct (line 78) | def prompt_construct(sample):
  class MultiroundDataset (line 88) | class MultiroundDataset(BaseDataset):
    method load (line 90) | def load(

FILE: opencompass/datasets/subjective/subjective_cmp.py
  class SubjectiveCmpDataset (line 13) | class SubjectiveCmpDataset(BaseDataset):
    method load (line 15) | def load(self, path: str, name: str, *args, **kwargs):

FILE: opencompass/datasets/subjective/utils.py
  function get_judgeanswer_and_reference (line 2) | def get_judgeanswer_and_reference(result, filename, post_process):

FILE: opencompass/datasets/subjective/wildbench.py
  function parse_conversation (line 189) | def parse_conversation(conversation):
  class WildBenchDataset (line 217) | class WildBenchDataset(BaseDataset):
    method load (line 219) | def load(self, path: str, K=-1, eval_mode='pair', *args, **kwargs):
  function post_process_wildbench_pair (line 276) | def post_process_wildbench_pair(judgement: dict):
  function post_process_wildbench_single (line 286) | def post_process_wildbench_single(judgement: dict):
  function wildbench_postprocess (line 304) | def wildbench_postprocess(
  function wildbench_bradleyterry_postprocess (line 362) | def wildbench_bradleyterry_postprocess(

FILE: opencompass/datasets/subjective/writingbench.py
  class WritingBenchDataset (line 47) | class WritingBenchDataset(BaseDataset):
    method load (line 49) | def load(self, path: str, name: str, *args, **kwargs):
  function post_process_writingbench (line 81) | def post_process_writingbench(judgement: dict):
  function writingbench_postprocess (line 96) | def writingbench_postprocess(output: dict, output_path: str) -> dict:

FILE: opencompass/datasets/summedits.py
  class SummeditsDataset_V2 (line 13) | class SummeditsDataset_V2(BaseDataset):
    method load (line 16) | def load(path: str):

FILE: opencompass/datasets/summscreen.py
  class SummScreenDataset (line 10) | class SummScreenDataset(BaseDataset):
    method load (line 13) | def load(path: str):

FILE: opencompass/datasets/supergpqa/supergpqa.py
  function _parse (line 16) | def _parse(item, template, prompt_mode):
  class SuperGPQADataset (line 29) | class SuperGPQADataset(BaseDataset):
    method load (line 32) | def load(path: str,
  class SuperGPQAEvaluator (line 70) | class SuperGPQAEvaluator(BaseEvaluator):
    method __init__ (line 72) | def __init__(self):
    method score (line 75) | def score(self, predictions, references, test_set):
  function _generic_llmjudge_postprocess (line 203) | def _generic_llmjudge_postprocess(judgement: str):
  function supergpqa_llmjudge_postprocess (line 210) | def supergpqa_llmjudge_postprocess(

FILE: opencompass/datasets/supergpqa/supergpqa_dataset_config/config_wrapper.py
  class ConfigWrapper (line 4) | class ConfigWrapper:
    method __init__ (line 6) | def __init__(self, config_path):
    method __setattr__ (line 13) | def __setattr__(self, key, value):
    method __getattr__ (line 20) | def __getattr__(self, key):
    method get_id (line 26) | def get_id(self, data):
    method print_all_keys (line 35) | def print_all_keys(self):
  function initialize_config (line 44) | def initialize_config(config_path):
  function get_config_wrapper (line 49) | def get_config_wrapper():

FILE: opencompass/datasets/supergpqa/supergpqa_eval.py
  function safe_regex_search (line 8) | def safe_regex_search(pattern, text, flags=0):
  function extract_option_labels (line 19) | def extract_option_labels(text, options='ABCDEFGHIJ'):
  function extract_option_content (line 60) | def extract_option_content(text, options_content=None):

FILE: opencompass/datasets/supergpqa/supergpqa_utils.py
  function load_yaml (line 10) | def load_yaml(yaml_path):
  function load_json_or_jsonl (line 18) | def load_json_or_jsonl(file_path):
  function find_file (line 30) | def find_file(base_path, sub_path, extensions=('json', 'jsonl')):
  function load_json_or_jsonl_with_idx (line 39) | def load_json_or_jsonl_with_idx(data_path, split='', idx=None):
  function load_split_data (line 65) | def load_split_data(base_path, split_name):
  function process_mixed_data (line 77) | def process_mixed_data(base_path, mode):
  class ConfigWrapper (line 99) | class ConfigWrapper:
    method __init__ (line 101) | def __init__(self, config_path):
    method __setattr__ (line 108) | def __setattr__(self, key, value):
    method __getattr__ (line 115) | def __getattr__(self, key):
    method get_id (line 121) | def get_id(self, data):
    method print_all_keys (line 130) | def print_all_keys(self):
  function initialize_config (line 139) | def initialize_config(config_path):
  function get_config_wrapper (line 144) | def get_config_wrapper():
  function read_yaml (line 188) | def read_yaml(config='default'):
  function write_jsonl_lines (line 197) | def write_jsonl_lines(file, data):
  function print_info (line 208) | def print_info(info):
  function read_json_or_jsonl (line 222) | def read_json_or_jsonl(data_path, split='', mapping_key=None):
  function read_json_or_jsonl_with_idx (line 248) | def read_json_or_jsonl_with_idx(data_path, split='', idx=None):
  function clean_json_string (line 293) | def clean_json_string(json_str):
  function is_in_idx_ranges (line 298) | def is_in_idx_ranges(idx, idx_ranges):
  function extract_json (line 305) | def extract_json(text):
  function extract_all_responses_from_json (line 319) | def extract_all_responses_from_json(response_json):
  function clean_latex (line 326) | def clean_latex(latex_expr):
  function extract_text_from_brackets (line 336) | def extract_text_from_brackets(text, clean_level='basic'):
  function extract_inner_text_from_brackets (line 358) | def extract_inner_text_from_brackets(text):
  function extract_numbers (line 366) | def extract_numbers(str):
  function extract_and_sort_inequalities (line 372) | def extract_and_sort_inequalities(latex_expr):
  function rule5_normalize_content (line 380) | def rule5_normalize_content(content):
  function normalize_string (line 386) | def normalize_string(s):
  function remove_commas_and_spaces (line 393) | def remove_commas_and_spaces(s):
  function remove_non_alphanumeric (line 397) | def remove_non_alphanumeric(s):
  function contains_or (line 401) | def contains_or(answer):
  function compare_multi_results (line 405) | def compare_multi_results(response, answer):
  function split_or_expression (line 422) | def split_or_expression(expression):
  function compare_math_expressions (line 426) | def compare_math_expressions(response, answer):
  function method_equal (line 457) | def method_equal(response_text, answer):
  function method_1 (line 461) | def method_1(response_text, answer):
  function method_2 (line 469) | def method_2(response_text, answer):
  function method_3 (line 476) | def method_3(response_text, answer):
  function method_4 (line 486) | def method_4(response_text, answer):
  function method_5 (line 492) | def method_5(response_text, answer):
  function method_9 (line 501) | def method_9(response_text, answer):
  function method_10 (line 523) | def method_10(response_text, answer):
  function method_18 (line 539) | def method_18(response_text, answer):
  function method_general (line 545) | def method_general(response_text, answer):
  function evaluate_response_vs_answer (line 563) | def evaluate_response_vs_answer(response, answer, question_type, rule_id...
  function compute_one_mixed_question_pass_rate (line 611) | def compute_one_mixed_question_pass_rate(idx,
  function evaluate_responses (line 662) | def evaluate_responses(data, mode, base_path=None):

FILE: opencompass/datasets/svamp.py
  class SVAMPDataset (line 12) | class SVAMPDataset(BaseDataset):
    method load (line 15) | def load(path):

FILE: opencompass/datasets/tabmwp.py
  function get_table_text (line 17) | def get_table_text(problem):
  function get_question_text (line 25) | def get_question_text(problem, option_inds='ABCDEFGH'):
  function get_answer (line 43) | def get_answer(problem):
  function get_choices (line 47) | def get_choices(problem):
  function get_unit (line 51) | def get_unit(problem):
  function get_solution_text (line 55) | def get_solution_text(problem):
  function normalize_answer (line 61) | def normalize_answer(text, unit):
  function score_string_similarity (line 91) | def score_string_similarity(str1, str2):
  function extract_prediction (line 106) | def extract_prediction(output, options=None, option_inds='ABCDEFGH'):
  class TabMWPEvaluator (line 174) | class TabMWPEvaluator(AccEvaluator):
    method _preprocess (line 177) | def _preprocess(self, predictions: List, references: List) -> dict:
  class TabMWPDataset (line 201) | class TabMWPDataset(BaseDataset):
    method load (line 211) | def load(path: str):

FILE: opencompass/datasets/taco.py
  class TACODataset (line 38) | class TACODataset(BaseDataset):
    method load (line 41) | def load(path: str, num_repeats: int = 1, difficulty='ALL'):
  class TACOEvaluator (line 101) | class TACOEvaluator(BaseEvaluator):
    method post_process (line 103) | def post_process(self, text):
    method check_correctness (line 116) | def check_correctness(self, sample, generation, timeout, debug=True):
    method evaluate_generations (line 142) | def evaluate_generations(self,
    method estimate_pass_at_k (line 187) | def estimate_pass_at_k(self, num_samples, num_correct, k):
    method compute_metrics (line 207) | def compute_metrics(self, results, k_list=[1, 10, 100]):
    method score (line 238) | def score(self, predictions, references, test_set):
  class CODE_TYPE (line 253) | class CODE_TYPE(Enum):
  class TimeoutException (line 259) | class TimeoutException(Exception):
  function timeout_handler (line 263) | def timeout_handler(signum, frame):
  class Capturing (line 279) | class Capturing(list):
    method __enter__ (line 281) | def __enter__(self):
    method __exit__ (line 288) | def __exit__(self, *args):
  function run_test (line 294) | def run_test(sample, test=None, debug=False):
  function custom_compare_ (line 701) | def custom_compare_(output, ground_truth):
  function stripped_string_compare (line 717) | def stripped_string_compare(s1, s2):
  function call_method (line 723) | def call_method(method, inputs):
  function reliability_guard (line 750) | def reliability_guard(maximum_memory_bytes=None):

FILE: opencompass/datasets/teval/__init__.py
  class TEvalDataset (line 14) | class TEvalDataset(BaseDataset):
    method __init__ (line 16) | def __init__(self, reader_cfg: Optional[Dict] = {}, **kwargs):
    method load (line 19) | def load(self, path: str, name: str):
  function teval_postprocess (line 46) | def teval_postprocess(text: str) -> str:

FILE: opencompass/datasets/teval/evaluators/instruct_evaluator.py
  class InstructEvaluator (line 10) | class InstructEvaluator:
    method __init__ (line 18) | def __init__(
    method _load_dataset (line 25) | def _load_dataset(self):
    method _process_response (line 39) | def _process_response(
    method _evaluate (line 63) | def _evaluate(self, data_sample: dict) -> dict:
    method compute_args_em_metric (line 85) | def compute_args_em_metric(self, gt_action, pred_action, gt_args, pred...
    method string_format_parse (line 96) | def string_format_parse(self, data_sample):
    method json_format_parse (line 119) | def json_format_parse(self, data_sample):
    method evaluate (line 133) | def evaluate(self):
    method _post_process (line 141) | def _post_process(self, results_list):

FILE: opencompass/datasets/teval/evaluators/planning_evaluator.py
  class PlanningEvaluator (line 15) | class PlanningEvaluator:
    method __init__ (line 26) | def __init__(
    method _load_dataset (line 49) | def _load_dataset(self):
    method format_load (line 66) | def format_load(self, data):
    method _process_response (line 87) | def _process_response(
    method _evaluate (line 218) | def _evaluate(self, data_sample) -> dict:
    method evaluate (line 233) | def evaluate(self):
    method permutation_match (line 242) | def permutation_match(self, pred_plan, gt_plan) -> dict:
    method bertscore_match (line 302) | def bertscore_match(self, pred_plan, gt_plan) -> dict:
    method _post_process (line 388) | def _post_process(self, results_list):

FILE: opencompass/datasets/teval/evaluators/reason_retrieve_understand_evaluator.py
  function input_postprocess (line 14) | def input_postprocess(text: str) -> str:
  class ReasonRetrieveUnderstandEvaluator (line 26) | class ReasonRetrieveUnderstandEvaluator:
    method __init__ (line 33) | def __init__(
    method _load_dataset (line 50) | def _load_dataset(self):
    method format_load (line 67) | def format_load(self, data):
    method _process_response (line 103) | def _process_response(
    method _evaluate (line 148) | def _evaluate(self, data_sample):
    method evaluate (line 154) | def evaluate(self):
    method find_a_dot_b_structure (line 163) | def find_a_dot_b_structure(self, text):
    method find_FinishAction (line 168) | def find_FinishAction(self, text):
    method _post_process (line 173) | def _post_process(self, results_list):
  class ReasonRetrieveUnderstandEvaluatorNoBatch (line 252) | class ReasonRetrieveUnderstandEvaluatorNoBatch:
    method __init__ (line 259) | def __init__(
    method _load_dataset (line 274) | def _load_dataset(self):
    method format_load (line 291) | def format_load(self, data):
    method _process_response (line 328) | def _process_response(
    method _evaluate (line 380) | def _evaluate(self, data_sample) -> dict:
    method evaluate (line 432) | def evaluate(self):
    method _post_process (line 441) | def _post_process(self, results_list):

FILE: opencompass/datasets/teval/evaluators/review_evaluator.py
  class ReviewEvaluator (line 8) | class ReviewEvaluator:
    method __init__ (line 16) | def __init__(
    method _load_dataset (line 26) | def _load_dataset(self):
    method _process_response (line 40) | def _process_response(
    method _evaluate (line 71) | def _evaluate(self, data_sample) -> dict:
    method json_format_parse (line 90) | def json_format_parse(self, pred_data):
    method evaluate (line 103) | def evaluate(self):
    method _post_process (line 112) | def _post_process(self, results_list):

FILE: opencompass/datasets/teval/schema.py
  class ResponseDataSample (line 6) | class ResponseDataSample:

FILE: opencompass/datasets/teval/utils/convert_results.py
  function parse_args (line 7) | def parse_args():
  function convert_results (line 13) | def convert_results(result_path):

FILE: opencompass/datasets/teval/utils/format_load.py
  function format_load (line 3) | def format_load(raw_data: str, start_character: str = '', end_character:...

FILE: opencompass/datasets/teval/utils/template.py
  function format_string (line 5) | def format_string(template: str, input_data: dict) -> str:
  function parse_string (line 20) | def parse_string(template: str, input_string: str, allow_newline: bool=F...

FILE: opencompass/datasets/tnews.py
  class TNewsDataset (line 12) | class TNewsDataset(BaseDataset):
    method load (line 15) | def load(**kwargs):
  class TNewsDatasetV2 (line 50) | class TNewsDatasetV2(BaseDataset):
    method load (line 53) | def load(path):

FILE: opencompass/datasets/triviaqa.py
  class TriviaQADataset (line 17) | class TriviaQADataset(BaseDataset):
    method load (line 20) | def load(path: str):
  class TriviaQADatasetV2 (line 59) | class TriviaQADatasetV2(BaseDataset):
    method load (line 62) | def load(path: str):
  class TriviaQADatasetV3 (line 95) | class TriviaQADatasetV3(BaseDataset):
    method load (line 98) | def load(path: str):
  class TriviaQAEvaluator (line 108) | class TriviaQAEvaluator(BaseEvaluator):
    method score (line 110) | def score(self, predictions, references):

FILE: opencompass/datasets/triviaqarc.py
  class TriviaQArcDataset (line 10) | class TriviaQArcDataset(BaseDataset):
    method load (line 13) | def load(path: str):

FILE: opencompass/datasets/truthfulqa.py
  class TruthfulQADataset (line 23) | class TruthfulQADataset(BaseDataset):
    method load (line 26) | def load(**kwargs):
  class TruthfulQAEvaluator (line 48) | class TruthfulQAEvaluator(BaseEvaluator):
    method __init__ (line 86) | def __init__(
    method score (line 117) | def score(self, predictions, references):
    method basic_score (line 126) | def basic_score(self, predictions, references):
    method prompt (line 184) | def prompt(self, pred, refer, metric):
    method postprocess (line 190) | def postprocess(self, generated_token):
    method api_score (line 195) | def api_score(self, predictions, references):

FILE: opencompass/datasets/tydiqa.py
  class TydiQADataset (line 16) | class TydiQADataset(BaseDataset):
    method load (line 19) | def load(path, lang):
  class TydiQAEvaluator (line 42) | class TydiQAEvaluator(BaseEvaluator):
    method f1_score (line 45) | def f1_score(self, prediction, ground_truth):
    method exact_match_score (line 57) | def exact_match_score(self, prediction, ground_truth):
    method metric_max_over_ground_truths (line 61) | def metric_max_over_ground_truths(self, metric_fn, prediction,
    method score (line 69) | def score(self, predictions, references):

FILE: opencompass/datasets/wic.py
  class WiCDataset (line 12) | class WiCDataset(BaseDataset):
    method load (line 15) | def load(**kwargs):
  class WiCDatasetV2 (line 35) | class WiCDatasetV2(BaseDataset):
    method load (line 38) | def load(path):

FILE: opencompass/datasets/wikibench.py
  function get_number (line 12) | def get_number(options):
  class WikiBenchDataset (line 21) | class WikiBenchDataset(BaseDataset):
    method load (line 24) | def load(path: str, filename: str, name: str):

FILE: opencompass/datasets/winograd.py
  class WinogradDataset (line 9) | class WinogradDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: opencompass/datasets/winogrande.py
  class WinograndeDataset (line 14) | class WinograndeDataset(BaseDataset):
    method load (line 18) | def load(path):
  class WinograndeDatasetV2 (line 63) | class WinograndeDatasetV2(BaseDataset):
    method load (line 67) | def load(path):
  class WinograndeDatasetV3 (line 116) | class WinograndeDatasetV3(BaseDataset):
    method load (line 120) | def load(path):

FILE: opencompass/datasets/wnli.py
  class wnliDataset (line 9) | class wnliDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: opencompass/datasets/wsc.py
  class WSCDataset (line 12) | class WSCDataset(BaseDataset):
    method load (line 15) | def load(**kwargs):
  class WSCDatasetV2 (line 43) | class WSCDatasetV2(BaseDataset):
    method load (line 46) | def load(path):
  class WSCDatasetV3 (line 66) | class WSCDatasetV3(BaseDataset):
    method load (line 69) | def load(path):

FILE: opencompass/datasets/xcopa.py
  class XCOPADataset (line 9) | class XCOPADataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: opencompass/datasets/xiezhi.py
  class XiezhiDataset (line 14) | class XiezhiDataset(BaseDataset):
    method load (line 17) | def load(path: str, name: str):
  class XiezhiRetriever (line 56) | class XiezhiRetriever(BaseRetriever):
    method __init__ (line 58) | def __init__(self,
    method retrieve (line 65) | def retrieve(self):

FILE: opencompass/datasets/xlsum.py
  class XLSUMDataset (line 9) | class XLSUMDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: opencompass/datasets/xsum.py
  class XsumDataset (line 13) | class XsumDataset(BaseDataset):
    method load (line 16) | def load(path: str):
  function Xsum_postprocess (line 52) | def Xsum_postprocess(text: str) -> str:

FILE: opencompass/evaluator/cascade_evaluator.py
  class CascadeEvaluator (line 13) | class CascadeEvaluator(BaseEvaluator):
    method __init__ (line 30) | def __init__(
    method sample_score (line 62) | def sample_score(self,
    method _get_llm_correctness (line 101) | def _get_llm_correctness(self, llm_detail):
    method score (line 122) | def score(

FILE: opencompass/evaluator/generic_llm_evaluator.py
  class GenericLLMEvaluator (line 22) | class GenericLLMEvaluator(BaseEvaluator):
    method __init__ (line 35) | def __init__(
    method build_inferencer (line 61) | def build_inferencer(self):
    method score (line 86) | def score(
    method pred_postprocess (line 168) | def pred_postprocess(self, predictions: List) -> Dict:
    method output_postprocess (line 176) | def output_postprocess(self, output: Dict, dataset=None) -> Dict:
    method default_judge_cfg (line 196) | def default_judge_cfg(self):

FILE: opencompass/evaluator/math_evaluator.py
  class MATHVerifyEvaluator (line 6) | class MATHVerifyEvaluator(BaseEvaluator):
    method score (line 8) | def score(self, predictions, references, test_set=None):
  function print_result (line 107) | def print_result(expr: str, result: list):

FILE: opencompass/lagent/actions/ipython_interpreter.py
  class TimeoutError (line 35) | class TimeoutError(Exception):
  class IPythonInterpreter (line 39) | class IPythonInterpreter(BaseAction):
    method __init__ (line 66) | def __init__(self,
    method start_kernel (line 100) | def start_kernel():
    method initialize (line 107) | def initialize(self):
    method reset (line 117) | def reset(self):
    method _call (line 125) | def _call(self,
    method __call__ (line 232) | def __call__(self,
  function extract_code (line 255) | def extract_code(text):
  function escape_ansi (line 273) | def escape_ansi(line):
  function publish_image_to_local (line 278) | def publish_image_to_local(image_base64: str):
  function get_multiline_input (line 291) | def get_multiline_input(hint):

FILE: opencompass/lagent/actions/python_interpreter.py
  class GenericRuntime (line 13) | class GenericRuntime:
    method __init__ (line 18) | def __init__(self):
    method exec_code (line 26) | def exec_code(self, code_piece: str) -> None:
    method eval_code (line 29) | def eval_code(self, expr: str) -> Any:
  class PythonInterpreter (line 51) | class PythonInterpreter(BaseAction):
    method __init__ (line 71) | def __init__(self,
    method extract_code (line 88) | def extract_code(command: str) -> str:
    method __call__ (line 96) | def __call__(self, command: str) -> ActionReturn:
    method _call (line 139) | def _call(self, command: str) -> ActionReturn:

FILE: opencompass/lagent/agents/react.py
  class ReActProtocol (line 10) | class ReActProtocol(_ReActProtocol):
    method __init__ (line 12) | def __init__(self, **kwargs) -> None:
    method format (line 19) | def format(self,
  class ReAct (line 75) | class ReAct(_ReAct):
    method __init__ (line 77) | def __init__(self,
    method chat (line 95) | def chat(self, message: str) -> AgentReturn:
  class CIReAct (line 134) | class CIReAct(ReAct):
    method reset (line 149) | def reset(self):
    method chat (line 158) | def chat(self, message: str) -> AgentReturn:
  class CIReActMergeRole (line 206) | class CIReActMergeRole(CIReAct):
    method chat (line 210) | def chat(self, message: str) -> AgentReturn:
    method merge_role (line 258) | def merge_role(self, inputs):

FILE: opencompass/metrics/dump_results.py
  class DumpResults (line 11) | class DumpResults(BaseMetric):
    method __init__ (line 25) | def __init__(self,
    method process (line 34) | def process(self, data_batch, data_samples) -> None:
    method compute_metrics (line 49) | def compute_metrics(self, results: list) -> dict:

FILE: opencompass/metrics/mme_score.py
  class MMEMetric (line 10) | class MMEMetric(BaseMetric):
    method __init__ (line 34) | def __init__(self,
    method process (line 39) | def process(self, data_batch, data_samples) -> None:
    method compute_metrics (line 48) | def compute_metrics(self, results: list) -> dict:

FILE: opencompass/metrics/seedbench.py
  class SEEDBenchAcc (line 23) | class SEEDBenchAcc(BaseMetric):
    method process (line 26) | def process(self, data_batch, data_samples) -> None:
    method compute_metrics (line 40) | def compute_metrics(self, results: list) -> dict:

FILE: opencompass/models/accessory.py
  class LLaMA2AccessoryModel (line 14) | class LLaMA2AccessoryModel(BaseModel):
    method __init__ (line 29) | def __init__(self,
    method _load_model (line 44) | def _load_model(self, from_pretrained_kwargs):
    method _load_tokenizer (line 57) | def _load_tokenizer(self, from_pretrained_kwargs):
    method generate (line 71) | def generate(self, inputs: List[str], max_out_len: int) -> List[str]:
    method get_ppl (line 79) | def get_ppl(self,
    method get_token_len (line 87) | def get_token_len(self, prompt: str) -> int:

FILE: opencompass/models/ai360_api.py
  class AI360GPT (line 13) | class AI360GPT(BaseAPIModel):
    method __init__ (line 31) | def __init__(
    method generate (line 60) | def generate(
    method _generate (line 83) | def _generate(

FILE: opencompass/models/alaya.py
  class AlayaLM (line 14) | class AlayaLM(BaseModel):
    method __init__ (line 36) | def __init__(self,
    method do_inference (line 80) | def do_inference(self, instruction, history=[]):
    method generate (line 115) | def generate(
    method get_token_len (line 127) | def get_token_len(self, prompt: str) -> int:
    method get_ppl (line 131) | def get_ppl(self,

FILE: opencompass/models/baichuan_api.py
  class BaiChuan (line 14) | class BaiChuan(BaseAPIModel):
    method __init__ (line 33) | def __init__(
    method generate (line 57) | def generate(
    method _generate (line 80) | def _generate(

FILE: opencompass/models/baidu_api.py
  class ERNIEBot (line 14) | class ERNIEBot(BaseAPIModel):
    method __init__ (line 35) | def __init__(self,
    method _generate_access_token (line 63) | def _generate_access_token(self):
    method generate (line 91) | def generate(
    method _generate (line 114) | def _generate(

FILE: opencompass/models/bailing_api_oc.py
  class HTTPAdapterWithSocketOptions (line 26) | class HTTPAdapterWithSocketOptions(HTTPAdapter):
    method __init__ (line 28) | def __init__(self, *args, **kwargs):
    method init_poolmanager (line 37) | def init_poolmanager(self, *args, **kwargs):
  class BailingAPI (line 44) | class BailingAPI(BaseAPIModel):
    method __init__ (line 55) | def __init__(
    method generate (line 103) | def generate(
    method _generate (line 160) | def _generate(
    method _infer_result (line 223) | def _infer_result(self, request, sess):

FILE: opencompass/models/base.py
  class BaseModel (line 13) | class BaseModel:
    method __init__ (line 34) | def __init__(self,
    method generate (line 53) | def generate(self, inputs: List[str], max_out_len: int) -> List[str]:
    method get_ppl (line 68) | def get_ppl(self,
    method get_ppl_tokenwise (line 89) | def get_ppl_tokenwise(
    method encode (line 111) | def encode(self, prompt: str) -> torch.Tensor:
    method decode (line 125) | def decode(self, tokens: torch.Tensor) -> str:
    method get_token_len (line 139) | def get_token_len(self, prompt: str) -> int:
    method parse_template (line 149) | def parse_template(self, prompt_template: PromptType, mode: str) -> str:
    method get_ppl_from_template (line 163) | def get_ppl_from_template(self,
    method get_ppl_tokenwise_from_template (line 176) | def get_ppl_tokenwise_from_template(self,
    method generate_from_template (line 190) | def generate_from_template(self, templates: List[PromptType],
    method get_token_len_from_template (line 203) | def get_token_len_from_template(
    method sync_inputs (line 228) | def sync_inputs(self, inputs: str) -> str:
    method to (line 261) | def to(self, device):
  class LMTemplateParser (line 265) | class LMTemplateParser:
    method __init__ (line 272) | def __init__(self, meta_template: Optional[Dict] = None):
    method parse_template (line 300) | def parse_template(self, prompt_template: PromptType, mode: str) -> str:
    method _split_rounds (line 396) | def _split_rounds(
    method _update_role_dict (line 423) | def _update_role_dict(self, prompt: Union[List, str,
    method _prompt2str (line 443) | def _prompt2str(self,
    method _role2str (line 475) | def _role2str(self,
    method _encode_speical_tokens (line 502) | def _encode_speical_tokens(self, prompt: List[Union[str, int]]) -> str:

FILE: opencompass/models/base_api.py
  class BaseAPIModel (line 23) | class BaseAPIModel(BaseModel):
    method __init__ (line 42) | def __init__(self,
    method generate (line 63) | def generate(self, inputs: List[PromptType],
    method flush (line 80) | def flush(self):
    method acquire (line 92) | def acquire(self):
    method release (line 103) | def release(self):
    method get_ppl (line 113) | def get_ppl(self,
    method get_token_len (line 133) | def get_token_len(self, prompt: str) -> int:
    method wait (line 156) | def wait(self):
    method to (line 163) | def to(self, device):
  class APITemplateParser (line 167) | class APITemplateParser:
    method __init__ (line 174) | def __init__(self, meta_template: Optional[Dict] = None):
    method parse_template (line 203) | def parse_template(self, prompt_template: PromptType,
    method _update_role_dict (line 337) | def _update_role_dict(self, prompts: Union[List, str]) -> Dict[str, Di...
    method _split_rounds (line 355) | def _split_rounds(
    method _prompt2api (line 388) | def _prompt2api(self,
    method _role2api_role (line 427) | def _role2api_role(self,
  class TokenBucket (line 458) | class TokenBucket:
    method __init__ (line 465) | def __init__(self, rate, verbose=False):
    method _add_tokens (line 473) | def _add_tokens(self):
    method get_token (line 480) | def get_token(self):

FILE: opencompass/models/bluelm_api.py
  class BlueLMAPI (line 15) | class BlueLMAPI(BaseAPIModel):
    method __init__ (line 33) | def __init__(
    method generate (line 60) | def generate(
    method get_streaming_response (line 83) | def get_streaming_response(self, response: requests.Response):
    method split_think (line 91) | def split_think(self, text: str) -> str:
    method _generate (line 100) | def _generate(

FILE: opencompass/models/bytedance_api.py
  class ByteDance (line 17) | class ByteDance(BaseAPIModel):
    method __init__ (line 36) | def __init__(self,
    method generate (line 65) | def generate(
    method _generate (line 88) | def _generate(

FILE: opencompass/models/claude_allesapin.py
  class ClaudeAllesAPIN (line 17) | class ClaudeAllesAPIN(BaseAPIModel):
    method __init__ (line 35) | def __init__(self,
    method generate (line 54) | def generate(self,
    method _generate (line 75) | def _generate(self, input: PromptType, max_out_len: int) -> str:

FILE: opencompass/models/claude_api/claude_api.py
  class Claude (line 13) | class Claude(BaseAPIModel):
    method __init__ (line 28) | def __init__(
    method generate (line 53) | def generate(
    method _generate (line 75) | def _generate(

FILE: opencompass/models/claude_api/postprocessors.py
  function gsm8k_postprocess (line 9) | def gsm8k_postprocess(text: str) -> str:
  function humaneval_postprocess (line 29) | def humaneval_postprocess(text: str) -> str:
  function lcsts_postprocess (line 53) | def lcsts_postprocess(text: str) -> str:
  function mbpp_postprocess (line 61) | def mbpp_postprocess(text: str) -> str:
  function strategyqa_pred_postprocess (line 75) | def strategyqa_pred_postprocess(text: str) -> str:
  function flores_postprocess (line 85) | def flores_postprocess(text: str) -> str:
  function flores_postprocess_chinese (line 90) | def flores_postprocess_chinese(text: str) -> str:
  function record_postprocess (line 99) | def record_postprocess(text: str) -> str:
  function humaneval_claude2_postprocess (line 108) | def humaneval_claude2_postprocess(text: str) -> str:
  function xsum_postprocess (line 114) | def xsum_postprocess(text: str) -> str:
  function yes_no_postprocess (line 120) | def yes_no_postprocess(text: str) -> str:

FILE: opencompass/models/claude_sdk_api.py
  class ClaudeSDK (line 13) | class ClaudeSDK(BaseAPIModel):
    method __init__ (line 28) | def __init__(
    method generate (line 55) | def generate(
    method _generate (line 77) | def _generate(

FILE: opencompass/models/deepseek_api.py
  class DeepseekAPI (line 14) | class DeepseekAPI(BaseAPIModel):
    method __init__ (line 32) | def __init__(
    method generate (line 56) | def generate(
    method _generate (line 79) | def _generate(

FILE: opencompass/models/doubao.py
  class Doubao (line 12) | class Doubao(BaseAPIModel):
    method __init__ (line 14) | def __init__(
    method generate (line 44) | def generate(self,
    method _generate (line 65) | def _generate(self, input: PromptType, max_out_len: int = 512) -> str:

FILE: opencompass/models/doubao_api.py
  class Doubao (line 16) | class Doubao(BaseAPIModel):
    method __init__ (line 36) | def __init__(self,
    method generate (line 62) | def generate(
    method _generate (line 85) | def _generate(

FILE: opencompass/models/gemini_api.py
  class Gemini (line 17) | class Gemini(BaseAPIModel):
    method __init__ (line 35) | def __init__(
    method generate (line 68) | def generate(
    method _generate (line 91) | def _generate(

FILE: opencompass/models/glm.py
  class GLM130B (line 16) | class GLM130B(BaseModel):
    method __init__ (line 18) | def __init__(self,
    method _load_model (line 34) | def _load_model(self, **kwargs):
    method get_token_len (line 121) | def get_token_len(self, prompt: str) -> int:
    method choice (line 132) | def choice(self, inputs, choices):
    method generate (line 166) | def generate(self, inputs: List[str], max_out_len: int) -> List[str]:
    method get_logits (line 287) | def get_logits(self, inputs: List[str]):
    method get_ppl (line 368) | def get_ppl(self,

FILE: opencompass/models/huggingface.py
  class MultiTokenEOSCriteria (line 17) | class MultiTokenEOSCriteria(transformers.StoppingCriteria):
    method __init__ (line 20) | def __init__(
    method __call__ (line 33) | def __call__(self, input_ids, scores, **kwargs) -> bool:
  class HuggingFace (line 45) | class HuggingFace(BaseModel):
    method __init__ (line 91) | def __init__(self,
    method _load_tokenizer (line 131) | def _load_tokenizer(self, path: str, tokenizer_path: Optional[str],
    method _set_model_kwargs_torch_dtype (line 181) | def _set_model_kwargs_torch_dtype(self, model_kwargs):
    method _load_model (line 196) | def _load_model(self,
    method generate (line 223) | def generate(self,
    method _batch_generate (line 256) | def _batch_generate(self,
    method _single_generate (line 340) | def _single_generate(self,
    method get_logits (line 431) | def get_logits(self, inputs: List[str]):
    method get_ppl (line 458) | def get_ppl(self,
    method _get_ppl (line 484) | def _get_ppl(self,
    method get_loglikelihood (line 525) | def get_loglikelihood(
    method _get_loglikelihood (line 556) | def _get_loglikelihood(self, inputs: str, conts: str) -> float:
    method get_mink_percent (line 604) | def get_mink_percent(self, inputs: List[str], k: int = 20) -> List[flo...
    method _get_mink_percent (line 615) | def _get_mink_percent(self, inputs: List[str], k: int = 20) -> List[fl...
    method get_token_len (line 635) | def get_token_len(self, prompt: str) -> int:
  class HuggingFaceCausalLM (line 648) | class HuggingFaceCausalLM(HuggingFace):
    method _load_model (line 674) | def _load_model(self,
  class HuggingFaceChatGLM3 (line 691) | class HuggingFaceChatGLM3(HuggingFace):
    method __init__ (line 698) | def __init__(self,
    method generate (line 732) | def generate(self,
    method get_token_len (line 790) | def get_token_len(self, prompt: str) -> int:

FILE: opencompass/models/huggingface_above_v4_33.py
  function _get_stopping_criteria (line 17) | def _get_stopping_criteria(stop_words, tokenizer, batch_size):
  function _get_possible_max_seq_len (line 45) | def _get_possible_max_seq_len(max_seq_len, path):
  function _convert_chat_messages (line 62) | def _convert_chat_messages(inputs, merge_role=True, skip_empty_prompt=Tr...
  function _format_with_fast_chat_template (line 92) | def _format_with_fast_chat_template(inputs: List[str], name: str='vicuna'):
  function _get_meta_template (line 115) | def _get_meta_template(meta_template):
  function _set_model_kwargs_torch_dtype (line 127) | def _set_model_kwargs_torch_dtype(model_kwargs):
  class HuggingFacewithChatTemplate (line 145) | class HuggingFacewithChatTemplate(BaseModel):
    method __init__ (line 154) | def __init__(self,
    method _load_tokenizer (line 190) | def _load_tokenizer(self, path: Optional[str], kwargs: dict, pad_token...
    method _load_model (line 220) | def _load_model(self, path: str, kwargs: dict, peft_path: Optional[str...
    method get_ppl_tokenwise (line 245) | def get_ppl_tokenwise(self, inputs: List[str], label: List[List[int]],...
    method _get_potential_stop_words (line 403) | def _get_potential_stop_words(self, path: Optional[str]):
    method generate (line 423) | def generate(self,
    method get_token_len (line 491) | def get_token_len(self, prompt: str) -> int:
  function _convert_base_messages (line 496) | def  _convert_base_messages(inputs):
  class HuggingFaceBaseModel (line 509) | class HuggingFaceBaseModel(HuggingFacewithChatTemplate):
    method __init__ (line 511) | def __init__(self,
    method generate (line 542) | def generate(self,
    method get_ppl (line 593) | def get_ppl(self, inputs: List[str], mask_length: Optional[List[int]] ...
    method get_loglikelihood (line 656) | def get_loglikelihood(self, inputs: List[str], conts:  List[str]) -> L...
    method get_token_len (line 660) | def get_token_len(self, prompt: str, add_special_tokens: bool=True) ->...

FILE: opencompass/models/hunyuan_api.py
  class Hunyuan (line 13) | class Hunyuan(BaseAPIModel):
    method __init__ (line 15) | def __init__(
    method generate (line 53) | def generate(self,
    method _generate (line 74) | def _generate(self, input: PromptType, max_out_len: int = 512) -> str:

FILE: opencompass/models/intern_model.py
  class InternLM (line 9) | class InternLM(BaseModel):
    method __init__ (line 11) | def __init__(self,
    method _load_model (line 34) | def _load_model(self,
    method _load_tokenizer (line 52) | def _load_tokenizer(self, tokenizer_path: str, tokenizer_type: str,
    method get_token_len (line 63) | def get_token_len(self, prompt: str) -> int:
    method generate (line 75) | def generate(self, inputs: List[str], max_out_len: int) -> List[str]:
    method get_ppl (line 91) | def get_ppl(self,

FILE: opencompass/models/interntrain.py
  class InternTrainManager (line 16) | class InternTrainManager:
    method __init__ (line 18) | def __init__(self, module_path):
    method build (line 22) | def build(module_path):
  class CurrentInternTrainManager (line 32) | class CurrentInternTrainManager(InternTrainManager):
    method load_config (line 34) | def load_config(self, path, model_config=None):
    method initialize_model (line 54) | def initialize_model(self):
  class LegacyInternTrainManager (line 63) | class LegacyInternTrainManager(InternTrainManager):
    method load_config (line 65) | def load_config(self, path, model_config=None):
    method initialize_model (line 81) | def initialize_model(self):
  class InternTrain (line 89) | class InternTrain(BaseModel):
    method __init__ (line 135) | def __init__(self,
    method _load_model (line 208) | def _load_model(self,
    method _load_tokenizer (line 275) | def _load_tokenizer(self, tokenizer_path: str, tokenizer_type: str):
    method _convert_dtype (line 290) | def _convert_dtype(self, default_dtype, model_dtype=None):
    method get_token_len (line 308) | def get_token_len(self, prompt: str, use_bos=None, use_eos=None) -> int:
    method generate (line 320) | def generate(self,
    method get_ppl (line 365) | def get_ppl(self,
    method get_loglikelihood (line 401) | def get_loglikelihood(self, input_texts: List[str],
    method get_mink_percent (line 426) | def get_mink_percent(self,
    method get_logits (line 447) | def get_logits(self, input_texts: Union[str, List[str]]):
    method batch_encode (line 455) | def batch_encode(self,
    method batch_decode (line 480) | def batch_decode(self,

FILE: opencompass/models/krgpt_api.py
  class KrGPT (line 17) | class KrGPT(BaseAPIModel):
    method __init__ (line 20) | def __init__(
    method generate (line 41) | def generate(self, inputs: List[str], max_out_len: int,
    method _generate (line 61) | def _generate(self,

FILE: opencompass/models/lagent.py
  class LagentAgent (line 9) | class LagentAgent:
    method __init__ (line 16) | def __init__(self, agent_type, llm, actions=None, protocol=None, **kwa...
    method reset (line 36) | def reset(self):
    method set_history (line 42) | def set_history(self, history):
    method gt_response (line 45) | def gt_response(self, prompt):
    method template_parser (line 67) | def template_parser(self):
    method template_parser (line 71) | def template_parser(self, value):
    method chat (line 74) | def chat(self,
  class CodeAgent (line 148) | class CodeAgent(LagentAgent):
    method __init__ (line 151) | def __init__(self, llm, **kwargs):

FILE: opencompass/models/langchain.py
  class LangchainAgent (line 8) | class LangchainAgent:
    method __init__ (line 14) | def __init__(self, agent_type, llm, tools) -> None:
    method chat (line 24) | def chat(self, user_input, ice=None) -> Tuple[str, List[dict]]:

FILE: opencompass/models/lightllm_api.py
  class LightllmAPI (line 21) | class LightllmAPI(BaseModel):
    method __init__ (line 25) | def __init__(
    method generate (line 48) | def generate(self, inputs: List[str], max_out_len: int,
    method _generate (line 69) | def _generate(self, input: str, max_out_len: int) -> str:
    method get_ppl (line 101) | def get_ppl(self, inputs: List[str], max_out_len: int,
    method _get_ppl (line 122) | def _get_ppl(self, input: str, max_out_len: int) -> float:
    method wait (line 167) | def wait(self):
    method get_token_len (line 174) | def get_token_len(self, prompt: str) -> int:
  class LightllmChatAPI (line 198) | class LightllmChatAPI(BaseAPIModel):
    method __init__ (line 216) | def __init__(
    method generate (line 233) | def generate(
    method _generate (line 256) | def _generate(

FILE: opencompass/models/llama2.py
  class Llama2 (line 13) | class Llama2(BaseModel):
    method __init__ (line 26) | def __init__(
    method _load_model (line 46) | def _load_model(self,
    method _load_tokenizer (line 57) | def _load_tokenizer(self, tokenizer_path: str):
    method generate (line 61) | def generate(self, inputs: List[str], max_out_len: int) -> List[str]:
    method get_ppl (line 75) | def get_ppl(self,
    method get_loglikelihood (line 103) | def get_loglikelihood(
    method get_token_len (line 139) | def get_token_len(self, prompt: str) -> int:
  class Llama2Chat (line 143) | class Llama2Chat(BaseModel):
    method __init__ (line 157) | def __init__(
    method _load_model (line 179) | def _load_model(self,
    method _load_tokenizer (line 197) | def _load_tokenizer(self, tokenizer_path: str):
    method generate (line 201) | def generate(self,
    method get_token_len (line 256) | def get_token_len(self, prompt: str) -> int:

FILE: opencompass/models/minimax_api.py
  class MiniMax (line 14) | class MiniMax(BaseAPIModel):
    method __init__ (line 35) | def __init__(
    method generate (line 61) | def generate(
    method _generate (line 84) | def _generate(
  class MiniMaxChatCompletionV2 (line 185) | class MiniMaxChatCompletionV2(BaseAPIModel):
    method __init__ (line 203) | def __init__(
    method generate (line 225) | def generate(
    method _generate (line 248) | def _generate(

FILE: opencompass/models/mistral_api.py
  class Mistral (line 13) | class Mistral(BaseAPIModel):
    method __init__ (line 15) | def __init__(
    method generate (line 37) | def generate(self,
    method _generate (line 58) | def _generate(self, input: PromptType, max_out_len: int = 512) -> str:

FILE: opencompass/models/mixtral.py
  class Mixtral (line 13) | class Mixtral(BaseModel):
    method __init__ (line 25) | def __init__(
    method _load_model (line 47) | def _load_model(self,
    method _load_tokenizer (line 62) | def _load_tokenizer(self, tokenizer_path: str):
    method generate (line 66) | def generate(self, inputs: List[str], max_out_len: int) -> List[str]:
    method get_ppl (line 80) | def get_ppl(self,
    method get_token_len (line 108) | def get_token_len(self, prompt: str) -> int:

FILE: opencompass/models/modelscope.py
  class ModelScope (line 12) | class ModelScope(HuggingFace):
    method __init__ (line 52) | def __init__(self,
    method _load_tokenizer (line 82) | def _load_tokenizer(self, path: str, tokenizer_path: Optional[str],
    method _set_model_kwargs_torch_dtype (line 132) | def _set_model_kwargs_torch_dtype(self, model_kwargs):
    method _load_model (line 147) | def _load_model(self,
  class ModelScopeCausalLM (line 175) | class ModelScopeCausalLM(ModelScope):
    method _load_model (line 201) | def _load_model(self,

FILE: opencompass/models/moonshot_api.py
  class MoonShot (line 14) | class MoonShot(BaseAPIModel):
    method __init__ (line 32) | def __init__(
    method generate (line 56) | def generate(
    method _generate (line 79) | def _generate(

FILE: opencompass/models/nanbeige_api.py
  class Nanbeige (line 14) | class Nanbeige(BaseAPIModel):
    method __init__ (line 32) | def __init__(self,
    method generate (line 53) | def generate(
    method _generate (line 76) | def _generate(

FILE: opencompass/models/openai_api.py
  class OpenAI (line 34) | class OpenAI(BaseAPIModel):
    method __init__ (line 83) | def __init__(
    method _next_valid_key (line 164) | def _next_valid_key(self):
    method generate (line 181) | def generate(
    method _generate (line 224) | def _generate(self, input: PromptType, max_out_len: int,
    method _init_tokenizer (line 392) | def _init_tokenizer(self):
    method get_token_len (line 435) | def get_token_len(self, prompt: str) -> int:
    method _bin_trim (line 453) | def _bin_trim(self, prompt: str, num_token: int, mode: str) -> str:
    method _preprocess_messages (line 499) | def _preprocess_messages(
  class OpenAISDK (line 596) | class OpenAISDK(OpenAI):
    method __init__ (line 598) | def __init__(
    method _create_fresh_client (line 660) | def _create_fresh_client(self):
    method _generate (line 685) | def _generate(
  class OpenAISDKRollout (line 839) | class OpenAISDKRollout(OpenAI):
    method __init__ (line 841) | def __init__(
    method _generate (line 914) | def _generate(

FILE: opencompass/models/openai_streaming.py
  class OpenAISDKStreaming (line 18) | class OpenAISDKStreaming(OpenAISDK):
    method __init__ (line 31) | def __init__(self,
    method _generate (line 88) | def _generate(
    method _handle_stream_response (line 197) | def _handle_stream_response(self, response_stream, thread_id=None) -> ...
    method estimate_token_count (line 325) | def estimate_token_count(self, text: str) -> int:

FILE: opencompass/models/pangu_api.py
  class PanGu (line 14) | class PanGu(BaseAPIModel):
    method __init__ (line 34) | def __init__(
    method generate (line 68) | def generate(
    method _get_token (line 91) | def _get_token(self):
    method _generate (line 118) | def _generate(

FILE: opencompass/models/qwen_api.py
  class Qwen (line 12) | class Qwen(BaseAPIModel):
    method __init__ (line 31) | def __init__(self,
    method generate (line 49) | def generate(
    method _generate (line 72) | def _generate(

FILE: opencompass/models/rendu_api.py
  class Rendu (line 14) | class Rendu(BaseAPIModel):
    method __init__ (line 33) | def __init__(self,
    method generate (line 60) | def generate(
    method _generate (line 83) | def _generate(

FILE: opencompass/models/sensetime_api.py
  class SenseTime (line 16) | class SenseTime(BaseAPIModel):
    method __init__ (line 32) | def __init__(
    method generate (line 62) | def generate(
    method _generate (line 85) | def _generate(

FILE: opencompass/models/stepfun_api.py
  class StepFun (line 14) | class StepFun(BaseAPIModel):
    method __init__ (line 32) | def __init__(
    method generate (line 56) | def generate(
    method _generate (line 79) | def _generate(

FILE: opencompass/models/telechat_api/telechat_api.py
  class TeleChat (line 19) | class TeleChat(BaseAPIModel):
    method __init__ (line 34) | def __init__(self,
    method _get_auth_headers (line 74) | def _get_auth_headers(self):
    method generate (line 88) | def generate(self,
    method _generate (line 115) | def _generate(

FILE: opencompass/models/telechat_api/telechat_api_streaming.py
  class TeleChatStream (line 20) | class TeleChatStream(BaseAPIModel):
    method __init__ (line 25) | def __init__(
    method _get_auth_headers (line 62) | def _get_auth_headers(self) -> Dict:
    method generate (line 81) | def generate(
    method _generate (line 101) | def _generate(self, input: PromptType, max_out_len: int) -> str:
    method _build_messages (line 144) | def _build_messages(self, input: PromptType) -> List[Dict]:
    method _parse_sse_stream (line 155) | def _parse_sse_stream(self, response: requests.Response) -> str:
    method _sse_event_iterator (line 190) | def _sse_event_iterator(self, response: requests.Response):
    method _merge_output (line 203) | def _merge_output(self, content, reasoning):

FILE: opencompass/models/telechat_api/telechat_auth_sdk.py
  class Authorization (line 9) | class Authorization:
    method __init__ (line 11) | def __init__(self):
    method normalize (line 16) | def normalize(self, string, encodingSlash=True):
    method generate_canonical_uri (line 22) | def generate_canonical_uri(self, url):
    method generate_canonical_headers (line 29) | def generate_canonical_headers(self, headers):
    method generate_signature (line 40) | def generate_signature(self,
    method generate_signature_all (line 62) | def generate_signature_all(self,
    method get_content_length (line 81) | def get_content_length(self, data):

FILE: opencompass/models/turbomind.py
  function valid_str (line 15) | def valid_str(string, coding='utf-8'):
  class TurboMindModel (line 25) | class TurboMindModel(BaseModel):
    method __init__ (line 48) | def __init__(self,
    method generate (line 78) | def generate(self,
    method get_token_len (line 147) | def get_token_len(self, prompt: str) -> int:
    method wait (line 151) | def wait(self):
    method get_ppl (line 158) | def get_ppl(self,
    method get_loglikelihood (line 201) | def get_loglikelihood(
    method _build_pipe (line 231) | def _build_pipe(self, model_path, backend, engine_config):

FILE: opencompass/models/turbomind_api.py
  function valid_str (line 12) | def valid_str(string, coding='utf-8'):
  class TurboMindAPIModel (line 22) | class TurboMindAPIModel(BaseModel):
    method __init__ (line 41) | def __init__(self,
    method generate (line 65) | def generate(
    method get_token_len (line 101) | def get_token_len(self, prompt: str) -> int:
    method wait (line 105) | def wait(self):
    method _generate (line 112) | def _generate(self, prompt: PromptType, max_out_len: int,

FILE: opencompass/models/turbomind_with_tf_above_v4_33.py
  function valid_str (line 22) | def valid_str(string, coding='utf-8'):
  class TurboMindModelwithChatTemplate (line 32) | class TurboMindModelwithChatTemplate(BaseModel):
    method __init__ (line 33) | def __init__(
    method _get_potential_stop_words (line 75) | def _get_potential_stop_words(self, path: Optional[str]):
    method generate (line 99) | def generate(self,
    method get_token_len (line 192) | def get_token_len(self, prompt: str) -> int:
    method _build_pipe (line 205) | def _build_pipe(self, model_path, backend, engine_config):

FILE: opencompass/models/unigpt_api.py
  function get_sign (line 16) | def get_sign(appkey, udid, timestamp, secret):
  class UniGPT (line 31) | class UniGPT(BaseAPIModel):
    method __init__ (line 33) | def __init__(
    method generate (line 60) | def generate(self,
    method _generate (line 81) | def _generate(self, input: PromptType, max_out_len: int = 512) -> str:

FILE: opencompass/models/vllm.py
  class VLLM (line 17) | class VLLM(BaseModel):
    method __init__ (line 20) | def __init__(
    method _load_model (line 49) | def _load_model(self,
    method generate (line 64) | def generate(self,
    method get_ppl (line 119) | def get_ppl(self,
    method get_loglikelihood (line 144) | def get_loglikelihood(self, inputs: List[str],
    method get_token_len (line 151) | def get_token_len(self,

FILE: opencompass/models/vllm_with_tf_above_v4_33.py
  class VLLMwithChatTemplate (line 21) | class VLLMwithChatTemplate(BaseModel):
    method __init__ (line 28) | def __init__(
    method _load_model (line 81) | def _load_model(self, path: str, added_model_kwargs: dict = dict()):
    method _get_potential_stop_words (line 93) | def _get_potential_stop_words(self, path: Optional[str]):
    method generate (line 113) | def generate(self, inputs: List[str], max_out_len: int, stopping_crite...
    method get_token_len (line 165) | def get_token_len(self, prompt: str) -> int:

FILE: opencompass/models/xunfei_api.py
  class XunFei (line 14) | class XunFei(BaseAPIModel):
    method __init__ (line 32) | def __init__(self,
    method get_url (line 70) | def get_url(self):
    method generate (line 101) | def generate(
    method _generate (line 124) | def _generate(
  class XunFeiSpark (line 228) | class XunFeiSpark(BaseAPIModel):
    method __init__ (line 246) | def __init__(
    method generate (line 274) | def generate(
    method _generate (line 282) | def _generate(

FILE: opencompass/models/yayi_api.py
  function generate_random_string (line 20) | def generate_random_string(length=16):
  function get_current_time (line 31) | def get_current_time(format='%Y-%m-%d %H:%M:%S'):
  function get_current_timestamp (line 42) | def get_current_timestamp():
  function encode_base64_string (line 48) | def encode_base64_string(s):
  function get_current_time_gmt_format (line 58) | def get_current_time_gmt_format():
  class Yayi (line 66) | class Yayi(BaseAPIModel):
    method __init__ (line 82) | def __init__(
    method generate_signature (line 112) | def generate_signature(self, method, accept, content_type, date, url_p...
    method generate_header (line 130) | def generate_header(self, content_type, accept, date, signature):
    method generate (line 149) | def generate(
    method _generate (line 172) | def _generate(

FILE: opencompass/models/yi_api.py
  class YiAPI (line 14) | class YiAPI(BaseAPIModel):
    method __init__ (line 32) | def __init__(
    method generate (line 56) | def generate(
    method _generate (line 79) | def _generate(

FILE: opencompass/models/zhipuai_api.py
  class ZhiPuAI (line 11) | class ZhiPuAI(BaseAPIModel):
    method __init__ (line 26) | def __init__(
    method generate (line 45) | def generate(
    method _generate (line 68) | def _generate(

FILE: opencompass/models/zhipuai_v2_api.py
  class ZhiPuV2AI (line 18) | class ZhiPuV2AI(BaseAPIModel):
    method __init__ (line 33) | def __init__(self,
    method generate (line 58) | def generate(
    method _generate (line 81) | def _generate(

FILE: opencompass/openicl/icl_dataset_reader.py
  class DatasetReader (line 17) | class DatasetReader:
    method __init__ (line 58) | def __init__(self,
    method generate_input_field_prompt (line 99) | def generate_input_field_prompt(self, entry: Dict) -> str:
    method generate_input_field_corpus (line 117) | def generate_input_field_corpus(self,
    method generate_output_field_prompt (line 140) | def generate_output_field_prompt(self, entry: Dict) -> str:
    method generate_output_field_corpus (line 158) | def generate_output_field_corpus(self,
    method generate_input_output_field_prompt (line 181) | def generate_input_output_field_prompt(self, entry: Dict) -> str:
    method _check_dataset_reader (line 200) | def _check_dataset_reader(obj) -> 'DatasetReader':
    method __len__ (line 206) | def __len__(self):
    method __getitem__ (line 209) | def __getitem__(self, idx):
    method __repr__ (line 212) | def __repr__(self):
  function load_partial_dataset (line 218) | def load_partial_dataset(
  class DatasetEncoder (line 247) | class DatasetEncoder(torch.utils.data.Dataset):
    method __init__ (line 249) | def __init__(self,
    method init_dataset (line 267) | def init_dataset(self):
    method __len__ (line 285) | def __len__(self):
    method __getitem__ (line 288) | def __getitem__(self, idx):

FILE: opencompass/openicl/icl_evaluator/code_evaluator.py
  class CodeEvaluator (line 18) | class CodeEvaluator(BaseEvaluator):
    method __init__ (line 26) | def __init__(self,
    method _extract_code (line 42) | def _extract_code(self, text: str) -> str:
    method _code_eval_service (line 56) | def _code_eval_service(
    method _process_completions (line 119) | def _process_completions(self, completion: str) -> list:
    method _evaluate (line 130) | def _evaluate(
    method _process_results (line 160) | def _process_results(self, outputs: List, prompts: List,
    method score (line 185) | def score(self, predictions: List, references: List,

FILE: opencompass/openicl/icl_evaluator/hf_metrics/accuracy.py
  class Accuracy (line 81) | class Accuracy(evaluate.Metric):
    method _info (line 82) | def _info(self):
    method _compute (line 101) | def _compute(self, predictions, references, normalize=True, sample_wei...

FILE: opencompass/openicl/icl_evaluator/hf_metrics/rouge.py
  class Tokenizer (line 83) | class Tokenizer:
    method __init__ (line 86) | def __init__(self, tokenizer_func):
    method tokenize (line 89) | def tokenize(self, text):
  class Rouge (line 94) | class Rouge(evaluate.Metric):
    method _info (line 95) | def _info(self):
    method _compute (line 121) | def _compute(

FILE: opencompass/openicl/icl_evaluator/hf_metrics/sacrebleu.py
  class Sacrebleu (line 106) | class Sacrebleu(evaluate.Metric):
    method _info (line 107) | def _info(self):
    method _compute (line 140) | def _compute(

FILE: opencompass/openicl/icl_evaluator/hf_metrics/squad.py
  class Squad (line 69) | class Squad(evaluate.Metric):
    method _info (line 70) | def _info(self):
    method _compute (line 93) | def _compute(self, predictions, references):

FILE: opencompass/openicl/icl_evaluator/icl_agent_evaluator.py
  function extract_answer (line 36) | def extract_answer(result: dict):
  class PassRateEvaluator (line 80) | class PassRateEvaluator(BaseEvaluator):
    method __init__ (line 84) | def __init__(self, fail_words=DEFAULT_FAIL_WORDS) -> None:
    method score (line 88) | def score(self, predictions: List, references: List = None) -> dict:
    method check_real_valid (line 98) | def check_real_valid(self, answer):
  class WinRateEvaluator (line 103) | class WinRateEvaluator(BaseEvaluator):
    method __init__ (line 121) | def __init__(self,
    method score (line 128) | def score(self, predictions: List, references: List, origin_prompt: List,
    method check_solve_query (line 164) | def check_solve_query(self, query: str, answer: str) -> bool:
    method select_best_final_answer (line 196) | def select_best_final_answer(self, query: str, answers: list) -> int:
    method compare_steps (line 236) | def compare_steps(self, steps_list: list) -> int:
    method _openai_function (line 255) | def _openai_function(self, msg: str, max_out_len: int, functions: dict,

FILE: opencompass/openicl/icl_evaluator/icl_aucroc_evaluator.py
  class AUCROCEvaluator (line 12) | class AUCROCEvaluator(BaseEvaluator):
    method __init__ (line 21) | def __init__(self) -> None:
    method score (line 24) | def score(self, predictions: List, references: List) -> dict:

FILE: opencompass/openicl/icl_evaluator/icl_base_evaluator.py
  function compute_pass_at_k (line 17) | def compute_pass_at_k(n, c, k):
  function _compute_g_pass_at_k (line 23) | def _compute_g_pass_at_k(n, c, k, m):
  function compute_g_pass_at_k (line 29) | def compute_g_pass_at_k(n, c, k, t):
  function compute_mg_pass_at_k (line 34) | def compute_mg_pass_at_k(n, c, k):
  class BaseEvaluator (line 45) | class BaseEvaluator:
    method __init__ (line 47) | def __init__(self, pred_postprocessor=None) -> None:
    method output_dir (line 52) | def output_dir(self):
    method dataset_replica_idx (line 57) | def dataset_replica_idx(self):
    method group (line 60) | def group(self, n: int, details: List[Dict[str, Any]],
    method reduce (line 73) | def reduce(self, details: List[Dict[str, Any]]) -> Dict[str, Any]:
    method pred_postprocess (line 95) | def pred_postprocess(self, predictions: List) -> Dict:
    method evaluate (line 104) | def evaluate(
    method score (line 235) | def score(self):
    method is_num_equal (line 239) | def is_num_equal(predictions, references):

FILE: opencompass/openicl/icl_evaluator/icl_bpc_evaluator.py
  class BPCEvaluator (line 11) | class BPCEvaluator(BaseEvaluator):
    method score (line 13) | def score(self, loss: List[float], total_chr_num: List[float]):

FILE: opencompass/openicl/icl_evaluator/icl_circular_evaluator.py
  class CircularEvaluator (line 9) | class CircularEvaluator(BaseEvaluator):
    method __init__ (line 12) | def __init__(self) -> None:
    method score (line 17) | def score(self, predictions, references):

FILE: opencompass/openicl/icl_evaluator/icl_em_evaluator.py
  class EMEvaluator (line 8) | class EMEvaluator(BaseEvaluator):
    method __init__ (line 11) | def __init__(self) -> None:
    method score (line 14) | def score(self, predictions, references):

FILE: opencompass/openicl/icl_evaluator/icl_hf_evaluator.py
  class HuggingfaceEvaluator (line 15) | class HuggingfaceEvaluator(BaseEvaluator):
    method __init__ (line 27) | def __init__(self,
    method _preprocess (line 35) | def _preprocess(self, predictions: List, references: List) -> dict:
    method _postprocess (line 50) | def _postprocess(self, scores: dict) -> dict:
    method score (line 61) | def score(self,
  class AccEvaluator (line 101) | class AccEvaluator(HuggingfaceEvaluator):
    method __init__ (line 104) | def __init__(self,
    method _preprocess (line 109) | def _preprocess(self,
    method _postprocess (line 137) | def _postprocess(self, scores: dict) -> dict:
  class AccContaminationEvaluator (line 151) | class AccContaminationEvaluator(AccEvaluator):
    method score (line 154) | def score(self, predictions: List, references: List,
  class RougeEvaluator (line 198) | class RougeEvaluator(HuggingfaceEvaluator):
    method __init__ (line 204) | def __init__(self,
    method _postprocess (line 208) | def _postprocess(self, scores: dict) -> dict:
  class BleuEvaluator (line 221) | class BleuEvaluator(HuggingfaceEvaluator):
    method __init__ (line 224) | def __init__(self,
  class BleuFloresEvaluator (line 230) | class BleuFloresEvaluator(HuggingfaceEvaluator):
    method __init__ (line 233) | def __init__(self) -> None:
    method _preprocess (line 236) | def _preprocess(self, predictions: List, references: List) -> dict:
  class MccEvaluator (line 245) | class MccEvaluator(AccEvaluator):
    method __init__ (line 248) | def __init__(self) -> None:
    method _postprocess (line 251) | def _postprocess(self, scores: dict) -> dict:
  class SquadEvaluator (line 265) | class SquadEvaluator(HuggingfaceEvaluator):
    method __init__ (line 268) | def __init__(self) -> None:
    method _preprocess (line 271) | def _preprocess(self, predictions: List, references: List) -> dict:
    method _postprocess (line 297) | def _postprocess(self, scores: dict) -> dict:
  class EDAccEvaluator (line 310) | class EDAccEvaluator(AccEvaluator):
    method __init__ (line 327) | def __init__(self) -> None:
    method _preprocess (line 332) | def _preprocess(self, predictions: List, references: List) -> dict:
  class AccwithDetailsEvaluator (line 365) | class AccwithDetailsEvaluator(BaseEvaluator):
    method score (line 367) | def score(self, predictions, references, origin_prompt) -> dict:

FILE: opencompass/openicl/icl_evaluator/icl_jieba_rouge_evaluator.py
  class JiebaRougeEvaluator (line 11) | class JiebaRougeEvaluator(BaseEvaluator):
    method __init__ (line 18) | def __init__(self) -> None:
    method score (line 21) | def score(self, predictions, references):

FILE: opencompass/openicl/icl_evaluator/icl_judge_evaluator.py
  class JudgeEvaluator (line 10) | class JudgeEvaluator(BaseEvaluator):
    method score (line 12) | def score(self, predictions, references):
  class RMBEvaluator (line 36) | class RMBEvaluator(BaseEvaluator):
    method calculate_pair_accuracy (line 38) | def calculate_pair_accuracy(self, data):
    method calculate_bon_accuracy (line 51) | def calculate_bon_accuracy(self, data):
    method score (line 69) | def score(self, predictions, references):
  class Judgerbenchv2Evaluator (line 255) | class Judgerbenchv2Evaluator(BaseEvaluator):
    method get_rank_dict (line 257) | def get_rank_dict(self, score_dict):
    method extract_winner (line 264) | def extract_winner(self, s, lan):
    method score (line 272) | def score(self, predictions, references):

FILE: opencompass/openicl/icl_evaluator/icl_korbench_evaluator.py
  function read_json_or_jsonl (line 11) | def read_json_or_jsonl(data_path, split='', mapping_key=None):
  function read_json_or_jsonl_with_idx (line 37) | def read_json_or_jsonl_with_idx(data_path, split='', idx=None):
  class korbenchEvaluator (line 63) | class korbenchEvaluator(BaseEvaluator):
    method __init__ (line 71) | def __init__(self, question_type, mode):
    method score (line 101) | def score(self, predictions, references):
    method evaluate_responses (line 132) | def evaluate_responses(self, data, question_type, mode):
    method extract_text_from_brackets (line 166) | def extract_text_from_brackets(self, text, clean_level='basic'):
    method clean_latex (line 199) | def clean_latex(self, latex_expr):
    method evaluate_response_vs_answer (line 216) | def evaluate_response_vs_answer(self, response, answer, question_type,
    method rule5_normalize_content (line 246) | def rule5_normalize_content(self, content):

FILE: opencompass/openicl/icl_evaluator/icl_misc_evaluator.py
  class AveragePPLEvaluator (line 7) | class AveragePPLEvaluator(BaseEvaluator):
    method score (line 9) | def score(self, ppl):
  class AverageMinKEvaluator (line 15) | class AverageMinKEvaluator(BaseEvaluator):
    method score (line 17) | def score(self, mink):
  class AverageInferencePPLEvaluator (line 23) | class AverageInferencePPLEvaluator(BaseEvaluator):
    method score (line 25) | def score(self, ppl, token_len):

FILE: opencompass/openicl/icl_evaluator/icl_plugin_evaluator.py
  class TEvalEvaluator (line 10) | class TEvalEvaluator(BaseEvaluator):
    method __init__ (line 26) | def __init__(self, subset) -> None:
    method score (line 80) | def score(self, predictions, references):

FILE: opencompass/openicl/icl_evaluator/icl_toxic_evaluator.py
  class PerspectiveAPIClient (line 19) | class PerspectiveAPIClient:
    method __init__ (line 31) | def __init__(self, key: str, batch_size: int, max_length: int = 20480):
    method _initialize (line 47) | def _initialize(self) -> None:
    method create_request_body (line 60) | def create_request_body(text: str) -> dict:
    method extract_toxicity_attributes (line 79) | def extract_toxicity_attributes(self, response: dict) -> dict:
    method get_toxicity_scores (line 98) | def get_toxicity_scores(self, predictions: List) -> dict:
  class ToxicEvaluator (line 153) | class ToxicEvaluator(BaseEvaluator):
    method __init__ (line 166) | def __init__(self,
    method get_scores (line 174) | def get_scores(self, predictions: List) -> dict:
    method get_metrics (line 185) | def get_metrics(self, scores: dict) -> dict:
    method score (line 216) | def score(self, predictions: List, references: List) -> dict:

FILE: opencompass/openicl/icl_evaluator/lm_evaluator.py
  function extract_dicts (line 18) | def extract_dicts(data):
  function order_preds_and_record_references (line 29) | def order_preds_and_record_references(
  function count_chinese_characters (line 92) | def count_chinese_characters(text):
  function count_english_words (line 97) | def count_english_words(text):
  class LMEvaluator (line 102) | class LMEvaluator:
    method __init__ (line 122) | def __init__(
    method score (line 163) | def score(
    method postprocess (line 359) | def postprocess(self, output: Dict) -> Dict:

FILE: opencompass/openicl/icl_evaluator/pi_llm_evaluator.py
  class PILLMEvaluator (line 12) | class PILLMEvaluator(BaseEvaluator):
    method __init__ (line 28) | def __init__(self, log_base: float = 1.5) -> None:
    method score (line 32) | def score(self,
    method compute_pi_auc_score (line 70) | def compute_pi_auc_score(self, results, log_base=1.5):
    method extract_pieces_response_to_dict (line 125) | def extract_pieces_response_to_dict(self,
    method _extract_verbal_matches (line 159) | def _extract_verbal_matches(self,
    method _extract_colon_matches (line 191) | def _extract_colon_matches(self, model_output: str):
    method grade_pi_response (line 207) | def grade_pi_response(self, response, answer_formatted):

FILE: opencompass/openicl/icl_inferencer/icl_agent_inferencer.py
  class AgentInferencerOutputHandler (line 16) | class AgentInferencerOutputHandler:
    method __init__ (line 18) | def __init__(self) -> None:
    method write_to_json (line 21) | def write_to_json(self, save_dir: str, filename: str):
    method save_results (line 25) | def save_results(self,
    method save_multiround_results (line 41) | def save_multiround_results(self,
  function model_adapter (line 60) | def model_adapter(model):
  class AgentInferencer (line 77) | class AgentInferencer(ChatInferencer):
    method __init__ (line 80) | def __init__(self, model, **kwargs) -> None:
    method infer_last (line 85) | def infer_last(self, chat: List[dict], index: int, output_handler):
    method infer_every (line 102) | def infer_every(self, chat: List[dict], index: int, output_handler):
    method infer_every_with_gt (line 121) | def infer_every_with_gt(self, chat: List[dict], index: int,

FILE: opencompass/openicl/icl_inferencer/icl_attack_inferencer.py
  class AttackInferencer (line 24) | class AttackInferencer(BaseInferencer):
    method __init__ (line 50) | def __init__(
    method predict (line 85) | def predict(self, adv_prompt) -> List:
    method get_generation_prompt_list_from_retriever_indices (line 181) | def get_generation_prompt_list_from_retriever_indices(

FILE: opencompass/openicl/icl_inferencer/icl_base_inferencer.py
  class BaseInferencer (line 18) | class BaseInferencer:
    method __init__ (line 34) | def __init__(
    method inference (line 59) | def inference(self,
    method get_dataloader (line 91) | def get_dataloader(datalist: List[List], batch_size: int) -> DataLoader:
  function dump_results_dict (line 99) | def dump_results_dict(results_dict, filename):
  class GenInferencerOutputHandler (line 104) | class GenInferencerOutputHandler:
    method __init__ (line 110) | def __init__(self) -> None:
    method write_to_jsonl (line 114) | def write_to_jsonl(self, save_dir: str, filename: str):
    method restore_from_jsonl (line 130) | def restore_from_jsonl(self, save_dir: str, filename: str) -> dict:
    method write_to_json (line 152) | def write_to_json(self, save_dir: str, filename: str):
    method save_results (line 156) | def save_results(self,
  class ChatOutputHandler (line 175) | class ChatOutputHandler:
    method __init__ (line 177) | def __init__(self) -> None:
    method write_to_jsonl (line 181) | def write_to_jsonl(self, save_dir: str, filename: str):
    method restore_from_jsonl (line 197) | def restore_from_jsonl(self, save_dir: str, filename: str) -> dict:
    method write_to_json (line 219) | def write_to_json(self, save_dir: str, filename: str):
    method save_results (line 223) | def save_results(self,
    method save_multiround_results (line 237) | def save_multiround_results(self,
  class PPLInferencerOutputHandler (line 253) | class PPLInferencerOutputHandler:
    method __init__ (line 256) | def __init__(self) -> None:
    method write_to_json (line 259) | def write_to_json(self, save_dir: str, filename: str):
    method save_ice (line 263) | def save_ice(self, ice):
    method save_predictions (line 269) | def save_predictions(self, predictions):
    method save_prompt_and_ppl (line 275) | def save_prompt_and_ppl(self, label, input, prompt, ppl, idx):
    method save_golds (line 287) | def save_golds(self, golds):
  class CLPInferencerOutputHandler (line 294) | class CLPInferencerOutputHandler:
    method __init__ (line 297) | def __init__(self) -> None:
    method write_to_json (line 300) | def write_to_json(self, save_dir: str, filename: str):
    method save_ice (line 304) | def save_ice(self, ice):
    method save_prompt_and_condprob (line 310) | def save_prompt_and_condprob(self,

FILE: opencompass/openicl/icl_inferencer/icl_chat_inferencer.py
  function promptlist_to_openai (line 24) | def promptlist_to_openai(prompt: Union[str, PromptList]):
  class LMTemplateParser (line 43) | class LMTemplateParser:
    method __init__ (line 46) | def __init__(self, meta_template: Optional[dict] = None):
    method parse_template (line 62) | def parse_template(self, chat: List[dict], mode='gen') -> str:
  class APITemplateParser (line 85) | class APITemplateParser:
    method __init__ (line 88) | def __init__(self, meta_template: Optional[dict] = None):
    method parse_template (line 110) | def parse_template(self, chat: List[dict], mode='gen') -> str:
  class ChatInferencer (line 127) | class ChatInferencer(BaseInferencer):
    method __init__ (line 130) | def __init__(
    method _set_meta_template (line 156) | def _set_meta_template(self, model):
    method inference (line 163) | def inference(self,
    method get_chat_list (line 228) | def get_chat_list(self,
    method infer_last (line 284) | def infer_last(self, chat: List[dict], index: int, output_handler):
    method infer_every (line 299) | def infer_every(self, chat: List[dict], index: int, output_handler):
    method infer_every_with_gt (line 337) | def infer_every_with_gt(self, chat: List[dict], index: int,

FILE: opencompass/openicl/icl_inferencer/icl_chat_inferencer_parallel.py
  class ParallelChatInferencer (line 19) | class ParallelChatInferencer(ChatInferencer):
    method __init__ (line 22) | def __init__(
    method _resolve_max_workers (line 44) | def _resolve_max_workers(self) -> int:
    method _progress_update (line 53) | def _progress_update(self, count: int = 1) -> None:
    method inference (line 57) | def inference(self,

FILE: opencompass/openicl/icl_inferencer/icl_chatml_inferencer.py
  class ChatMLInferencer (line 25) | class ChatMLInferencer(BaseInferencer):
    method __init__ (line 27) | def __init__(
    method _get_prompt_list_and_gold_ans (line 59) | def _get_prompt_list_and_gold_ans(
    method inference (line 111) | def inference(self,
    method get_generation_prompt_list_from_retriever_indices (line 219) | def get_generation_prompt_list_from_retriever_indices(

FILE: opencompass/openicl/icl_inferencer/icl_chatml_inferencer_parallel.py
  class ParallelChatMLInferencer (line 24) | class ParallelChatMLInferencer(ChatMLInferencer):
    method __init__ (line 27) | def __init__(
    method _resolve_max_workers (line 56) | def _resolve_max_workers(self) -> int:
    method _progress_update (line 65) | def _progress_update(self, count: int = 1) -> None:
    method inference (line 69) | def inference(self,

FILE: opencompass/openicl/icl_inferencer/icl_clp_inferencer.py
  class CLPInferencer (line 22) | class CLPInferencer(BaseInferencer):
    method __init__ (line 50) | def __init__(
    method inference (line 72) | def inference(self,
    method _get_cond_prob (line 243) | def _get_cond_prob(self, input_texts: List[str], target_pos: List[int],

FILE: opencompass/openicl/icl_inferencer/icl_gen_inferencer.py
  class GenInferencer (line 28) | class GenInferencer(BaseInferencer):
    method __init__ (line 51) | def __init__(
    method inference (line 86) | def inference(self,
    method get_generation_prompt_list_from_retriever_indices (line 261) | def get_generation_prompt_list_from_retriever_indices(
  class GLMChoiceInferencer (line 298) | class GLMChoiceInferencer(GenInferencer):
    method __init__ (line 300) | def __init__(self, *args, choices=['A', 'B', 'C', 'D'], **kwargs):
    method inference (line 304) | def inference(self,

FILE: opencompass/openicl/icl_inferencer/icl_gen_inferencer_parallel.py
  class ParallelGenInferencer (line 25) | class ParallelGenInferencer(GenInferencer):
    method __init__ (line 28) | def __init__(
    method _resolve_max_workers (line 57) | def _resolve_max_workers(self) -> int:
    method _progress_update (line 66) | def _progress_update(self, count: int = 1) -> None:
    method inference (line 70) | def inference(self,

FILE: opencompass/openicl/icl_inferencer/icl_inference_ppl_only_inferencer.py
  class InferencePPLOnlyInferencer (line 22) | class InferencePPLOnlyInferencer(BaseInferencer):
    method __init__ (line 39) | def __init__(
    method inference (line 59) | def inference(self,
    method get_generation_prompt_list_from_retriever_indices (line 151) | def get_generation_prompt_list_from_retriever_indices(
    method get_generation_prompt_list_and_label (line 185) | def get_generation_prompt_list_and_label(
  class InferencePPLOnlyInferencerOutputHandler (line 222) | class InferencePPLOnlyInferencerOutputHandler:
    method __init__ (line 227) | def __init__(self) -> None:
    method write_to_json (line 230) | def write_to_json(self, save_dir: str, filename: str):
    method save_results (line 234) | def save_results(self, origin_prompt, ppl, token_len, idx):

FILE: opencompass/openicl/icl_inferencer/icl_ll_inferencer.py
  class LLInferencer (line 23) | class LLInferencer(BaseInferencer):
    method __init__ (line 38) | def __init__(
    method inference (line 58) | def inference(self,
  class LLInferencerOutputHandler (line 159) | class LLInferencerOutputHandler:
    method __init__ (line 162) | def __init__(self) -> None:
    method write_to_json (line 165) | def write_to_json(self, save_dir: str, filename: str):
    method save_ice (line 169) | def save_ice(self, ice):
    method save_predictions (line 175) | def save_predictions(self, predictions):
    method save_prompt_and_loglikelihood (line 181) | def save_prompt_and_loglikelihood(self, label, input, prompt,
    method save_golds (line 193) | def save_golds(self, golds):

FILE: opencompass/openicl/icl_inferencer/icl_mink_percent_inferencer.py
  class MinKPercentInferencer (line 22) | class MinKPercentInferencer(BaseInferencer):
    method __init__ (line 38) | def __init__(
    method inference (line 58) | def inference(self,
    method get_generation_prompt_list_from_retriever_indices (line 140) | def get_generation_prompt_list_from_retriever_indices(
  class PPLOnlyInferencerOutputHandler (line 173) | class PPLOnlyInferencerOutputHandler:
    method __init__ (line 178) | def __init__(self) -> None:
    method write_to_json (line 181) | def write_to_json(self, save_dir: str, filename: str):
    method save_results (line 185) | def save_results(self, origin_prompt, mink, idx):

FILE: opencompass/openicl/icl_inferencer/icl_ppl_inferencer.py
  class PPLInferencer (line 23) | class PPLInferencer(BaseInferencer):
    method __init__ (line 38) | def __init__(
    method inference (line 58) | def inference(self,

FILE: opencompass/openicl/icl_inferencer/icl_ppl_only_inferencer.py
  class PPLOnlyInferencer (line 22) | class PPLOnlyInferencer(BaseInferencer):
    method __init__ (line 38) | def __init__(
    method inference (line 58) | def inference(self,
    method get_generation_prompt_list_from_retriever_indices (line 139) | def get_generation_prompt_list_from_retriever_indices(
  class PPLOnlyInferencerOutputHandler (line 172) | class PPLOnlyInferencerOutputHandler:
    method __init__ (line 177) | def __init__(self) -> None:
    method write_to_json (line 180) | def write_to_json(self, save_dir: str, filename: str):
    method save_results (line 184) | def save_results(self, origin_prompt, ppl, idx):

FILE: opencompass/openicl/icl_inferencer/icl_sc_inferencer.py
  class SCInferencer (line 21) | class SCInferencer(BaseInferencer):
    method __init__ (line 45) | def __init__(
    method inference (line 77) | def inference(self,
    method get_generation_prompt_list_from_retriever_indices (line 173) | def get_generation_prompt_list_from_retriever_indices(

FILE: opencompass/openicl/icl_inferencer/icl_sw_ce_loss_inferencer.py
  class SWCELossInferencer (line 26) | class SWCELossInferencer(BaseInferencer):
    method __init__ (line 48) | def __init__(
    method inference (line 73) | def inference(self,
    method get_encoding_from_retriever_indices (line 191) | def get_encoding_from_retriever_indices(
    method _get_cross_entropy (line 235) | def _get_cross_entropy(self,
  class SlidingWindowEvalDataset (line 265) | class SlidingWindowEvalDataset(Dataset):
    method __init__ (line 267) | def __init__(self,
    method _prepare (line 289) | def _prepare(self):
    method __len__ (line 307) | def __len__(self):
    method __getitem__ (line 310) | def __getitem__(self, item):
    method total_chr_num (line 330) | def total_chr_num(self):
  class SWCELossInferencerOutputHandler (line 334) | class SWCELossInferencerOutputHandler:
    method __init__ (line 339) | def __init__(self) -> None:
    method write_to_json (line 342) | def write_to_json(self, save_dir: str, filename: str):
    method save_results (line 346) | def save_results(self, loss: float, total_chr_num: int,

FILE: opencompass/openicl/icl_inferencer/icl_tot_inferencer.py
  class ToTInferencer (line 25) | class ToTInferencer(GenInferencer):
    method __init__ (line 68) | def __init__(
    method get_value (line 113) | def get_value(self,
    method get_values (line 143) | def get_values(self,
    method get_votes (line 174) | def get_votes(self, x: str, ys: List[str],
    method get_proposals (line 196) | def get_proposals(self, x: str, y: str) -> List[str]:
    method get_samples (line 215) | def get_samples(self, x: str, y: str, n_generate_sample: int,
    method tot_solve (line 242) | def tot_solve(self, x: str) -> str:
    method inference (line 304) | def inference(self,

FILE: opencompass/openicl/icl_prompt_template.py
  class PromptTemplate (line 13) | class PromptTemplate:
    method __init__ (line 32) | def __init__(
    method _check_template_legacy (line 46) | def _check_template_legacy(self):
    method generate_ice_item (line 71) | def generate_ice_item(self, entry: Dict, label: Hashable) -> PromptType:
    method generate_label_prompt_item (line 105) | def generate_label_prompt_item(self,
    method generate_item (line 146) | def generate_item(
    method _check_prompt_template (line 199) | def _check_prompt_template(obj) -> 'PromptTemplate':
    method __repr__ (line 205) | def __repr__(self):
    method _encode_template (line 209) | def _encode_template(self, prompt_template: Union[List[Union[str, Dict]],

FILE: opencompass/openicl/icl_raw_prompt_template.py
  class RawPromptTemplate (line 10) | class RawPromptTemplate:
    method __init__ (line 25) | def __init__(
    method _validate_messages (line 39) | def _validate_messages(self, messages: List[Dict]) -> None:
    method generate_item (line 57) | def generate_item(
    method generate_ice_item (line 84) | def generate_ice_item(self,
    method generate_label_prompt_item (line 94) | def generate_label_prompt_item(
    method __repr__ (line 104) | def __repr__(self):

FILE: opencompass/openicl/icl_retriever/icl_base_retriever.py
  class BaseRetriever (line 11) | class BaseRetriever:
    method __init__ (line 30) | def __init__(self,
    method retrieve (line 44) | def retrieve(self) -> List[List[int]]:
    method get_labels (line 47) | def get_labels(
    method generate_ice (line 73) | def generate_ice(self,
    method generate_label_prompt (line 114) | def generate_label_prompt(self,
    method generate_prompt_for_generate_task (line 157) | def generate_prompt_for_generate_task(
    method generate_prompt_and_label_for_generate_task (line 210) | def generate_prompt_and_label_for_generate_task(
    method generate_prompt_for_adv_generate_task (line 263) | def generate_prompt_for_adv_generate_task(

FILE: opencompass/openicl/icl_retriever/icl_bm25_retriever.py
  class BM25Retriever (line 18) | class BM25Retriever(BaseRetriever):
    method __init__ (line 47) | def __init__(self,
    method retrieve (line 63) | def retrieve(self) -> List[List]:

FILE: opencompass/openicl/icl_retriever/icl_dpp_retriever.py
  class DPPRetriever (line 15) | class DPPRetriever(TopkRetriever):
    method __init__ (line 25) | def __init__(self,
    method dpp_search (line 44) | def dpp_search(self):
    method retrieve (line 74) | def retrieve(self):
    method get_kernel (line 77) | def get_kernel(self, embed, candidates):
  function fast_map_dpp (line 103) | def fast_map_dpp(kernel_matrix, max_length):

FILE: opencompass/openicl/icl_retriever/icl_fix_k_retriever.py
  class FixKRetriever (line 15) | class FixKRetriever(BaseRetriever):
    method __init__ (line 34) | def __init__(self,
    method retrieve (line 43) | def retrieve(self):

FILE: opencompass/openicl/icl_retriever/icl_mdl_retriever.py
  class MDLRetriever (line 19) | class MDLRetriever(TopkRetriever):
    method __init__ (line 57) | def __init__(self,
    method topk_search (line 87) | def topk_search(self):
    method retrieve (line 140) | def retrieve(self):
    method cal_ce (line 145) | def cal_ce(self, input_texts: List[str], mask_length=None):
  function entropy (line 184) | def entropy(probs: np.array, label_dim: int = 0, mask=None):

FILE: opencompass/openicl/icl_retriever/icl_random_retriever.py
  class RandomRetriever (line 14) | class RandomRetriever(BaseRetriever):
    method __init__ (line 22) | def __init__(self,
    method retrieve (line 31) | def retrieve(self):

FILE: opencompass/openicl/icl_retriever/icl_sliding_k_retriever.py
  class SlidingWindowRetriever (line 15) | class SlidingWindowRetriever(BaseRetriever):
    method __init__ (line 38) | def __init__(self,
    method retrieve (line 47) | def retrieve(self):

FILE: opencompass/openicl/icl_retriever/icl_topk_retriever.py
  class TopkRetriever (line 23) | class TopkRetriever(BaseRetriever):
    method __init__ (line 48) | def __init__(self,
    method create_index (line 86) | def create_index(self):
    method knn_search (line 108) | def knn_search(self, ice_num):
    method forward (line 121) | def forward(self, dataloader, process_bar=False, information=''):
    method retrieve (line 142) | def retrieve(self):
  class ListWrapper (line 147) | class ListWrapper:
    method __init__ (line 149) | def __init__(self, data: List[Any]):
    method to (line 152) | def to(self, device):
  function ignore_pad_dict (line 156) | def ignore_pad_dict(features):
  class DataCollatorWithPaddingAndCuda (line 165) | class DataCollatorWithPaddingAndCuda:
    method __call__ (line 172) | def __call__(

FILE: opencompass/openicl/icl_retriever/icl_votek_retriever.py
  class VotekRetriever (line 15) | class VotekRetriever(TopkRetriever):
    method __init__ (line 22) | def __init__(self,
    method votek_select (line 37) | def votek_select(self,
    method vote_k_search (line 91) | def vote_k_search(self):
    method retrieve (line 98) | def retrieve(self):

FILE: opencompass/openicl/icl_retriever/icl_zero_retriever.py
  class ZeroRetriever (line 11) | class ZeroRetriever(BaseRetriever):
    method __init__ (line 22) | def __init__(self, dataset, ice_eos_token: Optional[str] = '') -> None:
    method retrieve (line 25) | def retrieve(self, id_list: List[int] = None) -> List[List]:

FILE: opencompass/openicl/utils/logging.py
  function get_logger (line 10) | def get_logger(name, level=LOG_LEVEL, log_file=None, file_mode='w'):

FILE: opencompass/partitioners/base.py
  class BasePartitioner (line 13) | class BasePartitioner:
    method __init__ (line 27) | def __init__(self, out_dir: str, keep_keys: Optional[List[str]] = None):
    method __call__ (line 40) | def __call__(self, cfg: ConfigDict) -> List[Dict]:
    method parse_model_dataset_args (line 104) | def parse_model_dataset_args(self, cfg: ConfigDict):
    method partition (line 158) | def partition(self,

FILE: opencompass/partitioners/naive.py
  class NaivePartitioner (line 13) | class NaivePartitioner(BasePartitioner):
    method __init__ (line 24) | def __init__(self,
    method partition (line 31) | def partition(self,

FILE: opencompass/partitioners/num_worker.py
  class NumWorkerPartitioner (line 17) | class NumWorkerPartitioner(BasePartitioner):
    method __init__ (line 32) | def __init__(self,
    method partition (line 55) | def partition(self,
    method dataset_size (line 110) | def dataset_size(self):
    method split_dataset (line 118) | def split_dataset(self, dataset_cfg: ConfigDict) -> List[ConfigDict]:
    method get_size (line 134) | def get_size(self, dataset: ConfigDict) -> int:

FILE: opencompass/partitioners/size.py
  class SizePartitioner (line 18) | class SizePartitioner(BasePartitioner):
    method __init__ (line 37) | def __init__(self,
    method partition (line 53) | def partition(self,
    method dataset_size (line 139) | def dataset_size(self):
    method split_dataset (line 147) | def split_dataset(self, dataset_cfg: ConfigDict) -> List[ConfigDict]:
    method get_factor (line 164) | def get_factor(self, dataset: ConfigDict) -> int:
    method get_cost (line 188) | def get_cost(self,

FILE: opencompass/partitioners/sub_naive.py
  function remove_duplicate_pairs (line 16) | def remove_duplicate_pairs(model_combinations):
  function replicate_tasks_with_judge_models (line 31) | def replicate_tasks_with_judge_models(tasks, judge_models, meta_judge_mo...
  function remove_already_tasks (line 52) | def remove_already_tasks(tasks, work_dir, meta_judge_model):
  function get_model_combinations (line 107) | def get_model_combinations(
  class SubjectiveNaivePartitioner (line 127) | class SubjectiveNaivePartitioner(NaivePartitioner):
    method __init__ (line 137) | def __init__(
    method partition (line 157) | def partition(self,

FILE: opencompass/partitioners/sub_num_worker.py
  class SubjectiveNumWorkerPartitioner (line 20) | class SubjectiveNumWorkerPartitioner(SubjectiveNaivePartitioner):
    method __init__ (line 32) | def __init__(self,
    method partition (line 68) | def partition(self,
    method dataset_size (line 164) | def dataset_size(self):
    method split_dataset (line 172) | def split_dataset(self, dataset_cfg: ConfigDict) -> List[ConfigDict]:
    method get_size (line 188) | def get_size(self, dataset: ConfigDict) -> int:

FILE: opencompass/partitioners/sub_size.py
  class SubjectiveSizePartitioner (line 21) | class SubjectiveSizePartitioner(SubjectiveNaivePartitioner):
    method __init__ (line 40) | def __init__(
    method partition (line 73) | def partition(self,
    method dataset_size (line 199) | def dataset_size(self):
    method split_dataset (line 207) | def split_dataset(self, dataset_cfg: ConfigDict) -> List[ConfigDict]:
    method get_factor (line 224) | def get_factor(self, dataset: ConfigDict) -> int:
    method get_cost (line 248) | def get_cost(self,

FILE: opencompass/registry.py
  class Registry (line 7) | class Registry(OriginalRegistry):
    method register_module (line 10) | def register_module(
  function build_from_cfg (line 53) | def build_from_cfg(cfg):

FILE: opencompass/runners/base.py
  class BaseRunner (line 10) | class BaseRunner:
    method __init__ (line 20) | def __init__(self,
    method __call__ (line 31) | def __call__(self, tasks: List[Dict[str, Any]]):
    method launch (line 43) | def launch(self, tasks: List[Dict[str, Any]]) -> List[Tuple[str, int]]:
    method summarize (line 54) | def summarize(self, status: List[Tuple[str, int]]) -> None:

FILE: opencompass/runners/dlc.py
  class DLCRunner (line 24) | class DLCRunner(BaseRunner):
    method __init__ (line 38) | def __init__(
    method launch (line 67) | def launch(self, tasks: List[Dict[str, Any]]) -> List[Tuple[str, int]]:
    method _launch (line 89) | def _launch(self, cfg: ConfigDict, random_sleep: Optional[bool] = None):
    method _job_failed (line 356) | def _job_failed(self, return_code: int, output_paths: List[str]) -> bool:

FILE: opencompass/runners/local.py
  function get_command_template (line 24) | def get_command_template(gpu_ids: List[int]) -> str:
  class LocalRunner (line 40) | class LocalRunner(BaseRunner):
    method __init__ (line 53) | def __init__(self,
    method launch (line 69) | def launch(self, tasks: List[Dict[str, Any]]) -> List[Tuple[str, int]]:
    method _launch (line 198) | def _launch(self, task, gpu_ids, index):

FILE: opencompass/runners/local_api.py
  function monkey_run (line 26) | def monkey_run(self, tokens: SyncManager.Semaphore):
  function redirect_std_to_file (line 59) | def redirect_std_to_file(filename: str):
  function reset_std (line 77) | def reset_std():
  function launch (line 95) | def launch(task: BaseTask, tokens: SyncManager.Semaphore):
  function submit (line 139) | def submit(task, type, tokens):
  class LocalAPIRunner (line 149) | class LocalAPIRunner(BaseRunner):
    method __init__ (line 166) | def __init__(self,
    method launch (line 180) | def launch(self, tasks: List[Dict[str, Any]]) -> List[Tuple[str, int]]:

FILE: opencompass/runners/rjob.py
  class RJOBRunner (line 21) | class RJOBRunner(BaseRunner):
    method __init__ (line 36) | def __init__(
    method launch (line 54) | def launch(self, tasks: List[Dict[str, Any]]) -> List[Tuple[str, int]]:
    method _run_task (line 67) | def _run_task(self, task_name, log_path, poll_interval=60):
    method _launch (line 159) | def _launch(self, cfg: ConfigDict, random_sleep: Optional[bool] = None):
    method _job_failed (line 287) | def _job_failed(self, return_code: int, output_paths: List[str]) -> bool:

FILE: opencompass/runners/slurm.py
  class SlurmRunner (line 20) | class SlurmRunner(BaseRunner):
    method __init__ (line 38) | def __init__(self,
    method launch (line 59) | def launch(self, tasks: List[Dict[str, Any]]) -> List[Tuple[str, int]]:
    method _launch (line 79) | def _launch(self, cfg: ConfigDict, random_sleep: bool = True):
    method _job_failed (line 160) | def _job_failed(self, return_code: int, output_paths: List[str]) -> bool:

FILE: opencompass/runners/slurm_sequential.py
  class SlurmSequentialRunner (line 22) | class SlurmSequentialRunner(BaseRunner):
    method __init__ (line 52) | def __init__(self,
    method launch (line 83) | def launch(self, tasks: List[Dict[str, Any]]) -> List[Tuple[str, int]]:
    method _launch_wo_debug (line 89) | def _launch_wo_debug(self,
    method _launch (line 166) | def _launch(self, cfg: ConfigDict, child_conn: Pipe = None):
    method _job_failed (line 271) | def _job_failed(self, return_code: int, output_paths: List[str]) -> bool:

FILE: opencompass/runners/volc.py
  class VOLCRunner (line 22) | class VOLCRunner(BaseRunner):
    method __init__ (line 41) | def __init__(self,
    method launch (line 61) | def launch(self, tasks: List[Dict[str, Any]]) -> List[Tuple[str, int]]:
    method _launch (line 81) | def _launch(self, task_cfg: ConfigDict, random_sleep: bool = True):
    method _run_task (line 207) | def _run_task(self, cmd, log_path, poll_interval):
    method _job_failed (line 250) | def _job_failed(self, task_status: str, output_paths: List[str]) -> bool:
    method _choose_flavor (line 254) | def _choose_flavor(self, num_gpus):

FILE: opencompass/summarizers/circular.py
  class CircularSummarizer (line 11) | class CircularSummarizer(DefaultSummarizer):
    method __init__ (line 13) | def __init__(self,
    method _format_table (line 22) | def _format_table(self, parsed_results, dataset_metrics,

FILE: opencompass/summarizers/default.py
  function model_abbr_from_cfg_used_in_summarizer (line 22) | def model_abbr_from_cfg_used_in_summarizer(model):
  class DefaultSummarizer (line 29) | class DefaultSummarizer:
    method __init__ (line 41) | def __init__(self, config: ConfigDict, dataset_abbrs: Optional[List[st...
    method _pick_up_results (line 67) | def _pick_up_results(self):
    method _calculate_group_metrics (line 137) | def _calculate_group_metrics(self, raw_results, parsed_results, datase...
    method _format_table (line 247) | def _format_table(self, parsed_results, dataset_metrics, dataset_eval_...
    method _format_raw_txt (line 299) | def _format_raw_txt(self, raw_results):
    method _format_md_table (line 316) | def _format_md_table(table):
    method _output_to_file (line 327) | def _output_to_file(self, output_path, time_str, table, raw_txts):
    method summarize (line 376) | def summarize(

FILE: opencompass/summarizers/default_subjective.py
  function model_abbr_from_cfg_used_in_summarizer (line 23) | def model_abbr_from_cfg_used_in_summarizer(model):
  class DefaultSubjectiveSummarizer (line 30) | class DefaultSubjectiveSummarizer:
    method __init__ (line 42) | def __init__(self, config: ConfigDict, dataset_abbrs: Optional[List[st...
    method _pick_up_results (line 73) | def _pick_up_results(self, judge_abbr):
    method _calculate_group_metrics (line 175) | def _calculate_group_metrics(self, raw_results, parsed_results, datase...
    method _format_table (line 272) | def _format_table(self, parsed_results, dataset_metrics, dataset_eval_...
    method _format_raw_txt (line 324) | def _format_raw_txt(self, raw_results):
    method _output_to_file (line 340) | def _output_to_file(self, output_path, time_str, table, raw_txts, judg...
    method summarize (line 377) | def summarize(

FILE: opencompass/summarizers/llm_compression.py
  class LLMCompressionSummarizer (line 16) | class LLMCompressionSummarizer(DefaultSummarizer):
    method __init__ (line 18) | def __init__(self,
    method _format_table (line 27) | def _format_table(self, parsed_results, dataset_metrics,
    method _format_table_pivot (line 91) | def _format_table_pivot(self, table: List[List], decimals: int = 4):
    method _output_df_to_file (line 137) | def _output_df_to_file(self, output_path: str, timestamp: str,
    method summarize (line 158) | def summarize(

FILE: opencompass/summarizers/multi_faceted.py
  class MultiFacetedSummarizer (line 14) | class MultiFacetedSummarizer(DefaultSummarizer):
    method __init__ (line 16) | def __init__(self, config: ConfigDict, dataset_abbrs_list: Optional[Di...
    method summarize (line 20) | def summarize(self, output_path: str = None, time_str: str = datetime....

FILE: opencompass/summarizers/multi_model.py
  function bold (line 24) | def bold(text):
  function green_bold (line 28) | def green_bold(text):
  function format_float (line 32) | def format_float(v):
  function to_float (line 36) | def to_float(text: str):
  function is_section_row (line 43) | def is_section_row(row: List[str]) -> bool:
  function average_rows (line 48) | def average_rows(name, rows: List[List[str]]) -> List[str]:
  function create_section_row (line 63) | def create_section_row(row_i: int, row: List[str], table) -> List[str]:
  function create_win_row (line 75) | def create_win_row(rows: List[List[str]]) -> List[str]:
  function highlight (line 89) | def highlight(row: List[str], meta_col_count: int = META_COL_COUNT) -> L...
  class MultiModelSummarizer (line 98) | class MultiModelSummarizer:
    method __init__ (line 113) | def __init__(self, config: ConfigDict, dataset_abbrs: Optional[List[st...
    method load (line 125) | def load( self ):  # noqa
    method merge (line 274) | def merge(self, summarizer: 'MultiModelSummarizer'):
    method summarize (line 285) | def summarize(self):
    method show_group (line 331) | def show_group(self, group: str):

FILE: opencompass/summarizers/needlebench.py
  function calculate_elementwise_average (line 111) | def calculate_elementwise_average(model_name, merged_df, mean=False):
  function read_after_specific_line_except_last (line 137) | def read_after_specific_line_except_last(file_name, keyword, offset):
  function create_model_dataframe (line 150) | def create_model_dataframe(nested_dict, model_name, dataset_abbr, parall...
  function convert_to_k (line 175) | def convert_to_k(value):
  function parse_model_scores (line 181) | def parse_model_scores(text):
  function remove_empty_subfolders (line 198) | def remove_empty_subfolders(plot_path):
  function save_results_to_plots (line 206) | def save_results_to_plots(txt_results_save_path, mean=False):
  function visualize (line 274) | def visualize(df_raw, save_path: str,model_name: str ,dataset_type:str):
  function ensure_directory (line 386) | def ensure_directory(path):
  function get_dict_model_names (line 390) | def get_dict_model_names(nested_dict):
  function merge_dataframes (line 396) | def merge_dataframes(model_name, dataset_abbrs, parsed_data):
  class NeedleBenchSummarizer (line 417) | class NeedleBenchSummarizer(DefaultSummarizer):
    method _format_table (line 428) | def _format_table(self, parsed_results, dataset_metrics, dataset_eval_...
    method _format_raw_txt (line 491) | def _format_raw_txt(self, raw_results):
    method _output_to_file (line 507) | def _output_to_file(self, output_path, time_str, table, raw_txts):
    method summarize (line 540) | def summarize(
  class NeedleBenchSummarizerV2 (line 562) | class NeedleBenchSummarizerV2(NeedleBenchSummarizer):
    method summarize (line 576) | def summarize(
  class NeedleBenchATCSummarizer (line 598) | class NeedleBenchATCSummarizer(DefaultSummarizer):
    method _format_table (line 610) | def _format_table(self, parsed_results, dataset_metrics, dataset_eval_...
    method _read_and_sort_dataframe (line 675) | def _read_and_sort_dataframe(self, file_path):
    method _output_to_file (line 726) | def _output_to_file(self, output_path, time_str, table, raw_txts):
    method summarize (line 766) | def summarize(

FILE: opencompass/summarizers/subjective/alignmentbench.py
  function detect_mapping (line 40) | def detect_mapping(text):
  function extract_missing_rating (line 53) | def extract_missing_rating(text, search_type):
  function extract_rating_plus (line 71) | def extract_rating_plus(text):
  function extract_rating (line 89) | def extract_rating(text):
  function check_rating (line 103) | def check_rating(rating, all_dimensions):
  function post_process_alignbench_plus (line 115) | def post_process_alignbench_plus(judgement: str,
  function post_process_alignbench (line 159) | def post_process_alignbench(judgement: str,
  function get_dimension_results (line 203) | def get_dimension_results(judged_answers, references, fout, fout_flag, m...
  function get_capability_results (line 238) | def get_capability_results(judged_answers,
  class AlignmentBenchSummarizer (line 310) | class AlignmentBenchSummarizer:
    method __init__ (line 318) | def __init__(self, config: ConfigDict, judge_type='general') -> None:
    method summarize (line 339) | def summarize(self,

FILE: opencompass/summarizers/subjective/all_obj.py
  function post_process_allobj (line 18) | def post_process_allobj(judgement: str):
  function get_capability_results (line 35) | def get_capability_results(
  class AllObjSummarizer (line 63) | class AllObjSummarizer:
    method __init__ (line 71) | def __init__(self, config: ConfigDict, judge_type='single') -> None:
    method summarize (line 89) | def summarize(self,

FILE: opencompass/summarizers/subjective/alpacaeval.py
  function post_process_alpacav1 (line 21) | def post_process_alpacav1(completion: str):
  function post_process_alpacav2 (line 50) | def post_process_alpacav2(completion: str):
  class AlpacaSummarizer (line 74) | class AlpacaSummarizer:
    method __init__ (line 82) | def __init__(self, config: ConfigDict, judge_type='v2') -> None:
    method summarize (line 96) | def summarize(self,

FILE: opencompass/summarizers/subjective/arenahard.py
  function compute_mle_elo (line 31) | def compute_mle_elo(df, SCALE=400, BASE=10, INIT_RATING=1000):
  function get_bootstrap_result (line 64) | def get_bootstrap_result(battles, func_compute_elo, num_round):
  function preety_print_two_ratings (line 72) | def preety_print_two_ratings(ratings_1, ratings_2, column_names):
  function visualize_bootstrap_scores (line 82) | def visualize_bootstrap_scores(df, title):
  function predict_win_rate (line 98) | def predict_win_rate(elo_ratings, SCALE=400, BASE=10, INIT_RATING=1000):
  function model_abbr_from_cfg_used_in_summarizer (line 118) | def model_abbr_from_cfg_used_in_summarizer(model):
  function post_process_compass_arena (line 124) | def post_process_compass_arena(s):
  function get_win_rate_column (line 130) | def get_win_rate_column(df, column, baseline='gpt4-0314'):
  function load_model_preds (line 136) | def load_model_preds(filename):
  function get_battles_from_judgment (line 157) | def get_battles_from_judgment(dataset, subdir_path, post_process, WEIGHT...
  class ArenaHardSummarizer (line 220) | class ArenaHardSummarizer:
    method __init__ (line 228) | def __init__(self,
    method get_score (line 246) | def get_score(self, time_str):
    method summarize (line 330) | def summarize(

FILE: opencompass/summarizers/subjective/charm.py
  function post_process_charm_mem (line 22) | def post_process_charm_mem(judgement: str):
  function get_judgeanswer_and_reference_charm_mem (line 39) | def get_judgeanswer_and_reference_charm_mem(dataset, subdir_path,
  function get_accuracy (line 102) | def get_accuracy(judged_answers):
  class CharmMemSummarizer (line 113) | class CharmMemSummarizer:
    method __init__ (line 121) | def __init__(self, config: ConfigDict, judge_type='single') -> None:
    method summarize (line 138) | def summarize(self,

FILE: opencompass/summarizers/subjective/common_summarizer.py
  function model_abbr_from_cfg_used_in_summarizer (line 20) | def model_abbr_from_cfg_used_in_summarizer(model):
  function post_process_single_rate (line 26) | def post_process_single_rate(judgement: str):
  function get_capability_results (line 40) | def get_capability_results(
  class CommonSummarizer (line 79) | class CommonSummarizer(CompassArenaSummarizer):
    method __init__ (line 87) | def __init__(self, config: ConfigDict, judge_type='single_rate') -> None:
    method summarize (line 99) | def summarize(self, time_str: str = datetime.now().strftime('%Y%m%d_%H...

FILE: opencompass/summarizers/subjective/compass_arena.py
  function model_abbr_from_cfg_used_in_summarizer (line 20) | def model_abbr_from_cfg_used_in_summarizer(model):
  function post_process_compass_arena (line 26) | def post_process_compass_arena(s):
  function check_position_bias (line 33) | def check_position_bias(judged_answers, references, banned_choice=['C']):
  class CompassArenaSummarizer (line 58) | class CompassArenaSummarizer:
    method __init__ (line 66) | def __init__(self,
    method get_score (line 84) | def get_score(self, time_str):
    method summarize (line 161) | def summarize(

FILE: opencompass/summarizers/subjective/compass_arena_bradley_terry.py
  function get_matchups_models (line 41) | def get_matchups_models(df):
  function preprocess_for_elo (line 50) | def preprocess_for_elo(df):
  function preprocess_for_bt (line 63) | def preprocess_for_bt(df):
  function preprocess_for_style (line 87) | def preprocess_for_style(
  function fit_vectorized_elo (line 172) | def fit_vectorized_elo(
  function compute_elo (line 202) | def compute_elo(
  function compute_bootstrap_elo (line 224) | def compute_bootstrap_elo(
  function bt_loss_and_grad (line 242) | def bt_loss_and_grad(ratings, matchups, outcomes, weights, alpha=1.0):
  function fit_bt (line 260) | def fit_bt(matchups, outcomes, weights, n_models, alpha, tol=1e-6):
  function scale_and_offset (line 277) | def scale_and_offset(
  function compute_bt (line 298) | def compute_bt(
  function compute_bootstrap_bt (line 323) | def compute_bootstrap_bt(
  function contextual_bt_loss_and_grad (line 374) | def contextual_bt_loss_and_grad(
  function fit_contextual_bt (line 411) | def fit_contextual_bt(
  function compute_style_control (line 446) | def compute_style_control(
  function compute_bootstrap_style_control (line 516) | def compute_bootstrap_style_control(
  class CompassArenaBradleyTerrySummarizer (line 601) | class CompassArenaBradleyTerrySummarizer(DefaultSubjectiveSummarizer):
    method __init__ (line 620) | def __init__(
    method _pick_up_results (line 648) | def _pick_up_results(self, judge_abbr):
    method _calculate_ratings (line 745) | def _calculate_ratings(
    method _output_to_file (line 847) | def _output_to_file(
    method flip_dict_levels (line 931) | def flip_dict_levels(self, original_dict: Dict):
    method predict_win_rate (line 950) | def predict_win_rate(
    method summarize (line 999) | def summarize(

FILE: opencompass/summarizers/subjective/compassbench.py
  function post_process_wildbench_pair (line 24) | def post_process_wildbench_pair(judgement: str):
  class CompassBenchSummarizer (line 41) | class CompassBenchSummarizer:
    method __init__ (line 49) | def __init__(self, config: ConfigDict, check_pos_bias=False) -> None:
    method get_score (line 61) | def get_score(self, time_str):
    method summarize (line 137) | def summarize(

FILE: opencompass/summarizers/subjective/compassbench_v13.py
  function post_process_wildbench_pair (line 24) | def post_process_wildbench_pair(judgement: str):
  class CompassBenchSummarizer (line 38) | class CompassBenchSummarizer:
    method __init__ (line 46) | def __init__(self, config: ConfigDict, check_pos_bias=False) -> None:
    method get_score (line 57) | def get_score(self, time_str):
    method summarize (line 124) | def summarize(

FILE: opencompass/summarizers/subjective/corev2.py
  function match_general_answer (line 22) | def match_general_answer(s):
  function match_GPT4_answer (line 30) | def match_GPT4_answer(s):
  function call_function (line 40) | def call_function(name, arg):
  class Corev2Summarizer (line 47) | class Corev2Summarizer:
    method __init__ (line 55) | def __init__(self, config: ConfigDict, match_method='smart') -> None:
    method summarize (line 63) | def summarize(self,

FILE: opencompass/summarizers/subjective/creationbench.py
  function post_process_creationbench (line 34) | def post_process_creationbench(judgement: str,
  class CreationBenchSummarizer (line 45) | class CreationBenchSummarizer(AlignmentBenchSummarizer):
    method __init__ (line 53) | def __init__(self, config: ConfigDict, judge_type: str) -> None:
    method summarize (line 63) | def summarize(self,

FILE: opencompass/summarizers/subjective/flames.py
  function post_process_flames (line 19) | def post_process_flames(judgement: str):
  class FlamesSummarizer (line 35) | class FlamesSummarizer:
    method __init__ (line 43) | def __init__(self, config: ConfigDict, judge_type='general') -> None:
    method summarize (line 60) | def summarize(self,

FILE: opencompass/summarizers/subjective/fofo.py
  function post_process_fofo (line 26) | def post_process_fofo(judgement: str):
  class FofoSummarizer (line 40) | class FofoSummarizer:
    method __init__ (line 48) | def __init__(self, config: ConfigDict, judge_type='single') -> None:
    method get_score (line 62) | def get_score(self, time_str):
    method summarize (line 103) | def summarize(self,

FILE: opencompass/summarizers/subjective/followbench.py
  function post_process_followbench (line 24) | def post_process_followbench(item):
  function get_scores (line 62) | def get_scores(judged_answers, references):
  class FollowBenchSummarizer (line 92) | class FollowBenchSummarizer:
    method __init__ (line 100) | def __init__(self, config: ConfigDict) -> None:
    method summarize (line 111) | def summarize(self,

FILE: opencompass/summarizers/subjective/mtbench.py
  function model_abbr_from_cfg_used_in_summarizer (line 21) | def model_abbr_from_cfg_used_in_summarizer(model):
  function post_process_mtbench_pair (line 27) | def post_process_mtbench_pair(judgement: str):
  function post_process_mtbench_single (line 40) | def post_process_mtbench_single(judgement: str):
  function get_capability_results (line 54) | def get_capability_results(
  class MTBenchSummarizer (line 87) | class MTBenchSummarizer(CompassArenaSummarizer):
    method __init__ (line 95) | def __init__(self, config: ConfigDict, judge_type='single') -> None:
    method summarize (line 111) | def summarize(self, time_str: str = datetime.now().strftime('%Y%m%d_%H...

FILE: opencompass/summarizers/subjective/mtbench101.py
  function post_process_mtbench_pair (line 25) | def post_process_mtbench_pair(judgement: str):
  function post_process_mtbench101 (line 38) | def post_process_mtbench101(judgement: str):
  function get_final_results (line 53) | def get_final_results(judged_answers, references, output_dir, fout_flag,...
  class MTBench101Summarizer (line 92) | class MTBench101Summarizer(CompassArenaSummarizer):
    method __init__ (line 100) | def __init__(self, config: ConfigDict, judge_type='single') -> None:
    method summarize (line 114) | def summarize(self,

FILE: opencompass/summarizers/subjective/multiround.py
  function post_process_multiround (line 27) | def post_process_multiround(judgement: str):
  function get_capability_results (line 52) | def get_capability_results(judged_answers,
  class MultiroundSummarizer (line 116) | class MultiroundSummarizer:
    method __init__ (line 124) | def __init__(self, config: ConfigDict) -> None:
    method summarize (line 134) | def summarize(self,

FILE: opencompass/summarizers/subjective/qacompassbench.py
  function post_process_wildbench_pair (line 20) | def post_process_wildbench_pair(judgement: str):
  class QaCompassBenchSummarizer (line 30) | class QaCompassBenchSummarizer:
    method __init__ (line 38) | def __init__(self, config: ConfigDict, check_pos_bias=False) -> None:
    method get_score (line 50) | def get_score(self, time_str):
    method summarize (line 128) | def summarize(

FILE: opencompass/summarizers/subjective/subjective.py
  function flatten_data (line 13) | def flatten_data(data):
  class SubjectiveSummarizer (line 50) | class SubjectiveSummarizer:
    method __init__ (line 58) | def __init__(self, config: ConfigDict, function: str) -> None:
    method summarize (line 62) | def summarize(

FILE: opencompass/summarizers/subjective/subjective_post_process.py
  function post_process_autoj (line 4) | def post_process_autoj(judgement: str):
  function post_process_judgelm (line 18) | def post_process_judgelm(judgement: str):

FILE: opencompass/summarizers/subjective/utils.py
  function get_outdir (line 9) | def get_outdir(cfg, time_str):
  function get_judgeanswer_and_reference (line 24) | def get_judgeanswer_and_reference(dataset, subdir_path, post_process):
  function get_judgeanswer_and_reference_update (line 78) | def get_judgeanswer_and_reference_update(dataset, subdir_path, post_proc...

FILE: opencompass/summarizers/subjective/wildbench.py
  function post_process_wildbench_pair (line 37) | def post_process_wildbench_pair(judgement: str):
  function post_process_wildbench_single (line 46) | def post_process_wildbench_single(judgement: str):
  function get_capability_results (line 62) | def get_capability_results(
  class WildBenchSingleSummarizer (line 97) | class WildBenchSingleSummarizer(CompassArenaSummarizer):
    method __init__ (line 105) | def __init__(self, config: ConfigDict) -> None:
    method summarize (line 114) | def summarize(self, time_str: str = datetime.now().strftime('%Y%m%d_%H...
  class WildBenchPairSummarizer (line 147) | class WildBenchPairSummarizer(CompassArenaSummarizer):
    method __init__ (line 155) | def __init__(self, config: ConfigDict, check_pos_bias=False) -> None:
    method get_score (line 167) | def get_score(self, time_str):
    method summarize (line 237) | def summarize(

FILE: opencompass/summarizers/summarizer_pretrain.py
  class PretrainSummarizer (line 21) | class PretrainSummarizer:
    method __init__ (line 24) | def __init__(self, config: ConfigDict, dataset_abbrs: Optional[List[st...
    method summarize (line 34) | def summarize(

FILE: opencompass/tasks/base.py
  function extract_role_pred (line 12) | def extract_role_pred(s: str, begin_str: Optional[str],
  class BaseTask (line 43) | class BaseTask:
    method __init__ (line 60) | def __init__(self, cfg: ConfigDict):
    method run (line 68) | def run(self):
    method get_command (line 72) | def get_command(self, cfg_path, template) -> str:
    method name (line 82) | def name(self) -> str:
    method __repr__ (line 89) | def __repr__(self) -> str:
    method get_log_path (line 92) | def get_log_path(self, file_extension: str = 'json') -> str:
    method get_output_paths (line 103) | def get_output_paths(self, file_extension: str = 'json') -> List[str]:

FILE: opencompass/tasks/llm_eval.py
  class ModelEvaluator (line 12) | class ModelEvaluator:
    method __init__ (line 15) | def __init__(
    method parse_cfg (line 28) | def parse_cfg(self, cfg: ConfigDict):
    method evaluate (line 34) | def evaluate(self):
    method _load_dataset (line 46) | def _load_dataset(self, dataset_abbr: str):
    method _evaluate_dataset (line 61) | def _evaluate_dataset(self, dataset_abbr: str):
    method _make_prompt (line 73) | def _make_prompt(self, question: str, responses: List[str]) -> str:
    method _rank_models (line 85) | def _rank_models(self, output: str,

FILE: opencompass/tasks/openicl_attack.py
  class OpenICLAttackTask (line 20) | class OpenICLAttackTask(BaseTask):
    method __init__ (line 30) | def __init__(self, cfg: ConfigDict):
    method get_command (line 37) | def get_command(self, cfg_path, template):
    method prompt_selection (line 58) | def prompt_selection(self, inferencer, prompts):
    method run (line 71) | def run(self):
    method _inference (line 94) | def _inference(self):
    method _set_default_value (line 187) | def _set_default_value(self, cfg: ConfigDict, key: str, value: Any):
  function parse_args (line 193) | def parse_args():

FILE: opencompass/tasks/openicl_eval.py
  class OpenICLEvalTask (line 29) | class OpenICLEvalTask(BaseTask):
    method __init__ (line 40) | def __init__(self, cfg: ConfigDict):
    method get_command (line 66) | def get_command(self, cfg_path, template):
    method run (line 80) | def run(self):
    method _score (line 100) | def _score(self):
    method _load_and_preprocess_test_data (line 133) | def _load_and_preprocess_test_data(self):
    method _load_predictions (line 152) | def _load_predictions(self):
    method _process_predictions (line 195) | def _process_predictions(self, pred_strs):
    method _evaluate_predictions (line 254) | def _evaluate_predictions(
    method _sum_rollout (line 348) | def _sum_rollout(
    method _save_results (line 424) | def _save_results(self, result):
    method extract_rate (line 434) | def extract_rate(self, results):
    method format_details (line 453) | def format_details(
    method calculate_bpb (line 512) | def calculate_bpb(self, pred_dicts: List):
  function parse_args (line 555) | def parse_args():

FILE: opencompass/tasks/openicl_eval_watch.py
  class OpenICLEvalWatchTask (line 19) | class OpenICLEvalWatchTask(OpenICLEvalTask):
    method __init__ (line 24) | def __init__(self, cfg: ConfigDict):
    method get_command (line 32) | def get_command(self, cfg_path, template):
    method run (line 46) | def run(self):
    method _is_ready (line 103) | def _is_ready(self, model_cfg, dataset_cfg, status_index: dict) -> bool:
  function parse_args (line 113) | def parse_args():

FILE: opencompass/tasks/openicl_infer.py
  class OpenICLInferTask (line 21) | class OpenICLInferTask(BaseTask):
    method __init__ (line 31) | def __init__(self, cfg: ConfigDict):
    method get_command (line 40) | def get_command(self, cfg_path, template):
    method run (line 67) | def run(self, cur_model=None, cur_model_abbr=None):
    method _inference (line 105) | def _inference(self):
    method _set_default_value (line 160) | def _set_default_value(self, cfg: ConfigDict, key: str, value: Any):
  function parse_args (line 165) | def parse_args():

FILE: opencompass/tasks/openicl_infer_concurrent.py
  class _ProgressTracker (line 29) | class _ProgressTracker:
    method set_total (line 35) | def set_total(self, total: int) -> None:
    method set_completed (line 39) | def set_completed(self, completed: int) -> None:
    method incr (line 43) | def incr(self, count: int = 1) -> None:
    method remaining (line 47) | def remaining(self) -> Optional[int]:
  class _RunningTask (line 55) | class _RunningTask:
  class OpenICLInferConcurrentTask (line 62) | class OpenICLInferConcurrentTask(BaseTask):
    method __init__ (line 69) | def __init__(self, cfg: ConfigDict):
    method get_command (line 80) | def get_command(self, cfg_path, template):
    method run (line 100) | def run(self, cur_model=None, cur_model_abbr=None):
    method _default_max_workers (line 133) | def _default_max_workers(self) -> int:
    method _inferencer_name (line 137) | def _inferencer_name(self, inferencer_type) -> str:
    method _set_default_value (line 142) | def _set_default_value(self, cfg: ConfigDict, key: str, value: Any):
    method _build_inferencer (line 146) | def _build_inferencer(self, model, model_cfg, dataset_cfg, max_workers):
    method _run_dataset_task (line 184) | def _run_dataset_task(self, model_cfg, dataset_cfg, tokens, max_workers,
    method _remaining_total (line 238) | def _remaining_total(self, running: List[_RunningTask],
    method _run_task_group (line 248) | def _run_task_group(self, model_cfg, tasks, tokens, max_workers):
  function parse_args (line 334) | def parse_args():

FILE: opencompass/tasks/outer_eval/alpacaeval.py
  class PredictionMerger (line 15) | class PredictionMerger:
    method __init__ (line 18) | def __init__(self, cfg: ConfigDict) -> None:
    method run (line 26) | def run(self):
  class AlpacaEvalTask (line 77) | class AlpacaEvalTask(BaseTask):
    method __init__ (line 91) | def __init__(self, cfg: ConfigDict):
    method get_command (line 101) | def get_command(self, cfg_path, template):
    method run (line 144) | def run(self):

FILE: opencompass/tasks/subjective_eval.py
  class SubjectiveEvalTask (line 23) | class SubjectiveEvalTask(BaseTask):
    method __init__ (line 37) | def __init__(self, cfg: ConfigDict):
    method get_command (line 61) | def get_command(self, cfg_path, template):
    method run (line 82) | def run(self):
    method name (line 100) | def name(self) -> str:
    method _load_model_pred (line 108) | def _load_model_pred(
    method _load_model_judgements (line 228) | def _load_model_judgements(
    method _score (line 357) | def _score(self,
    method get_output_paths (line 421) | def get_output_paths(self, file_extension: str = 'json') -> List[str]:
  function parse_args (line 453) | def parse_args():

FILE: opencompass/utils/abbr.py
  function model_abbr_from_cfg (line 7) | def model_abbr_from_cfg(cfg: Union[ConfigDict, List[ConfigDict]]) -> str:
  function dataset_abbr_from_cfg (line 19) | def dataset_abbr_from_cfg(cfg: ConfigDict) -> str:
  function task_abbr_from_cfg (line 30) | def task_abbr_from_cfg(task: Dict) -> str:
  function get_infer_output_path (line 40) | def get_infer_output_path(model_cfg: ConfigDict,
  function deal_with_judge_model_abbr (line 51) | def deal_with_judge_model_abbr(model_cfg, judge_model_cfg, meta=False):

FILE: opencompass/utils/auxiliary.py
  function batched (line 8) | def batched(iterable, n):

FILE: opencompass/utils/build.py
  function build_dataset_from_cfg (line 8) | def build_dataset_from_cfg(dataset_cfg: ConfigDict):
  function build_model_from_cfg (line 15) | def build_model_from_cfg(model_cfg: ConfigDict):

FILE: opencompass/utils/collect_env.py
  function collect_env (line 7) | def collect_env():

FILE: opencompass/utils/datasets.py
  function get_data_path (line 10) | def get_data_path(dataset_id: str, local_mode: bool = False):
  function download_dataset (line 57) | def download_dataset(data_path, cache_dir, remove_finished=True):

FILE: opencompass/utils/dependency.py
  function satisfy_requirement (line 7) | def satisfy_requirement(dep):

FILE: opencompass/utils/dict_postprocessors.py
  function base_postprocess (line 5) | def base_postprocess(output: dict) -> dict:

FILE: opencompass/utils/file.py
  function match_files (line 6) | def match_files(path: str,

FILE: opencompass/utils/fileio.py
  class JSONToolkit (line 25) | class JSONToolkit:
    method read_json (line 29) | def read_json(file_path: Union[str, Path]) -> Dict[str, Any]:
    method read_jsonl (line 54) | def read_jsonl(file_path: Union[str, Path]) -> List[Dict[str, Any]]:
    method save_json (line 87) | def save_json(data: Dict[str, Any],
    method save_jsonl (line 113) | def save_jsonl(data: List[Dict[str, Any]], file_path: Union[str,
    method jsonl_writer (line 139) | def jsonl_writer(file_path: Union[str, Path]):
  function patch_func (line 165) | def patch_func(module, fn_name_to_wrap):
  function patch_fileio (line 180) | def patch_fileio(global_vars=None):
  function patch_hf_auto_model (line 275) | def patch_hf_auto_model(cache_dir=None):
  function calculate_md5 (line 328) | def calculate_md5(fpath: str, chunk_size: int = 1024 * 1024):
  function check_md5 (line 341) | def check_md5(fpath, md5, **kwargs):
  function check_integrity (line 345) | def check_integrity(fpath, md5=None):
  function download_url_to_file (line 353) | def download_url_to_file(url, dst, hash_prefix=None, progress=True):
  function download_url (line 420) | def download_url(url, root, filename=None, md5=None):
  function _is_tarxz (line 456) | def _is_tarxz(filename):
  function _is_tar (line 460) | def _is_tar(filename):
  function _is_targz (line 464) | def _is_targz(filename):
  function _is_tgz (line 468) | def _is_tgz(filename):
  function _is_gzip (line 472) | def _is_gzip(filename):
  function _is_zip (line 476) | def _is_zip(filename):
  function extract_archive (line 480) | def extract_archive(from_path, to_path=None, remove_finished=False):
  function download_and_extract_archive (line 509) | def download_and_extract_archive(url,

FILE: opencompass/utils/heartbeat.py
  class HeartBeatManager (line 8) | class HeartBeatManager:
    method __init__ (line 10) | def __init__(self, work_dir: str | Path, fname: str = 'infer_heartbeat'):
    method start_heartbeat (line 14) | def start_heartbeat(self, write_interval: float = 5.):
    method last_heartbeat (line 31) | def last_heartbeat(self) -> float:

FILE: opencompass/utils/infer_status.py
  class InferStatus (line 13) | class InferStatus(TypedDict):
  function safe_read (line 19) | def safe_read(file: Path, work_dir: Path):
  function safe_write (line 26) | def safe_write(file: Path, content: str, work_dir: Path):
  class InferStatusManager (line 32) | class InferStatusManager:
    method __init__ (line 34) | def __init__(
    method update (line 56) | def update(
    method _maybe_write (line 72) | def _maybe_write(self, entry: dict) -> None:
    method write_task_status (line 77) | def write_task_status(self):
    method get_task_status (line 83) | def get_task_status(self) -> dict[str, InferStatus]:

FILE: opencompass/utils/lark.py
  class LarkReporter (line 8) | class LarkReporter:
    method __init__ (line 10) | def __init__(self, url: str):
    method post (line 13) | def post(self,
  function parse_args (line 43) | def parse_args():

FILE: opencompass/utils/logging.py
  function get_logger (line 20) | def get_logger(log_level='INFO', filter_duplicate_level=None) -> MMLogger:
  class FilterDuplicateMessage (line 47) | class FilterDuplicateMessage(logging.Filter):
    method __init__ (line 54) | def __init__(self, name, filter_duplicate_level):
    method filter (line 68) | def filter(self, record: logging.LogRecord) -> bool:
  function pretty_print_config (line 86) | def pretty_print_config(cfg):

FILE: opencompass/utils/menu.py
  class Menu (line 12) | class Menu:
    method __init__ (line 23) | def __init__(self, lists, prompts=None):
    method draw_menu (line 29) | def draw_menu(self, stdscr, selected_row_idx, offset, max_rows):
    method run (line 45) | def run(self):
    method main_loop (line 49) | def main_loop(self, stdscr):

FILE: opencompass/utils/network.py
  function setup_proxies (line 14) | def setup_proxies(proxy_env_name):
  function check_network_connectivity (line 43) | def check_network_connectivity(
  function check_url_accessibility (line 119) | def check_url_accessibility(

FILE: opencompass/utils/prompt.py
  function safe_format (line 11) | def safe_format(input_str: str, **kwargs) -> str:
  function get_prompt_hash (line 38) | def get_prompt_hash(dataset_cfg: Union[ConfigDict, List[ConfigDict]]) ->...
  class PromptList (line 79) | class PromptList(list):
    method format (line 82) | def format(self, **kwargs) -> PromptList:
    method replace (line 107) | def replace(self, src: str, dst: Union[str, PromptList]) -> PromptList:
    method __add__ (line 149) | def __add__(self, other: Union[str, PromptList]) -> PromptList:
    method __radd__ (line 165) | def __radd__(self, other: Union[str, PromptList]) -> PromptList:
    method __iadd__ (line 182) | def __iadd__(self, other: Union[str, PromptList]) -> PromptList:
    method __str__ (line 199) | def __str__(self) -> str:

FILE: opencompass/utils/result_station.py
  function save_to_station (line 11) | def save_to_station(cfg, args):
  function read_from_station (line 241) | def read_from_station(cfg, args):
  function find_files_by_regex (line 378) | def find_files_by_regex(directory, pattern):
  function check_filenames (line 390) | def check_filenames(x, filenames):

FILE: opencompass/utils/run.py
  function match_cfg_file (line 23) | def match_cfg_file(workdir: Union[str, List[str]],
  function try_fill_in_custom_cfgs (line 75) | def try_fill_in_custom_cfgs(config):
  function get_config_from_arg (line 88) | def get_config_from_arg(args) -> Config:
  function change_accelerator (line 242) | def change_accelerator(models, accelerator):
  function get_config_type (line 363) | def get_config_type(obj) -> str:
  function fill_infer_cfg (line 367) | def fill_infer_cfg(cfg, args):
  function fill_eval_cfg (line 395) | def fill_eval_cfg(cfg, args):
  function consturct_chatml_datasets (line 421) | def consturct_chatml_datasets(custom_cfg: List[Dict[str, Any]]):

FILE: opencompass/utils/text_postprocessors.py
  function general_postprocess (line 8) | def general_postprocess(text: str) -> str:
  function general_cn_postprocess (line 28) | def general_cn_postprocess(text: str) -> str:
  function first_capital_postprocess (line 46) | def first_capital_postprocess(text: str) -> str:
  function last_capital_postprocess (line 54) | def last_capital_postprocess(text: str) -> str:
  function think_pred_postprocess (line 62) | def think_pred_postprocess(
  function first_option_postprocess (line 73) | def first_option_postprocess(text: str, options: str, cushion=True) -> str:
  function first_capital_postprocess_multi (line 155) | def first_capital_postprocess_multi(text: str) -> str:
  function last_option_postprocess (line 162) | def last_option_postprocess(text: str, options: str) -> str:
  function first_number_postprocess (line 169) | def first_number_postprocess(text: str) -> float:
  function multiple_select_postprocess (line 182) | def multiple_select_postprocess(text: str) -> str:
  function xml_tag_postprocessor (line 188) | def xml_tag_postprocessor(text, tag):
  function general_eval_wrapper_postprocess (line 215) | def general_eval_wrapper_postprocess(text: str,
  function match_answer_pattern (line 242) | def match_answer_pattern(response_text: str, answer_pattern: str):
  function extract_non_reasoning_content (line 249) | def extract_non_reasoning_content(

FILE: opencompass/utils/types.py
  function get_type_from_cfg (line 9) | def get_type_from_cfg(cfg: Union[Config, Dict]) -> Any:
  function _check_type_list (line 22) | def _check_type_list(obj, typelist: List):
  function _check_dataset (line 34) | def _check_dataset(obj) -> Union[Dataset, DatasetDict]:
  function _check_list (line 43) | def _check_list(obj) -> List:
  function _check_str (line 50) | def _check_str(obj) -> str:
  function _check_dict (line 57) | def _check_dict(obj) -> Dict:

FILE: setup.py
  class DownloadNLTK (line 5) | class DownloadNLTK(install):
    method run (line 7) | def run(self):
  function readme (line 14) | def readme():
  function parse_requirements (line 20) | def parse_requirements(fname='requirements.txt', with_version=True):
  function get_version (line 100) | def get_version():
  function do_setup (line 107) | def do_setup():

FILE: tests/datasets/test_aime2025.py
  class CustomDataset (line 20) | class CustomDataset:
    method load (line 24) | def load(path, **kwargs):
    method __init__ (line 35) | def __init__(self, **kwargs):
  class TestAime2025Dataset (line 59) | class TestAime2025Dataset(unittest.TestCase):
    method setUp (line 62) | def setUp(self):
    method _create_temp_jsonl_file (line 76) | def _create_temp_jsonl_file(self, data):
    method test_load_reads_jsonl_file (line 87) | def test_load_reads_jsonl_file(self):
    method test_load_returns_dataset (line 110) | def test_load_returns_dataset(self):
    method test_load_preserves_all_columns (line 125) | def test_load_preserves_all_columns(self):
    method test_load_with_empty_file (line 153) | def test_load_with_empty_file(self):
    method test_dataset_initialization (line 172) | def test_dataset_initialization(self):
    method test_dataset_reader_config (line 193) | def test_dataset_reader_config(self):
    method test_load_handles_unicode (line 215) | def test_load_handles_unicode(self):
    method test_load_with_real_data (line 239) | def test_load_with_real_data(self):
    method test_dataset_initialization_with_real_data (line 277) | def test_dataset_initialization_with_real_data(self):

FILE: tests/datasets/test_aime2025_eval.py
  class TestAime2025EvalResultValidation (line 12) | class TestAime2025EvalResultValidation(unittest.TestCase):
    method setUp (line 15) | def setUp(self):
    method _create_test_dataset (line 39) | def _create_test_dataset(self):
    method test_result_structure (line 50) | def test_result_structure(self):
    method test_accuracy_calculation (line 103) | def test_accuracy_calculation(self):
    method test_rule_evaluator_result (line 122) | def test_rule_evaluator_result(self):
    method test_llm_evaluator_result (line 138) | def test_llm_evaluator_result(self):
    method test_cascade_mode_result (line 151) | def test_cascade_mode_result(self):
    method test_parallel_mode_result (line 206) | def test_parallel_mode_result(self):
    method test_result_statistics_consistency (line 243) | def test_result_statistics_consistency(self):
    method test_details_count_matches_total_samples (line 290) | def test_details_count_matches_total_samples(self):
    method test_llm_prediction_format (line 304) | def test_llm_prediction_format(self):
    method test_boxed_extraction (line 326) | def test_boxed_extraction(self):
    method test_mathematical_equivalence (line 347) | def test_mathematical_equivalence(self):
    method test_edge_cases (line 370) | def test_edge_cases(self):
    method test_result_metrics_completeness (line 415) | def test_result_metrics_completeness(self):

FILE: tests/datasets/test_beyondaime.py
  function _load_beyondaime_module (line 17) | def _load_beyondaime_module():
  class TestBeyondAIMEDataset (line 34) | class TestBeyondAIMEDataset(unittest.TestCase):
    method setUp (line 37) | def setUp(self):
    method test_load_renames_problem_to_question (line 48) | def test_load_renames_problem_to_question(self, mock_load_dataset):
    method test_load_returns_dataset (line 68) | def test_load_returns_dataset(self, mock_load_dataset):
    method test_load_preserves_other_columns (line 79) | def test_load_preserves_other_columns(self, mock_load_dataset):
    method test_load_with_empty_dataset (line 100) | def test_load_with_empty_dataset(self, mock_load_dataset):
    method test_load_with_kwargs (line 122) | def test_load_with_kwargs(self, mock_load_dataset):
    method test_dataset_initialization (line 134) | def test_dataset_initialization(self, mock_load):
    method test_dataset_reader_config (line 152) | def test_dataset_reader_config(self, mock_load):

FILE: tests/datasets/test_humaneval.py
  function humaneval_postprocess (line 6) | def humaneval_postprocess(text: str) -> str:
  function run_humaneval_check (line 141) | def run_humaneval_check(completion):
  class TestHumaneval (line 153) | class TestHumaneval(unittest.TestCase):
    method test_vanilla (line 155) | def test_vanilla(self):
    method test_python_quote (line 159) | def test_python_quote(self):
    method test_bare_quote (line 168) | def test_bare_quote(self):
    method test_error_space_quote (line 177) | def test_error_space_quote(self):
    method test_import_1 (line 186) | def test_import_1(self):
    method test_import_2 (line 198) | def test_import_2(self):
    method test_import_3 (line 209) | def test_import_3(self):
    method test_comment (line 220) | def test_comment(self):
    method test_additional (line 232) | def test_additional(self):

FILE: tests/datasets/test_local_datasets.py
  function reload_datasets (line 22) | def reload_datasets():
  function load_datasets_conf (line 301) | def load_datasets_conf(source):
  function load_datasets (line 307) | def load_datasets(source, conf):
  function clean_string (line 332) | def clean_string(value):
  class TestingLocalDatasets (line 343) | class TestingLocalDatasets(unittest.TestCase):
    method test_datasets (line 345) | def test_datasets(self):
  function _check_data (line 464) | def _check_data(dataset1: Dataset | DatasetDict,

FILE: tests/models/test_base_model.py
  class TestBaseModel (line 9) | class TestBaseModel(unittest.TestCase):
    method test_is_api_attribute (line 12) | def test_is_api_attribute(self):
    method test_initialization_signature (line 16) | def test_initialization_signature(self):
    method test_initialization_with_meta_template (line 32) | def test_initialization_with_meta_template(self, mock_parser):

FILE: tests/models/test_huggingface.py
  class TestHuggingFace (line 10) | class TestHuggingFace(unittest.TestCase):
    method test_initialization_basic (line 15) | def test_initialization_basic(self, mock_model_class,
    method test_generate_basic (line 43) | def test_generate_basic(self, mock_torch, mock_transformers,
    method test_get_token_len (line 89) | def test_get_token_len(self, mock_model_class, mock_tokenizer_class):
    method test_generate_with_mid_mode (line 115) | def test_generate_with_mid_mode(self, mock_torch, mock_transformers,
    method test_generate_with_batch_padding (line 170) | def test_generate_with_batch_padding(self, mock_transformers,
    method test_get_ppl (line 211) | def test_get_ppl(self, mock_model_class, mock_tokenizer_class):

FILE: tests/models/test_huggingface_above_v4_33.py
  class TestHuggingFacewithChatTemplate (line 10) | class TestHuggingFacewithChatTemplate(unittest.TestCase):
    method test_initialization_basic (line 18) | def test_initialization_basic(self, mock_get_meta_template,
    method test_initialization_tokenizer_only (line 49) | def test_initialization_tokenizer_only(self, mock_get_meta_template,
    method test_generate_basic (line 77) | def test_generate_basic(self, mock_get_stopping_criteria,
    method test_get_token_len (line 135) | def test_get_token_len(self, mock_convert_messages, mock_get_meta_temp...
  class TestHuggingFaceBaseModel (line 166) | class TestHuggingFaceBaseModel(unittest.TestCase):
    method test_initialization_basic (line 173) | def test_initialization_basic(self, mock_get_max_seq_len, mock_model_c...
    method test_generate_basic (line 204) | def test_generate_basic(self, mock_get_stopping_criteria,
    method test_get_token_len (line 251) | def test_get_token_len(self, mock_convert_base_messages,

FILE: tests/models/test_openai_api.py
  function setup_tiktoken_mock (line 10) | def setup_tiktoken_mock(mock_tiktoken):
  class TestOpenAI (line 21) | class TestOpenAI(unittest.TestCase):
    method test_initialization_basic (line 26) | def test_initialization_basic(self, mock_tiktoken):
    method test_initialization_with_key_list (line 43) | def test_initialization_with_key_list(self, mock_tiktoken):
    method test_generate_basic (line 59) | def test_generate_basic(self, mock_requests, mock_tiktoken):
    method test_get_token_len (line 89) | def test_get_token_len(self, mock_tiktoken):
    method test_generate_with_reasoning_content (line 105) | def test_generate_with_reasoning_content(self, mock_requests,
    method test_generate_with_retry (line 138) | def test_generate_with_retry(self, mock_requests, mock_tiktoken):
  class TestOpenAISDK (line 173) | class TestOpenAISDK(unittest.TestCase):
    method test_initialization_basic (line 180) | def test_initialization_basic(self, mock_httpx_client, mock_openai_class,
    method test_generate_basic (line 202) | def test_generate_basic(self, mock_httpx_client, mock_openai_class,
    method test_generate_with_reasoning_content (line 233) | def test_generate_with_reasoning_content(self, mock_httpx_client,
    method test_generate_with_o1_model (line 264) | def test_generate_with_o1_model(self, mock_httpx_client, mock_openai_c...
    method test_generate_with_retry (line 297) | def test_generate_with_retry(self, mock_httpx_client, mock_openai_class,

FILE: tests/models/test_openai_streaming.py
  function setup_tiktoken_mock (line 10) | def setup_tiktoken_mock(mock_tiktoken):
  class TestOpenAISDKStreaming (line 21) | class TestOpenAISDKStreaming(unittest.TestCase):
    method test_initialization_basic (line 27) | def test_initialization_basic(self, mock_openai_class, mock_tiktoken):
    method test_initialization_with_stream_false (line 49) | def test_initialization_with_stream_false(self, mock_openai_class,
    method test_generate_with_streaming (line 68) | def test_generate_with_streaming(self, mock_openai_class, mock_tiktoken):
    method test_generate_with_streaming_reasoning_content (line 129) | def test_generate_with_streaming_reasoning_content(self, mock_openai_c...
    method test_create_fresh_client (line 191) | def test_create_fresh_client(self, mock_openai_class, mock_tiktoken):
    method test_estimate_token_count (line 211) | def test_estimate_token_count(self, mock_openai_class, mock_tiktoken):

FILE: tests/models/test_turbomind.py
  class TestTurboMindModel (line 9) | class TestTurboMindModel(unittest.TestCase):
    method test_initialization_basic (line 16) | def test_initialization_basic(self, mock_get_max_seq_len, mock_pipeline,
    method test_initialization_with_drop_middle (line 44) | def test_initialization_with_drop_middle(self, mock_get_max_seq_len,
    method test_generate_basic (line 67) | def test_generate_basic(self, mock_gen_config_class, mock_get_max_seq_...
    method test_generate_with_stop_words (line 103) | def test_generate_with_stop_words(self, mock_gen_config_class,
    method test_generate_with_drop_middle (line 140) | def test_generate_with_drop_middle(self, mock_gen_config_class,
    method test_get_token_len (line 178) | def test_get_token_len(self, mock_get_max_seq_len, mock_pipeline,
    method test_generate_with_old_version (line 204) | def test_generate_with_old_version(self, mock_gen_config_class,
    method test_generate_with_do_sample (line 240) | def test_generate_with_do_sample(self, mock_gen_config_class,
    method test_build_pipe_turbomind_backend (line 282) | def test_build_pipe_turbomind_backend(self, mock_pytorch_config,
    method test_build_pipe_pytorch_backend (line 312) | def test_build_pipe_pytorch_backend(self, mock_pytorch_config,
    method test_build_pipe_invalid_backend (line 340) | def test_build_pipe_invalid_backend(self, mock_get_max_seq_len,

FILE: tests/models/test_turbomind_with_tf_above_v4_33.py
  class TestTurboMindModelwithChatTemplate (line 12) | class TestTurboMindModelwithChatTemplate(unittest.TestCase):
    method test_initialization_with_dict_engine_config (line 23) | def test_initialization_with_dict_engine_config(self,
    method test_initialization_with_configdict_engine_config (line 59) | def test_initialization_with_configdict_engine_config(
    method test_initialization_tokenizer_only (line 90) | def test_initialization_tokenizer_only(self, mock_get_meta_template,
    method test_initialization_invalid_engine_config_type (line 116) | def test_initialization_invalid_engine_config_type(self,
    method test_generate_basic (line 148) | def test_generate_basic(self, mock_gen_config_class, mock_convert_mess...
    method test_generate_with_stop_words (line 201) | def test_generate_with_stop_words(self, mock_gen_config_class,
    method test_generate_with_drop_middle (line 257) | def test_generate_with_drop_middle(self, mock_gen_config_class,
    method test_get_token_len (line 316) | def test_get_token_len(self, mock_convert_messages, mock_get_meta_temp...
    method test_get_potential_stop_words (line 358) | def test_get_potential_stop_words(self, mock_gen_config_class,
    method test_generate_with_do_sample_false (line 400) | def test_generate_with_do_sample_false(self, mock_gen_config_class,
    method test_generate_with_old_version (line 454) | def test_generate_with_old_version(self, mock_gen_config_class,
    method test_generate_with_min_out_len (line 508) | def test_generate_with_min_out_len(self, mock_gen_config_class,
    method test_generate_with_bos_token_removal (line 562) | def test_generate_with_bos_token_removal(self, mock_gen_config_class,
    method test_build_pipe_turbomind_backend (line 617) | def test_build_pipe_turbomind_backend(self, mock_pytorch_config,
    method test_build_pipe_pytorch_backend (line 665) | def test_build_pipe_pytorch_backend(self, mock_pytorch_config,
    method test_build_pipe_invalid_backend (line 700) | def test_build_pipe_invalid_backend(self, mock_get_meta_template,
    method test_gen_config_override_in_generate (line 730) | def test_gen_config_override_in_generate(self, mock_gen_config_class,
    method test_gen_config_max_out_len_override (line 789) | def test_gen_config_max_out_len_override(self, mock_gen_config_class,
    method test_gen_config_min_out_len_override (line 845) | def test_gen_config_min_out_len_override(self, mock_gen_config_class,
    method test_gen_config_sampling_params_override (line 901) | def test_gen_config_sampling_params_override(self, mock_gen_config_class,

FILE: tests/models/test_vllm.py
  class TestVLLM (line 9) | class TestVLLM(unittest.TestCase):
    method test_initialization_basic (line 14) | def test_initialization_basic(self, mock_ray, mock_llm_class):
    method test_generate_basic (line 38) | def test_generate_basic(self, mock_sampling_params_class, mock_ray,
    method test_generate_with_mid_mode (line 70) | def test_generate_with_mid_mode(self, mock_sampling_params_class, mock...
    method test_get_token_len (line 103) | def test_get_token_len(self, mock_ray, mock_llm_class):
    method test_generate_with_lora (line 128) | def test_generate_with_lora(self, mock_lora_request_class,
    method test_generate_with_stop_words (line 166) | def test_generate_with_stop_words(self, mock_sampling_params_class,

FILE: tests/models/test_vllm_with_tf_above_v4_33.py
  class TestVLLMwithChatTemplate (line 9) | class TestVLLMwithChatTemplate(unittest.TestCase):
    method test_initialization_basic (line 19) | def test_initialization_basic(self, mock_ray, mock_get_meta_template,
    method test_initialization_tokenizer_only (line 52) | def test_initialization_tokenizer_only(self, mock_get_meta_template,
    method test_generate_basic (line 82) | def test_generate_basic(self, mock_ray, mock_sampling_params_class,
    method test_get_token_len (line 129) | def test_get_token_len(self, mock_convert_messages, mock_get_meta_temp...
    method test_generate_with_stop_words (line 175) | def test_generate_with_stop_words(self, mock_ray,
    method test_generate_with_lora (line 232) | def test_generate_with_lora(self, mock_lora_request_class, mock_ray,

FILE: tests/openicl/test_icl_chat_inferencer_parallel.py
  class TestParallelChatInferencer (line 12) | class TestParallelChatInferencer(unittest.TestCase):
    method setUp (line 15) | def setUp(self):
    method test_initialization (line 23) | def test_initialization(self):
    method test_initialization_defaults (line 32) | def test_initialization_defaults(self):
    method test_resolve_max_workers_from_config (line 39) | def test_resolve_max_workers_from_config(self):
    method test_resolve_max_workers_from_model (line 47) | def test_resolve_max_workers_from_model(self):
    method test_resolve_max_workers_default (line 61) | def test_resolve_max_workers_default(self, mock_getattr, mock_cpu_count):
    method test_resolve_max_workers_max_limit (line 86) | def test_resolve_max_workers_max_limit(self, mock_getattr, mock_cpu_co...
    method test_progress_update (line 105) | def test_progress_update(self):
    method test_progress_update_no_tracker (line 115) | def test_progress_update_no_tracker(self):
    method test_inference_basic (line 131) | def test_inference_basic(self, mock_resolve, mock_makedirs, mock_exists):
    method test_inference_with_progress_tracker (line 167) | def test_inference_with_progress_tracker(self, mock_resolve, mock_make...
    method test_inference_with_resume (line 208) | def test_inference_with_resume(self, mock_resolve, mock_remove,
    method tearDown (line 244) | def tearDown(self):

FILE: tests/openicl/test_icl_chatml_inferencer_parallel.py
  class TestParallelChatMLInferencer (line 12) | class TestParallelChatMLInferencer(unittest.TestCase):
    method setUp (line 15) | def setUp(self):
    method test_initialization (line 28) | def test_initialization(self):
    method test_initialization_defaults (line 37) | def test_initialization_defaults(self):
    method test_resolve_max_workers_from_config (line 44) | def test_resolve_max_workers_from_config(self):
    method test_resolve_max_workers_from_model (line 52) | def test_resolve_max_workers_from_model(self):
    method test_resolve_max_workers_default (line 66) | def test_resolve_max_workers_default(self, mock_getattr, mock_cpu_count):
    method test_resolve_max_workers_max_limit (line 91) | def test_resolve_max_workers_max_limit(self, mock_getattr, mock_cpu_co...
    method test_progress_update (line 110) | def test_progress_update(self):
    method test_progress_update_no_tracker (line 120) | def test_progress_update_no_tracker(self):
    method test_inference_basic (line 136) | def test_inference_basic(self, mock_resolve, mock_makedirs, mock_exists):
    method test_inference_with_progress_tracker (line 168) | def test_inference_with_progress_tracker(self, mock_resolve, mock_make...
    method test_inference_with_resume (line 208) | def test_inference_with_resume(self, mock_resolve, mock_makedirs,
    method tearDown (line 245) | def tearDown(self):

FILE: tests/openicl/test_icl_gen_inferencer_parallel.py
  class TestParallelGenInferencer (line 12) | class TestParallelGenInferencer(unittest.TestCase):
    method setUp (line 15) | def setUp(self):
    method test_initialization (line 28) | def test_initialization(self):
    method test_initialization_defaults (line 37) | def test_initialization_defaults(self):
    method test_resolve_max_workers_from_config (line 44) | def test_resolve_max_workers_from_config(self):
    method test_resolve_max_workers_from_model (line 52) | def test_resolve_max_workers_from_model(self):
    method test_resolve_max_workers_default (line 66) | def test_resolve_max_workers_default(self, mock_getattr, mock_cpu_count):
    method test_resolve_max_workers_max_limit (line 91) | def test_resolve_max_workers_max_limit(self, mock_getattr, mock_cpu_co...
    method test_progress_update (line 110) | def test_progress_update(self):
    method test_progress_update_no_tracker (line 120) | def test_progress_update_no_tracker(self):
    method test_inference_basic (line 136) | def test_inference_basic(self, mock_resolve, mock_makedirs, mock_exists):
    method test_inference_with_progress_tracker (line 168) | def test_inference_with_progress_tracker(self, mock_resolve, mock_make...
    method test_inference_with_resume (line 209) | def test_inference_with_resume(self, mock_resolve, mock_makedirs,
    method tearDown (line 246) | def tearDown(self):

FILE: tests/openicl/test_prompt_template.py
  class TestPromptTemplate (line 7) | class TestPromptTemplate(unittest.TestCase):
    method setUp (line 9) | def setUp(self) -> None:
    method test_init (line 28) | def test_init(self):
    method test_generate_ice_item (line 34) | def test_generate_ice_item(self):
    method test_generate_label_prompt_item (line 87) | def test_generate_label_prompt_item(self):
    method test_generate_item (line 172) | def test_generate_item(self):

FILE: tests/openicl/test_raw_prompt_template.py
  class TestRawPromptTemplate (line 7) | class TestRawPromptTemplate(unittest.TestCase):
    method setUp (line 9) | def setUp(self) -> None:
    method test_init (line 49) | def test_init(self):
    method test_init_with_format_variables_false (line 59) | def test_init_with_format_variables_false(self):
    method test_validation_not_list (line 65) | def test_validation_not_list(self):
    method test_validation_item_not_dict (line 71) | def test_validation_item_not_dict(self):
    method test_validation_missing_role (line 80) | def test_validation_missing_role(self):
    method test_validation_missing_content (line 86) | def test_validation_missing_content(self):
    method test_validation_invalid_role (line 92) | def test_validation_invalid_role(self):
    method test_generate_item (line 98) | def test_generate_item(self):
    method test_generate_item_no_format (line 119) | def test_generate_item_no_format(self):
    method test_generate_item_does_not_modify_original (line 128) | def test_generate_item_does_not_modify_original(self):
    method test_generate_item_with_output_field (line 137) | def test_generate_item_with_output_field(self):
    method test_generate_ice_item (line 148) | def test_generate_ice_item(self):
    method test_generate_label_prompt_item (line 169) | def test_generate_label_prompt_item(self):
    method test_repr (line 192) | def test_repr(self):

FILE: tests/partitioners/test_base_partitioner.py
  class TestBasePartitioner (line 11) | class TestBasePartitioner(unittest.TestCase):
    method setUp (line 14) | def setUp(self):
    method test_initialization (line 18) | def test_initialization(self):
    method test_initialization_with_custom_keep_keys (line 24) | def test_initialization_with_custom_keep_keys(self):
    method test_parse_model_dataset_args (line 31) | def test_parse_model_dataset_args(self):
    method test_parse_model_dataset_args_with_combinations (line 46) | def test_parse_model_dataset_args_with_combinations(self):

FILE: tests/partitioners/test_naive.py
  class TestNaivePartitioner (line 12) | class TestNaivePartitioner(unittest.TestCase):
    method setUp (line 15) | def setUp(self):
    method test_initialization (line 29) | def test_initialization(self):
    method test_initialization_with_custom_n (line 35) | def test_initialization_with_custom_n(self):
    method test_partition_creates_tasks (line 41) | def test_partition_creates_tasks(self, mock_exists):
    method test_partition_with_n_greater_than_one (line 64) | def test_partition_with_n_greater_than_one(self, mock_exists):
    method test_partition_skips_existing_files (line 95) | def test_partition_skips_existing_files(self, mock_exists):
    method test_partition_with_add_cfg (line 128) | def test_partition_with_add_cfg(self):

FILE: tests/prompt/test_api_template_parser.py
  class TestAPITemplateParser (line 7) | class TestAPITemplateParser(unittest.TestCase):
    method setUp (line 9) | def setUp(self):
    method test_parse_template_str_input (line 77) | def test_parse_template_str_input(self):
    method test_parse_template_list_input (line 83) | def test_parse_template_list_input(self):
    method test_parse_template_PromptList_input_no_meta_template (line 89) | def test_parse_template_PromptList_input_no_meta_template(self):
    method test_parse_template_PromptList_input_with_meta_template (line 97) | def test_parse_template_PromptList_input_with_meta_template(self):

FILE: tests/prompt/test_lm_template_parser.py
  class TestLMTemplateParser (line 7) | class TestLMTemplateParser(unittest.TestCase):
    method setUp (line 9) | def setUp(self):
    method test_parse_template_str_input (line 78) | def test_parse_template_str_input(self):
    method test_parse_template_list_input (line 84) | def test_parse_template_list_input(self):
    method test_parse_template_PromptList_input_no_meta_template (line 90) | def test_parse_template_PromptList_input_no_meta_template(self):
    method test_parse_template_PromptList_input_with_meta_template (line 98) | def test_parse_template_PromptList_input_with_meta_template(self):

FILE: tests/prompt/test_prompt_list.py
  class TestPromptList (line 6) | class TestPromptList(unittest.TestCase):
    method test_initialization (line 8) | def test_initialization(self):
    method test_format (line 15) | def test_format(self):
    method test_replace (line 26) | def test_replace(self):
    method test_add (line 55) | def test_add(self):
    method test_str (line 70) | def test_str(self):

FILE: tests/summarizers/test_default.py
  class TestDefaultSummarizer (line 12) | class TestDefaultSummarizer(unittest.TestCase):
    method setUp (line 15) | def setUp(self):
    method test_initialization (line 24) | def test_initialization(self):
    method test_initialization_with_dataset_abbrs (line 30) | def test_initialization_with_dataset_abbrs(self):
    method test_initialization_with_summary_groups (line 37) | def test_initialization_with_summary_groups(self):
    method test_initialization_deprecates_prompt_db (line 47) | def test_initialization_deprecates_prompt_db(self):

FILE: tests/tasks/test_base_task.py
  class TestExtractRolePred (line 11) | class TestExtractRolePred(unittest.TestCase):
    method deperacated_test_extract_role_pred_function_source (line 14) | def deperacated_test_extract_role_pred_function_source(self):
    method deperacated_test_extract_role_pred_with_begin_and_end (line 39) | def deperacated_test_extract_role_pred_with_begin_and_end(self):
    method deperacated_test_extract_role_pred_with_begin_only (line 50) | def deperacated_test_extract_role_pred_with_begin_only(self):
    method deperacated_test_extract_role_pred_with_end_only (line 59) | def deperacated_test_extract_role_pred_with_end_only(self):
    method test_extract_role_pred_without_markers (line 68) | def test_extract_role_pred_without_markers(self):
    method test_extract_role_pred_with_whitespace_begin (line 75) | def test_extract_role_pred_with_whitespace_begin(self):
  class TestBaseTask (line 84) | class TestBaseTask(unittest.TestCase):
    method setUp (line 87) | def setUp(self):
    method test_initialization (line 96) | def test_initialization(self):

FILE: tests/tasks/test_openicl_eval_watch.py
  class TestOpenICLEvalWatchTask (line 14) | class TestOpenICLEvalWatchTask(unittest.TestCase):
    method setUp (line 17) | def setUp(self):
    method test_initialization (line 44) | def test_initialization(self):
    method test_initialization_with_defaults (line 52) | def test_initialization_with_defaults(self):
    method test_get_command_single_gpu (line 82) | def test_get_command_single_gpu(self):
    method test_get_command_multi_gpu (line 97) | def test_get_command_multi_gpu(self, mock_randint):
    method test_run_skips_finished_tasks (line 129) | def test_run_skips_finished_tasks(self, mock_get_path):
    method test_is_ready_all_done (line 148) | def test_is_ready_all_done(self, mock_status_manager, mock_get_path):
    method test_is_ready_not_all_done (line 175) | def test_is_ready_not_all_done(self, mock_status_manager, mock_get_path):
    method test_is_ready_no_status (line 202) | def test_is_ready_no_status(self, mock_status_manager, mock_get_path):
    method test_run_with_ready_tasks (line 223) | def test_run_with_ready_tasks(self, mock_sleep, mock_status_manager,
    method test_run_heartbeat_timeout (line 263) | def test_run_heartbeat_timeout(self, mock_time, mock_sleep,
    method tearDown (line 300) | def tearDown(self):

FILE: tests/tasks/test_openicl_infer_concurrent.py
  class TestProgressTracker (line 17) | class TestProgressTracker(unittest.TestCase):
    method test_initialization (line 20) | def test_initialization(self):
    method test_set_total (line 28) | def test_set_total(self):
    method test_set_completed (line 34) | def test_set_completed(self):
    method test_incr (line 40) | def test_incr(self):
    method test_remaining_without_total (line 48) | def test_remaining_without_total(self):
    method test_remaining_with_total (line 53) | def test_remaining_with_total(self):
    method test_remaining_negative_protection (line 60) | def test_remaining_negative_protection(self):
    method test_thread_safety (line 67) | def test_thread_safety(self):
  class TestRunningTask (line 86) | class TestRunningTask(unittest.TestCase):
    method test_initialization (line 89) | def test_initialization(self):
  class TestOpenICLInferConcurrentTask (line 99) | class TestOpenICLInferConcurrentTask(unittest.TestCase):
    method setUp (line 102) | def setUp(self):
    method test_initialization (line 142) | def test_initialization(self):
    method test_initialization_with_defaults (line 152) | def test_initialization_with_defaults(self):
    method test_get_command_single_gpu (line 174) | def test_get_command_single_gpu(self):
    method test_get_command_multi_gpu (line 189) | def test_get_command_multi_gpu(self, mock_randint):
    method test_get_command_with_backend (line 217) | def test_get_command_with_backend(self):
    method test_default_max_workers (line 240) | def test_default_max_workers(self):
    method test_inferencer_name_from_string (line 250) | def test_inferencer_name_from_string(self):
    method test_inferencer_name_from_class (line 256) | def test_inferencer_name_from_class(self):
    method test_set_default_value (line 266) | def test_set_default_value(self):
    method test_set_default_value_existing (line 273) | def test_set_default_value_existing(self):
    method test_build_inferencer_gen (line 281) | def test_build_inferencer_gen(self, mock_registry):
    method test_build_inferencer_chat (line 306) | def test_build_inferencer_chat(self, mock_registry):
    method test_build_inferencer_unsupported (line 330) | def test_build_inferencer_unsupported(self, mock_registry):
    method test_remaining_total (line 346) | def test_remaining_total(self):
    method test_remaining_total_with_none (line 366) | def test_remaining_total_with_none(self):
    method test_run_dataset_task_success (line 383) | def test_run_dataset_task_success(self, mock_mkdir, mock_get_path,
    method test_run_dataset_task_non_api_model (line 439) | def test_run_dataset_task_non_api_model(self, mock_build_dataset):
    method test_run_dataset_task_missing_template (line 469) | def test_run_dataset_task_missing_template(self, mock_build_dataset):
    method test_run_skips_existing_output (line 507) | def test_run_skips_existing_output(self, mock_task_abbr, mock_get_path,
    method test_run_with_cur_model (line 536) | def test_run_with_cur_model(self, mock_model_abbr, mock_task_abbr,
    method test_run_with_max_workers_config (line 556) | def test_run_with_max_workers_config(self):
    method test_build_inferencer_with_max_infer_workers (line 575) | def test_build_inferencer_with_max_infer_workers(self, mock_registry):
    method test_build_inferencer_with_model_config (line 596) | def test_build_inferencer_with_model_config(self, mock_registry):
    method tearDown (line 624) | def tearDown(self):

FILE: tests/utils/test_heartbeat.py
  class TestHeartBeatManager (line 12) | class TestHeartBeatManager(unittest.TestCase):
    method setUp (line 15) | def setUp(self):
    method test_initialization (line 19) | def test_initialization(self):
    method test_initialization_with_custom_fname (line 25) | def test_initialization_with_custom_fname(self):
    method test_last_heartbeat_nonexistent (line 30) | def test_last_heartbeat_nonexistent(self):
    method test_last_heartbeat_existing (line 36) | def test_last_heartbeat_existing(self):
    method test_start_heartbeat (line 54) | def test_start_heartbeat(self):
    method test_start_heartbeat_writes_periodically (line 72) | def test_start_heartbeat_writes_periodically(self):
    method test_start_heartbeat_daemon_thread (line 90) | def test_start_heartbeat_daemon_thread(self):
    method test_last_heartbeat_with_invalid_isoformat (line 100) | def test_last_heartbeat_with_invalid_isoformat(self):
    method test_heartbeat_thread_stops_on_event (line 111) | def test_heartbeat_thread_stops_on_event(self):
    method tearDown (line 126) | def tearDown(self):

FILE: tests/utils/test_infer_status.py
  class TestSafeReadWrite (line 15) | class TestSafeReadWrite(unittest.TestCase):
    method setUp (line 18) | def setUp(self):
    method test_safe_read (line 24) | def test_safe_read(self):
    method test_safe_write (line 29) | def test_safe_write(self):
    method tearDown (line 35) | def tearDown(self):
  class TestInferStatusManager (line 42) | class TestInferStatusManager(unittest.TestCase):
    method setUp (line 45) | def setUp(self):
    method test_initialization (line 51) | def test_initialization(self):
    method test_update_status (line 63) | def test_update_status(self):
    method test_update_total (line 70) | def test_update_total(self):
    method test_update_completed (line 77) | def test_update_completed(self):
    method test_update_multiple (line 84) | def test_update_multiple(self):
    method test_update_partial (line 93) | def test_update_partial(self):
    method test_write_task_status (line 103) | def test_write_task_status(self):
    method test_get_task_status_existing (line 118) | def test_get_task_status_existing(self):
    method test_get_task_status_nonexistent (line 130) | def test_get_task_status_nonexistent(self):
    method test_get_task_status_with_children (line 137) | def test_get_task_status_with_children(self):
    method test_maybe_write_only_on_change (line 162) | def test_maybe_write_only_on_change(self):
    method tearDown (line 182) | def tearDown(self):

FILE: tests/utils/test_text_postprocessors.py
  class TestTextPostprocessors (line 9) | class TestTextPostprocessors(unittest.TestCase):
    method test_general_postprocess (line 12) | def test_general_postprocess(self):
    method test_general_cn_postprocess (line 17) | def test_general_cn_postprocess(self, mock_cut):
    method test_first_capital_postprocess (line 23) | def test_first_capital_postprocess(self):
    method test_last_capital_postprocess (line 27) | def test_last_capital_postprocess(self):
    method test_think_pred_postprocess (line 31) | def test_think_pred_postprocess(self):
    method test_first_option_postprocess (line 38) | def test_first_option_postprocess(self):
    method test_last_option_postprocess (line 42) | def test_last_option_postprocess(self):
    method test_first_number_postprocess (line 46) | def test_first_number_postprocess(self):
    method test_multiple_select_postprocess (line 50) | def test_multiple_select_postprocess(self):
    method test_xml_tag_postprocessor (line 54) | def test_xml_tag_postprocessor(self):
    method test_general_eval_wrapper_postprocess (line 58) | def test_general_eval_wrapper_postprocess(self):
    method test_match_answer_pattern (line 64) | def test_match_answer_pattern(self):
    method test_extract_non_reasoning_content (line 69) | def test_extract_non_reasoning_content(self):

FILE: tools/case_analyzer.py
  function parse_args (line 15) | def parse_args():
  class BadcaseShower (line 37) | class BadcaseShower:
    method __init__ (line 40) | def __init__(self, cfg: ConfigDict) -> None:
    method run (line 51) | def run(self):
  function dispatch_tasks (line 168) | def dispatch_tasks(cfg, force=False):
  function main (line 182) | def main():

FILE: tools/chatml_format_test.py
  function parse_args (line 8) | def parse_args():
  function collect_file_paths (line 17) | def collect_file_paths(path):
  function main (line 37) | def main():

FILE: tools/collect_code_preds.py
  function parse_args (line 16) | def parse_args():
  function gpt_python_postprocess (line 46) | def gpt_python_postprocess(ori_prompt: str, text: str) -> str:
  function wizardcoder_postprocess (line 73) | def wizardcoder_postprocess(text: str) -> str:
  function collect_preds (line 91) | def collect_preds(filename: str):
  function main (line 127) | def main():

FILE: tools/compare_configs.py
  function get_files (line 9) | def get_files(folder, extensions, ignore_folder=[]):
  function compare_folders (line 34) | def compare_folders(folder1, folder2, extensions, ignore_folder):
  function main (line 64) | def main():

FILE: tools/convert_alignmentbench.py
  function extract_predictions_from_json (line 10) | def extract_predictions_from_json(input_folder):
  function process_jsonl (line 48) | def process_jsonl(file_path):
  function save_as_json (line 66) | def save_as_json(data, output_file='./alignment_bench.json'):
  function parse_args (line 71) | def parse_args():

FILE: tools/list_configs.py
  function parse_args (line 8) | def parse_args():
  function main (line 20) | def main():

FILE: tools/prediction_merger.py
  function parse_args (line 12) | def parse_args():
  class PredictionMerger (line 24) | class PredictionMerger:
    method __init__ (line 26) | def __init__(self, cfg: ConfigDict) -> None:
    method run (line 32) | def run(self):
  function dispatch_tasks (line 75) | def dispatch_tasks(cfg):
  function main (line 87) | def main():

FILE: tools/prompt_viewer.py
  function parse_args (line 18) | def parse_args():
  function parse_model_cfg (line 37) | def parse_model_cfg(model_cfg: ConfigDict) -> Dict[str, ConfigDict]:
  function parse_dataset_cfg (line 44) | def parse_dataset_cfg(dataset_cfg: ConfigDict) -> Dict[str, ConfigDict]:
  function print_prompts (line 51) | def print_prompts(model_cfg, dataset_cfg, count=1):
  function main (line 177) | def main():

FILE: tools/test_api_model.py
  function test_model (line 156) | def test_model(model_cfg: ConfigDict):
  function parse_args (line 173) | def parse_args():
  function parse_model_cfg (line 182) | def parse_model_cfg(model_cfg: ConfigDict) -> Dict[str, ConfigDict]:
  function main (line 189) | def main():

FILE: tools/update_dataset_suffix.py
  function get_prompt_hash (line 16) | def get_prompt_hash(dataset_cfg: Union[ConfigDict, List[ConfigDict]]) ->...
  function get_hash (line 67) | def get_hash(path):
  function check_and_rename (line 76) | def check_and_rename(filepath):
  function main (line 114) | def main():

FILE: tools/viz_multi_model.py
  function main (line 15) | def main(