SYMBOL INDEX (1323 symbols across 246 files)

FILE: eval/docs/en/conf.py
  function get_version (line 33) | def get_version():

FILE: eval/docs/zh_cn/conf.py
  function get_version (line 33) | def get_version():
  function builder_inited_handler (line 224) | def builder_inited_handler(app):
  function setup (line 228) | def setup(app):

FILE: eval/opencompass/datasets/GaokaoBench.py
  class GaokaoBenchDataset (line 13) | class GaokaoBenchDataset(BaseDataset):
    method load (line 16) | def load(path: str):
  class GaokaoBenchEvaluator (line 28) | class GaokaoBenchEvaluator(BaseEvaluator):
    method __init__ (line 30) | def __init__(self, question_type) -> None:
    method do_predictions_postprocess (line 35) | def do_predictions_postprocess(self, model_output, answer_lenth=None):
    method ensure_same_length (line 82) | def ensure_same_length(self, pred, refr):
    method score (line 87) | def score(self, predictions, references):
  function _gaokao_register (line 126) | def _gaokao_register(question_type):

FILE: eval/opencompass/datasets/TheoremQA.py
  class TheoremQADataset (line 11) | class TheoremQADataset(BaseDataset):
    method load (line 14) | def load(path: str):
  function TheoremQA_postprocess (line 19) | def TheoremQA_postprocess(text: str) -> str:

FILE: eval/opencompass/datasets/advglue.py
  class AdvDataset (line 11) | class AdvDataset(BaseDataset):
    method __init__ (line 21) | def __init__(
    method aug_with_original_data (line 33) | def aug_with_original_data(self, dataset):
    method load (line 51) | def load(self, path):
  class AdvSst2Dataset (line 78) | class AdvSst2Dataset(AdvDataset):
    method __init__ (line 81) | def __init__(self, **kwargs):
  class AdvQqpDataset (line 89) | class AdvQqpDataset(AdvDataset):
    method __init__ (line 92) | def __init__(self, **kwargs):
  class AdvMnliDataset (line 102) | class AdvMnliDataset(AdvDataset):
    method __init__ (line 105) | def __init__(self, **kwargs):
  class AdvMnliMMDataset (line 115) | class AdvMnliMMDataset(AdvDataset):
    method __init__ (line 118) | def __init__(self, **kwargs):
  class AdvQnliDataset (line 127) | class AdvQnliDataset(AdvDataset):
    method __init__ (line 130) | def __init__(self, **kwargs):
  class AdvRteDataset (line 139) | class AdvRteDataset(AdvDataset):
    method __init__ (line 142) | def __init__(self, **kwargs):
  class AccDropEvaluator (line 149) | class AccDropEvaluator(AccEvaluator):
    method __init__ (line 152) | def __init__(self) -> None:
    method score (line 155) | def score(self, predictions: List, references: List) -> dict:

FILE: eval/opencompass/datasets/afqmcd.py
  class AFQMCDataset_V2 (line 11) | class AFQMCDataset_V2(BaseDataset):
    method load (line 14) | def load(path):

FILE: eval/opencompass/datasets/agieval/agieval.py
  class AGIEvalDataset (line 15) | class AGIEvalDataset(BaseDataset):
    method load (line 18) | def load(path: str, name: str, setting_name: str):
  class AGIEvalDataset_v2 (line 36) | class AGIEvalDataset_v2(BaseDataset):
    method load (line 39) | def load(path: str, name: str, setting_name: str):
  class AGIEvalEvaluator (line 63) | class AGIEvalEvaluator(BaseEvaluator):
    method score (line 65) | def score(self, predictions, references):

FILE: eval/opencompass/datasets/agieval/constructions.py
  class TaskSchema (line 5) | class TaskSchema(object):
    method __init__ (line 7) | def __init__(self,
    method to_dict (line 21) | def to_dict(self):
  class AgiInstance (line 33) | class AgiInstance(object):
    method __init__ (line 35) | def __init__(self, task_description, data_source, task_schema, output,
    method to_dict (line 44) | def to_dict(self):
  class ChatGPTSchema (line 55) | class ChatGPTSchema(object):
    method __init__ (line 57) | def __init__(self, context=None, metadata=''):
    method to_dict (line 61) | def to_dict(self):
  class ResultsForHumanSchema (line 65) | class ResultsForHumanSchema(object):
    method __init__ (line 67) | def __init__(self,
    method to_dict (line 87) | def to_dict(self):
    method to_tsv (line 101) | def to_tsv(result_list, path):

FILE: eval/opencompass/datasets/agieval/dataset_loader.py
  function convert_zero_shot (line 30) | def convert_zero_shot(line, dataset_name):
  function convert_zero_shot_CoT_stage1 (line 65) | def convert_zero_shot_CoT_stage1(line, dataset_name):
  function combine_prompt (line 94) | def combine_prompt(prompt_path,
  function _lazy_load_enc (line 169) | def _lazy_load_enc():
  function concat_prompt (line 176) | def concat_prompt(demos,
  function concat_prompt_chat_mode (line 209) | def concat_prompt_chat_mode(demos,
  function convert_few_shot (line 241) | def convert_few_shot(line, dataset_name, demo, n_shot, chat_mode=False):
  function load_dataset (line 274) | def load_dataset(dataset_name,
  function generate_second_stage_input (line 322) | def generate_second_stage_input(dataset_name,
  function load_dataset_as_result_schema (line 358) | def load_dataset_as_result_schema(dataset_name, parent_path):

FILE: eval/opencompass/datasets/agieval/evaluation.py
  function convert_to_set (line 6) | def convert_to_set(item):
  function evaluate_single_sample (line 16) | def evaluate_single_sample(dataset_name, prediction, label):

FILE: eval/opencompass/datasets/agieval/math_equivalence.py
  function _fix_fracs (line 5) | def _fix_fracs(string):
  function _fix_a_slash_b (line 37) | def _fix_a_slash_b(string):
  function _remove_right_units (line 52) | def _remove_right_units(string):
  function _fix_sqrt (line 62) | def _fix_sqrt(string):
  function _strip_string (line 77) | def _strip_string(string):
  function is_equiv (line 147) | def is_equiv(str1, str2, verbose=False):

FILE: eval/opencompass/datasets/agieval/post_process.py
  function extract_last_line (line 8) | def extract_last_line(string):
  function remove_few_shot_prefix (line 17) | def remove_few_shot_prefix(string: str):
  function try_parse_few_shot_qa_single_answer (line 29) | def try_parse_few_shot_qa_single_answer(string, setting_name, language='...
  function try_parse_few_shot_pattern (line 46) | def try_parse_few_shot_pattern(string: str, dataset_name, setting_name):
  function parse_few_shot_qa_single_answer (line 64) | def parse_few_shot_qa_single_answer(string, setting_name, language='en'):
  function find_first_capital_letter (line 73) | def find_first_capital_letter(answer):
  function extract_answer_in_bracket (line 82) | def extract_answer_in_bracket(answer, prefix='【', suffix='】'):
  function parse_math_answer (line 92) | def parse_math_answer(setting_name, raw_string):
  function parse_qa_multiple_answer (line 170) | def parse_qa_multiple_answer(string, setting_name):
  function post_process (line 180) | def post_process(dataset_name, setting_name, prediction):

FILE: eval/opencompass/datasets/agieval/utils.py
  function read_jsonl (line 5) | def read_jsonl(path):
  function save_jsonl (line 21) | def save_jsonl(lines, directory):
  function extract_answer (line 27) | def extract_answer(js):

FILE: eval/opencompass/datasets/anli.py
  class AnliDataset (line 8) | class AnliDataset(BaseDataset):
    method load (line 11) | def load(path: str):

FILE: eval/opencompass/datasets/arc.py
  class ARCDataset (line 11) | class ARCDataset(BaseDataset):
    method load (line 14) | def load(path: str):

FILE: eval/opencompass/datasets/ax.py
  class AXDataset_V2 (line 11) | class AXDataset_V2(BaseDataset):
    method load (line 14) | def load(path: str):

FILE: eval/opencompass/datasets/base.py
  class BaseDataset (line 9) | class BaseDataset:
    method __init__ (line 11) | def __init__(self, reader_cfg: Optional[Dict] = {}, **kwargs):
    method _init_reader (line 15) | def _init_reader(self, **kwargs):
    method train (line 19) | def train(self):
    method test (line 23) | def test(self):
    method load (line 27) | def load(**kwargs) -> Union[Dataset, DatasetDict]:

FILE: eval/opencompass/datasets/bbh.py
  class BBHDataset (line 15) | class BBHDataset(BaseDataset):
    method load (line 18) | def load(path: str, name: str):
  function bbh_mcq_postprocess (line 26) | def bbh_mcq_postprocess(text: str) -> str:
  function bbh_freeform_postprocess (line 41) | def bbh_freeform_postprocess(text: str) -> str:
  class BBHEvaluator (line 53) | class BBHEvaluator(BaseEvaluator):
    method score (line 55) | def score(self, predictions, references):

FILE: eval/opencompass/datasets/boolq.py
  class BoolQDataset (line 11) | class BoolQDataset(BaseDataset):
    method load (line 14) | def load(**kwargs):
  class BoolQDataset_V2 (line 29) | class BoolQDataset_V2(BaseDataset):
    method load (line 32) | def load(path):
  class BoolQDataset_V3 (line 43) | class BoolQDataset_V3(BaseDataset):
    method load (line 46) | def load(path):

FILE: eval/opencompass/datasets/bustum.py
  class bustumDataset_V2 (line 11) | class bustumDataset_V2(BaseDataset):
    method load (line 14) | def load(path):

FILE: eval/opencompass/datasets/c3.py
  class C3Dataset (line 11) | class C3Dataset(BaseDataset):
    method load (line 14) | def load(path: str):
  class C3Dataset_V2 (line 57) | class C3Dataset_V2(BaseDataset):
    method load (line 60) | def load(path: str):

FILE: eval/opencompass/datasets/cb.py
  class CBDataset_V2 (line 11) | class CBDataset_V2(BaseDataset):
    method load (line 14) | def load(path):

FILE: eval/opencompass/datasets/ceval.py
  class CEvalDataset (line 11) | class CEvalDataset(BaseDataset):
    method load (line 14) | def load(path: str, name: str):

FILE: eval/opencompass/datasets/chid.py
  class CHIDDataset (line 11) | class CHIDDataset(BaseDataset):
    method load (line 14) | def load(**kwargs):
  class CHIDDataset_V2 (line 29) | class CHIDDataset_V2(BaseDataset):
    method load (line 32) | def load(path):

FILE: eval/opencompass/datasets/civilcomments.py
  class CivilCommentsDataset (line 9) | class CivilCommentsDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/cluewsc.py
  class CluewscDataset (line 11) | class CluewscDataset(BaseDataset):
    method load (line 14) | def load(**kwargs):
  class CluewscDataset_V2 (line 39) | class CluewscDataset_V2(BaseDataset):
    method load (line 42) | def load(path):

FILE: eval/opencompass/datasets/cmb.py
  class CMBDataset (line 12) | class CMBDataset(BaseDataset):
    method load (line 15) | def load(path: str):

FILE: eval/opencompass/datasets/cmmlu.py
  class CMMLUDataset (line 12) | class CMMLUDataset(BaseDataset):
    method load (line 15) | def load(path: str, name: str):

FILE: eval/opencompass/datasets/cmnli.py
  class cmnliDataset_V2 (line 11) | class cmnliDataset_V2(BaseDataset):
    method load (line 14) | def load(path):

FILE: eval/opencompass/datasets/cmrc.py
  class CMRCDataset (line 11) | class CMRCDataset(BaseDataset):
    method load (line 14) | def load(path: str):
  function cmrc_postprocess (line 44) | def cmrc_postprocess(text: str) -> str:

FILE: eval/opencompass/datasets/commonsenseqa.py
  class commonsenseqaDataset (line 9) | class commonsenseqaDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/copa.py
  class COPADataset_V2 (line 11) | class COPADataset_V2(BaseDataset):
    method load (line 14) | def load(path):

FILE: eval/opencompass/datasets/crowspairs.py
  class crowspairsDataset (line 13) | class crowspairsDataset(BaseDataset):
    method load (line 16) | def load(**kwargs):
  class crowspairsDataset_V2 (line 28) | class crowspairsDataset_V2(BaseDataset):
    method load (line 31) | def load(**kwargs):
  function crowspairs_postprocess (line 41) | def crowspairs_postprocess(text: str) -> str:
  class CrowspairsEvaluator (line 60) | class CrowspairsEvaluator(BaseEvaluator):
    method __init__ (line 64) | def __init__(self) -> None:
    method score (line 67) | def score(self, predictions: List, references: List) -> dict:

FILE: eval/opencompass/datasets/csl.py
  class CslDataset (line 11) | class CslDataset(BaseDataset):
    method load (line 14) | def load(**kwargs):
  class CslDataset_V2 (line 29) | class CslDataset_V2(BaseDataset):
    method load (line 32) | def load(path):

FILE: eval/opencompass/datasets/cvalues.py
  class CValuesDataset (line 11) | class CValuesDataset(BaseDataset):
    method load (line 14) | def load(path):

FILE: eval/opencompass/datasets/drcd.py
  class DRCDDataset (line 11) | class DRCDDataset(BaseDataset):
    method load (line 14) | def load(path: str):
  function drcd_postprocess (line 44) | def drcd_postprocess(text: str) -> str:

FILE: eval/opencompass/datasets/drop.py
  class dropDataset (line 9) | class dropDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/ds1000.py
  class DS1000Dataset (line 25) | class DS1000Dataset(BaseDataset):
    method get_data (line 42) | def get_data(self, problem_path: str) -> dict:
    method load (line 79) | def load(self,
  function ds1000_postprocess (line 117) | def ds1000_postprocess(text: str) -> str:
  function ds1000_matplotlib_postprocess (line 138) | def ds1000_matplotlib_postprocess(text: str) -> str:
  class DS1000Evaluator (line 160) | class DS1000Evaluator(BaseEvaluator):
    method __init__ (line 163) | def __init__(self, num_workers=16) -> None:
    method score_single (line 166) | def score_single(self, pred, refer):
    method score (line 247) | def score(self, predictions, references):
  class Command (line 258) | class Command(object):
    method __init__ (line 261) | def __init__(self, cmd):
    method run (line 265) | def run(self, timeout):
  function import_source_file (line 296) | def import_source_file(fname, modname):

FILE: eval/opencompass/datasets/eprstmt.py
  class eprstmtDataset_V2 (line 11) | class eprstmtDataset_V2(BaseDataset):
    method load (line 14) | def load(path):

FILE: eval/opencompass/datasets/flores.py
  class FloresFirst100Dataset (line 11) | class FloresFirst100Dataset(BaseDataset):
    method load (line 14) | def load(name):
  function flores_postprocess (line 26) | def flores_postprocess(text: str) -> str:
  function flores_postprocess_chinese (line 32) | def flores_postprocess_chinese(text: str) -> str:

FILE: eval/opencompass/datasets/game24.py
  function get_current_numbers (line 156) | def get_current_numbers(y: str) -> str:
  class Game24Dataset (line 161) | class Game24Dataset(BaseDataset):
    method load (line 164) | def load(path: str):
  class Game24PromptWrapper (line 170) | class Game24PromptWrapper:
    method __init__ (line 181) | def __init__(self):
    method standard_prompt_wrap (line 187) | def standard_prompt_wrap(x: str, y: str = '') -> str:
    method cot_prompt_wrap (line 191) | def cot_prompt_wrap(x: str, y: str = '') -> str:
    method propose_prompt_wrap (line 195) | def propose_prompt_wrap(x: str, y: str = '') -> str:
    method value_prompt_wrap (line 204) | def value_prompt_wrap(x: str, y: str) -> str:
    method value_outputs_unwrap (line 213) | def value_outputs_unwrap(x: str, y: str, value_outputs: list) -> float:
  function game24_postprocess (line 227) | def game24_postprocess(output: str):
  class Game24Evaluator (line 233) | class Game24Evaluator(BaseEvaluator):
    method __init__ (line 235) | def __init__(self) -> None:
    method check_nums (line 240) | def check_nums(self, prediction, reference):
    method score (line 250) | def score(self, predictions: List, references: List) -> dict:

FILE: eval/opencompass/datasets/govrepcrs.py
  class GovRepcrsDataset (line 9) | class GovRepcrsDataset(BaseDataset):
    method load (line 12) | def load(path: str):

FILE: eval/opencompass/datasets/gsm8k.py
  function gsm8k_dataset_postprocess (line 5) | def gsm8k_dataset_postprocess(text: str) -> str:
  function gsm8k_postprocess (line 10) | def gsm8k_postprocess(text: str) -> str:

FILE: eval/opencompass/datasets/hellaswag.py
  class hellaswagDataset (line 11) | class hellaswagDataset(BaseDataset):
    method load (line 14) | def load(**kwargs):
  class hellaswagDataset_V2 (line 27) | class hellaswagDataset_V2(BaseDataset):
    method load (line 30) | def load(**kwargs):
  class hellaswagDataset_V3 (line 47) | class hellaswagDataset_V3(BaseDataset):
    method load (line 50) | def load(path):

FILE: eval/opencompass/datasets/huggingface.py
  class HFDataset (line 9) | class HFDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/humaneval.py
  class HumanEvaluator (line 9) | class HumanEvaluator(BaseEvaluator):
    method __init__ (line 12) | def __init__(self, k: List[int] = [1, 10, 100]) -> None:
    method score (line 26) | def score(self, predictions, references):
  function humaneval_postprocess (line 43) | def humaneval_postprocess(text: str) -> str:
  function humaneval_gpt_postprocess (line 67) | def humaneval_gpt_postprocess(text: str) -> str:

FILE: eval/opencompass/datasets/humanevalx.py
  class HumanevalXDataset (line 27) | class HumanevalXDataset(BaseDataset):
    method load (line 30) | def load(path, language, **kwargs):
    method _stream_jsonl_all (line 38) | def _stream_jsonl_all(filename: str) -> Iterable[Dict]:
  class HumanevalXEvaluator (line 52) | class HumanevalXEvaluator(BaseEvaluator):
    method __init__ (line 72) | def __init__(self,
    method score (line 87) | def score(self, predictions, references):
    method _code_eval_service (line 119) | def _code_eval_service(self, file_path):
  function _clean_up_code (line 145) | def _clean_up_code(text: str, language_type: str) -> str:

FILE: eval/opencompass/datasets/iwslt2017.py
  class IWSLT2017Dataset (line 9) | class IWSLT2017Dataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/jigsawmultilingual.py
  class JigsawMultilingualDataset (line 11) | class JigsawMultilingualDataset(BaseDataset):
    method load (line 14) | def load(path, label, lang):

FILE: eval/opencompass/datasets/lambada.py
  class lambadaDataset (line 14) | class lambadaDataset(BaseDataset):
    method load (line 17) | def load(**kwargs):
  class LambadaEvaluator (line 31) | class LambadaEvaluator(BaseEvaluator):
    method __init__ (line 33) | def __init__(self) -> None:
    method score (line 36) | def score(self, predictions, references):

FILE: eval/opencompass/datasets/lcsts.py
  class LCSTSDataset (line 11) | class LCSTSDataset(BaseDataset):
    method load (line 14) | def load(path: str):
  function lcsts_postprocess (line 35) | def lcsts_postprocess(text: str) -> str:

FILE: eval/opencompass/datasets/leval/evaluators.py
  class LEvalGPTEvaluator (line 10) | class LEvalGPTEvaluator(BaseEvaluator):
    method __init__ (line 22) | def __init__(self,
    method run_judge_pair (line 29) | def run_judge_pair(self, prompt_template, system_prompt, question,
    method score (line 63) | def score(self, predictions: List, references: List) -> dict:

FILE: eval/opencompass/datasets/leval/leval_coursera.py
  class LEvalCourseraDataset (line 9) | class LEvalCourseraDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_financial_qa.py
  class LEvalFinancialQADataset (line 9) | class LEvalFinancialQADataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_gov_report_summ.py
  class LEvalGovReportSummDataset (line 9) | class LEvalGovReportSummDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_gsm100.py
  function gsm100_dataset_postprocess (line 9) | def gsm100_dataset_postprocess(text: str) -> str:
  function gsm100_postprocess (line 14) | def gsm100_postprocess(text: str) -> str:
  class LEvalGSM100Dataset (line 40) | class LEvalGSM100Dataset(BaseDataset):
    method load (line 43) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_legal_contract_qa.py
  class LEvalLegalContractQADataset (line 9) | class LEvalLegalContractQADataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_meeting_summ.py
  class LEvalMeetingSummDataset (line 9) | class LEvalMeetingSummDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_multidoc_qa.py
  class LEvalMultidocQADataset (line 9) | class LEvalMultidocQADataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_narrattive_qa.py
  class LEvalNarrativeQADataset (line 9) | class LEvalNarrativeQADataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_natural_question.py
  class LEvalNaturalQuestionDataset (line 9) | class LEvalNaturalQuestionDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_news_summ.py
  class LEvalNewsSummDataset (line 9) | class LEvalNewsSummDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_paper_assistant.py
  class LEvalPaperAssistantDataset (line 9) | class LEvalPaperAssistantDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_patent_summ.py
  class LEvalPatentSummDataset (line 9) | class LEvalPatentSummDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_quality.py
  class LEvalQualityDataset (line 9) | class LEvalQualityDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_review_summ.py
  class LEvalReviewSummDataset (line 9) | class LEvalReviewSummDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_scientific_qa.py
  class LEvalScientificQADataset (line 9) | class LEvalScientificQADataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_topic_retrieval.py
  class LEvalTopicRetrievalDataset (line 9) | class LEvalTopicRetrievalDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_tpo.py
  class LEvalTPODataset (line 9) | class LEvalTPODataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/leval/leval_tvshow_summ.py
  class LEvalTVShowSummDataset (line 9) | class LEvalTVShowSummDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/evaluators.py
  function normalize_answer (line 15) | def normalize_answer(s):
  function normalize_zh_answer (line 34) | def normalize_zh_answer(s):
  class LongBenchF1Evaluator (line 54) | class LongBenchF1Evaluator(BaseEvaluator):
    method __init__ (line 56) | def __init__(self, language: str = 'en') -> None:
    method score (line 61) | def score(self, predictions: List, references: List) -> dict:
  class LongBenchCountEvaluator (line 116) | class LongBenchCountEvaluator(BaseEvaluator):
    method score (line 118) | def score(self, predictions: List, references: List) -> dict:
  class LongBenchRetrievalEvaluator (line 137) | class LongBenchRetrievalEvaluator(BaseEvaluator):
    method __init__ (line 139) | def __init__(self, language: str = 'en') -> None:
    method score (line 144) | def score(self, predictions: List, references: List) -> dict:
  class LongBenchRougeEvaluator (line 171) | class LongBenchRougeEvaluator(BaseEvaluator):
    method __init__ (line 173) | def __init__(self, language: str = 'en') -> None:
    method score (line 178) | def score(self, predictions: List, references: List) -> dict:
  class LongBenchCodeSimEvaluator (line 206) | class LongBenchCodeSimEvaluator(BaseEvaluator):
    method score (line 208) | def score(self, predictions: List, references: List) -> dict:
  class LongBenchClassificationEvaluator (line 233) | class LongBenchClassificationEvaluator(BaseEvaluator):
    method score (line 235) | def score(self, predictions: List, references: List) -> dict:

FILE: eval/opencompass/datasets/longbench/longbench_2wikim_qa.py
  class LongBench2wikimqaDataset (line 9) | class LongBench2wikimqaDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_dureader.py
  class LongBenchdureaderDataset (line 9) | class LongBenchdureaderDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_gov_report.py
  class LongBenchgov_reportDataset (line 9) | class LongBenchgov_reportDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_hotpot_qa.py
  class LongBenchhotpotqaDataset (line 9) | class LongBenchhotpotqaDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_lcc.py
  class LongBenchlccDataset (line 9) | class LongBenchlccDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_lsht.py
  class LongBenchlshtDataset (line 9) | class LongBenchlshtDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_multifieldqa_en.py
  class LongBenchmultifieldqa_enDataset (line 9) | class LongBenchmultifieldqa_enDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_multifieldqa_zh.py
  class LongBenchmultifieldqa_zhDataset (line 9) | class LongBenchmultifieldqa_zhDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_musique.py
  class LongBenchmusiqueDataset (line 9) | class LongBenchmusiqueDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_narrative_qa.py
  class LongBenchnarrativeqaDataset (line 9) | class LongBenchnarrativeqaDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_nq.py
  class LongBenchnqDataset (line 9) | class LongBenchnqDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_passage_count.py
  class LongBenchpassage_countDataset (line 9) | class LongBenchpassage_countDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_passage_retrieval_en.py
  class LongBenchpassage_retrieval_enDataset (line 9) | class LongBenchpassage_retrieval_enDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_passage_retrieval_zh.py
  class LongBenchpassage_retrieval_zhDataset (line 9) | class LongBenchpassage_retrieval_zhDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_qasper.py
  class LongBenchqasperDataset (line 9) | class LongBenchqasperDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_qmsum.py
  class LongBenchqmsumDataset (line 9) | class LongBenchqmsumDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_repobench.py
  class LongBenchrepobenchDataset (line 9) | class LongBenchrepobenchDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_trec.py
  class LongBenchtrecDataset (line 9) | class LongBenchtrecDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_trivia_qa.py
  class LongBenchtriviaqaDataset (line 9) | class LongBenchtriviaqaDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/longbench/longbench_vcsum.py
  class LongBenchvcsumDataset (line 9) | class LongBenchvcsumDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/math.py
  class MATHDataset (line 13) | class MATHDataset(BaseDataset):
    method load (line 16) | def load(path: str):
  function math_postprocess (line 70) | def math_postprocess(text: str) -> str:
  class MATHEvaluator (line 141) | class MATHEvaluator(BaseEvaluator):
    method score (line 143) | def score(self, predictions, references):
    method _fix_fracs (line 158) | def _fix_fracs(self, string):
    method _fix_a_slash_b (line 189) | def _fix_a_slash_b(self, string):
    method _remove_right_units (line 203) | def _remove_right_units(self, string):
    method _fix_sqrt (line 213) | def _fix_sqrt(self, string):
    method _strip_string (line 227) | def _strip_string(self, string):
    method is_equiv (line 294) | def is_equiv(self, str1, str2, verbose=False):

FILE: eval/opencompass/datasets/mbpp.py
  class MBPPDataset (line 15) | class MBPPDataset(BaseDataset):
    method load (line 18) | def load(path: str):
  class TimeOutException (line 33) | class TimeOutException(Exception):
  class MBPPEvaluator (line 38) | class MBPPEvaluator(BaseEvaluator):
    method score (line 40) | def score(self, predictions, references):
    method _process_answer (line 65) | def _process_answer(self, text):
    method _process_test (line 80) | def _process_test(self, test_case, pred):
    method swallow_io (line 86) | def swallow_io(self):
    method time_limit (line 94) | def time_limit(self, seconds: float):
    class WriteOnlyStringIO (line 106) | class WriteOnlyStringIO(io.StringIO):
      method read (line 109) | def read(self, *args, **kwargs):
      method readline (line 112) | def readline(self, *args, **kwargs):
      method readlines (line 115) | def readlines(self, *args, **kwargs):
      method readable (line 118) | def readable(self, *args, **kwargs):
    class redirect_stdin (line 122) | class redirect_stdin(contextlib._RedirectStream):  # type: ignore
  class MBPPEvaluator2 (line 127) | class MBPPEvaluator2(MBPPEvaluator):
    method _process_answer (line 130) | def _process_answer(self, text):

FILE: eval/opencompass/datasets/mmlu.py
  class MMLUDataset (line 12) | class MMLUDataset(BaseDataset):
    method load (line 15) | def load(path: str, name: str):

FILE: eval/opencompass/datasets/multirc.py
  class MultiRCDataset (line 11) | class MultiRCDataset(BaseDataset):
    method load (line 14) | def load(path: str):
  class MultiRCDataset_V2 (line 44) | class MultiRCDataset_V2(BaseDataset):
    method load (line 47) | def load(path: str):

FILE: eval/opencompass/datasets/music_theory_bench.py
  function convert_dataset (line 5) | def convert_dataset(dataset_item):
  class MusicTheoryBenchDataset (line 23) | class MusicTheoryBenchDataset(BaseDataset):
    method load (line 25) | def load(path: str, name: str):

FILE: eval/opencompass/datasets/narrativeqa.py
  class NarrativeQADataset (line 9) | class NarrativeQADataset(BaseDataset):
    method load (line 12) | def load(path: str):

FILE: eval/opencompass/datasets/natural_question.py
  class NaturalQuestionDataset (line 14) | class NaturalQuestionDataset(BaseDataset):
    method load (line 17) | def load(path: str):
  class NQEvaluator (line 37) | class NQEvaluator(BaseEvaluator):
    method score (line 39) | def score(self, predictions, references):

FILE: eval/opencompass/datasets/obqa.py
  class OBQADataset (line 9) | class OBQADataset(BaseDataset):
    method load (line 12) | def load(**kwargs):
  class OBQADataset_V2 (line 25) | class OBQADataset_V2(BaseDataset):
    method load (line 28) | def load(**kwargs):

FILE: eval/opencompass/datasets/piqa.py
  class piqaDataset_V2 (line 9) | class piqaDataset_V2(BaseDataset):
    method load (line 12) | def load(**kwargs):
  class piqaDataset_V3 (line 29) | class piqaDataset_V3(BaseDataset):
    method load (line 32) | def load(**kwargs):

FILE: eval/opencompass/datasets/qasper.py
  class QASPERDataset (line 9) | class QASPERDataset(BaseDataset):
    method load (line 12) | def load(path: str):

FILE: eval/opencompass/datasets/qaspercut.py
  class QASPERCUTDataset (line 9) | class QASPERCUTDataset(BaseDataset):
    method load (line 12) | def load(path: str):

FILE: eval/opencompass/datasets/race.py
  class RaceDataset (line 9) | class RaceDataset(BaseDataset):
    method load (line 12) | def load(path: str, name: str):

FILE: eval/opencompass/datasets/realtoxicprompts.py
  class RealToxicPromptsDataset (line 9) | class RealToxicPromptsDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/record.py
  class ReCoRDDataset (line 11) | class ReCoRDDataset(BaseDataset):
    method load (line 14) | def load(path: str):
  class ReCoRDDataset_V2 (line 46) | class ReCoRDDataset_V2(BaseDataset):
    method load (line 49) | def load(path: str):
  function ReCoRD_postprocess (line 74) | def ReCoRD_postprocess(text: str) -> str:

FILE: eval/opencompass/datasets/safety.py
  class SafetyDataset (line 9) | class SafetyDataset(BaseDataset):
    method load (line 12) | def load(path):

FILE: eval/opencompass/datasets/siqa.py
  class siqaDataset_V2 (line 9) | class siqaDataset_V2(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/squad20.py
  class SQuAD20Dataset (line 11) | class SQuAD20Dataset(BaseDataset):
    method load (line 14) | def load(path: str):
  class SQuAD20Evaluator (line 43) | class SQuAD20Evaluator(BaseEvaluator):
    method score (line 45) | def score(self, predictions, references):

FILE: eval/opencompass/datasets/storycloze.py
  class storyclozeDataset (line 9) | class storyclozeDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):
  class storyclozeDataset_V2 (line 29) | class storyclozeDataset_V2(BaseDataset):
    method load (line 32) | def load(**kwargs):

FILE: eval/opencompass/datasets/strategyqa.py
  function strategyqa_pred_postprocess (line 7) | def strategyqa_pred_postprocess(text: str) -> str:
  function strategyqa_dataset_postprocess (line 17) | def strategyqa_dataset_postprocess(text: str) -> str:

FILE: eval/opencompass/datasets/summedits.py
  class SummeditsDataset_V2 (line 11) | class SummeditsDataset_V2(BaseDataset):
    method load (line 14) | def load(path: str):

FILE: eval/opencompass/datasets/summscreen.py
  class SummScreenDataset (line 9) | class SummScreenDataset(BaseDataset):
    method load (line 12) | def load(path: str):

FILE: eval/opencompass/datasets/tnews.py
  class TNewsDataset (line 11) | class TNewsDataset(BaseDataset):
    method load (line 14) | def load(**kwargs):
  class TNewsDataset_V2 (line 47) | class TNewsDataset_V2(BaseDataset):
    method load (line 50) | def load(path):

FILE: eval/opencompass/datasets/triviaqa.py
  class TriviaQADataset (line 14) | class TriviaQADataset(BaseDataset):
    method load (line 17) | def load(path: str):
  class TriviaQAEvaluator (line 36) | class TriviaQAEvaluator(BaseEvaluator):
    method score (line 38) | def score(self, predictions, references):

FILE: eval/opencompass/datasets/triviaqarc.py
  class TriviaQArcDataset (line 9) | class TriviaQArcDataset(BaseDataset):
    method load (line 12) | def load(path: str):

FILE: eval/opencompass/datasets/truthfulqa.py
  class TruthfulQADataset (line 15) | class TruthfulQADataset(BaseDataset):
    method load (line 18) | def load(**kwargs):
  class TruthfulQAEvaluator (line 37) | class TruthfulQAEvaluator(BaseEvaluator):
    method __init__ (line 72) | def __init__(self,
    method score (line 109) | def score(self, predictions, references):
    method basic_score (line 118) | def basic_score(self, predictions, references):
    method prompt (line 176) | def prompt(self, pred, refer, metric):
    method api_score (line 182) | def api_score(self, predictions, references):

FILE: eval/opencompass/datasets/tydiqa.py
  class TydiQADataset (line 12) | class TydiQADataset(BaseDataset):
    method load (line 15) | def load(**kwargs):
  class TydiQAEvaluator (line 26) | class TydiQAEvaluator(BaseEvaluator):
    method f1_score (line 29) | def f1_score(self, prediction, ground_truth):
    method exact_match_score (line 41) | def exact_match_score(self, prediction, ground_truth):
    method metric_max_over_ground_truths (line 45) | def metric_max_over_ground_truths(self, metric_fn, prediction,
    method score (line 53) | def score(self, predictions, references):

FILE: eval/opencompass/datasets/wic.py
  class WiCDataset (line 11) | class WiCDataset(BaseDataset):
    method load (line 14) | def load(**kwargs):
  class WiCDataset_V2 (line 31) | class WiCDataset_V2(BaseDataset):
    method load (line 34) | def load(path):

FILE: eval/opencompass/datasets/winograd.py
  class winogradDataset (line 9) | class winogradDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/winogrande.py
  class winograndeDataset (line 9) | class winograndeDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):
  class winograndeDataset_V2 (line 26) | class winograndeDataset_V2(BaseDataset):
    method load (line 29) | def load(**kwargs):

FILE: eval/opencompass/datasets/wnli.py
  class wnliDataset (line 9) | class wnliDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/wsc.py
  class WSCDataset (line 11) | class WSCDataset(BaseDataset):
    method load (line 14) | def load(**kwargs):
  class WSCDataset_V2 (line 40) | class WSCDataset_V2(BaseDataset):
    method load (line 43) | def load(path):
  class WSCDataset_V3 (line 62) | class WSCDataset_V3(BaseDataset):
    method load (line 65) | def load(path):

FILE: eval/opencompass/datasets/xcopa.py
  class XCOPADataset (line 9) | class XCOPADataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/xiezhi.py
  class XiezhiDataset (line 13) | class XiezhiDataset(BaseDataset):
    method load (line 16) | def load(path: str, name: str):
  class XiezhiRetriever (line 54) | class XiezhiRetriever(BaseRetriever):
    method __init__ (line 56) | def __init__(self,
    method retrieve (line 63) | def retrieve(self):

FILE: eval/opencompass/datasets/xlsum.py
  class XLSUMDataset (line 9) | class XLSUMDataset(BaseDataset):
    method load (line 12) | def load(**kwargs):

FILE: eval/opencompass/datasets/xsum.py
  class XsumDataset (line 11) | class XsumDataset(BaseDataset):
    method load (line 14) | def load(path: str):
  function Xsum_postprocess (line 34) | def Xsum_postprocess(text: str) -> str:

FILE: eval/opencompass/metrics/dump_results.py
  class DumpResults (line 11) | class DumpResults(BaseMetric):
    method __init__ (line 25) | def __init__(self,
    method process (line 34) | def process(self, data_batch, data_samples) -> None:
    method compute_metrics (line 49) | def compute_metrics(self, results: list) -> dict:

FILE: eval/opencompass/metrics/mme_score.py
  class MMEMetric (line 10) | class MMEMetric(BaseMetric):
    method __init__ (line 34) | def __init__(self,
    method process (line 39) | def process(self, data_batch, data_samples) -> None:
    method compute_metrics (line 48) | def compute_metrics(self, results: list) -> dict:

FILE: eval/opencompass/metrics/seedbench.py
  class SEEDBenchAcc (line 23) | class SEEDBenchAcc(BaseMetric):
    method process (line 26) | def process(self, data_batch, data_samples) -> None:
    method compute_metrics (line 40) | def compute_metrics(self, results: list) -> dict:

FILE: eval/opencompass/models/base.py
  class BaseModel (line 10) | class BaseModel:
    method __init__ (line 26) | def __init__(self,
    method generate (line 41) | def generate(self, inputs: List[str], max_out_len: int) -> List[str]:
    method get_ppl (line 53) | def get_ppl(self,
    method get_token_len (line 71) | def get_token_len(self, prompt: str) -> int:
    method parse_template (line 81) | def parse_template(self, prompt_template: PromptType, mode: str) -> str:
    method get_ppl_from_template (line 95) | def get_ppl_from_template(self,
    method generate_from_template (line 108) | def generate_from_template(self, templates: List[PromptType],
    method get_token_len_from_template (line 119) | def get_token_len_from_template(
    method to (line 144) | def to(self, device):
  class LMTemplateParser (line 148) | class LMTemplateParser:
    method __init__ (line 155) | def __init__(self, meta_template: Optional[Dict] = None):
    method parse_template (line 183) | def parse_template(self, prompt_template: PromptType, mode: str) -> str:
    method _split_rounds (line 278) | def _split_rounds(
    method _update_role_dict (line 305) | def _update_role_dict(self, prompt: Union[List, str,
    method _prompt2str (line 325) | def _prompt2str(self,
    method _role2str (line 357) | def _role2str(self,
    method _encode_speical_tokens (line 384) | def _encode_speical_tokens(self, prompt: List[Union[str, int]]) -> str:

FILE: eval/opencompass/models/base_api.py
  class BaseAPIModel (line 17) | class BaseAPIModel(BaseModel):
    method __init__ (line 34) | def __init__(self,
    method generate (line 50) | def generate(self, inputs: List[PromptType],
    method get_ppl (line 65) | def get_ppl(self,
    method get_token_len (line 82) | def get_token_len(self, prompt: str) -> int:
    method wait (line 105) | def wait(self):
    method to (line 112) | def to(self, device):
  class APITemplateParser (line 116) | class APITemplateParser:
    method __init__ (line 123) | def __init__(self, meta_template: Optional[Dict] = None):
    method parse_template (line 145) | def parse_template(self, prompt_template: PromptType,
    method _update_role_dict (line 254) | def _update_role_dict(self, prompts: Union[List, str]) -> Dict[str, Di...
    method _split_rounds (line 272) | def _split_rounds(
    method _prompt2api (line 305) | def _prompt2api(self,
    method _role2api_role (line 344) | def _role2api_role(self,
  class TokenBucket (line 375) | class TokenBucket:
    method __init__ (line 382) | def __init__(self, rate):
    method _add_tokens (line 387) | def _add_tokens(self):
    method get_token (line 394) | def get_token(self):

FILE: eval/opencompass/models/claude_api/claude_api.py
  class Claude (line 13) | class Claude(BaseAPIModel):
    method __init__ (line 28) | def __init__(
    method generate (line 53) | def generate(
    method _generate (line 75) | def _generate(

FILE: eval/opencompass/models/claude_api/postprocessors.py
  function gsm8k_postprocess (line 9) | def gsm8k_postprocess(text: str) -> str:
  function humaneval_postprocess (line 29) | def humaneval_postprocess(text: str) -> str:
  function lcsts_postprocess (line 53) | def lcsts_postprocess(text: str) -> str:
  function mbpp_postprocess (line 61) | def mbpp_postprocess(text: str) -> str:
  function strategyqa_pred_postprocess (line 75) | def strategyqa_pred_postprocess(text: str) -> str:
  function record_postprocess (line 85) | def record_postprocess(text: str) -> str:
  function humaneval_claude2_postprocess (line 94) | def humaneval_claude2_postprocess(text: str) -> str:
  function xsum_postprocess (line 100) | def xsum_postprocess(text: str) -> str:
  function yes_no_postprocess (line 106) | def yes_no_postprocess(text: str) -> str:

FILE: eval/opencompass/models/glm.py
  class GLM130B (line 16) | class GLM130B(BaseModel):
    method __init__ (line 18) | def __init__(self,
    method _load_model (line 34) | def _load_model(self, **kwargs):
    method get_token_len (line 121) | def get_token_len(self, prompt: str) -> int:
    method choice (line 132) | def choice(self, inputs, choices):
    method generate (line 166) | def generate(self, inputs: List[str], max_out_len: int) -> List[str]:
    method get_logits (line 287) | def get_logits(self, inputs: List[str]):
    method get_ppl (line 368) | def get_ppl(self,

FILE: eval/opencompass/models/huggingface.py
  class HuggingFace (line 16) | class HuggingFace(BaseModel):
    method __init__ (line 53) | def __init__(self,
    method _load_tokenizer (line 86) | def _load_tokenizer(self, path: str, tokenizer_path: Optional[str],
    method _load_model (line 128) | def _load_model(self,
    method generate (line 154) | def generate(self, inputs: List[str], max_out_len: int,
    method _batch_generate (line 174) | def _batch_generate(self, inputs: List[str], max_out_len: int,
    method _single_generate (line 217) | def _single_generate(self, inputs: List[str], max_out_len: int,
    method get_logits (line 255) | def get_logits(self, inputs: List[str]):
    method get_ppl (line 282) | def get_ppl(self,
    method _get_ppl (line 308) | def _get_ppl(self,
    method get_token_len (line 349) | def get_token_len(self, prompt: str) -> int:
  class HuggingFaceCausalLM (line 362) | class HuggingFaceCausalLM(HuggingFace):
    method _load_model (line 388) | def _load_model(self,

FILE: eval/opencompass/models/intern_model.py
  class InternLM (line 9) | class InternLM(BaseModel):
    method __init__ (line 11) | def __init__(self,
    method _load_model (line 34) | def _load_model(self,
    method _load_tokenizer (line 52) | def _load_tokenizer(self, tokenizer_path: str, tokenizer_type: str,
    method get_token_len (line 63) | def get_token_len(self, prompt: str) -> int:
    method generate (line 75) | def generate(self, inputs: List[str], max_out_len: int) -> List[str]:
    method get_ppl (line 91) | def get_ppl(self,

FILE: eval/opencompass/models/llama2.py
  class Llama2 (line 13) | class Llama2(BaseModel):
    method __init__ (line 26) | def __init__(
    method _load_model (line 46) | def _load_model(self,
    method _load_tokenizer (line 57) | def _load_tokenizer(self, tokenizer_path: str):
    method generate (line 61) | def generate(self, inputs: List[str], max_out_len: int) -> List[str]:
    method get_ppl (line 75) | def get_ppl(self,
    method get_token_len (line 103) | def get_token_len(self, prompt: str) -> int:
  class Llama2Chat (line 107) | class Llama2Chat(BaseModel):
    method __init__ (line 120) | def __init__(
    method _load_model (line 140) | def _load_model(self,
    method _load_tokenizer (line 151) | def _load_tokenizer(self, tokenizer_path: str):
    method generate (line 155) | def generate(self,
    method get_token_len (line 208) | def get_token_len(self, prompt: str) -> int:

FILE: eval/opencompass/models/openai_api.py
  class OpenAI (line 22) | class OpenAI(BaseAPIModel):
    method __init__ (line 57) | def __init__(self,
    method generate (line 98) | def generate(
    method _generate (line 129) | def _generate(self, input: str or PromptList, max_out_len: int,
    method get_token_len (line 254) | def get_token_len(self, prompt: str) -> int:
    method bin_trim (line 268) | def bin_trim(self, prompt: str, num_token: int) -> str:

FILE: eval/opencompass/models/turbomind.py
  function valid_str (line 14) | def valid_str(string, coding='utf-8'):
  class TurboMindModel (line 24) | class TurboMindModel(BaseModel):
    method __init__ (line 43) | def __init__(
    method generate (line 73) | def generate(
    method wait (line 102) | def wait(self):
    method _generate (line 109) | def _generate(self, input: str or PromptList, max_out_len: int,

FILE: eval/opencompass/multimodal/datasets/mmbench.py
  function decode_base64_to_image (line 13) | def decode_base64_to_image(base64_string) -> Image:
  class MMBenchDataset (line 21) | class MMBenchDataset(Dataset):
    method __init__ (line 32) | def __init__(self,
    method __len__ (line 40) | def __len__(self) -> None:
    method __getitem__ (line 43) | def __getitem__(self, idx: int) -> dict:
    method load_from_df (line 75) | def load_from_df(self, idx: int, key: str) -> Optional[str]:

FILE: eval/opencompass/multimodal/datasets/mme.py
  class MMEDataset (line 11) | class MMEDataset(Dataset):
    method __init__ (line 26) | def __init__(self, data_dir: str, pipeline: List[dict]) -> None:
    method load_data (line 30) | def load_data(self, data_dir: str):
    method __len__ (line 68) | def __len__(self) -> None:
    method __getitem__ (line 71) | def __getitem__(self, idx: int) -> dict:

FILE: eval/opencompass/multimodal/datasets/seedbench.py
  class SEEDBenchDataset (line 17) | class SEEDBenchDataset(Dataset):
    method __init__ (line 32) | def __init__(
    method __len__ (line 60) | def __len__(self) -> None:
    method __getitem__ (line 63) | def __getitem__(self, idx: str) -> dict:
    method get_index (line 163) | def get_index(self, num_frames, num_segments):

FILE: eval/opencompass/multimodal/models/instructblip/blip2_vicuna_instruct.py
  class InstructBlipInferencer (line 16) | class InstructBlipInferencer(Blip2Base):
    method __init__ (line 18) | def __init__(
    method forward (line 106) | def forward(self, batch):
    method concat_text_input_output (line 112) | def concat_text_input_output(self, input_ids, input_atts, output_ids,
    method pack_inputs (line 134) | def pack_inputs(self, batch):
    method generate (line 142) | def generate(

FILE: eval/opencompass/multimodal/models/instructblip/post_processor.py
  class InstructBlipMMBenchPostProcessor (line 7) | class InstructBlipMMBenchPostProcessor:
    method __init__ (line 10) | def __init__(self) -> None:
    method __call__ (line 13) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:
    method _extract_key_words (line 21) | def _extract_key_words(self, output_text: str) -> str:
  class InstructBlipCOCOCaptionPostProcessor (line 35) | class InstructBlipCOCOCaptionPostProcessor:
    method __init__ (line 38) | def __init__(self) -> None:
    method __call__ (line 41) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:
  class InstructBlipVQAPostProcessor (line 54) | class InstructBlipVQAPostProcessor:
    method __init__ (line 57) | def __init__(self) -> None:
    method __call__ (line 60) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:
  class InstructBlipScienceQAPostProcessor (line 72) | class InstructBlipScienceQAPostProcessor:
    method __init__ (line 75) | def __init__(self) -> None:
    method __call__ (line 78) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:
  class InstructBlipVSRPostProcessor (line 97) | class InstructBlipVSRPostProcessor:
    method __init__ (line 100) | def __init__(self) -> None:
    method __call__ (line 103) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:

FILE: eval/opencompass/multimodal/models/instructblip/prompt_constructor.py
  class InstructBlipMMBenchPromptConstructor (line 6) | class InstructBlipMMBenchPromptConstructor:
    method __init__ (line 14) | def __init__(self, image_prompt: str = '', reply_prompt: str = '') -> ...
    method __call__ (line 18) | def __call__(self, inputs: dict) -> dict:
    method _process (line 33) | def _process(self, data_samples: List[DataSample]) -> str:
  class InstructBlipCOCOCaotionPromptConstructor (line 58) | class InstructBlipCOCOCaotionPromptConstructor(
    method _process (line 62) | def _process(self, data_samples: List[DataSample]) -> str:
  class InstructBlipVQAPromptConstructor (line 68) | class InstructBlipVQAPromptConstructor(InstructBlipMMBenchPromptConstruc...
    method _process (line 71) | def _process(self, data_samples: List[DataSample]) -> str:
  class InstructBlipScienceQAPromptConstructor (line 81) | class InstructBlipScienceQAPromptConstructor(
    method _process (line 87) | def _process(self, data_samples: List[DataSample]) -> str:
  class InstructBlipVSRPromptConstructor (line 112) | class InstructBlipVSRPromptConstructor(InstructBlipMMBenchPromptConstruc...
    method _process (line 115) | def _process(self, data_samples: List[DataSample]) -> str:

FILE: eval/opencompass/multimodal/models/llama_adapter_v2_multimodal/llama_adapter.py
  class LLaMA_adapter (line 18) | class LLaMA_adapter(nn.Module):
    method __init__ (line 20) | def __init__(self,
    method clip_encode_image (line 93) | def clip_encode_image(self, x):
    method forward_visual (line 119) | def forward_visual(self, imgs):
    method forward (line 136) | def forward(self, visual_query, tokens, start_pos: int):
    method pack_inputs (line 165) | def pack_inputs(self, batch):
    method generate (line 173) | def generate(self, batch):
  class LLaMA_adapter_v2 (line 258) | class LLaMA_adapter_v2(nn.Module):
    method __init__ (line 260) | def __init__(self,
    method forward (line 304) | def forward(self, batch):

FILE: eval/opencompass/multimodal/models/llama_adapter_v2_multimodal/post_processor.py
  class LlamaAadapterMMBenchPostProcessor (line 4) | class LlamaAadapterMMBenchPostProcessor:
    method __init__ (line 7) | def __init__(self) -> None:
    method __call__ (line 10) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:

FILE: eval/opencompass/multimodal/models/llama_adapter_v2_multimodal/prompt_constructor.py
  class LlamaAadapterMMBenchPromptConstructor (line 6) | class LlamaAadapterMMBenchPromptConstructor:
    method __init__ (line 14) | def __init__(self, image_prompt: str = '', reply_prompt: str = '') -> ...
    method __call__ (line 18) | def __call__(self, inputs: dict) -> dict:
    method _process (line 33) | def _process(self, data_samples: List[DataSample]) -> str:

FILE: eval/opencompass/multimodal/models/llava/llava.py
  function load_package (line 16) | def load_package():
  class KeywordsStoppingCriteria (line 25) | class KeywordsStoppingCriteria(StoppingCriteria):
    method __init__ (line 28) | def __init__(self, keywords, tokenizer, input_ids):
    method __call__ (line 34) | def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTe...
  class LLaVA (line 49) | class LLaVA(nn.Module):
    method __init__ (line 61) | def __init__(
    method generate (line 107) | def generate(self, batch):
    method forward (line 155) | def forward(self, batch):

FILE: eval/opencompass/multimodal/models/llava/post_processor.py
  class LLaVABasePostProcessor (line 1) | class LLaVABasePostProcessor:
    method __init__ (line 4) | def __init__(self) -> None:
    method __call__ (line 7) | def __call__(self, outputs: str, stop_str: str) -> str:
  class LLaVAVSRPostProcessor (line 15) | class LLaVAVSRPostProcessor(LLaVABasePostProcessor):
    method __init__ (line 18) | def __init__(self) -> None:
    method __call__ (line 21) | def __call__(self, outputs: str, stop_str: str) -> str:

FILE: eval/opencompass/multimodal/models/llava/prompt_constructor.py
  class LLaVABasePromptConstructor (line 9) | class LLaVABasePromptConstructor:
    method __init__ (line 19) | def __init__(self,
    method __call__ (line 30) | def __call__(self, inputs: dict) -> tuple:
    method _build_prompt (line 57) | def _build_prompt(self, data_sample):
  class LLaVAMMBenchPromptConstructor (line 61) | class LLaVAMMBenchPromptConstructor(LLaVABasePromptConstructor):
    method __init__ (line 71) | def __init__(self,
    method _build_prompt (line 77) | def _build_prompt(self, data_sample):
  class LLaVAVQAPromptConstructor (line 89) | class LLaVAVQAPromptConstructor(LLaVABasePromptConstructor):
    method __init__ (line 99) | def __init__(self,
    method _build_prompt (line 105) | def _build_prompt(self, data_sample):
  class LLaVAScienceQAPromptConstructor (line 111) | class LLaVAScienceQAPromptConstructor(LLaVABasePromptConstructor):
    method __init__ (line 123) | def __init__(self,
    method _build_prompt (line 129) | def _build_prompt(self, data_sample):

FILE: eval/opencompass/multimodal/models/minigpt_4/minigpt_4.py
  class LayerNorm (line 15) | class LayerNorm(nn.LayerNorm):
    method forward (line 18) | def forward(self, x: torch.Tensor):
  function load_package (line 24) | def load_package():
  class MiniGPT4Inferencer (line 41) | class MiniGPT4Inferencer(MiniGPT4):
    method __init__ (line 57) | def __init__(self,
    method forward (line 91) | def forward(self, batch):
    method encode_img (line 99) | def encode_img(self, image):
    method pack_inputs (line 151) | def pack_inputs(self, batch):
    method generate (line 158) | def generate(self, batch):
    method loss (line 207) | def loss(self, batch):

FILE: eval/opencompass/multimodal/models/minigpt_4/post_processor.py
  class MiniGPT4MMBenchPostProcessor (line 7) | class MiniGPT4MMBenchPostProcessor:
    method __init__ (line 10) | def __init__(self) -> None:
    method __call__ (line 13) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:
    method _extract_key_words (line 24) | def _extract_key_words(self, output_text: str) -> str:
  class MiniGPT4COCOCaptionPostProcessor (line 38) | class MiniGPT4COCOCaptionPostProcessor:
    method __init__ (line 41) | def __init__(self) -> None:
    method __call__ (line 44) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:
  class MiniGPT4ScienceQAPostProcessor (line 60) | class MiniGPT4ScienceQAPostProcessor:
    method __init__ (line 63) | def __init__(self) -> None:
    method __call__ (line 66) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:
  class MiniGPT4VQAPostProcessor (line 85) | class MiniGPT4VQAPostProcessor:
    method __init__ (line 88) | def __init__(self) -> None:
    method __call__ (line 91) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:
  class MiniGPT4VSRPostProcessor (line 104) | class MiniGPT4VSRPostProcessor:
    method __init__ (line 107) | def __init__(self) -> None:
    method __call__ (line 110) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:
  class MiniGPT4MMEPostProcessor (line 124) | class MiniGPT4MMEPostProcessor(MiniGPT4MMBenchPostProcessor):
    method __init__ (line 127) | def __init__(self) -> None:
    method __call__ (line 130) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:

FILE: eval/opencompass/multimodal/models/minigpt_4/prompt_constructor.py
  class MiniGPT4MMBenchPromptConstructor (line 6) | class MiniGPT4MMBenchPromptConstructor:
    method __init__ (line 14) | def __init__(self, image_prompt: str = '', reply_prompt: str = '') -> ...
    method __call__ (line 18) | def __call__(self, inputs: dict) -> dict:
    method _process (line 33) | def _process(self, data_samples: List[DataSample]) -> str:
  class MiniGPT4COCOCaotionPromptConstructor (line 58) | class MiniGPT4COCOCaotionPromptConstructor(MiniGPT4MMBenchPromptConstruc...
    method _process (line 61) | def _process(self, data_samples: List[DataSample]) -> str:
  class MiniGPT4ScienceQAPromptConstructor (line 67) | class MiniGPT4ScienceQAPromptConstructor(MiniGPT4MMBenchPromptConstructor):
    method _process (line 72) | def _process(self, data_samples: List[DataSample]) -> str:
  class MiniGPT4VQAPromptConstructor (line 97) | class MiniGPT4VQAPromptConstructor(MiniGPT4MMBenchPromptConstructor):
    method _process (line 100) | def _process(self, data_samples: List[DataSample]) -> str:
  class MiniGPT4VSRPromptConstructor (line 110) | class MiniGPT4VSRPromptConstructor(MiniGPT4MMBenchPromptConstructor):
    method _process (line 113) | def _process(self, data_samples: List[DataSample]) -> str:
  class MiniGPT4SEEDBenchPromptConstructor (line 123) | class MiniGPT4SEEDBenchPromptConstructor(MiniGPT4MMBenchPromptConstructor):
    method _process (line 125) | def _process(self, data_samples: List[DataSample]) -> str:
  class MiniGPT4MMEPromptConstructor (line 143) | class MiniGPT4MMEPromptConstructor:
    method __init__ (line 151) | def __init__(self) -> None:
    method __call__ (line 158) | def __call__(self, inputs: dict) -> dict:
    method _process (line 173) | def _process(self, data_samples: List[DataSample]) -> str:

FILE: eval/opencompass/multimodal/models/minigpt_4/utils.py
  class StoppingCriteriaSub (line 11) | class StoppingCriteriaSub(StoppingCriteria):
    method __init__ (line 13) | def __init__(self, stops=[], encounters=1):
    method __call__ (line 17) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
  function download_cached_file (line 25) | def download_cached_file(url, check_hash=True, progress=False):
  function is_url (line 50) | def is_url(input_url):

FILE: eval/opencompass/multimodal/models/mplug_owl/mplug_owl.py
  class MplugOwl (line 15) | class MplugOwl(nn.Module):
    method __init__ (line 17) | def __init__(self,
    method forward (line 49) | def forward(self, batch):
    method generate (line 53) | def generate(self, batch):

FILE: eval/opencompass/multimodal/models/mplug_owl/post_processor.py
  class MplugOwlMMBenchPostProcessor (line 6) | class MplugOwlMMBenchPostProcessor:
    method __init__ (line 9) | def __init__(self) -> None:
    method __call__ (line 12) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:

FILE: eval/opencompass/multimodal/models/mplug_owl/prompt_constructor.py
  class MplugOwlMMBenchPromptConstructor (line 6) | class MplugOwlMMBenchPromptConstructor:
    method __init__ (line 14) | def __init__(self, image_prompt: str = '', reply_prompt: str = '') -> ...
    method __call__ (line 18) | def __call__(self, inputs: dict) -> dict:
    method _process (line 33) | def _process(self, data_samples: List[DataSample]) -> str:

FILE: eval/opencompass/multimodal/models/openflamingo/openflamingo.py
  class OpenFlamingoInferencer (line 12) | class OpenFlamingoInferencer(Flamingo):
    method __init__ (line 23) | def __init__(self,
    method preprocess_text (line 37) | def preprocess_text(self, data_samples: List[DataSample],
    method forward (line 70) | def forward(self, batch: dict) -> Union[DataSample, List[DataSample]]:
    method generate (line 77) | def generate(self, batch: dict) -> Union[DataSample, List[DataSample]]:

FILE: eval/opencompass/multimodal/models/otter/otter.py
  class Otter (line 11) | class Otter(nn.Module):
    method __init__ (line 24) | def __init__(self, model_path, load_bit, prompt_constructor,
    method forward (line 39) | def forward(self, batch):
    method generate (line 47) | def generate(self, batch):

FILE: eval/opencompass/multimodal/models/otter/post_processor.py
  class OTTERMMBenchPostProcessor (line 7) | class OTTERMMBenchPostProcessor:
    method __init__ (line 10) | def __init__(self) -> None:
    method __call__ (line 13) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:
    method _extract_key_words (line 24) | def _extract_key_words(self, output_text: str) -> str:
  class OTTERCOCOCaptionPostProcessor (line 34) | class OTTERCOCOCaptionPostProcessor:
    method __init__ (line 37) | def __init__(self) -> None:
    method __call__ (line 40) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:
  class OTTERScienceQAPostProcessor (line 57) | class OTTERScienceQAPostProcessor:
    method __init__ (line 60) | def __init__(self) -> None:
    method __call__ (line 63) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:
  class OTTERVQAPostProcessor (line 82) | class OTTERVQAPostProcessor:
    method __init__ (line 85) | def __init__(self) -> None:
    method __call__ (line 88) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:
  class OTTERVSRPostProcessor (line 101) | class OTTERVSRPostProcessor:
    method __init__ (line 104) | def __init__(self) -> None:
    method __call__ (line 107) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:
  class OTTERMMEPostProcessor (line 121) | class OTTERMMEPostProcessor(OTTERMMBenchPostProcessor):
    method __init__ (line 124) | def __init__(self) -> None:
    method __call__ (line 127) | def __call__(self, output_token: torch.tensor, tokenizer) -> str:

FILE: eval/opencompass/multimodal/models/otter/prompt_constructor.py
  class OTTERMMBenchPromptConstructor (line 7) | class OTTERMMBenchPromptConstructor:
    method __init__ (line 15) | def __init__(self, user_label: str = '', model_label: str = '') -> None:
    method __call__ (line 21) | def __call__(self, inputs: dict) -> dict:
    method _process (line 40) | def _process(self, data_samples: List[DataSample]) -> str:
  class OTTERCOCOCaotionPromptConstructor (line 63) | class OTTERCOCOCaotionPromptConstructor(OTTERMMBenchPromptConstructor):
    method _process (line 66) | def _process(self, data_samples: List[DataSample]) -> str:
  class OTTERScienceQAPromptConstructor (line 72) | class OTTERScienceQAPromptConstructor(OTTERMMBenchPromptConstructor):
    method _process (line 77) | def _process(self, data_samples: List[DataSample]) -> str:
  class OTTERVQAPromptConstructor (line 102) | class OTTERVQAPromptConstructor(OTTERMMBenchPromptConstructor):
    method _process (line 105) | def _process(self, data_samples: List[DataSample]) -> str:
  class OTTERVSRPromptConstructor (line 115) | class OTTERVSRPromptConstructor(OTTERMMBenchPromptConstructor):
    method _process (line 118) | def _process(self, data_samples: List[DataSample]) -> str:
  class OTTERSEEDBenchPromptConstructor (line 128) | class OTTERSEEDBenchPromptConstructor(OTTERMMBenchPromptConstructor):
    method _process (line 130) | def _process(self, data_samples: List[DataSample]) -> str:
  class OTTERMMEPromptConstructor (line 148) | class OTTERMMEPromptConstructor(OTTERMMBenchPromptConstructor):
    method _process (line 156) | def _process(self, data_samples: List[DataSample]) -> str:

FILE: eval/opencompass/multimodal/models/qwen/generation_utils.py
  function pad_batch (line 18) | def pad_batch(batch: BatchTokensType, pad_id: int,
  function get_ltor_masks_and_position_ids (line 27) | def get_ltor_masks_and_position_ids(
  function get_batch (line 91) | def get_batch(context_tokens: torch.LongTensor, eod_id: int):
  function get_stop_words_ids (line 106) | def get_stop_words_ids(chat_format: str, tokenizer: PreTrainedTokenizer):
  function make_context (line 116) | def make_context(
  function _decode_default (line 187) | def _decode_default(
  function _decode_chatml (line 220) | def _decode_chatml(tokens: List[int],
  function decode_tokens (line 256) | def decode_tokens(

FILE: eval/opencompass/multimodal/models/qwen/post_processor.py
  class QwenVLBasePostProcessor (line 6) | class QwenVLBasePostProcessor:
    method __init__ (line 9) | def __init__(self) -> None:
    method __call__ (line 12) | def __call__(self, pred: torch.tensor, tokenizer: Any,

FILE: eval/opencompass/multimodal/models/qwen/prompt_constructor.py
  class QwenVLMMBenchPromptConstructor (line 1) | class QwenVLMMBenchPromptConstructor:
    method __init__ (line 7) | def __init__(self) -> None:
    method __call__ (line 10) | def __call__(self, inputs: dict) -> str:

FILE: eval/opencompass/multimodal/models/qwen/qwen.py
  class QwenVLBase (line 18) | class QwenVLBase(nn.Module):
    method __init__ (line 33) | def __init__(
    method _build_embeds (line 62) | def _build_embeds(self, images, input_ids):
    method generate (line 80) | def generate(self, batch):
    method forward (line 104) | def forward(self, batch):
  class QwenVLChat (line 109) | class QwenVLChat(QwenVLBase):
    method __init__ (line 121) | def __init__(self,
    method generate (line 129) | def generate(self, batch):
  function forward_hack (line 165) | def forward_hack(self,

FILE: eval/opencompass/multimodal/models/visualglm/post_processor.py
  class VisualGLMBasePostProcessor (line 6) | class VisualGLMBasePostProcessor:
    method __init__ (line 9) | def __init__(self) -> None:
    method __call__ (line 12) | def __call__(self, output_token: torch.tensor, tokenizer: Any,
  class VisualGLMVSRPostProcessor (line 17) | class VisualGLMVSRPostProcessor(VisualGLMBasePostProcessor):
    method __init__ (line 20) | def __init__(self) -> None:
    method __call__ (line 23) | def __call__(self, output_token: torch.tensor, tokenizer: Any,

FILE: eval/opencompass/multimodal/models/visualglm/prompt_constructor.py
  class VisualGLMMMBenchPromptConstructor (line 4) | class VisualGLMMMBenchPromptConstructor:
    method __init__ (line 16) | def __init__(self,
    method __call__ (line 26) | def __call__(self, batch: dict) -> tuple:
  class VisualGLMBasePromptConstructor (line 58) | class VisualGLMBasePromptConstructor:
    method __init__ (line 66) | def __init__(self, system_prompt='') -> None:
    method __call__ (line 69) | def __call__(self, batch: dict) -> tuple:
  class VisualGLMVQAPromptConstructor (line 93) | class VisualGLMVQAPromptConstructor(VisualGLMBasePromptConstructor):
    method __init__ (line 101) | def __init__(self, system_prompt='') -> None:
    method __call__ (line 104) | def __call__(self, batch: dict) -> tuple:
  class VisualGLMScienceQAPromptConstructor (line 129) | class VisualGLMScienceQAPromptConstructor(VisualGLMBasePromptConstructor):
    method __init__ (line 139) | def __init__(self, system_prompt='') -> None:
    method __call__ (line 142) | def __call__(self, batch: dict) -> tuple:
  class VisualGLMIconQAPromptConstructor (line 181) | class VisualGLMIconQAPromptConstructor(VisualGLMBasePromptConstructor):
    method __init__ (line 189) | def __init__(self, system_prompt='') -> None:
    method __call__ (line 192) | def __call__(self, batch: dict) -> tuple:

FILE: eval/opencompass/multimodal/models/visualglm/visualglm.py
  class VisualGLM (line 13) | class VisualGLM(nn.Module):
    method __init__ (line 27) | def __init__(self,
    method encode_by_tokenizer (line 49) | def encode_by_tokenizer(self, multi_prompts, image_position):
    method generate (line 73) | def generate(self, batch):
    method forward (line 106) | def forward(self, batch):

FILE: eval/opencompass/openicl/icl_dataset_reader.py
  class DatasetReader (line 17) | class DatasetReader:
    method __init__ (line 58) | def __init__(self,
    method generate_input_field_prompt (line 97) | def generate_input_field_prompt(self, entry: Dict) -> str:
    method generate_input_field_corpus (line 115) | def generate_input_field_corpus(self,
    method generate_output_field_prompt (line 138) | def generate_output_field_prompt(self, entry: Dict) -> str:
    method generate_output_field_corpus (line 156) | def generate_output_field_corpus(self,
    method generate_input_output_field_prompt (line 179) | def generate_input_output_field_prompt(self, entry: Dict) -> str:
    method _check_dataset_reader (line 198) | def _check_dataset_reader(obj) -> 'DatasetReader':
    method __len__ (line 204) | def __len__(self):
    method __getitem__ (line 207) | def __getitem__(self, idx):
    method __repr__ (line 210) | def __repr__(self):
  function load_partial_dataset (line 216) | def load_partial_dataset(
  class DatasetEncoder (line 245) | class DatasetEncoder(torch.utils.data.Dataset):
    method __init__ (line 247) | def __init__(self,
    method init_dataset (line 265) | def init_dataset(self):
    method __len__ (line 283) | def __len__(self):
    method __getitem__ (line 286) | def __getitem__(self, idx):

FILE: eval/opencompass/openicl/icl_evaluator/icl_aucroc_evaluator.py
  class AUCROCEvaluator (line 12) | class AUCROCEvaluator(BaseEvaluator):
    method __init__ (line 21) | def __init__(self) -> None:
    method score (line 24) | def score(self, predictions: List, references: List) -> dict:

FILE: eval/opencompass/openicl/icl_evaluator/icl_base_evaluator.py
  class BaseEvaluator (line 4) | class BaseEvaluator:
    method __init__ (line 6) | def __init__(self) -> None:
    method score (line 9) | def score(self):

FILE: eval/opencompass/openicl/icl_evaluator/icl_em_evaluator.py
  class EMEvaluator (line 8) | class EMEvaluator(BaseEvaluator):
    method __init__ (line 11) | def __init__(self) -> None:
    method score (line 14) | def score(self, predictions, references):

FILE: eval/opencompass/openicl/icl_evaluator/icl_hf_evaluator.py
  class HuggingfaceEvaluator (line 12) | class HuggingfaceEvaluator(BaseEvaluator):
    method __init__ (line 22) | def __init__(self, metric: str, seed: int = 0) -> None:
    method _preprocess (line 27) | def _preprocess(self, predictions: List, references: List) -> dict:
    method _postprocess (line 42) | def _postprocess(self, scores: dict) -> dict:
    method score (line 53) | def score(self, predictions: List, references: List) -> dict:
  class AccEvaluator (line 84) | class AccEvaluator(HuggingfaceEvaluator):
    method __init__ (line 87) | def __init__(self) -> None:
    method _preprocess (line 90) | def _preprocess(self, predictions: List, references: List) -> dict:
    method _postprocess (line 115) | def _postprocess(self, scores: dict) -> dict:
  class RougeEvaluator (line 129) | class RougeEvaluator(HuggingfaceEvaluator):
    method __init__ (line 132) | def __init__(self) -> None:
    method _postprocess (line 135) | def _postprocess(self, scores: dict) -> dict:
  class BleuEvaluator (line 148) | class BleuEvaluator(HuggingfaceEvaluator):
    method __init__ (line 151) | def __init__(self) -> None:
  class MccEvaluator (line 156) | class MccEvaluator(AccEvaluator):
    method __init__ (line 159) | def __init__(self) -> None:
    method _postprocess (line 162) | def _postprocess(self, scores: dict) -> dict:
  class SquadEvaluator (line 176) | class SquadEvaluator(HuggingfaceEvaluator):
    method __init__ (line 179) | def __init__(self) -> None:
    method _preprocess (line 182) | def _preprocess(self, predictions: List, references: List) -> dict:
    method _postprocess (line 208) | def _postprocess(self, scores: dict) -> dict:
  class EDAccEvaluator (line 221) | class EDAccEvaluator(AccEvaluator):
    method __init__ (line 238) | def __init__(self) -> None:
    method _preprocess (line 243) | def _preprocess(self, predictions: List, references: List) -> dict:

FILE: eval/opencompass/openicl/icl_evaluator/icl_toxic_evaluator.py
  class PerspectiveAPIClient (line 19) | class PerspectiveAPIClient:
    method __init__ (line 31) | def __init__(self, key: str, batch_size: int, max_length: int = 20480):
    method _initialize (line 47) | def _initialize(self) -> None:
    method create_request_body (line 60) | def create_request_body(text: str) -> dict:
    method extract_toxicity_attributes (line 79) | def extract_toxicity_attributes(self, response: dict) -> dict:
    method get_toxicity_scores (line 98) | def get_toxicity_scores(self, predictions: List) -> dict:
  class ToxicEvaluator (line 153) | class ToxicEvaluator(BaseEvaluator):
    method __init__ (line 166) | def __init__(self,
    method get_scores (line 174) | def get_scores(self, predictions: List) -> dict:
    method get_metrics (line 185) | def get_metrics(self, scores: dict) -> dict:
    method score (line 216) | def score(self, predictions: List, references: List) -> dict:

FILE: eval/opencompass/openicl/icl_inferencer/icl_attack_inferencer.py
  class AttackInferencer (line 24) | class AttackInferencer(BaseInferencer):
    method __init__ (line 50) | def __init__(
    method predict (line 87) | def predict(self, adv_prompt) -> List:
    method get_generation_prompt_list_from_retriever_indices (line 171) | def get_generation_prompt_list_from_retriever_indices(

FILE: eval/opencompass/openicl/icl_inferencer/icl_base_inferencer.py
  class BaseInferencer (line 15) | class BaseInferencer:
    method __init__ (line 34) | def __init__(
    method inference (line 53) | def inference(self,
    method get_dataloader (line 85) | def get_dataloader(datalist: List[List], batch_size: int) -> DataLoader:
  function dump_results_dict (line 93) | def dump_results_dict(results_dict, filename):
  class GenInferencerOutputHandler (line 98) | class GenInferencerOutputHandler:
    method __init__ (line 104) | def __init__(self) -> None:
    method write_to_json (line 107) | def write_to_json(self, save_dir: str, filename: str):
    method save_results (line 111) | def save_results(self, origin_prompt, prediction, idx):
  class PPLInferencerOutputHandler (line 118) | class PPLInferencerOutputHandler:
    method __init__ (line 121) | def __init__(self) -> None:
    method write_to_json (line 124) | def write_to_json(self, save_dir: str, filename: str):
    method save_ice (line 128) | def save_ice(self, ice):
    method save_predictions (line 134) | def save_predictions(self, predictions):
    method save_prompt_and_ppl (line 140) | def save_prompt_and_ppl(self, label, input, prompt, ppl, idx):
  class CLPInferencerOutputHandler (line 151) | class CLPInferencerOutputHandler:
    method __init__ (line 154) | def __init__(self) -> None:
    method write_to_json (line 157) | def write_to_json(self, save_dir: str, filename: str):
    method save_ice (line 161) | def save_ice(self, ice):
    method save_prompt_and_condprob (line 167) | def save_prompt_and_condprob(self, input, prompt, cond_prob, idx, choi...

FILE: eval/opencompass/openicl/icl_inferencer/icl_clp_inferencer.py
  class CLPInferencer (line 22) | class CLPInferencer(BaseInferencer):
    method __init__ (line 50) | def __init__(
    method inference (line 74) | def inference(self,
    method __get_cond_prob (line 207) | def __get_cond_prob(self,

FILE: eval/opencompass/openicl/icl_inferencer/icl_gen_inferencer.py
  class GenInferencer (line 23) | class GenInferencer(BaseInferencer):
    method __init__ (line 44) | def __init__(
    method inference (line 73) | def inference(self,
    method get_generation_prompt_list_from_retriever_indices (line 150) | def get_generation_prompt_list_from_retriever_indices(
  class GLMChoiceInferencer (line 187) | class GLMChoiceInferencer(GenInferencer):
    method __init__ (line 189) | def __init__(self, *args, choices=['A', 'B', 'C', 'D'], **kwargs):
    method inference (line 193) | def inference(self,

FILE: eval/opencompass/openicl/icl_inferencer/icl_ppl_inferencer.py
  class PPLInferencer (line 21) | class PPLInferencer(BaseInferencer):
    method __init__ (line 36) | def __init__(
    method inference (line 58) | def inference(self,

FILE: eval/opencompass/openicl/icl_inferencer/icl_sc_inferencer.py
  class SCInferencer (line 21) | class SCInferencer(BaseInferencer):
    method __init__ (line 45) | def __init__(
    method inference (line 79) | def inference(self,
    method get_generation_prompt_list_from_retriever_indices (line 163) | def get_generation_prompt_list_from_retriever_indices(

FILE: eval/opencompass/openicl/icl_inferencer/icl_tot_inferencer.py
  class ToTInferencer (line 25) | class ToTInferencer(GenInferencer):
    method __init__ (line 69) | def __init__(
    method get_value (line 116) | def get_value(self,
    method get_values (line 146) | def get_values(self,
    method get_votes (line 177) | def get_votes(self, x: str, ys: List[str],
    method get_proposals (line 199) | def get_proposals(self, x: str, y: str) -> List[str]:
    method get_samples (line 218) | def get_samples(self, x: str, y: str, n_generate_sample: int,
    method tot_solve (line 245) | def tot_solve(self, x: str) -> str:
    method inference (line 307) | def inference(self,

FILE: eval/opencompass/openicl/icl_prompt_template.py
  class PromptTemplate (line 13) | class PromptTemplate:
    method __init__ (line 32) | def __init__(
    method _check_template_legacy (line 46) | def _check_template_legacy(self):
    method generate_ice_item (line 71) | def generate_ice_item(self, entry: Dict, label: Hashable) -> PromptType:
    method generate_label_prompt_item (line 105) | def generate_label_prompt_item(self,
    method generate_item (line 146) | def generate_item(
    method _check_prompt_template (line 199) | def _check_prompt_template(obj) -> 'PromptTemplate':
    method __repr__ (line 205) | def __repr__(self):
    method _encode_template (line 209) | def _encode_template(self, prompt_template: Union[List[Union[str, Dict]],

FILE: eval/opencompass/openicl/icl_retriever/icl_base_retriever.py
  class BaseRetriever (line 11) | class BaseRetriever:
    method __init__ (line 30) | def __init__(self,
    method retrieve (line 44) | def retrieve(self) -> List[List[int]]:
    method get_labels (line 47) | def get_labels(
    method generate_ice (line 73) | def generate_ice(self,
    method generate_label_prompt (line 114) | def generate_label_prompt(self,
    method generate_prompt_for_generate_task (line 157) | def generate_prompt_for_generate_task(
    method generate_prompt_for_adv_generate_task (line 210) | def generate_prompt_for_adv_generate_task(

FILE: eval/opencompass/openicl/icl_retriever/icl_bm25_retriever.py
  class BM25Retriever (line 18) | class BM25Retriever(BaseRetriever):
    method __init__ (line 47) | def __init__(self,
    method retrieve (line 63) | def retrieve(self) -> List[List]:

FILE: eval/opencompass/openicl/icl_retriever/icl_dpp_retriever.py
  class DPPRetriever (line 15) | class DPPRetriever(TopkRetriever):
    method __init__ (line 25) | def __init__(self,
    method dpp_search (line 44) | def dpp_search(self):
    method retrieve (line 74) | def retrieve(self):
    method get_kernel (line 77) | def get_kernel(self, embed, candidates):
  function fast_map_dpp (line 103) | def fast_map_dpp(kernel_matrix, max_length):

FILE: eval/opencompass/openicl/icl_retriever/icl_fix_k_retriever.py
  class FixKRetriever (line 15) | class FixKRetriever(BaseRetriever):
    method __init__ (line 32) | def __init__(self,
    method retrieve (line 39) | def retrieve(self, id_list: List[int]):

FILE: eval/opencompass/openicl/icl_retriever/icl_mdl_retriever.py
  class MDLRetriever (line 19) | class MDLRetriever(TopkRetriever):
    method __init__ (line 57) | def __init__(self,
    method topk_search (line 87) | def topk_search(self):
    method retrieve (line 140) | def retrieve(self):
    method cal_ce (line 145) | def cal_ce(self, input_texts: List[str], mask_length=None):
  function entropy (line 184) | def entropy(probs: np.array, label_dim: int = 0, mask=None):

FILE: eval/opencompass/openicl/icl_retriever/icl_random_retriever.py
  class RandomRetriever (line 14) | class RandomRetriever(BaseRetriever):
    method __init__ (line 22) | def __init__(self,
    method retrieve (line 31) | def retrieve(self):

FILE: eval/opencompass/openicl/icl_retriever/icl_topk_retriever.py
  class TopkRetriever (line 25) | class TopkRetriever(BaseRetriever):
    method __init__ (line 50) | def __init__(self,
    method create_index (line 86) | def create_index(self):
    method knn_search (line 106) | def knn_search(self, ice_num):
    method forward (line 119) | def forward(self, dataloader, process_bar=False, information=''):
    method retrieve (line 140) | def retrieve(self):
  class ListWrapper (line 145) | class ListWrapper:
    method __init__ (line 147) | def __init__(self, data: List[Any]):
    method to (line 150) | def to(self, device):
  function ignore_pad_dict (line 154) | def ignore_pad_dict(features):
  class DataCollatorWithPaddingAndCuda (line 163) | class DataCollatorWithPaddingAndCuda:
    method __call__ (line 170) | def __call__(

FILE: eval/opencompass/openicl/icl_retriever/icl_votek_retriever.py
  class VotekRetriever (line 15) | class VotekRetriever(TopkRetriever):
    method __init__ (line 22) | def __init__(self,
    method votek_select (line 37) | def votek_select(self,
    method vote_k_search (line 91) | def vote_k_search(self):
    method retrieve (line 98) | def retrieve(self):

FILE: eval/opencompass/openicl/icl_retriever/icl_zero_retriever.py
  class ZeroRetriever (line 11) | class ZeroRetriever(BaseRetriever):
    method __init__ (line 22) | def __init__(self, dataset, ice_eos_token: Optional[str] = '') -> None:
    method retrieve (line 25) | def retrieve(self, id_list: List[int] = None) -> List[List]:

FILE: eval/opencompass/openicl/utils/logging.py
  function get_logger (line 10) | def get_logger(name, level=LOG_LEVEL, log_file=None, file_mode='w'):

FILE: eval/opencompass/partitioners/base.py
  class BasePartitioner (line 10) | class BasePartitioner:
    method __init__ (line 18) | def __init__(self, out_dir: str):
    method __call__ (line 22) | def __call__(self, cfg: ConfigDict) -> List[Dict]:
    method partition (line 57) | def partition(self, models: List[ConfigDict], datasets: List[ConfigDict],

FILE: eval/opencompass/partitioners/mm_naive.py
  class MultimodalNaivePartitioner (line 12) | class MultimodalNaivePartitioner(BasePartitioner):
    method partition (line 22) | def partition(self, models: List[ConfigDict], datasets: List[ConfigDict],
    method __call__ (line 75) | def __call__(self, cfg: ConfigDict) -> List[Dict]:

FILE: eval/opencompass/partitioners/naive.py
  class NaivePartitioner (line 13) | class NaivePartitioner(BasePartitioner):
    method partition (line 21) | def partition(self, models: List[ConfigDict], datasets: List[ConfigDict],

FILE: eval/opencompass/partitioners/size.py
  class SizePartitioner (line 18) | class SizePartitioner(BasePartitioner):
    method __init__ (line 30) | def __init__(self,
    method partition (line 40) | def partition(self, models: List[ConfigDict], datasets: List[ConfigDict],
    method dataset_size (line 113) | def dataset_size(self):
    method split_dataset (line 121) | def split_dataset(self, dataset_cfg: ConfigDict) -> List[ConfigDict]:
    method get_factor (line 138) | def get_factor(self, dataset: ConfigDict) -> int:
    method get_cost (line 162) | def get_cost(self,

FILE: eval/opencompass/runners/base.py
  class BaseRunner (line 10) | class BaseRunner:
    method __init__ (line 20) | def __init__(self,
    method __call__ (line 31) | def __call__(self, tasks: List[Dict[str, Any]]):
    method launch (line 42) | def launch(self, tasks: List[Dict[str, Any]]) -> List[Tuple[str, int]]:
    method summarize (line 53) | def summarize(self, status: List[Tuple[str, int]]) -> None:

FILE: eval/opencompass/runners/dlc.py
  class DLCRunner (line 20) | class DLCRunner(BaseRunner):
    method __init__ (line 34) | def __init__(self,
    method launch (line 46) | def launch(self, tasks: List[Dict[str, Any]]) -> List[Tuple[str, int]]:
    method _launch (line 66) | def _launch(self, task_cfg: ConfigDict, random_sleep: bool = True):
    method _job_failed (line 155) | def _job_failed(self, return_code: int, output_paths: List[str]) -> bool:

FILE: eval/opencompass/runners/local.py
  class LocalRunner (line 23) | class LocalRunner(BaseRunner):
    method __init__ (line 36) | def __init__(self,
    method launch (line 46) | def launch(self, tasks: List[Dict[str, Any]]) -> List[Tuple[str, int]]:
    method _launch (line 131) | def _launch(self, task, gpu_ids, index):

FILE: eval/opencompass/runners/slurm.py
  class SlurmRunner (line 20) | class SlurmRunner(BaseRunner):
    method __init__ (line 36) | def __init__(self,
    method launch (line 52) | def launch(self, tasks: List[Dict[str, Any]]) -> List[Tuple[str, int]]:
    method _launch (line 72) | def _launch(self, task_cfg: ConfigDict, random_sleep: bool = True):
    method _job_failed (line 154) | def _job_failed(self, return_code: int, output_paths: List[str]) -> bool:

FILE: eval/opencompass/tasks/base.py
  class BaseTask (line 11) | class BaseTask:
    method __init__ (line 28) | def __init__(self, cfg: ConfigDict):
    method run (line 36) | def run(self):
    method get_command (line 40) | def get_command(self, cfg_path, template) -> str:
    method name (line 50) | def name(self) -> str:
    method __repr__ (line 57) | def __repr__(self) -> str:
    method get_log_path (line 60) | def get_log_path(self, file_extension: str = 'json') -> str:
    method get_output_paths (line 71) | def get_output_paths(self, file_extension: str = 'json') -> List[str]:

FILE: eval/opencompass/tasks/llm_eval.py
  class ModelEvaluator (line 12) | class ModelEvaluator:
    method __init__ (line 15) | def __init__(
    method parse_cfg (line 28) | def parse_cfg(self, cfg: ConfigDict):
    method evaluate (line 34) | def evaluate(self):
    method _load_dataset (line 46) | def _load_dataset(self, dataset_abbr: str):
    method _evaluate_dataset (line 61) | def _evaluate_dataset(self, dataset_abbr: str):
    method _make_prompt (line 73) | def _make_prompt(self, question: str, responses: List[str]) -> str:
    method _rank_models (line 85) | def _rank_models(self, output: str,

FILE: eval/opencompass/tasks/mm_infer.py
  function build_model (line 25) | def build_model(cfg):
  class MultimodalInferTask (line 46) | class MultimodalInferTask:
    method __init__ (line 52) | def __init__(self, cfg: ConfigDict):
    method name (line 62) | def name(self) -> str:
    method get_log_path (line 68) | def get_log_path(self, file_extension: str = 'json') -> str:
    method get_output_paths (line 82) | def get_output_paths(self, file_extension: str = 'json') -> List[str]:
    method get_command (line 98) | def get_command(self, cfg_path, template):
    method run (line 117) | def run(self):
  function parse_args (line 144) | def parse_args():

FILE: eval/opencompass/tasks/openicl_attack.py
  class OpenICLAttackTask (line 19) | class OpenICLAttackTask(BaseTask):
    method __init__ (line 29) | def __init__(self, cfg: ConfigDict):
    method get_command (line 36) | def get_command(self, cfg_path, template):
    method prompt_selection (line 55) | def prompt_selection(self, inferencer, prompts):
    method run (line 68) | def run(self):
    method _inference (line 91) | def _inference(self):
    method _set_default_value (line 184) | def _set_default_value(self, cfg: ConfigDict, key: str, value: Any):
  function parse_args (line 190) | def parse_args():

FILE: eval/opencompass/tasks/openicl_eval.py
  class OpenICLEvalTask (line 21) | class OpenICLEvalTask(BaseTask):
    method __init__ (line 32) | def __init__(self, cfg: ConfigDict):
    method get_command (line 37) | def get_command(self, cfg_path, template):
    method run (line 42) | def run(self):
    method _score (line 70) | def _score(self):
    method _extract_role_pred (line 175) | def _extract_role_pred(self, s: str, begin_str: Optional[str],
  function parse_args (line 206) | def parse_args():

FILE: eval/opencompass/tasks/openicl_infer.py
  class OpenICLInferTask (line 19) | class OpenICLInferTask(BaseTask):
    method __init__ (line 29) | def __init__(self, cfg: ConfigDict):
    method get_command (line 36) | def get_command(self, cfg_path, template):
    method run (line 55) | def run(self):
    method _inference (line 78) | def _inference(self):
    method _set_default_value (line 129) | def _set_default_value(self, cfg: ConfigDict, key: str, value: Any):
  function parse_args (line 135) | def parse_args():

FILE: eval/opencompass/utils/abbr.py
  function model_abbr_from_cfg (line 7) | def model_abbr_from_cfg(cfg: ConfigDict) -> str:
  function dataset_abbr_from_cfg (line 17) | def dataset_abbr_from_cfg(cfg: ConfigDict) -> str:
  function task_abbr_from_cfg (line 28) | def task_abbr_from_cfg(task: Dict) -> str:
  function get_infer_output_path (line 38) | def get_infer_output_path(model_cfg: ConfigDict,

FILE: eval/opencompass/utils/build.py
  function build_dataset_from_cfg (line 8) | def build_dataset_from_cfg(dataset_cfg: ConfigDict) -> ConfigDict:
  function build_model_from_cfg (line 16) | def build_model_from_cfg(model_cfg: ConfigDict) -> ConfigDict:

FILE: eval/opencompass/utils/collect_env.py
  function collect_env (line 7) | def collect_env():

FILE: eval/opencompass/utils/dependency.py
  function satisfy_requirement (line 7) | def satisfy_requirement(dep):

FILE: eval/opencompass/utils/file.py
  function match_files (line 6) | def match_files(path: str,

FILE: eval/opencompass/utils/fileio.py
  function patch_func (line 8) | def patch_func(module, fn_name_to_wrap):
  function patch_fileio (line 23) | def patch_fileio(global_vars=None):
  function patch_hf_auto_model (line 118) | def patch_hf_auto_model(cache_dir=None):

FILE: eval/opencompass/utils/lark.py
  class LarkReporter (line 8) | class LarkReporter:
    method __init__ (line 10) | def __init__(self, url: str):
    method post (line 13) | def post(self,
  function parse_args (line 43) | def parse_args():

FILE: eval/opencompass/utils/logging.py
  function get_logger (line 18) | def get_logger(log_level='INFO', filter_duplicate_level=None) -> MMLogger:
  class FilterDuplicateMessage (line 45) | class FilterDuplicateMessage(logging.Filter):
    method __init__ (line 52) | def __init__(self, name, filter_duplicate_level):
    method filter (line 66) | def filter(self, record: logging.LogRecord) -> bool:

FILE: eval/opencompass/utils/menu.py
  class Menu (line 4) | class Menu:
    method __init__ (line 15) | def __init__(self, lists, prompts=None):
    method draw_menu (line 21) | def draw_menu(self, stdscr, selected_row_idx, offset, max_rows):
    method run (line 37) | def run(self):
    method main_loop (line 41) | def main_loop(self, stdscr):

FILE: eval/opencompass/utils/prompt.py
  function safe_format (line 11) | def safe_format(input_str: str, **kwargs) -> str:
  function get_prompt_hash (line 27) | def get_prompt_hash(dataset_cfg: Union[ConfigDict, List[ConfigDict]]) ->...
  class PromptList (line 64) | class PromptList(list):
    method format (line 67) | def format(self, **kwargs) -> PromptList:
    method replace (line 92) | def replace(self, src: str, dst: Union[str, PromptList]) -> PromptList:
    method __add__ (line 134) | def __add__(self, other: Union[str, PromptList]) -> PromptList:
    method __radd__ (line 150) | def __radd__(self, other: Union[str, PromptList]) -> PromptList:
    method __iadd__ (line 167) | def __iadd__(self, other: Union[str, PromptList]) -> PromptList:
    method __str__ (line 184) | def __str__(self) -> str:

FILE: eval/opencompass/utils/run.py
  function match_cfg_file (line 12) | def match_cfg_file(workdir: str, pattern: Union[str, List[str]]) -> List...
  function get_config_from_arg (line 49) | def get_config_from_arg(args) -> Config:
  function exec_mm_infer_runner (line 110) | def exec_mm_infer_runner(tasks, args, cfg):
  function get_config_type (line 131) | def get_config_type(obj) -> str:
  function fill_infer_cfg (line 135) | def fill_infer_cfg(cfg, args):
  function fill_eval_cfg (line 164) | def fill_eval_cfg(cfg, args):

FILE: eval/opencompass/utils/summarizer.py
  class Summarizer (line 19) | class Summarizer:
    method __init__ (line 22) | def __init__(self, config: ConfigDict) -> None:
    method summarize (line 32) | def summarize(

FILE: eval/opencompass/utils/text_postprocessors.py
  function general_postprocess (line 7) | def general_postprocess(text: str) -> str:
  function general_cn_postprocess (line 27) | def general_cn_postprocess(text: str) -> str:
  function first_capital_postprocess (line 44) | def first_capital_postprocess(text: str) -> str:
  function first_option_postprocess (line 51) | def first_option_postprocess(text: str, options: str) -> str:
  function first_capital_postprocess_multi (line 77) | def first_capital_postprocess_multi(text: str) -> str:
  function last_option_postprocess (line 84) | def last_option_postprocess(text: str, options: str) -> str:

FILE: eval/opencompass/utils/types.py
  function _check_type_list (line 6) | def _check_type_list(obj, typelist: List):
  function _check_dataset (line 18) | def _check_dataset(obj) -> Union[Dataset, DatasetDict]:
  function _check_list (line 27) | def _check_list(obj) -> List:
  function _check_str (line 34) | def _check_str(obj) -> str:
  function _check_dict (line 41) | def _check_dict(obj) -> Dict:

FILE: eval/run.py
  function parse_args (line 17) | def parse_args():
  function parse_slurm_args (line 140) | def parse_slurm_args(slurm_parser):
  function parse_dlc_args (line 158) | def parse_dlc_args(dlc_parser):
  function parse_hf_args (line 166) | def parse_hf_args(hf_parser):
  function main (line 183) | def main():

FILE: eval/setup.py
  class DownloadNLTK (line 5) | class DownloadNLTK(install):
    method run (line 7) | def run(self):
  function readme (line 13) | def readme():
  function parse_requirements (line 19) | def parse_requirements(fname='requirements.txt', with_version=True):
  function get_version (line 98) | def get_version():
  function do_setup (line 105) | def do_setup():

FILE: eval/tests/dataset/test_humaneval.py
  function run_humaneval_check (line 6) | def run_humaneval_check(completion):
  class TestHumaneval (line 18) | class TestHumaneval(unittest.TestCase):
    method test_vanilla (line 20) | def test_vanilla(self):
    method test_python_quote (line 24) | def test_python_quote(self):
    method test_bare_quote (line 33) | def test_bare_quote(self):
    method test_error_space_quote (line 42) | def test_error_space_quote(self):
    method test_import_1 (line 51) | def test_import_1(self):
    method test_import_2 (line 63) | def test_import_2(self):
    method test_import_3 (line 74) | def test_import_3(self):
    method test_comment (line 85) | def test_comment(self):
    method test_additional (line 97) | def test_additional(self):

FILE: eval/tests/openicl/test_prompt_template.py
  class TestPromptTemplate (line 7) | class TestPromptTemplate(unittest.TestCase):
    method setUp (line 9) | def setUp(self) -> None:
    method test_init (line 28) | def test_init(self):
    method test_generate_ice_item (line 34) | def test_generate_ice_item(self):
    method test_generate_label_prompt_item (line 87) | def test_generate_label_prompt_item(self):
    method test_generate_item (line 172) | def test_generate_item(self):

FILE: eval/tests/prompt/test_api_template_parser.py
  class TestAPITemplateParser (line 7) | class TestAPITemplateParser(unittest.TestCase):
    method setUp (line 9) | def setUp(self):
    method test_parse_template_str_input (line 77) | def test_parse_template_str_input(self):
    method test_parse_template_list_input (line 83) | def test_parse_template_list_input(self):
    method test_parse_template_PromptList_input_no_meta_template (line 89) | def test_parse_template_PromptList_input_no_meta_template(self):
    method test_parse_template_PromptList_input_with_meta_template (line 97) | def test_parse_template_PromptList_input_with_meta_template(self):

FILE: eval/tests/prompt/test_lm_template_parser.py
  class TestLMTemplateParser (line 7) | class TestLMTemplateParser(unittest.TestCase):
    method setUp (line 9) | def setUp(self):
    method test_parse_template_str_input (line 78) | def test_parse_template_str_input(self):
    method test_parse_template_list_input (line 84) | def test_parse_template_list_input(self):
    method test_parse_template_PromptList_input_no_meta_template (line 90) | def test_parse_template_PromptList_input_no_meta_template(self):
    method test_parse_template_PromptList_input_with_meta_template (line 98) | def test_parse_template_PromptList_input_with_meta_template(self):

FILE: eval/tests/prompt/test_prompt_list.py
  class TestPromptList (line 6) | class TestPromptList(unittest.TestCase):
    method test_initialization (line 8) | def test_initialization(self):
    method test_format (line 15) | def test_format(self):
    method test_replace (line 26) | def test_replace(self):
    method test_add (line 55) | def test_add(self):
    method test_str (line 70) | def test_str(self):

FILE: eval/tools/case_analyzer.py
  function parse_args (line 15) | def parse_args():
  class BadcaseShower (line 37) | class BadcaseShower:
    method __init__ (line 40) | def __init__(self, cfg: ConfigDict) -> None:
    method run (line 51) | def run(self):
  function dispatch_tasks (line 168) | def dispatch_tasks(cfg, force=False):
  function main (line 182) | def main():

FILE: eval/tools/collect_code_preds.py
  function parse_args (line 16) | def parse_args():
  function gpt_python_postprocess (line 46) | def gpt_python_postprocess(ori_prompt: str, text: str) -> str:
  function wizardcoder_postprocess (line 73) | def wizardcoder_postprocess(text: str) -> str:
  function collect_preds (line 91) | def collect_preds(filename: str):
  function main (line 127) | def main():

FILE: eval/tools/eval_mmbench.py
  function double_log (line 20) | def double_log(msg, fout=None):
  function dump (line 27) | def dump(data, f):
  function load (line 59) | def load(f):
  function report_acc (line 95) | def report_acc(df, group='category'):
  function build_option_str (line 123) | def build_option_str(option_list):
  function extract_options (line 134) | def extract_options(item):
  function build_choices (line 144) | def build_choices(item):
  function build_prompt (line 152) | def build_prompt(question, options, prediction):
  function can_infer_option (line 174) | def can_infer_option(answer, num_choice=5):
  function can_infer_text (line 206) | def can_infer_text(answer, choices):
  function can_infer (line 221) | def can_infer(answer, choices):
  function prefetch_answer (line 226) | def prefetch_answer(item):
  function extract_answer_from_item (line 232) | def extract_answer_from_item(model, item):
  function eval_sub_data (line 273) | def eval_sub_data(model, sub_data, answer_map):
  function eval_result (line 296) | def eval_result(eval_file, eval_method, meta_file):
  function parse_args (line 383) | def parse_args():

FILE: eval/tools/list_configs.py
  function parse_args (line 8) | def parse_args():
  function main (line 20) | def main():

FILE: eval/tools/prediction_merger.py
  function parse_args (line 12) | def parse_args():
  class PredictionMerger (line 29) | class PredictionMerger:
    method __init__ (line 32) | def __init__(self, cfg: ConfigDict) -> None:
    method run (line 39) | def run(self):
  function dispatch_tasks (line 79) | def dispatch_tasks(cfg):
  function main (line 89) | def main():

FILE: eval/tools/prompt_viewer.py
  function parse_args (line 15) | def parse_args():
  function parse_model_cfg (line 34) | def parse_model_cfg(model_cfg: ConfigDict) -> Dict[str, ConfigDict]:
  function parse_dataset_cfg (line 41) | def parse_dataset_cfg(dataset_cfg: ConfigDict) -> Dict[str, ConfigDict]:
  function print_prompts (line 48) | def print_prompts(model_cfg, dataset_cfg, count=1):
  function main (line 175) | def main():

FILE: eval/tools/test_api_model.py
  function test_model (line 156) | def test_model(model_cfg: ConfigDict):
  function parse_args (line 173) | def parse_args():
  function parse_model_cfg (line 182) | def parse_model_cfg(model_cfg: ConfigDict) -> Dict[str, ConfigDict]:
  function main (line 189) | def main():

FILE: eval/tools/update_dataset_suffix.py
  function get_prompt_hash (line 16) | def get_prompt_hash(dataset_cfg: Union[ConfigDict, List[ConfigDict]]) ->...
  function get_hash (line 55) | def get_hash(path):
  function check_and_rename (line 64) | def check_and_rename(filepath):
  function update_imports (line 80) | def update_imports(data):
  function main (line 97) | def main():

FILE: model/infer/chatmusician_web_demo.py
  function _get_args (line 31) | def _get_args():
  function get_uuid (line 51) | def get_uuid():
  function _load_model_tokenizer (line 54) | def _load_model_tokenizer(args):
  function log_conversation (line 83) | def log_conversation(conversation_id, history, messages, response, gener...
  function _parse_text (line 96) | def _parse_text(text):
  function convert_history_to_text (line 128) | def convert_history_to_text(task_history):
  function postprocess_abc (line 137) | def postprocess_abc(text, conversation_id):
  function _launch_demo (line 165) | def _launch_demo(args, model, tokenizer):
  function main (line 344) | def main():

FILE: model/infer/predict.py
  function generate_prompt (line 45) | def generate_prompt(input_text):

FILE: model/train/data_preprocess.py
  function main (line 6) | def main(args):

FILE: model/train/llama/configuration_llama.py
  class LlamaConfig (line 31) | class LlamaConfig(PretrainedConfig):
    method __init__ (line 76) | def __init__(

FILE: model/train/llama/convert_llama_weights_to_hf.py
  function compute_intermediate_size (line 59) | def compute_intermediate_size(n):
  function read_json (line 63) | def read_json(path):
  function write_json (line 68) | def write_json(text, path):
  function write_model (line 73) | def write_model(model_path, input_base_path, model_size):
  function write_tokenizer (line 232) | def write_tokenizer(tokenizer_path, input_tokenizer_path):
  function main (line 240) | def main():

FILE: model/train/llama/modeling_llama.py
  function _make_causal_mask (line 42) | def _make_causal_mask(
  function _expand_mask (line 60) | def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Option...
  class LlamaRMSNorm (line 74) | class LlamaRMSNorm(nn.Module):
    method __init__ (line 75) | def __init__(self, hidden_size, eps=1e-6):
    method forward (line 83) | def forward(self, hidden_states):
  class LlamaRotaryEmbedding (line 94) | class LlamaRotaryEmbedding(torch.nn.Module):
    method __init__ (line 95) | def __init__(self, dim, max_position_embeddings=2048, base=10000, devi...
    method forward (line 109) | def forward(self, x, seq_len=None):
  function rotate_half (line 126) | def rotate_half(x):
  function apply_rotary_pos_emb (line 133) | def apply_rotary_pos_emb(q, k, cos, sin, position_ids):
  class LlamaMLP (line 143) | class LlamaMLP(nn.Module):
    method __init__ (line 144) | def __init__(
    method forward (line 156) | def forward(self, x):
  class LlamaAttention (line 160) | class LlamaAttention(nn.Module):
    method __init__ (line 163) | def __init__(self, config: LlamaConfig):
    method _shape (line 182) | def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
    method forward (line 185) | def forward(
  class LlamaDecoderLayer (line 251) | class LlamaDecoderLayer(nn.Module):
    method __init__ (line 252) | def __init__(self, config: LlamaConfig):
    method forward (line 264) | def forward(
  class LlamaPreTrainedModel (line 338) | class LlamaPreTrainedModel(PreTrainedModel):
    method _init_weights (line 345) | def _init_weights(self, module):
    method _set_gradient_checkpointing (line 356) | def _set_gradient_checkpointing(self, module, value=False):
  class LlamaModel (line 418) | class LlamaModel(LlamaPreTrainedModel):
    method __init__ (line 425) | def __init__(self, config: LlamaConfig):
    method get_input_embeddings (line 438) | def get_input_embeddings(self):
    method set_input_embeddings (line 441) | def set_input_embeddings(self, value):
    method _prepare_decoder_attention_mask (line 445) | def _prepare_decoder_attention_mask(self, attention_mask, input_shape,...
    method forward (line 469) | def forward(
  class LlamaForCausalLM (line 597) | class LlamaForCausalLM(LlamaPreTrainedModel):
    method __init__ (line 598) | def __init__(self, config):
    method get_input_embeddings (line 607) | def get_input_embeddings(self):
    method set_input_embeddings (line 610) | def set_input_embeddings(self, value):
    method get_output_embeddings (line 613) | def get_output_embeddings(self):
    method set_output_embeddings (line 616) | def set_output_embeddings(self, new_embeddings):
    method set_decoder (line 619) | def set_decoder(self, decoder):
    method get_decoder (line 622) | def get_decoder(self):
    method forward (line 627) | def forward(
    method prepare_inputs_for_generation (line 707) | def prepare_inputs_for_generation(
    method _reorder_cache (line 738) | def _reorder_cache(past_key_values, beam_idx):
  class LlamaForSequenceClassification (line 758) | class LlamaForSequenceClassification(LlamaPreTrainedModel):
    method __init__ (line 761) | def __init__(self, config):
    method get_input_embeddings (line 770) | def get_input_embeddings(self):
    method set_input_embeddings (line 773) | def set_input_embeddings(self, value):
    method forward (line 777) | def forward(

FILE: model/train/llama/tokenization_llama.py
  class LlamaTokenizer (line 49) | class LlamaTokenizer(PreTrainedTokenizer):
    method __init__ (line 62) | def __init__(
    method __getstate__ (line 97) | def __getstate__(self):
    method __setstate__ (line 102) | def __setstate__(self, d):
    method vocab_size (line 108) | def vocab_size(self):
    method get_vocab (line 112) | def get_vocab(self):
    method _tokenize (line 118) | def _tokenize(self, text):
    method _convert_token_to_id (line 122) | def _convert_token_to_id(self, token):
    method _convert_id_to_token (line 126) | def _convert_id_to_token(self, index):
    method convert_tokens_to_string (line 131) | def convert_tokens_to_string(self, tokens):
    method save_vocabulary (line 150) | def save_vocabulary(self, save_directory, filename_prefix: Optional[st...
    method build_inputs_with_special_tokens (line 175) | def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=No...
    method get_special_tokens_mask (line 186) | def get_special_tokens_mask(
    method create_token_type_ids_from_sequences (line 221) | def create_token_type_ids_from_sequences(

FILE: model/train/merge.py
  function set_args (line 7) | def set_args():

FILE: model/train/train.py
  function parse_args (line 26) | def parse_args():
  function main (line 66) | def main():

FILE: model/train/utils.py
  class DataCollator (line 8) | class DataCollator(object):
    method __init__ (line 9) | def __init__(self, tokenizer):
    method __call__ (line 13) | def __call__(self, batch):
  function print_trainable_parameters (line 32) | def print_trainable_parameters(model):
  function print_rank_0 (line 47) | def print_rank_0(msg, rank=0):
  function to_device (line 52) | def to_device(batch, device):
  function set_random_seed (line 62) | def set_random_seed(seed):
  function save_model (line 71) | def save_model(model, tokenizer, output_dir, model_name, state_dict=None):