SYMBOL INDEX (3516 symbols across 442 files)

FILE: data/add_aime.py
  function process_example (line 6) | def process_example(example):

FILE: data/bulk_inference.py
  class DataModuleConfigs (line 17) | class DataModuleConfigs:
  function shard_question (line 21) | def shard_question(chunk_size: int=10_000):
  function _qwen_forward (line 27) | def _qwen_forward(
  function difficulty_classification (line 56) | def difficulty_classification(shard_index: int, model_name: str):
  function assemble_output (line 70) | def assemble_output(model_name: str, upload: bool = False):
  function assemble_output_gemini (line 96) | def assemble_output_gemini():

FILE: data/collect_data.py
  function load_generic (line 108) | def load_generic(name, split, question_field="question", solution_field=...
  function load_math (line 118) | def load_math():
  function load_numinamath (line 132) | def load_numinamath():
  function load_olympic_arena (line 146) | def load_olympic_arena():
  function load_theoremqa (line 157) | def load_theoremqa():
  function load_scieval (line 164) | def load_scieval():
  function load_olympiad_bench (line 195) | def load_olympiad_bench():
  function load_jeebench (line 208) | def load_jeebench():
  function load_agieval (line 215) | def load_agieval():
  function load_statsqual (line 238) | def load_statsqual():
  function load_gpqa_extended (line 244) | def load_gpqa_extended():
  function load_xword (line 255) | def load_xword():
  function load_usaco (line 268) | def load_usaco():
  function load_quant (line 275) | def load_quant():
  function load_livecodebench (line 281) | def load_livecodebench():
  function select_examples_omni_math (line 303) | def select_examples_omni_math(clean_examples, n_examples):
  function select_examples_scieval (line 355) | def select_examples_scieval(ds, n_examples):
  function decontaminate_train_data (line 387) | def decontaminate_train_data(train_questions, test_questions, ds, ngram_...
  function is_unique (line 482) | def is_unique(elem, column, memory):

FILE: data/decontaminate_util.py
  function normalize_string (line 6) | def normalize_string(text):
  function word_ngrams (line 14) | def word_ngrams(text, n):
  function build_ngram_lookup (line 27) | def build_ngram_lookup(documents, ngram_size=13):
  function find_contaminated_questions (line 41) | def find_contaminated_questions(test_lookup, train_lookup):
  function upload_to_huggingface (line 61) | def upload_to_huggingface(selected_examples, repo_id="your-username/data...

FILE: data/featurization.py
  function _process_question (line 15) | def _process_question(question: str, msc_prompt: str, system_prompt: str):
  function do_domain_classification (line 31) | def do_domain_classification():
  function _process_example_grading (line 50) | def _process_example_grading(qhash: str,
  function do_grading (line 86) | def do_grading(response_dir: str = "Qwen_Qwen2_5_32B_Instruct"):
  function upload_grading (line 107) | def upload_grading(response_dir: str = "Qwen_Qwen2_5_32B_Instruct"):
  function upload_domain (line 142) | def upload_domain():
  function upload_length (line 163) | def upload_length():

FILE: data/fix_gpqa.py
  function process_example (line 15) | def process_example(example):
  function parse (line 41) | def parse():

FILE: data/gemini.py
  function gemini_qa (line 17) | def gemini_qa(prompt: str):
  function process_question (line 36) | def process_question(question: str, subdir: str):
  function generate_gemini1k (line 47) | def generate_gemini1k():
  function generate_gemini (line 56) | def generate_gemini():
  function upload_gemini (line 70) | def upload_gemini():

FILE: data/tokenization.py
  function preprocess (line 9) | def preprocess(text):
  function process_cot_example (line 18) | def process_cot_example(
  function mathcot_sft (line 36) | def mathcot_sft(upload_data_path: str, num_proc: int,

FILE: data/utils/inference_utils.py
  function calc_price (line 8) | def calc_price(model, usage):
  function _gptqa (line 34) | def _gptqa(prompt: str, openai_model_name: str, system_message: str, jso...
  function _claudeqa (line 66) | def _claudeqa(prompt: str, system_message: str):
  function apiqa (line 81) | def apiqa(prompt: str, model_name: str, system_message: str, json_format...
  function claude_multi_round (line 98) | def claude_multi_round(system_prompt: str, messages: List[Dict[str, str]]):

FILE: data/utils/io_utils.py
  function set_openai_private_key (line 10) | def set_openai_private_key():
  function set_anthropic_private_key (line 15) | def set_anthropic_private_key():
  function set_genmini_private_key (line 20) | def set_genmini_private_key():
  function _make_w_io_base (line 25) | def _make_w_io_base(f, mode: str):
  function _make_r_io_base (line 34) | def _make_r_io_base(f, mode: str):
  function jdump (line 40) | def jdump(obj, f: str, mode="w", indent=4, default=str):
  function jload_list (line 60) | def jload_list(f, mode="r"):
  function jload (line 69) | def jload(f, mode="r"):
  function tload (line 76) | def tload(f, mode="r"):
  function upload_to_huggingface (line 81) | def upload_to_huggingface(cot_dataset_str: str, repo_name: str):
  function write_to_memmap (line 87) | def write_to_memmap(dset: Dataset, filename: str):
  function save_dataset (line 105) | def save_dataset(dataset: Dataset, filename: str):
  function question_hash (line 109) | def question_hash(question: str) -> str:

FILE: data/utils/string_utils.py
  function extract_content (line 1) | def extract_content(input_string, start_token, end_token=None):
  function remove_special_tokens (line 9) | def remove_special_tokens(input_string):

FILE: eval/lm-evaluation-harness/lm_eval/__main__.py
  function _int_or_none_list_arg_type (line 16) | def _int_or_none_list_arg_type(
  function check_argument_types (line 51) | def check_argument_types(parser: argparse.ArgumentParser):
  function setup_parser (line 65) | def setup_parser() -> argparse.ArgumentParser:
  function parse_eval_args (line 275) | def parse_eval_args(parser: argparse.ArgumentParser) -> argparse.Namespace:
  function cli_evaluate (line 280) | def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:

FILE: eval/lm-evaluation-harness/lm_eval/api/filter.py
  class Filter (line 8) | class Filter(ABC):
    method __init__ (line 17) | def __init__(self, **kwargs) -> None:
    method apply (line 23) | def apply(self, resps: Union[List, Iterable], docs: List[dict]) -> Ite...
  class FilterEnsemble (line 34) | class FilterEnsemble:
    method apply (line 45) | def apply(self, instances: List[Instance]) -> None:

FILE: eval/lm-evaluation-harness/lm_eval/api/group.py
  class AggMetricConfig (line 8) | class AggMetricConfig(dict):
    method __post_init__ (line 15) | def __post_init__(self):
  class GroupConfig (line 26) | class GroupConfig(dict):
    method __getitem__ (line 37) | def __getitem__(self, item):
    method __setitem__ (line 40) | def __setitem__(self, item, value):
    method __post_init__ (line 43) | def __post_init__(self):
    method to_dict (line 53) | def to_dict(self, keep_callable: bool = False) -> dict:
    method serialize_function (line 70) | def serialize_function(
  class ConfigurableGroup (line 87) | class ConfigurableGroup(abc.ABC):
    method __init__ (line 88) | def __init__(
    method group (line 95) | def group(self):
    method group_alias (line 99) | def group_alias(self):
    method version (line 103) | def version(self):
    method config (line 107) | def config(self):
    method group_name (line 111) | def group_name(self) -> Any:
    method __repr__ (line 114) | def __repr__(self):

FILE: eval/lm-evaluation-harness/lm_eval/api/instance.py
  class Instance (line 11) | class Instance:
    method __post_init__ (line 27) | def __post_init__(self) -> None:
    method args (line 32) | def args(self):

FILE: eval/lm-evaluation-harness/lm_eval/api/metrics.py
  function bypass_agg (line 20) | def bypass_agg(arr):
  function mean_last30 (line 24) | def mean_last30(arr):
  function mean (line 31) | def mean(arr):
  function median (line 36) | def median(arr):
  function perplexity (line 43) | def perplexity(items):
  function weighted_perplexity (line 48) | def weighted_perplexity(items):
  function bits_per_byte (line 53) | def bits_per_byte(items):
  function f1_score (line 58) | def f1_score(items):
  function matthews_corrcoef (line 70) | def matthews_corrcoef(items):
  function bleu (line 80) | def bleu(items):
  function chrf (line 98) | def chrf(items):
  function ter (line 113) | def ter(items):
  function brier_score (line 129) | def brier_score(items):  # This is a passthrough function
  function brier_score_fn (line 144) | def brier_score_fn(items):  # This is a passthrough function
  function acc_fn (line 154) | def acc_fn(items):  # This is a passthrough function
  function acc_norm_fn (line 164) | def acc_norm_fn(items):  # This is a passthrough function
  function acc_mutual_info_fn (line 174) | def acc_mutual_info_fn(items):  # This is a passthrough function
  function exact_match_hf_evaluate (line 196) | def exact_match_hf_evaluate(
  function exact_match_fn (line 240) | def exact_match_fn(**kwargs):
  function perplexity_fn (line 250) | def perplexity_fn(items):  # This is a passthrough function
  function word_perplexity_fn (line 260) | def word_perplexity_fn(items):  # This is a passthrough function
  function byte_perplexity_fn (line 270) | def byte_perplexity_fn(items):  # This is a passthrough function
  function bits_per_byte_fn (line 280) | def bits_per_byte_fn(items):  # This is a passthrough function
  function pop_stddev (line 284) | def pop_stddev(arr):
  function sample_stddev (line 289) | def sample_stddev(arr):
  function mean_stderr (line 294) | def mean_stderr(arr):
  function bypass (line 304) | def bypass(items):
  function mcc_fn (line 314) | def mcc_fn(items):  # This is a passthrough function
  function f1_fn (line 324) | def f1_fn(items):  # This is a passthrough function
  function bleu_fn (line 334) | def bleu_fn(items):  # This is a passthrough function
  function chrf_fn (line 344) | def chrf_fn(items):  # This is a passthrough function
  function ter_fn (line 354) | def ter_fn(items):  # This is a passthrough function
  function acc_all (line 364) | def acc_all(items):
  function acc_all_stderr (line 383) | def acc_all_stderr(items):
  function metric_max_over_ground_truths (line 401) | def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
  function weighted_mean (line 410) | def weighted_mean(items):
  function is_non_str_iterable (line 415) | def is_non_str_iterable(obj):
  function _sacreformat (line 419) | def _sacreformat(refs, preds):
  class _bootstrap_internal (line 450) | class _bootstrap_internal:
    method __init__ (line 451) | def __init__(self, f, n) -> None:
    method __call__ (line 455) | def __call__(self, v):
  function bootstrap_stderr (line 465) | def bootstrap_stderr(f, xs, iters):
  function stderr_for_metric (line 494) | def stderr_for_metric(metric, bootstrap_iters: int):
  function pooled_sample_stderr (line 517) | def pooled_sample_stderr(stderrs: List[float], sizes: List[int]):
  function combined_sample_stderr (line 535) | def combined_sample_stderr(stderrs: List[float], sizes: List[int], metri...
  function aggregate_subtask_metrics (line 567) | def aggregate_subtask_metrics(metrics, sizes, weight_by_size=True):

FILE: eval/lm-evaluation-harness/lm_eval/api/model.py
  class LM (line 20) | class LM(abc.ABC):
    method __init__ (line 21) | def __init__(self) -> None:
    method loglikelihood (line 33) | def loglikelihood(self, requests) -> List[Tuple[float, bool]]:
    method loglikelihood_rolling (line 58) | def loglikelihood_rolling(self, requests) -> List[float]:
    method generate_until (line 100) | def generate_until(self, requests) -> List[str]:
    method apply_chat_template (line 116) | def apply_chat_template(self, chat_history: List[Dict[str, str]]) -> str:
    method create_from_arg_string (line 131) | def create_from_arg_string(
    method create_from_arg_obj (line 150) | def create_from_arg_obj(
    method rank (line 172) | def rank(self):
    method world_size (line 179) | def world_size(self):
    method tokenizer_name (line 186) | def tokenizer_name(self) -> str:
    method chat_template (line 195) | def chat_template(self, chat_template: Union[bool, str] = False) -> Op...
    method set_cache_hook (line 203) | def set_cache_hook(self, cache_hook) -> None:
  function hash_args (line 208) | def hash_args(attr, args):
  class CacheHook (line 213) | class CacheHook:
    method __init__ (line 214) | def __init__(self, cachinglm) -> None:
    method add_partial (line 221) | def add_partial(self, attr, req, res) -> None:
  class CachingLM (line 228) | class CachingLM:
    method __init__ (line 229) | def __init__(self, lm, cache_db) -> None:
    method __getattr__ (line 246) | def __getattr__(self, attr: str):
    method get_cache_hook (line 307) | def get_cache_hook(self):
  class TemplateLM (line 311) | class TemplateLM(LM):
    method eot_token_id (line 321) | def eot_token_id(self):
    method prefix_token_id (line 325) | def prefix_token_id(self):
    method tok_encode (line 330) | def tok_encode(self, string: str, **kwargs) -> List[int]:
    method _loglikelihood_tokens (line 337) | def _loglikelihood_tokens(self, requests, **kwargs) -> List[Tuple[floa...
    method _encode_pair (line 340) | def _encode_pair(
    method loglikelihood (line 362) | def loglikelihood(
    method loglikelihood_rolling (line 381) | def loglikelihood_rolling(
    method generate_until (line 387) | def generate_until(self, requests, disable_tqdm: bool = False) -> List...
    method chat_template (line 390) | def chat_template(self, chat_template: Union[bool, str] = False) -> Op...

FILE: eval/lm-evaluation-harness/lm_eval/api/registry.py
  function register_model (line 14) | def register_model(*names):
  function get_model (line 34) | def get_model(model_name):
  function register_task (line 49) | def register_task(name):
  function register_group (line 63) | def register_group(name):
  function register_metric (line 94) | def register_metric(**args):
  function get_metric (line 123) | def get_metric(name: str, hf_evaluate_metric=False) -> Callable:
  function register_aggregation (line 141) | def register_aggregation(name: str):
  function get_aggregation (line 153) | def get_aggregation(name: str) -> Callable[[], Dict[str, Callable]]:
  function get_metric_aggregation (line 160) | def get_metric_aggregation(name: str) -> Callable[[], Dict[str, Callable]]:
  function is_higher_better (line 167) | def is_higher_better(metric_name) -> bool:
  function register_filter (line 176) | def register_filter(name):
  function get_filter (line 188) | def get_filter(filter_name: str) -> type:

FILE: eval/lm-evaluation-harness/lm_eval/api/samplers.py
  class ContextSampler (line 6) | class ContextSampler:
    method __init__ (line 7) | def __init__(self, docs, task, fewshot_indices=None, rnd=None) -> None:
    method get_context (line 61) | def get_context(self, doc, num_fewshot):
    method get_chat_context (line 99) | def get_chat_context(
    method sample (line 151) | def sample(self, n):
  class FirstNSampler (line 159) | class FirstNSampler(ContextSampler):
    method sample (line 160) | def sample(self, n) -> None:
  class BalancedSampler (line 171) | class BalancedSampler(ContextSampler):
    method sample (line 172) | def sample(self, n) -> None:
  class ManualSampler (line 181) | class ManualSampler(ContextSampler):
    method sample (line 182) | def sample(self, n) -> None:
  function get_sampler (line 193) | def get_sampler(name):

FILE: eval/lm-evaluation-harness/lm_eval/api/task.py
  class TaskConfig (line 55) | class TaskConfig(dict):
    method __post_init__ (line 100) | def __post_init__(self) -> None:
    method __getitem__ (line 138) | def __getitem__(self, item):
    method __setitem__ (line 141) | def __setitem__(self, item, value):
    method to_dict (line 144) | def to_dict(self, keep_callable: bool = False) -> dict:
    method serialize_function (line 171) | def serialize_function(
  class Task (line 188) | class Task(abc.ABC):
    method __init__ (line 209) | def __init__(
    method download (line 249) | def download(
    method config (line 288) | def config(self) -> TaskConfig:
    method has_training_docs (line 293) | def has_training_docs(self):
    method has_validation_docs (line 298) | def has_validation_docs(self):
    method has_test_docs (line 303) | def has_test_docs(self):
    method training_docs (line 307) | def training_docs(self) -> Iterable:
    method validation_docs (line 314) | def validation_docs(self) -> Iterable:
    method test_docs (line 321) | def test_docs(self) -> Iterable:
    method fewshot_docs (line 328) | def fewshot_docs(self) -> Iterable:
    method _process_doc (line 344) | def _process_doc(self, doc: dict) -> dict:
    method instances (line 356) | def instances(self) -> List[Instance]:
    method fewshot_examples (line 362) | def fewshot_examples(self, k, rnd):
    method doc_to_decontamination_query (line 368) | def doc_to_decontamination_query(self, doc):
    method doc_to_text (line 374) | def doc_to_text(self, doc):
    method doc_to_target (line 378) | def doc_to_target(self, doc):
    method doc_to_image (line 382) | def doc_to_image(self, doc):
    method build_all_requests (line 385) | def build_all_requests(
    method construct_requests (line 493) | def construct_requests(self, doc, ctx, **kwargs):
    method process_results (line 514) | def process_results(self, doc, results):
    method aggregation (line 527) | def aggregation(self):
    method higher_is_better (line 536) | def higher_is_better(self):
    method get_config (line 544) | def get_config(self, key: str) -> Any:
    method count_bytes (line 548) | def count_bytes(cls, doc):
    method count_words (line 553) | def count_words(cls, doc):
    method fewshot_context (line 558) | def fewshot_context(
    method apply_filters (line 622) | def apply_filters(self) -> Optional[List[Instance]]:
    method dump_config (line 631) | def dump_config(self) -> dict:
    method set_config (line 637) | def set_config(self, key: str, value: Any, update: bool = False) -> None:
    method override_metric (line 652) | def override_metric(self, metric_name: str) -> None:
    method set_fewshot_seed (line 677) | def set_fewshot_seed(self, seed: Optional[int] = None) -> None:
    method eval_docs (line 683) | def eval_docs(self) -> Union[datasets.Dataset, List[dict]]:
    method doc_iterator (line 693) | def doc_iterator(
  class ConfigurableTask (line 709) | class ConfigurableTask(Task):
    method __init__ (line 714) | def __init__(
    method download (line 942) | def download(self, dataset_kwargs: Optional[Dict[str, Any]] = None) ->...
    method has_training_docs (line 950) | def has_training_docs(self) -> bool:
    method has_validation_docs (line 956) | def has_validation_docs(self) -> bool:
    method has_test_docs (line 962) | def has_test_docs(self) -> bool:
    method training_docs (line 968) | def training_docs(self) -> datasets.Dataset:
    method validation_docs (line 976) | def validation_docs(self) -> datasets.Dataset:
    method test_docs (line 984) | def test_docs(self) -> datasets.Dataset:
    method fewshot_docs (line 990) | def fewshot_docs(self):
    method append_target_question (line 1017) | def append_target_question(
    method fewshot_context (line 1038) | def fewshot_context(
    method apply_filters (line 1154) | def apply_filters(self):
    method should_decontaminate (line 1163) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 1166) | def doc_to_decontamination_query(self, doc):
    method _process_doc (line 1183) | def _process_doc(self, doc: dict) -> dict:
    method doc_to_text (line 1194) | def doc_to_text(self, doc, doc_to_text=None):
    method doc_to_target (line 1230) | def doc_to_target(self, doc: Mapping, doc_to_target=None) -> Union[int...
    method doc_to_choice (line 1276) | def doc_to_choice(self, doc: Any, doc_to_choice=None) -> List[str]:
    method doc_to_image (line 1302) | def doc_to_image(self, doc: Any, doc_to_image=None) -> Union[int, str,...
    method construct_requests (line 1325) | def construct_requests(
    method process_results (line 1399) | def process_results(self, doc, results):
    method aggregation (line 1590) | def aggregation(self) -> dict:
    method higher_is_better (line 1593) | def higher_is_better(self) -> dict:
    method get_config (line 1596) | def get_config(self, key: str) -> Any:
    method task_name (line 1600) | def task_name(self) -> Any:
    method __repr__ (line 1603) | def __repr__(self):
  class MultipleChoiceTask (line 1612) | class MultipleChoiceTask(Task):
    method doc_to_target (line 1615) | def doc_to_target(self, doc: dict) -> str:
    method construct_requests (line 1618) | def construct_requests(self, doc: dict, ctx: str, **kwargs) -> List[In...
    method process_results (line 1631) | def process_results(self, doc: dict, results: Iterable[Tuple[float, bo...
    method higher_is_better (line 1646) | def higher_is_better(self) -> dict:
    method aggregation (line 1652) | def aggregation(self) -> dict:
  class PerplexityTask (line 1659) | class PerplexityTask(Task):
    method has_training_docs (line 1662) | def has_training_docs(self) -> bool:
    method fewshot_examples (line 1665) | def fewshot_examples(self, k: int, rnd) -> List:
    method fewshot_context (line 1672) | def fewshot_context(self, doc: dict, num_fewshot: int) -> Literal[""]:
    method higher_is_better (line 1680) | def higher_is_better(self) -> dict:
    method doc_to_decontamination_query (line 1687) | def doc_to_decontamination_query(self, doc):
    method doc_to_text (line 1690) | def doc_to_text(self, doc) -> str:
    method doc_to_target (line 1693) | def doc_to_target(self, doc):
    method construct_requests (line 1696) | def construct_requests(self, doc: dict, ctx: Optional[str], **kwargs):
    method process_results (line 1708) | def process_results(self, doc: dict, results: Tuple[float]) -> dict:
    method aggregation (line 1718) | def aggregation(self) -> dict:
    method count_bytes (line 1726) | def count_bytes(cls, doc) -> int:
    method count_words (line 1730) | def count_words(cls, doc) -> int:

FILE: eval/lm-evaluation-harness/lm_eval/caching/cache.py
  function load_from_cache (line 24) | def load_from_cache(file_name):
  function save_to_cache (line 37) | def save_to_cache(file_name, obj):
  function delete_cache (line 49) | def delete_cache(key: str = ""):

FILE: eval/lm-evaluation-harness/lm_eval/decontamination/archiver.py
  function json_serial (line 14) | def json_serial(obj: Any) -> str:
  class Archive (line 23) | class Archive:
    method __init__ (line 24) | def __init__(self, file_path: str, compression_level: int = 3) -> None:
    method add_data (line 33) | def add_data(self, data, meta=None) -> None:
    method commit (line 43) | def commit(self) -> None:
  class Reader (line 50) | class Reader:
    method __init__ (line 51) | def __init__(self) -> None:
    method read (line 54) | def read(
  class TextArchive (line 84) | class TextArchive:
    method __init__ (line 85) | def __init__(self, file_path, mode: str = "rb+") -> None:
    method add_data (line 96) | def add_data(self, data) -> None:
    method commit (line 99) | def commit(self) -> None:
  class TextReader (line 104) | class TextReader:
    method __init__ (line 105) | def __init__(self, file_path) -> None:
    method read_tqdm (line 110) | def read_tqdm(self, update_frequency: int = 10000):
    method read_and_tell (line 131) | def read_and_tell(self):
    method read (line 142) | def read(self):
    method read_slow (line 149) | def read_slow(self):
  class ZStdTextReader (line 161) | class ZStdTextReader:
    method __init__ (line 162) | def __init__(self, file) -> None:
    method read_tqdm (line 165) | def read_tqdm(self):

FILE: eval/lm-evaluation-harness/lm_eval/decontamination/decontaminate.py
  function get_train_overlap_stub (line 14) | def get_train_overlap_stub(docs: dict, ngrams_path: str, ngrams_n_size: ...
  function get_train_overlap (line 37) | def get_train_overlap(docs_by_task_set: dict, ngrams_path: str, limit: i...

FILE: eval/lm-evaluation-harness/lm_eval/decontamination/janitor.py
  function form_ngrams (line 24) | def form_ngrams(sequence: Iterator[T], n: int) -> Iterator[Tuple[T, ...]]:
  function word_ngrams (line 41) | def word_ngrams(s: str, n: int) -> Iterator[str]:
  function split_indices (line 73) | def split_indices(s: str) -> Iterator[Tuple[str, Tuple[int, int]]]:
  function word_ngrams_indices (line 80) | def word_ngrams_indices(s: str, n: int) -> Iterator[Tuple[str, Tuple[int...
  class Janitor (line 108) | class Janitor:
    method __init__ (line 110) | def __init__(
    method save_contamination_ngrams (line 139) | def save_contamination_ngrams(self, filename: str) -> None:
    method load_contamination_ngrams (line 143) | def load_contamination_ngrams(self, filename: str) -> None:
    method register_contaminant (line 151) | def register_contaminant(self, dirt_string: str) -> None:
    method clean (line 160) | def clean(self, dirty_string: str) -> List[str]:
    method _split_chunks (line 170) | def _split_chunks(
    method register_contaminant_cpp (line 195) | def register_contaminant_cpp(self, dirt_string) -> None:
    method clean_cpp (line 200) | def clean_cpp(self, dirty_string: str) -> List[str]:
    method normalize_string (line 210) | def normalize_string(self, s: str) -> str:
    method register_contaminant_python (line 213) | def register_contaminant_python(self, dirt_string: str) -> None:
    method clean_python (line 218) | def clean_python(self, dirty_string: str) -> List[str]:

FILE: eval/lm-evaluation-harness/lm_eval/evaluator.py
  function simple_evaluate (line 47) | def simple_evaluate(
  function evaluate (line 362) | def evaluate(
  function request_caching_arg_to_dict (line 681) | def request_caching_arg_to_dict(cache_requests: str) -> dict:

FILE: eval/lm-evaluation-harness/lm_eval/evaluator_utils.py
  class TaskOutput (line 17) | class TaskOutput:
    method __init__ (line 44) | def __init__(
    method from_taskdict (line 71) | def from_taskdict(cls, task_name: str, task):
    method calculate_aggregate_metric (line 100) | def calculate_aggregate_metric(self, bootstrap_iters=100000) -> None:
    method __repr__ (line 121) | def __repr__(self):
  function get_task_list (line 132) | def get_task_list(task_dict: dict) -> List[TaskOutput]:
  function get_subtask_list (line 145) | def get_subtask_list(task_dict, task_root=None, depth=0):
  function print_writeout (line 192) | def print_writeout(task) -> None:
  function get_sample_size (line 203) | def get_sample_size(task, limit: Optional[int]) -> Union[int, None]:
  function prepare_print_tasks (line 211) | def prepare_print_tasks(
  function consolidate_results (line 303) | def consolidate_results(
  function consolidate_group_results (line 365) | def consolidate_group_results(
  function find_test_root (line 507) | def find_test_root(start_path: pathlib.Path) -> pathlib.Path:
  function run_task_tests (line 525) | def run_task_tests(task_list: List[str]):

FILE: eval/lm-evaluation-harness/lm_eval/filters/__init__.py
  function build_filter_ensemble (line 10) | def build_filter_ensemble(

FILE: eval/lm-evaluation-harness/lm_eval/filters/decontamination.py
  class DecontaminationFilter (line 6) | class DecontaminationFilter(Filter):
    method __init__ (line 13) | def __init__(self, path) -> None:
    method apply (line 21) | def apply(self, resps, docs) -> None:

FILE: eval/lm-evaluation-harness/lm_eval/filters/extraction.py
  class RegexFilter (line 10) | class RegexFilter(Filter):
    method __init__ (line 13) | def __init__(
    method apply (line 28) | def apply(self, resps, docs):
  class WhitespaceFilter (line 55) | class WhitespaceFilter(Filter):
    method __init__ (line 58) | def __init__(self) -> None:
    method apply (line 61) | def apply(self, resps, docs):
  class MultiChoiceRegexFilter (line 75) | class MultiChoiceRegexFilter(RegexFilter):
    method __init__ (line 83) | def __init__(
    method apply (line 106) | def apply(self, resps, docs):

FILE: eval/lm-evaluation-harness/lm_eval/filters/selection.py
  class TakeFirstFilter (line 13) | class TakeFirstFilter(Filter):
    method __init__ (line 14) | def __init__(self) -> None:
    method apply (line 19) | def apply(self, resps, docs):
  class TakeKFilter (line 27) | class TakeKFilter(Filter):
    method __init__ (line 28) | def __init__(self, **kwargs) -> None:
    method apply (line 33) | def apply(self, resps, docs):
  class MajorityVoteFilter (line 44) | class MajorityVoteFilter(Filter):
    method __init__ (line 45) | def __init__(self) -> None:
    method apply (line 50) | def apply(self, resps, docs):

FILE: eval/lm-evaluation-harness/lm_eval/filters/transformation.py
  class LowercaseFilter (line 6) | class LowercaseFilter(Filter):
    method __init__ (line 7) | def __init__(self) -> None:
    method apply (line 10) | def apply(self, resps, docs):
  class UppercaseFilter (line 18) | class UppercaseFilter(Filter):
    method __init__ (line 19) | def __init__(self) -> None:
    method apply (line 22) | def apply(self, resps, docs):
  class MapFilter (line 30) | class MapFilter(Filter):
    method __init__ (line 31) | def __init__(self, mapping_dict: dict = None, default_value=None) -> N...
    method apply (line 52) | def apply(self, resps, docs):

FILE: eval/lm-evaluation-harness/lm_eval/loggers/evaluation_tracker.py
  class GeneralConfigTracker (line 35) | class GeneralConfigTracker:
    method __init__ (line 60) | def __init__(self) -> None:
    method _get_model_name (line 65) | def _get_model_name(model_args: str) -> str:
    method log_experiment_args (line 80) | def log_experiment_args(
    method log_end_time (line 100) | def log_end_time(self) -> None:
  class EvaluationTracker (line 106) | class EvaluationTracker:
    method __init__ (line 112) | def __init__(
    method save_results_aggregated (line 190) | def save_results_aggregated(
    method save_results_samples (line 277) | def save_results_samples(
    method recreate_metadata_card (line 370) | def recreate_metadata_card(self) -> None:

FILE: eval/lm-evaluation-harness/lm_eval/loggers/utils.py
  function remove_none_pattern (line 16) | def remove_none_pattern(input_string: str) -> Tuple[str, bool]:
  function _handle_non_serializable (line 38) | def _handle_non_serializable(o: Any) -> Union[int, str, list]:
  function get_commit_from_path (line 57) | def get_commit_from_path(repo_path: Union[Path, str]) -> Optional[str]:
  function get_git_commit_hash (line 84) | def get_git_commit_hash():
  function add_env_info (line 98) | def add_env_info(storage: Dict[str, Any]):
  function add_tokenizer_info (line 115) | def add_tokenizer_info(storage: Dict[str, Any], lm):

FILE: eval/lm-evaluation-harness/lm_eval/loggers/wandb_logger.py
  function get_wandb_printer (line 16) | def get_wandb_printer() -> Literal["Printer"]:
  class WandbLogger (line 25) | class WandbLogger:
    method __init__ (line 26) | def __init__(self, **kwargs) -> None:
    method post_init (line 60) | def post_init(self, results: Dict[str, Any]) -> None:
    method _get_config (line 65) | def _get_config(self) -> Dict[str, Any]:
    method _sanitize_results_dict (line 76) | def _sanitize_results_dict(self) -> Tuple[Dict[str, str], Dict[str, An...
    method _log_results_as_table (line 112) | def _log_results_as_table(self) -> None:
    method _log_results_as_artifact (line 162) | def _log_results_as_artifact(self) -> None:
    method log_eval_result (line 174) | def log_eval_result(self) -> None:
    method _generate_dataset (line 190) | def _generate_dataset(
    method _log_samples_as_artifact (line 281) | def _log_samples_as_artifact(
    method log_eval_samples (line 301) | def log_eval_samples(self, samples: Dict[str, List[Dict[str, Any]]]) -...

FILE: eval/lm-evaluation-harness/lm_eval/models/anthropic_llms.py
  function anthropic_completion (line 17) | def anthropic_completion(
  function anthropic_chat (line 80) | def anthropic_chat(
  class AnthropicLM (line 145) | class AnthropicLM(LM):
    method __init__ (line 148) | def __init__(
    method eot_token_id (line 186) | def eot_token_id(self):
    method max_length (line 191) | def max_length(self) -> int:
    method max_gen_toks (line 195) | def max_gen_toks(self) -> int:
    method batch_size (line 199) | def batch_size(self):
    method device (line 204) | def device(self):
    method tok_encode (line 208) | def tok_encode(self, string: str) -> List[int]:
    method tok_decode (line 211) | def tok_decode(self, tokens: List[int]) -> str:
    method _loglikelihood_tokens (line 214) | def _loglikelihood_tokens(self, requests, disable_tqdm: bool = False):
    method generate_until (line 217) | def generate_until(self, requests, disable_tqdm: bool = False) -> List...
    method _model_call (line 261) | def _model_call(self, inps):
    method _model_generate (line 265) | def _model_generate(self, context, max_length, eos_token_id):
    method loglikelihood (line 269) | def loglikelihood(self, requests, disable_tqdm: bool = False):
    method loglikelihood_rolling (line 272) | def loglikelihood_rolling(self, requests, disable_tqdm: bool = False):
  class AnthropicChat (line 277) | class AnthropicChat(LocalCompletionsAPI):
    method __init__ (line 278) | def __init__(
    method api_key (line 297) | def api_key(self):
    method header (line 307) | def header(self):
    method _create_payload (line 313) | def _create_payload(
    method parse_generations (line 339) | def parse_generations(
    method tok_encode (line 350) | def tok_encode(
    method loglikelihood (line 359) | def loglikelihood(self, requests, **kwargs):

FILE: eval/lm-evaluation-harness/lm_eval/models/api_models.py
  class JsonChatStr (line 44) | class JsonChatStr(NamedTuple):
    method encode (line 47) | def encode(self, encoding):
  class TemplateAPI (line 54) | class TemplateAPI(TemplateLM):
    method __init__ (line 55) | def __init__(
    method _create_payload (line 168) | def _create_payload(
    method create_message (line 180) | def create_message(
    method parse_logprobs (line 212) | def parse_logprobs(
    method parse_generations (line 223) | def parse_generations(outputs: Union[Any, List[Any]], **kwargs) -> Lis...
    method api_key (line 228) | def api_key(self) -> str:
    method header (line 233) | def header(self) -> dict:
    method tokenizer_name (line 238) | def tokenizer_name(self) -> str:
    method apply_chat_template (line 245) | def apply_chat_template(
    method eot_token_id (line 258) | def eot_token_id(self) -> Optional[int]:
    method prefix_token_id (line 268) | def prefix_token_id(self) -> Optional[int]:
    method tok_encode (line 281) | def tok_encode(
    method decode_batch (line 318) | def decode_batch(self, tokens: List[List[int]]) -> List[str]:
    method model_call (line 324) | def model_call(
    method amodel_call (line 358) | async def amodel_call(
    method batch_logliklehood_requests (line 415) | def batch_logliklehood_requests(
    method get_batched_requests (line 434) | async def get_batched_requests(
    method _loglikelihood_tokens (line 472) | def _loglikelihood_tokens(self, requests, **kwargs) -> List[Tuple[floa...
    method generate_until (line 535) | def generate_until(
    method loglikelihood_rolling (line 614) | def loglikelihood_rolling(

FILE: eval/lm-evaluation-harness/lm_eval/models/dummy.py
  class DummyLM (line 10) | class DummyLM(LM):
    method __init__ (line 11) | def __init__(self) -> None:
    method create_from_arg_string (line 15) | def create_from_arg_string(cls, arg_string, additional_config=None):
    method loglikelihood (line 18) | def loglikelihood(self, requests, disable_tqdm: bool = False):
    method generate_until (line 26) | def generate_until(self, requests, disable_tqdm: bool = False):
    method loglikelihood_rolling (line 35) | def loglikelihood_rolling(self, requests, disable_tqdm: bool = False):

FILE: eval/lm-evaluation-harness/lm_eval/models/gemini.py
  function gemini_completion (line 16) | def gemini_completion(
  class Gemini (line 86) | class Gemini(LM):
    method __init__ (line 89) | def __init__(
    method eot_token_id (line 127) | def eot_token_id(self):
    method max_length (line 131) | def max_length(self) -> int:
    method max_gen_toks (line 135) | def max_gen_toks(self) -> int:
    method batch_size (line 139) | def batch_size(self):
    method device (line 144) | def device(self):
    method tok_encode (line 148) | def tok_encode(self, string: str) -> List[int]:
    method tok_decode (line 152) | def tok_decode(self, tokens: List[int]) -> str:
    method _loglikelihood_tokens (line 155) | def _loglikelihood_tokens(self, requests, disable_tqdm: bool = False):
    method generate_until (line 158) | def generate_until(self, requests, disable_tqdm: bool = False) -> List...
    method _model_call (line 200) | def _model_call(self, inps):
    method _model_generate (line 204) | def _model_generate(self, context, max_length, eos_token_id):
    method loglikelihood (line 208) | def loglikelihood(self, requests, disable_tqdm: bool = False):
    method loglikelihood_rolling (line 211) | def loglikelihood_rolling(self, requests, disable_tqdm: bool = False):

FILE: eval/lm-evaluation-harness/lm_eval/models/gguf.py
  function get_result (line 15) | def get_result(logprobs, context_length):
  class GGUFLM (line 37) | class GGUFLM(LM):
    method __init__ (line 38) | def __init__(self, base_url=None, max_length=2048, **kwargs):
    method gguf_completion (line 46) | def gguf_completion(
    method loglikelihood (line 73) | def loglikelihood(self, requests, disable_tqdm: bool = False):
    method generate_until (line 102) | def generate_until(self, requests, disable_tqdm: bool = False):
    method loglikelihood_rolling (line 127) | def loglikelihood_rolling(self, requests, disable_tqdm: bool = False):

FILE: eval/lm-evaluation-harness/lm_eval/models/hf_vlms.py
  class HFMultimodalLM (line 29) | class HFMultimodalLM(HFLM):
    method __init__ (line 37) | def __init__(
    method _create_tokenizer (line 91) | def _create_tokenizer(
    method tok_multimodal_encode (line 141) | def tok_multimodal_encode(
    method _encode_multimodal_pair (line 171) | def _encode_multimodal_pair(self, context, continuation, images):
    method apply_chat_template (line 201) | def apply_chat_template(self, chat_history: List[Dict[str, str]]) -> str:
    method chat_template (line 254) | def chat_template(self, chat_template: Union[bool, str] = False) -> Op...
    method tok_batch_multimodal_encode (line 266) | def tok_batch_multimodal_encode(
    method _model_multimodal_call (line 317) | def _model_multimodal_call(self, inps, imgs, attn_mask=None, labels=No...
    method _model_multimodal_generate (line 325) | def _model_multimodal_generate(self, inputs, max_length, stop, **gener...
    method _batch_images (line 351) | def _batch_images(self, image_encs):
    method loglikelihood_rolling (line 369) | def loglikelihood_rolling(self, requests: List[Instance]) -> List[float]:
    method loglikelihood (line 375) | def loglikelihood(
    method _loglikelihood_tokens (line 406) | def _loglikelihood_tokens(
    method generate_until (line 592) | def generate_until(

FILE: eval/lm-evaluation-harness/lm_eval/models/huggingface.py
  class HFLM (line 48) | class HFLM(TemplateLM):
    method __init__ (line 59) | def __init__(
    method _get_accelerate_args (line 293) | def _get_accelerate_args(
    method config (line 382) | def config(self):
    method model (line 387) | def model(self):
    method eot_token_id (line 395) | def eot_token_id(self):
    method prefix_token_id (line 400) | def prefix_token_id(self):
    method max_length (line 409) | def max_length(self):
    method max_gen_toks (line 423) | def max_gen_toks(self) -> int:
    method batch_size (line 427) | def batch_size(self):
    method device (line 431) | def device(self):
    method rank (line 435) | def rank(self):
    method world_size (line 439) | def world_size(self):
    method tokenizer_name (line 443) | def tokenizer_name(self) -> str:
    method _get_backend (line 446) | def _get_backend(
    method _get_config (line 507) | def _get_config(
    method _create_model (line 519) | def _create_model(
    method _create_tokenizer (line 658) | def _create_tokenizer(
    method _detect_batch_size (line 707) | def _detect_batch_size(self, requests=None, pos: int = 0):
    method tok_encode (line 764) | def tok_encode(
    method tok_batch_encode (line 790) | def tok_batch_encode(
    method tok_decode (line 821) | def tok_decode(self, tokens, skip_special_tokens=True):
    method _model_call (line 824) | def _model_call(self, inps, attn_mask=None, labels=None):
    method _model_generate (line 850) | def _model_generate(self, context, max_length, stop, **generation_kwar...
    method _select_cont_toks (line 881) | def _select_cont_toks(
    method loglikelihood_rolling (line 901) | def loglikelihood_rolling(
    method _batch_scheduler (line 964) | def _batch_scheduler(self, pos, n_reordered_requests):
    method _loglikelihood_tokens (line 981) | def _loglikelihood_tokens(
    method generate_until (line 1203) | def generate_until(
    method apply_chat_template (line 1342) | def apply_chat_template(self, chat_history: List[Dict[str, str]]) -> str:
    method get_model_info (line 1350) | def get_model_info(self) -> dict:

FILE: eval/lm-evaluation-harness/lm_eval/models/mamba_lm.py
  class MambaLMWrapper (line 11) | class MambaLMWrapper(HFLM):
    method __init__ (line 12) | def __init__(
    method _get_config (line 65) | def _get_config(
    method _create_model (line 80) | def _create_model(
    method _model_generate (line 105) | def _model_generate(self, context, max_length, stop, **generation_kwar...

FILE: eval/lm-evaluation-harness/lm_eval/models/nemo_lm.py
  function _patch_pretrained_cfg (line 37) | def _patch_pretrained_cfg(
  function _get_target_from_class (line 67) | def _get_target_from_class(target_class) -> str:
  function load_model (line 71) | def load_model(
  function setup_distributed_environment (line 140) | def setup_distributed_environment(trainer):
  class NeMoLM (line 163) | class NeMoLM(LM):
    method __init__ (line 164) | def __init__(
    method create_from_arg_string (line 270) | def create_from_arg_string(cls, arg_string, additional_config=None):
    method eot_token_id (line 278) | def eot_token_id(self):
    method max_length (line 285) | def max_length(self):
    method max_gen_toks (line 289) | def max_gen_toks(self):
    method batch_size (line 293) | def batch_size(self):
    method device (line 297) | def device(self):
    method rank (line 301) | def rank(self):
    method world_size (line 305) | def world_size(self):
    method accelerator (line 309) | def accelerator(self):
    class _Accelerator (line 312) | class _Accelerator:
      method __init__ (line 313) | def __init__(self, world_size):
      method wait_for_everyone (line 316) | def wait_for_everyone(self):
      method gather (line 319) | def gather(self, local_tensor):
    method tok_encode (line 327) | def tok_encode(self, string: str):
    method tok_decode (line 330) | def tok_decode(self, tokens):
    method _encode_pair (line 333) | def _encode_pair(self, context, continuation):
    method loglikelihood (line 344) | def loglikelihood(self, requests):
    method loglikelihood_rolling (line 360) | def loglikelihood_rolling(
    method _loglikelihood_tokens (line 394) | def _loglikelihood_tokens(self, requests, disable_tqdm=False):
    method generate_until (line 486) | def generate_until(self, requests):

FILE: eval/lm-evaluation-harness/lm_eval/models/neuralmagic.py
  class SparseMLLM (line 20) | class SparseMLLM(HFLM):
    method _create_model (line 31) | def _create_model(
    method _get_config (line 88) | def _get_config(self, pretrained: str, **kwargs) -> None:
    method _create_tokenizer (line 101) | def _create_tokenizer(
  class DeepSparseLM (line 147) | class DeepSparseLM(LM):
    method __init__ (line 156) | def __init__(
    method tok_encode (line 201) | def tok_encode(self, string: str) -> List[int]:
    method tok_decode (line 204) | def tok_decode(self, tokens: List[int]) -> str:
    method eot_token_id (line 208) | def eot_token_id(self):
    method prefix_token_id (line 213) | def prefix_token_id(self):
    method max_length (line 220) | def max_length(self) -> int:
    method max_gen_toks (line 224) | def max_gen_toks(self) -> int:
    method loglikelihood (line 227) | def loglikelihood(self, requests) -> List[Tuple[float, bool]]:
    method _loglikelihood_tokens (line 244) | def _loglikelihood_tokens(
    method loglikelihood_rolling (line 331) | def loglikelihood_rolling(self, requests: List[Instance]) -> List[float]:
    method generate_until (line 336) | def generate_until(self, requests: List[Instance]) -> List[str]:
    method _encode_pair (line 414) | def _encode_pair(

FILE: eval/lm-evaluation-harness/lm_eval/models/neuron_optimum.py
  class CustomNeuronModelForCausalLM (line 34) | class CustomNeuronModelForCausalLM(NeuronModelForCausalLM):
    method generate (line 37) | def generate(
  class NEURON_HF (line 123) | class NEURON_HF(TemplateLM):
    method __init__ (line 130) | def __init__(
    method config (line 247) | def config(self):
    method eot_token_id (line 252) | def eot_token_id(self):
    method prefix_token_id (line 257) | def prefix_token_id(self):
    method max_length (line 262) | def max_length(self):
    method max_gen_toks (line 266) | def max_gen_toks(self) -> int:
    method batch_size (line 270) | def batch_size(self):
    method device (line 274) | def device(self):
    method rank (line 279) | def rank(self):
    method world_size (line 283) | def world_size(self):
    method tok_encode (line 286) | def tok_encode(self, string: str, left_truncate_len=None, add_special_...
    method tok_batch_encode (line 299) | def tok_batch_encode(
    method tok_decode (line 328) | def tok_decode(self, tokens):
    method _model_generate (line 331) | def _model_generate(self, context, max_length, stop, **generation_kwar...
    method _select_cont_toks (line 355) | def _select_cont_toks(self, logits, contlen=None, inplen=None):
    method loglikelihood_rolling (line 365) | def loglikelihood_rolling(self, requests, disable_tqdm: bool = False):
    method _loglikelihood_tokens (line 418) | def _loglikelihood_tokens(
    method generate_until (line 567) | def generate_until(self, requests, disable_tqdm: bool = False):

FILE: eval/lm-evaluation-harness/lm_eval/models/openai_completions.py
  class LocalCompletionsAPI (line 11) | class LocalCompletionsAPI(TemplateAPI):
    method __init__ (line 12) | def __init__(
    method _create_payload (line 22) | def _create_payload(
    method parse_logprobs (line 59) | def parse_logprobs(
    method parse_generations (line 83) | def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> L...
    method api_key (line 93) | def api_key(self):
  class LocalChatCompletion (line 98) | class LocalChatCompletion(LocalCompletionsAPI):
    method __init__ (line 99) | def __init__(
    method _create_payload (line 121) | def _create_payload(
    method parse_generations (line 149) | def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> L...
    method tok_encode (line 158) | def tok_encode(
    method loglikelihood (line 167) | def loglikelihood(self, requests, **kwargs):
  class OpenAICompletionsAPI (line 176) | class OpenAICompletionsAPI(LocalCompletionsAPI):
    method __init__ (line 177) | def __init__(
    method api_key (line 188) | def api_key(self):
    method loglikelihood (line 197) | def loglikelihood(self, requests, **kwargs):
    method chat_template (line 207) | def chat_template(self, chat_template: Union[bool, str] = False) -> Op...
  class OpenAIChatCompletion (line 212) | class OpenAIChatCompletion(LocalChatCompletion):
    method __init__ (line 213) | def __init__(
    method api_key (line 228) | def api_key(self):
    method loglikelihood (line 237) | def loglikelihood(self, requests, **kwargs):

FILE: eval/lm-evaluation-harness/lm_eval/models/optimum_lm.py
  class OptimumLM (line 14) | class OptimumLM(HFLM):
    method __init__ (line 25) | def __init__(
    method _create_model (line 44) | def _create_model(

FILE: eval/lm-evaluation-harness/lm_eval/models/sglang.py
  class SGLang (line 46) | class SGLang(TemplateLM):
    method __init__ (line 49) | def __init__(
    method eot_token_id (line 159) | def eot_token_id(self):
    method prefix_token_id (line 164) | def prefix_token_id(self):
    method max_length (line 173) | def max_length(self):
    method max_gen_toks (line 190) | def max_gen_toks(self):
    method apply_chat_template (line 193) | def apply_chat_template(self, chat_history: List[Dict[str, str]]) -> str:
    method tokenizer_name (line 202) | def tokenizer_name(self) -> str:
    method tok_encode (line 205) | def tok_encode(
    method _model_generate (line 230) | def _model_generate(
    method loglikelihood_rolling (line 243) | def loglikelihood_rolling(
    method generate_until (line 279) | def generate_until(
    method _loglikelihood_tokens (line 378) | def _loglikelihood_tokens(
    method _parse_logprobs (line 435) | def _parse_logprobs(tokens: List, outputs, ctxlen: int) -> Tuple[float...
    method modify_gen_kwargs (line 498) | def modify_gen_kwargs(kwargs: dict) -> dict:

FILE: eval/lm-evaluation-harness/lm_eval/models/textsynth.py
  function textsynth_completion (line 29) | def textsynth_completion(**kwargs):
  class TextSynthLM (line 51) | class TextSynthLM(LM):
    method __init__ (line 52) | def __init__(self, engine, truncate: bool = False, **kwargs) -> None:
    method eot_token_id (line 68) | def eot_token_id(self):
    method max_length (line 73) | def max_length(self) -> int:
    method max_gen_toks (line 78) | def max_gen_toks(self) -> int:
    method batch_size (line 82) | def batch_size(self):
    method device (line 87) | def device(self):
    method tok_encode (line 91) | def tok_encode(self, string: str):
    method tok_decode (line 95) | def tok_decode(self, tokens):
    method loglikelihood (line 99) | def loglikelihood(self, requests, disable_tqdm: bool = False):
    method loglikelihood_rolling (line 123) | def loglikelihood_rolling(self, requests, disable_tqdm: bool = False):
    method generate_until (line 133) | def generate_until(self, requests, disable_tqdm: bool = False):
    method _model_call (line 166) | def _model_call(self, inps):
    method _model_generate (line 170) | def _model_generate(self, context, max_length, eos_token_id):

FILE: eval/lm-evaluation-harness/lm_eval/models/utils.py
  function chunks (line 33) | def chunks(iter, n: int = 0, fn=None):
  class MultiChoice (line 71) | class MultiChoice:
    method __init__ (line 72) | def __init__(self, choices) -> None:
    method __contains__ (line 76) | def __contains__(self, values) -> bool:
    method __iter__ (line 85) | def __iter__(self) -> Iterator:
  class Grouper (line 90) | class Grouper:
    method __init__ (line 97) | def __init__(self, arr, fn) -> None:
    method get_grouped (line 115) | def get_grouped(self):
    method get_original (line 126) | def get_original(self, grouped_dict):
  function pad_and_concat (line 148) | def pad_and_concat(
  function clear_torch_cache (line 199) | def clear_torch_cache() -> None:
  function get_dtype (line 204) | def get_dtype(dtype: Union[str, torch.dtype]) -> torch.dtype:
  class MultiTokenEOSCriteria (line 214) | class MultiTokenEOSCriteria(transformers.StoppingCriteria):
    method __init__ (line 217) | def __init__(
    method __call__ (line 240) | def __call__(self, input_ids, scores, **kwargs) -> bool:
  function stop_sequences_criteria (line 254) | def stop_sequences_criteria(
  function undistribute (line 272) | def undistribute(iterable):
  function retry_on_specific_exceptions (line 312) | def retry_on_specific_exceptions(
  class Collator (line 352) | class Collator:
    method __init__ (line 365) | def __init__(
    method _group_by_index (line 386) | def _group_by_index(self) -> None:
    method _group_by_context (line 392) | def _group_by_context(self) -> None:
    method get_batched (line 398) | def get_batched(self, n: int = 1, batch_fn: Optional[Callable] = None)...
    method get_cache (line 439) | def get_cache(
    method _reorder (line 501) | def _reorder(self, arr: Union[List, Tuple[Tuple[int, Any], ...]]) -> I...
    method get_original (line 517) | def get_original(self, newarr: List) -> List:
    method __len__ (line 538) | def __len__(self):
    method group (line 542) | def group(
    method get_chunks (line 585) | def get_chunks(_iter, n: int = 0, fn=None):
  function configure_pad_token (line 624) | def configure_pad_token(
  function replace_placeholders (line 669) | def replace_placeholders(

FILE: eval/lm-evaluation-harness/lm_eval/models/vllm_causallms.py
  class VLLM (line 36) | class VLLM(TemplateLM):
    method __init__ (line 39) | def __init__(
    method eot_token_id (line 148) | def eot_token_id(self):
    method prefix_token_id (line 153) | def prefix_token_id(self):
    method max_length (line 162) | def max_length(self):
    method max_gen_toks (line 179) | def max_gen_toks(self):
    method apply_chat_template (line 182) | def apply_chat_template(self, chat_history: List[Dict[str, str]]) -> str:
    method tokenizer_name (line 191) | def tokenizer_name(self) -> str:
    method tok_encode (line 194) | def tok_encode(
    method _model_generate (line 219) | def _model_generate(
    method loglikelihood_rolling (line 464) | def loglikelihood_rolling(
    method generate_until (line 500) | def generate_until(
    method _loglikelihood_tokens (line 611) | def _loglikelihood_tokens(
    method _parse_logprobs (line 668) | def _parse_logprobs(tokens: List, outputs, ctxlen: int) -> Tuple[float...
    method modify_gen_kwargs (line 731) | def modify_gen_kwargs(kwargs: dict) -> dict:

FILE: eval/lm-evaluation-harness/lm_eval/models/vllm_vlms.py
  class VLLM_VLM (line 28) | class VLLM_VLM(VLLM):
    method __init__ (line 31) | def __init__(
    method tok_batch_multimodal_encode (line 58) | def tok_batch_multimodal_encode(
    method _model_generate (line 76) | def _model_generate(
    method apply_chat_template (line 130) | def apply_chat_template(self, chat_history: List[Dict[str, str]]) -> str:
    method generate_until (line 183) | def generate_until(

FILE: eval/lm-evaluation-harness/lm_eval/prompts/__init__.py
  function get_prompt (line 21) | def get_prompt(prompt_id: str, dataset_name: str = None, subset_name: st...
  function load_prompt_list (line 70) | def load_prompt_list(
  class PromptString (line 111) | class PromptString:
    method __init__ (line 112) | def __init__(self, prompt_string):
    method apply (line 115) | def apply(self, doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/__init__.py
  class TaskManager (line 17) | class TaskManager:
    method __init__ (line 23) | def __init__(
    method initialize_tasks (line 55) | def initialize_tasks(
    method all_tasks (line 87) | def all_tasks(self):
    method all_groups (line 91) | def all_groups(self):
    method all_subtasks (line 95) | def all_subtasks(self):
    method all_tags (line 99) | def all_tags(self):
    method task_index (line 103) | def task_index(self):
    method list_all_tasks (line 106) | def list_all_tasks(
    method match_tasks (line 172) | def match_tasks(self, task_list):
    method _name_is_registered (line 175) | def _name_is_registered(self, name) -> bool:
    method _name_is_task (line 180) | def _name_is_task(self, name) -> bool:
    method _name_is_tag (line 185) | def _name_is_tag(self, name) -> bool:
    method _name_is_group (line 190) | def _name_is_group(self, name) -> bool:
    method _name_is_python_task (line 197) | def _name_is_python_task(self, name):
    method _config_is_task (line 204) | def _config_is_task(self, config) -> bool:
    method _config_is_group (line 209) | def _config_is_group(self, config) -> bool:
    method _config_is_python_task (line 214) | def _config_is_python_task(self, config) -> bool:
    method _get_yaml_path (line 219) | def _get_yaml_path(self, name):
    method _get_config (line 224) | def _get_config(self, name):
    method _get_tasklist (line 233) | def _get_tasklist(self, name):
    method _process_alias (line 238) | def _process_alias(self, config, group=None):
    method _class_has_config_in_constructor (line 247) | def _class_has_config_in_constructor(self, cls):
    method _load_individual_task_or_group (line 255) | def _load_individual_task_or_group(
    method load_task_or_group (line 401) | def load_task_or_group(self, task_list: Optional[Union[str, list]] = N...
    method load_config (line 418) | def load_config(self, config: Dict):
    method _get_task_and_group (line 421) | def _get_task_and_group(self, task_dir: str):
  function get_task_name_from_config (line 532) | def get_task_name_from_config(task_config: Dict[str, str]) -> str:
  function get_task_name_from_object (line 541) | def get_task_name_from_object(task_object):
  function _check_duplicates (line 554) | def _check_duplicates(task_dict: dict) -> List[str]:
  function get_task_dict (line 581) | def get_task_dict(

FILE: eval/lm-evaluation-harness/lm_eval/tasks/aclue/_generate_configs.py
  function parse_args (line 33) | def parse_args():

FILE: eval/lm-evaluation-harness/lm_eval/tasks/afrimgsm/utils.py
  function add_regex_pattern (line 75) | def add_regex_pattern(regex_pattern):
  function gen_lang_yamls (line 109) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 195) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 9) | def doc_to_text(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 9) | def doc_to_text(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/afrimmlu/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 9) | def doc_to_text(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/en-direct/utils.py
  function doc_to_target (line 4) | def doc_to_target(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/afrixnli/anli prompt/translate/utils.py
  function doc_to_target (line 4) | def doc_to_target(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/utils.py
  function doc_to_text (line 4) | def doc_to_text(doc):
  function doc_to_target (line 17) | def doc_to_target(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/utils.py
  function doc_to_text (line 4) | def doc_to_text(doc):
  function doc_to_target (line 17) | def doc_to_target(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/afrixnli/utils.py
  class FunctionTag (line 6) | class FunctionTag:
    method __init__ (line 7) | def __init__(self, value):
  function gen_lang_yamls (line 123) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 211) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/agieval/utils.py
  function parse_math_answer (line 10) | def parse_math_answer(raw_string):
  function _fix_fracs (line 82) | def _fix_fracs(string):
  function _fix_a_slash_b (line 114) | def _fix_a_slash_b(string):
  function _remove_right_units (line 129) | def _remove_right_units(string):
  function _fix_sqrt (line 139) | def _fix_sqrt(string):
  function _strip_string (line 154) | def _strip_string(string):
  function is_equiv (line 224) | def is_equiv(str1, str2, verbose=False):
  function process_results (line 243) | def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
  function process_results_mcqa (line 262) | def process_results_mcqa(doc, results):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/aime/utils.py
  function extract_answer_idx (line 126) | def extract_answer_idx(sampler, options: List[str], attempt: str):
  class ChatCompletionSampler (line 137) | class ChatCompletionSampler:
    method __init__ (line 142) | def __init__(
    method _handle_image (line 158) | def _handle_image(
    method _handle_text (line 169) | def _handle_text(self, text: str):
    method _pack_message (line 172) | def _pack_message(self, role: str, content):
    method __call__ (line 175) | def __call__(self, message_list) -> str:
  function doc_to_text (line 202) | def doc_to_text(doc: dict) -> str:
  function process_docs (line 205) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
  function process_results (line 222) | def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
  function last_boxed_only_string (line 304) | def last_boxed_only_string(string: str) -> Optional[str]:
  function remove_boxed (line 333) | def remove_boxed(s: str) -> str:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/utils.py
  function process_docs (line 15) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/utils.py
  function process_docs (line 15) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/utils.py
  function process_docs (line 7) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/utils.py
  function doc_to_text (line 7) | def doc_to_text(doc):
  function process_docs (line 24) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/utils.py
  function process_docs (line 15) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/utils.py
  function process_docs (line 15) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/utils.py
  function process_docs (line 7) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/utils.py
  function doc_to_text (line 7) | def doc_to_text(doc):
  function process_docs (line 24) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/utils.py
  function process_docs (line 5) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/_generate_configs.py
  function parse_args (line 60) | def parse_args():

FILE: eval/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/utils.py
  function doc_to_text (line 14) | def doc_to_text(doc):
  function doc_to_choice (line 43) | def doc_to_choice(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/basque_bench/flores_eu/create_yamls_flores_eu.py
  function doc_to_text (line 257) | def doc_to_text(src: str, tgt: str) -> str:
  function doc_to_target (line 265) | def doc_to_target(tgt: str) -> str:
  function gen_lang_yamls (line 272) | def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
  function main (line 316) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/basque_bench/utils.py
  function xcopa_doc_to_text (line 9) | def xcopa_doc_to_text(doc):
  function xcopa_doc_to_choice (line 14) | def xcopa_doc_to_choice(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/basqueglue/utils.py
  function general_detokenize (line 7) | def general_detokenize(string):
  function process_doc (line 16) | def process_doc(string):
  function process_wic_docs (line 22) | def process_wic_docs(dataset):
  function coref_doc_to_text (line 36) | def coref_doc_to_text(x):
  function micro_f1_score (line 62) | def micro_f1_score(items):
  function vaxx_f1_score (line 71) | def vaxx_f1_score(items):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/bbh/_generate_configs.py
  function parse_args (line 17) | def parse_args():

FILE: eval/lm-evaluation-harness/lm_eval/tasks/bbh/cot_zeroshot/utils.py
  class ExtendedRegexFilter (line 9) | class ExtendedRegexFilter(RegexFilter):
    method __init__ (line 14) | def __init__(
    method filter_ignores (line 28) | def filter_ignores(self, st):
    method find_match (line 41) | def find_match(self, regex, resp, convert_dict={}):
  class MapRegexFilter (line 53) | class MapRegexFilter(ExtendedRegexFilter):
    method __init__ (line 54) | def __init__(
    method apply (line 82) | def apply(self, resps, docs):
  class NumberParseRegexFilter (line 109) | class NumberParseRegexFilter(ExtendedRegexFilter):
    method apply (line 110) | def apply(self, resps, docs):
  class WordSortFilter (line 140) | class WordSortFilter(Filter):
    method apply (line 143) | def apply(self, resps, docs):
  class MultiChoiceRegexFilter (line 162) | class MultiChoiceRegexFilter(ExtendedRegexFilter):
    method __init__ (line 163) | def __init__(self, *args, **kwargs):
    method apply (line 175) | def apply(self, resps, docs):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/bbh/zeroshot/utils.py
  class ExtendedRegexFilter (line 9) | class ExtendedRegexFilter(RegexFilter):
    method __init__ (line 14) | def __init__(
    method filter_ignores (line 28) | def filter_ignores(self, st):
    method find_match (line 41) | def find_match(self, regex, resp, convert_dict={}):
  class MapRegexFilter (line 53) | class MapRegexFilter(ExtendedRegexFilter):
    method __init__ (line 54) | def __init__(
    method apply (line 82) | def apply(self, resps, docs):
  class NumberParseRegexFilter (line 109) | class NumberParseRegexFilter(ExtendedRegexFilter):
    method apply (line 110) | def apply(self, resps, docs):
  class WordSortFilter (line 140) | class WordSortFilter(Filter):
    method apply (line 143) | def apply(self, resps, docs):
  class MultiChoiceRegexFilter (line 162) | class MultiChoiceRegexFilter(ExtendedRegexFilter):
    method __init__ (line 163) | def __init__(self, *args, **kwargs):
    method apply (line 175) | def apply(self, resps, docs):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/belebele/_generate_configs.py
  function parse_args (line 18) | def parse_args():
  function query (line 41) | def query():

FILE: eval/lm-evaluation-harness/lm_eval/tasks/bigbench/generate_tasks.py
  function main (line 183) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/blimp/generate_configs.py
  function main (line 75) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/catalan_bench/flores_ca/create_yamls_flores_ca.py
  function code_to_language_name (line 246) | def code_to_language_name(code):
  function code_to_short_name (line 250) | def code_to_short_name(code):
  function jinja_var (line 254) | def jinja_var(s):
  function doc_to_text (line 258) | def doc_to_text(src: str, tgt: str) -> str:
  function doc_to_target (line 266) | def doc_to_target(tgt: str) -> str:
  function gen_lang_yamls (line 273) | def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
  function main (line 317) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/catalan_bench/utils.py
  function lowercase_first_letter (line 10) | def lowercase_first_letter(text):
  function process_doc_nli (line 14) | def process_doc_nli(dataset):
  function process_results_coqcat (line 38) | def process_results_coqcat(doc, results):
  function process_results_qa (line 72) | def process_results_qa(doc, results):
  function process_doc_cabreu (line 81) | def process_doc_cabreu(dataset):
  function process_docs_paraphrases (line 96) | def process_docs_paraphrases(dataset):
  function process_docs_copa_ca (line 119) | def process_docs_copa_ca(dataset):
  function rouge1 (line 128) | def rouge1(items):
  function rouge1_agg (line 135) | def rouge1_agg(items):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/ceval/_generate_configs.py
  function parse_args (line 70) | def parse_args():

FILE: eval/lm-evaluation-harness/lm_eval/tasks/cmmlu/_generate_configs.py
  function parse_args (line 85) | def parse_args():

FILE: eval/lm-evaluation-harness/lm_eval/tasks/code_x_glue/code-text/bleu.py
  function normalize (line 58) | def normalize(s):
  function count_ngrams (line 78) | def count_ngrams(words, n=4):
  function cook_refs (line 87) | def cook_refs(refs, n=4):
  function cook_test (line 101) | def cook_test(test, item, n=4):
  function score_cooked (line 132) | def score_cooked(allcomps, n=4, ground=0, smooth=1):
  function bleu (line 174) | def bleu(refs, candidate, ground=0, smooth=1):
  function splitPuncts (line 180) | def splitPuncts(line):
  function computeMaps (line 184) | def computeMaps(predictions, goldfile):
  function bleuFromMaps (line 210) | def bleuFromMaps(m1, m2):
  function smoothed_bleu_4 (line 222) | def smoothed_bleu_4(references, predictions, **kwargs):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/code_x_glue/code-text/utils.py
  function doc_to_text (line 1) | def doc_to_text(doc):
  function doc_to_target (line 8) | def doc_to_target(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/copal_id/utils.py
  function convert_choice (line 4) | def convert_choice(choice):
  function doc_to_text (line 8) | def doc_to_text(doc, connector):
  function doc_to_choice (line 13) | def doc_to_choice(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/coqa/utils.py
  function doc_to_text (line 6) | def doc_to_text(doc):
  function doc_to_target (line 19) | def doc_to_target(doc):
  function em (line 37) | def em(gold_list, pred):
  function compute_scores (line 51) | def compute_scores(gold_list, pred):
  function process_results (line 72) | def process_results(doc, results):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/crows_pairs/utils.py
  function process_results (line 4) | def process_results(doc, results):
  function doc_to_choice (line 19) | def doc_to_choice(doc):
  function filter_dataset (line 23) | def filter_dataset(dataset: datasets.Dataset, bias_type: str) -> dataset...
  function filter_race_color (line 27) | def filter_race_color(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_socio (line 31) | def filter_socio(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_gender (line 35) | def filter_gender(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_age (line 39) | def filter_age(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_religion (line 43) | def filter_religion(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_disability (line 47) | def filter_disability(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_orientation (line 51) | def filter_orientation(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_nationality (line 55) | def filter_nationality(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_appearance (line 59) | def filter_appearance(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_autre (line 63) | def filter_autre(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/csatqa/_generate_configs.py
  function parse_args (line 17) | def parse_args():

FILE: eval/lm-evaluation-harness/lm_eval/tasks/csatqa/utils.py
  function process_docs (line 4) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/drop/utils.py
  function process_docs (line 10) | def process_docs(dataset):
  function get_answers (line 22) | def get_answers(doc):
  function parse_answer (line 51) | def parse_answer(answer):
  function process_results (line 64) | def process_results(doc, results):
  function get_metrics (line 76) | def get_metrics(predicted, gold):
  function _answer_to_bags (line 100) | def _answer_to_bags(answer):
  function _align_bags (line 114) | def _align_bags(predicted, gold):
  function _compute_f1 (line 134) | def _compute_f1(predicted_bag, gold_bag):
  function _match_numbers_if_present (line 152) | def _match_numbers_if_present(gold_bag, predicted_bag):
  function _is_number (line 166) | def _is_number(text):
  function _remove_articles (line 174) | def _remove_articles(text):
  function _white_space_fix (line 178) | def _white_space_fix(text):
  function _remove_punc (line 182) | def _remove_punc(text):
  function _fix_number (line 190) | def _fix_number(text):
  function _tokenize (line 194) | def _tokenize(text):
  function _normalize (line 198) | def _normalize(answer):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/eq_bench/utils.py
  function calculate_score_fullscale (line 5) | def calculate_score_fullscale(docs, results):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/eus_exams/configs.py
  function gen_config_yamls (line 16) | def gen_config_yamls(output_dir: str, overwrite: bool) -> None:
  function main (line 49) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/eus_exams/utils.py
  function process_docs (line 4) | def process_docs(dataset: datasets.Dataset):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/eus_reading/utils.py
  function doc_to_text_context (line 7) | def doc_to_text_context(doc) -> str:
  function doc_to_choice (line 28) | def doc_to_choice(doc) -> List[str]:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/eus_trivia/utils.py
  function doc_to_text (line 7) | def doc_to_text(doc) -> str:
  function doc_to_choice (line 28) | def doc_to_choice(doc) -> List[str]:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/fda/task.py
  class FDA (line 10) | class FDA(ConfigurableTask):
    method __init__ (line 15) | def __init__(self, **kwargs):
    method has_training_docs (line 18) | def has_training_docs(self):
    method has_validation_docs (line 21) | def has_validation_docs(self):
    method has_test_docs (line 24) | def has_test_docs(self):
    method validation_docs (line 27) | def validation_docs(self):
    method doc_to_text (line 30) | def doc_to_text(self, doc):
    method doc_to_target (line 33) | def doc_to_target(self, doc):
    method construct_requests (line 36) | def construct_requests(self, doc, ctx, **kwargs):
    method process_results (line 58) | def process_results(self, doc, results):
    method aggregation (line 73) | def aggregation(self):
    method higher_is_better (line 83) | def higher_is_better(self):
  function contains_score (line 94) | def contains_score(prediction: str, labels: List[str]):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/french_bench/preprocess_wikitext.py
  function wikitext_detokenizer (line 4) | def wikitext_detokenizer(doc):
  function process_results (line 39) | def process_results(doc, results):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/french_bench/utils.py
  function normalize_answer (line 9) | def normalize_answer(s):
  function get_tokens (line 29) | def get_tokens(s):
  function exact (line 36) | def exact(predictions, references):
  function f1 (line 41) | def f1(predictions, references):
  function rouge1 (line 57) | def rouge1(items):
  function rouge1_agg (line 64) | def rouge1_agg(items):
  function is_included (line 74) | def is_included(items):
  function preprocess (line 83) | def preprocess(text):
  function process_docs (line 92) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/galician_bench/flores_gl/create_yamls_flores_gl.py
  function doc_to_text (line 257) | def doc_to_text(src: str, tgt: str) -> str:
  function doc_to_target (line 265) | def doc_to_target(tgt: str) -> str:
  function gen_lang_yamls (line 272) | def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
  function main (line 316) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/galician_bench/utils.py
  function lowercase_first_letter (line 14) | def lowercase_first_letter(text):
  function process_summarization (line 18) | def process_summarization(dataset):
  function process_docs_paraphrases (line 28) | def process_docs_paraphrases(dataset):
  function process_docs_paws (line 56) | def process_docs_paws(dataset):
  function rouge1 (line 84) | def rouge1(items):
  function rouge1_agg (line 91) | def rouge1_agg(items):
  function process_results_mc2 (line 102) | def process_results_mc2(doc, results):
  function process_docs_gen (line 115) | def process_docs_gen(dataset: datasets.Dataset) -> datasets.Dataset:
  function preprocess_function_gen (line 119) | def preprocess_function_gen(examples):
  function process_doc_nli (line 143) | def process_doc_nli(dataset):
  function process_results_gen (line 170) | def process_results_gen(doc, results):
  function bleu (line 241) | def bleu(refs, preds):
  function rouge (line 264) | def rouge(refs, preds):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/glianorex/preprocess_glianorex.py
  function doc_to_text (line 4) | def doc_to_text(doc) -> str:
  function doc_to_target (line 10) | def doc_to_target(doc) -> str:
  function filter_dataset (line 15) | def filter_dataset(dataset: datasets.Dataset, lang: str) -> datasets.Dat...
  function filter_french (line 19) | def filter_french(dataset: datasets.Dataset) -> datasets.Dataset:
  function filter_english (line 23) | def filter_english(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/glue/mnli/utils.py
  function doc_to_text (line 1) | def doc_to_text(doc) -> str:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/gpqa/cot_n_shot/_generate_configs.py
  function main (line 5) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/gpqa/cot_n_shot/utils.py
  function preprocess (line 7) | def preprocess(text):
  function process_docs (line 17) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/gpqa/cot_zeroshot/_generate_configs.py
  function main (line 5) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/gpqa/cot_zeroshot/utils.py
  function preprocess (line 7) | def preprocess(text):
  function process_docs (line 17) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/gpqa/generative/_generate_configs.py
  function main (line 5) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/gpqa/generative/utils.py
  function preprocess (line 7) | def preprocess(text):
  function process_docs (line 17) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/gpqa/n_shot/_generate_configs.py
  function main (line 5) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/gpqa/n_shot/utils.py
  function preprocess (line 7) | def preprocess(text):
  function process_docs (line 20) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/gpqa/openai/utils.py
  function extract_answer (line 86) | def extract_answer(sampler, question: str, attempt: str):
  class ChatCompletionSampler (line 91) | class ChatCompletionSampler:
    method __init__ (line 96) | def __init__(
    method _handle_image (line 112) | def _handle_image(
    method _handle_text (line 123) | def _handle_text(self, text: str):
    method _pack_message (line 126) | def _pack_message(self, role: str, content):
    method __call__ (line 129) | def __call__(self, message_list) -> str:
  function process_results (line 156) | def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
  function process_docs (line 221) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
  function last_boxed_only_string (line 244) | def last_boxed_only_string(string: str) -> Optional[str]:
  function remove_boxed (line 273) | def remove_boxed(s: str) -> str:
  function doc_to_text_gpqa (line 286) | def doc_to_text_gpqa(doc: dict) -> str:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/gpqa/zeroshot/_generate_configs.py
  function main (line 5) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/gpqa/zeroshot/utils.py
  function preprocess (line 7) | def preprocess(text):
  function process_docs (line 17) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/hellaswag/utils.py
  function preprocess (line 6) | def preprocess(text):
  function process_docs (line 15) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/hendrycks_ethics/utils.py
  function _preproc_doc (line 5) | def _preproc_doc(doc):
  function doc_to_text (line 18) | def doc_to_text(doc) -> str:
  function doc_to_target (line 23) | def doc_to_target(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/hendrycks_math/utils.py
  function process_docs (line 6) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
  function process_results (line 18) | def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
  function is_equiv (line 36) | def is_equiv(str1, str2, verbose=False):
  function remove_boxed (line 53) | def remove_boxed(s):
  function last_boxed_only_string (line 67) | def last_boxed_only_string(string):
  function fix_fracs (line 97) | def fix_fracs(string):
  function fix_a_slash_b (line 129) | def fix_a_slash_b(string):
  function remove_right_units (line 144) | def remove_right_units(string):
  function fix_sqrt (line 154) | def fix_sqrt(string):
  function strip_string (line 169) | def strip_string(string):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/ifeval/instructions.py
  class Instruction (line 110) | class Instruction:
    method __init__ (line 113) | def __init__(self, instruction_id):
    method build_description (line 116) | def build_description(self, **kwargs):
    method get_instruction_args (line 119) | def get_instruction_args(self):
    method get_instruction_args_keys (line 122) | def get_instruction_args_keys(self):
    method check_following (line 125) | def check_following(self, value):
  class ResponseLanguageChecker (line 129) | class ResponseLanguageChecker(Instruction):
    method build_description (line 132) | def build_description(self, *, language=None):
    method get_instruction_args (line 155) | def get_instruction_args(self):
    method get_instruction_args_keys (line 159) | def get_instruction_args_keys(self):
    method check_following (line 163) | def check_following(self, value):
  class NumberOfSentences (line 184) | class NumberOfSentences(Instruction):
    method build_description (line 187) | def build_description(self, *, num_sentences=None, relation=None):
    method get_instruction_args (line 225) | def get_instruction_args(self):
    method get_instruction_args_keys (line 232) | def get_instruction_args_keys(self):
    method check_following (line 236) | def check_following(self, value):
  class PlaceholderChecker (line 256) | class PlaceholderChecker(Instruction):
    method build_description (line 259) | def build_description(self, *, num_placeholders=None):
    method get_instruction_args (line 278) | def get_instruction_args(self):
    method get_instruction_args_keys (line 282) | def get_instruction_args_keys(self):
    method check_following (line 286) | def check_following(self, value):
  class BulletListChecker (line 301) | class BulletListChecker(Instruction):
    method build_description (line 304) | def build_description(self, *, num_bullets=None):
    method get_instruction_args (line 325) | def get_instruction_args(self):
    method get_instruction_args_keys (line 329) | def get_instruction_args_keys(self):
    method check_following (line 333) | def check_following(self, value):
  class ConstrainedResponseChecker (line 350) | class ConstrainedResponseChecker(Instruction):
    method build_description (line 353) | def build_description(self):
    method get_instruction_args (line 364) | def get_instruction_args(self):
    method get_instruction_args_keys (line 368) | def get_instruction_args_keys(self):
    method check_following (line 372) | def check_following(self, value):
  class ConstrainedStartChecker (line 389) | class ConstrainedStartChecker(Instruction):
    method build_description (line 392) | def build_description(self, *, starter=None):
    method get_instruction_args (line 411) | def get_instruction_args(self):
    method get_instruction_args_keys (line 415) | def get_instruction_args_keys(self):
    method check_following (line 419) | def check_following(self, value):
  class HighlightSectionChecker (line 436) | class HighlightSectionChecker(Instruction):
    method build_description (line 439) | def build_description(self, *, num_highlights=None):
    method get_instruction_args (line 460) | def get_instruction_args(self):
    method get_instruction_args_keys (line 464) | def get_instruction_args_keys(self):
    method check_following (line 468) | def check_following(self, value):
  class SectionChecker (line 492) | class SectionChecker(Instruction):
    method build_description (line 495) | def build_description(self, *, section_spliter=None, num_sections=None):
    method get_instruction_args (line 531) | def get_instruction_args(self):
    method get_instruction_args_keys (line 538) | def get_instruction_args_keys(self):
    method check_following (line 542) | def check_following(self, value):
  class ParagraphChecker (line 561) | class ParagraphChecker(Instruction):
    method build_description (line 564) | def build_description(self, *, num_paragraphs=None):
    method get_instruction_args (line 584) | def get_instruction_args(self):
    method get_instruction_args_keys (line 588) | def get_instruction_args_keys(self):
    method check_following (line 592) | def check_following(self, value):
  class PostscriptChecker (line 616) | class PostscriptChecker(Instruction):
    method build_description (line 619) | def build_description(self, *, postscript_marker=None):
    method get_instruction_args (line 644) | def get_instruction_args(self):
    method get_instruction_args_keys (line 648) | def get_instruction_args_keys(self):
    method check_following (line 652) | def check_following(self, value):
  class RephraseChecker (line 674) | class RephraseChecker(Instruction):
    method build_description (line 677) | def build_description(self, *, original_message):
    method get_instruction_args (line 703) | def get_instruction_args(self):
    method get_instruction_args_keys (line 707) | def get_instruction_args_keys(self):
    method check_following (line 711) | def check_following(self, value):
    method is_change (line 733) | def is_change(self, response):
    method strip_changes (line 737) | def strip_changes(self, response):
  class KeywordChecker (line 742) | class KeywordChecker(Instruction):
    method build_description (line 745) | def build_description(self, *, keywords=None):
    method get_instruction_args (line 768) | def get_instruction_args(self):
    method get_instruction_args_keys (line 772) | def get_instruction_args_keys(self):
    method check_following (line 776) | def check_following(self, value):
  class KeywordFrequencyChecker (line 784) | class KeywordFrequencyChecker(Instruction):
    method build_description (line 787) | def build_description(self, *, keyword=None, frequency=None, relation=...
    method get_instruction_args (line 833) | def get_instruction_args(self):
    method get_instruction_args_keys (line 841) | def get_instruction_args_keys(self):
    method check_following (line 845) | def check_following(self, value):
  class NumberOfWords (line 855) | class NumberOfWords(Instruction):
    method build_description (line 858) | def build_description(self, *, num_words=None, relation=None):
    method get_instruction_args (line 896) | def get_instruction_args(self):
    method get_instruction_args_keys (line 900) | def get_instruction_args_keys(self):
    method check_following (line 904) | def check_following(self, value):
  class JsonFormat (line 914) | class JsonFormat(Instruction):
    method build_description (line 917) | def build_description(self):
    method get_instruction_args (line 924) | def get_instruction_args(self):
    method get_instruction_args_keys (line 928) | def get_instruction_args_keys(self):
    method check_following (line 932) | def check_following(self, value):
  class ParagraphFirstWordCheck (line 949) | class ParagraphFirstWordCheck(Instruction):
    method build_description (line 952) | def build_description(
    method get_instruction_args (line 998) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1006) | def get_instruction_args_keys(self):
    method check_following (line 1010) | def check_following(self, value):
  class KeySentenceChecker (line 1056) | class KeySentenceChecker(Instruction):
    method build_description (line 1059) | def build_description(self, key_sentences=None, num_sentences=None):
    method get_instruction_args (line 1091) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1098) | def get_instruction_args_keys(self):
    method check_following (line 1102) | def check_following(self, value):
  class ForbiddenWords (line 1113) | class ForbiddenWords(Instruction):
    method build_description (line 1116) | def build_description(self, forbidden_words=None):
    method get_instruction_args (line 1140) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1144) | def get_instruction_args_keys(self):
    method check_following (line 1148) | def check_following(self, value):
  class RephraseParagraph (line 1156) | class RephraseParagraph(Instruction):
    method build_description (line 1159) | def build_description(self, *, original_paragraph, low, high):
    method get_instruction_args (line 1190) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1198) | def get_instruction_args_keys(self):
    method check_following (line 1202) | def check_following(self, value):
  class TwoResponsesChecker (line 1216) | class TwoResponsesChecker(Instruction):
    method build_description (line 1219) | def build_description(self):
    method get_instruction_args (line 1227) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1231) | def get_instruction_args_keys(self):
    method check_following (line 1235) | def check_following(self, value):
  class RepeatPromptThenAnswer (line 1258) | class RepeatPromptThenAnswer(Instruction):
    method build_description (line 1261) | def build_description(self, *, prompt_to_repeat=None):
    method get_instruction_args (line 1282) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1285) | def get_instruction_args_keys(self):
    method check_following (line 1289) | def check_following(self, value):
  class EndChecker (line 1295) | class EndChecker(Instruction):
    method build_description (line 1298) | def build_description(self, *, end_phrase=None):
    method get_instruction_args (line 1318) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1321) | def get_instruction_args_keys(self):
    method check_following (line 1325) | def check_following(self, value):
  class TitleChecker (line 1332) | class TitleChecker(Instruction):
    method build_description (line 1335) | def build_description(self):
    method get_instruction_args (line 1343) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1346) | def get_instruction_args_keys(self):
    method check_following (line 1350) | def check_following(self, value):
  class LetterFrequencyChecker (line 1362) | class LetterFrequencyChecker(Instruction):
    method build_description (line 1365) | def build_description(self, *, letter=None, let_frequency=None, let_re...
    method get_instruction_args (line 1417) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1425) | def get_instruction_args_keys(self):
    method check_following (line 1429) | def check_following(self, value):
  class CapitalLettersEnglishChecker (line 1440) | class CapitalLettersEnglishChecker(Instruction):
    method build_description (line 1443) | def build_description(self):
    method get_instruction_args (line 1450) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1453) | def get_instruction_args_keys(self):
    method check_following (line 1457) | def check_following(self, value):
  class LowercaseLettersEnglishChecker (line 1471) | class LowercaseLettersEnglishChecker(Instruction):
    method build_description (line 1474) | def build_description(self):
    method get_instruction_args (line 1482) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1485) | def get_instruction_args_keys(self):
    method check_following (line 1489) | def check_following(self, value):
  class CommaChecker (line 1503) | class CommaChecker(Instruction):
    method build_description (line 1506) | def build_description(self):
    method get_instruction_args (line 1513) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1516) | def get_instruction_args_keys(self):
    method check_following (line 1520) | def check_following(self, value):
  class CapitalWordFrequencyChecker (line 1525) | class CapitalWordFrequencyChecker(Instruction):
    method build_description (line 1528) | def build_description(
    method get_instruction_args (line 1566) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1573) | def get_instruction_args_keys(self):
    method check_following (line 1577) | def check_following(self, value):
  class QuotationChecker (line 1591) | class QuotationChecker(Instruction):
    method build_description (line 1594) | def build_description(self):
    method get_instruction_args (line 1601) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1605) | def get_instruction_args_keys(self):
    method check_following (line 1609) | def check_following(self, value):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/ifeval/instructions_registry.py
  function conflict_make (line 153) | def conflict_make(conflicts):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/ifeval/instructions_util.py
  function download_nltk_resources (line 34) | def download_nltk_resources():
  function split_into_sentences (line 1623) | def split_into_sentences(text):
  function count_words (line 1674) | def count_words(text):
  function _get_sentence_tokenizer (line 1683) | def _get_sentence_tokenizer():
  function count_sentences (line 1687) | def count_sentences(text):
  function generate_keywords (line 1694) | def generate_keywords(num_keywords):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/ifeval/utils.py
  class InputExample (line 9) | class InputExample:
  class OutputExample (line 17) | class OutputExample:
  function test_instruction_following_strict (line 25) | def test_instruction_following_strict(
  function test_instruction_following_loose (line 58) | def test_instruction_following_loose(
  function process_results (line 112) | def process_results(doc, results):
  function agg_inst_level_acc (line 132) | def agg_inst_level_acc(items):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/kobest/utils.py
  function copa_doc_to_text (line 4) | def copa_doc_to_text(doc: dict) -> str:
  function copa_doc_to_target (line 9) | def copa_doc_to_target(doc: dict) -> str:
  function copa_doc_to_choice (line 14) | def copa_doc_to_choice(doc: dict) -> list:
  function sentineg_doc_to_text (line 18) | def sentineg_doc_to_text(doc: dict):
  function wic_doc_to_text (line 22) | def wic_doc_to_text(doc: dict) -> str:
  function hellaswag_process_doc (line 26) | def hellaswag_process_doc(doc: Dataset) -> Dataset:
  function macro_f1_score (line 42) | def macro_f1_score(items):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/leaderboard/gpqa/utils.py
  function preprocess (line 7) | def preprocess(text):
  function process_docs (line 17) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/leaderboard/ifeval/instructions.py
  class Instruction (line 110) | class Instruction:
    method __init__ (line 113) | def __init__(self, instruction_id):
    method build_description (line 116) | def build_description(self, **kwargs):
    method get_instruction_args (line 119) | def get_instruction_args(self):
    method get_instruction_args_keys (line 122) | def get_instruction_args_keys(self):
    method check_following (line 125) | def check_following(self, value):
  class ResponseLanguageChecker (line 129) | class ResponseLanguageChecker(Instruction):
    method build_description (line 132) | def build_description(self, *, language=None):
    method get_instruction_args (line 155) | def get_instruction_args(self):
    method get_instruction_args_keys (line 159) | def get_instruction_args_keys(self):
    method check_following (line 163) | def check_following(self, value):
  class NumberOfSentences (line 184) | class NumberOfSentences(Instruction):
    method build_description (line 187) | def build_description(self, *, num_sentences=None, relation=None):
    method get_instruction_args (line 225) | def get_instruction_args(self):
    method get_instruction_args_keys (line 232) | def get_instruction_args_keys(self):
    method check_following (line 236) | def check_following(self, value):
  class PlaceholderChecker (line 256) | class PlaceholderChecker(Instruction):
    method build_description (line 259) | def build_description(self, *, num_placeholders=None):
    method get_instruction_args (line 278) | def get_instruction_args(self):
    method get_instruction_args_keys (line 282) | def get_instruction_args_keys(self):
    method check_following (line 286) | def check_following(self, value):
  class BulletListChecker (line 301) | class BulletListChecker(Instruction):
    method build_description (line 304) | def build_description(self, *, num_bullets=None):
    method get_instruction_args (line 325) | def get_instruction_args(self):
    method get_instruction_args_keys (line 329) | def get_instruction_args_keys(self):
    method check_following (line 333) | def check_following(self, value):
  class ConstrainedResponseChecker (line 350) | class ConstrainedResponseChecker(Instruction):
    method build_description (line 353) | def build_description(self):
    method get_instruction_args (line 364) | def get_instruction_args(self):
    method get_instruction_args_keys (line 368) | def get_instruction_args_keys(self):
    method check_following (line 372) | def check_following(self, value):
  class ConstrainedStartChecker (line 389) | class ConstrainedStartChecker(Instruction):
    method build_description (line 392) | def build_description(self, *, starter=None):
    method get_instruction_args (line 411) | def get_instruction_args(self):
    method get_instruction_args_keys (line 415) | def get_instruction_args_keys(self):
    method check_following (line 419) | def check_following(self, value):
  class HighlightSectionChecker (line 436) | class HighlightSectionChecker(Instruction):
    method build_description (line 439) | def build_description(self, *, num_highlights=None):
    method get_instruction_args (line 460) | def get_instruction_args(self):
    method get_instruction_args_keys (line 464) | def get_instruction_args_keys(self):
    method check_following (line 468) | def check_following(self, value):
  class SectionChecker (line 492) | class SectionChecker(Instruction):
    method build_description (line 495) | def build_description(self, *, section_spliter=None, num_sections=None):
    method get_instruction_args (line 531) | def get_instruction_args(self):
    method get_instruction_args_keys (line 538) | def get_instruction_args_keys(self):
    method check_following (line 542) | def check_following(self, value):
  class ParagraphChecker (line 561) | class ParagraphChecker(Instruction):
    method build_description (line 564) | def build_description(self, *, num_paragraphs=None):
    method get_instruction_args (line 584) | def get_instruction_args(self):
    method get_instruction_args_keys (line 588) | def get_instruction_args_keys(self):
    method check_following (line 592) | def check_following(self, value):
  class PostscriptChecker (line 616) | class PostscriptChecker(Instruction):
    method build_description (line 619) | def build_description(self, *, postscript_marker=None):
    method get_instruction_args (line 644) | def get_instruction_args(self):
    method get_instruction_args_keys (line 648) | def get_instruction_args_keys(self):
    method check_following (line 652) | def check_following(self, value):
  class RephraseChecker (line 674) | class RephraseChecker(Instruction):
    method build_description (line 677) | def build_description(self, *, original_message):
    method get_instruction_args (line 703) | def get_instruction_args(self):
    method get_instruction_args_keys (line 707) | def get_instruction_args_keys(self):
    method check_following (line 711) | def check_following(self, value):
    method is_change (line 733) | def is_change(self, response):
    method strip_changes (line 737) | def strip_changes(self, response):
  class KeywordChecker (line 742) | class KeywordChecker(Instruction):
    method build_description (line 745) | def build_description(self, *, keywords=None):
    method get_instruction_args (line 768) | def get_instruction_args(self):
    method get_instruction_args_keys (line 772) | def get_instruction_args_keys(self):
    method check_following (line 776) | def check_following(self, value):
  class KeywordFrequencyChecker (line 784) | class KeywordFrequencyChecker(Instruction):
    method build_description (line 787) | def build_description(self, *, keyword=None, frequency=None, relation=...
    method get_instruction_args (line 833) | def get_instruction_args(self):
    method get_instruction_args_keys (line 841) | def get_instruction_args_keys(self):
    method check_following (line 845) | def check_following(self, value):
  class NumberOfWords (line 855) | class NumberOfWords(Instruction):
    method build_description (line 858) | def build_description(self, *, num_words=None, relation=None):
    method get_instruction_args (line 896) | def get_instruction_args(self):
    method get_instruction_args_keys (line 900) | def get_instruction_args_keys(self):
    method check_following (line 904) | def check_following(self, value):
  class JsonFormat (line 914) | class JsonFormat(Instruction):
    method build_description (line 917) | def build_description(self):
    method get_instruction_args (line 924) | def get_instruction_args(self):
    method get_instruction_args_keys (line 928) | def get_instruction_args_keys(self):
    method check_following (line 932) | def check_following(self, value):
  class ParagraphFirstWordCheck (line 949) | class ParagraphFirstWordCheck(Instruction):
    method build_description (line 952) | def build_description(
    method get_instruction_args (line 998) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1006) | def get_instruction_args_keys(self):
    method check_following (line 1010) | def check_following(self, value):
  class KeySentenceChecker (line 1056) | class KeySentenceChecker(Instruction):
    method build_description (line 1059) | def build_description(self, key_sentences=None, num_sentences=None):
    method get_instruction_args (line 1091) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1098) | def get_instruction_args_keys(self):
    method check_following (line 1102) | def check_following(self, value):
  class ForbiddenWords (line 1113) | class ForbiddenWords(Instruction):
    method build_description (line 1116) | def build_description(self, forbidden_words=None):
    method get_instruction_args (line 1140) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1144) | def get_instruction_args_keys(self):
    method check_following (line 1148) | def check_following(self, value):
  class RephraseParagraph (line 1156) | class RephraseParagraph(Instruction):
    method build_description (line 1159) | def build_description(self, *, original_paragraph, low, high):
    method get_instruction_args (line 1190) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1198) | def get_instruction_args_keys(self):
    method check_following (line 1202) | def check_following(self, value):
  class TwoResponsesChecker (line 1216) | class TwoResponsesChecker(Instruction):
    method build_description (line 1219) | def build_description(self):
    method get_instruction_args (line 1227) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1231) | def get_instruction_args_keys(self):
    method check_following (line 1235) | def check_following(self, value):
  class RepeatPromptThenAnswer (line 1258) | class RepeatPromptThenAnswer(Instruction):
    method build_description (line 1261) | def build_description(self, *, prompt_to_repeat=None):
    method get_instruction_args (line 1282) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1285) | def get_instruction_args_keys(self):
    method check_following (line 1289) | def check_following(self, value):
  class EndChecker (line 1295) | class EndChecker(Instruction):
    method build_description (line 1298) | def build_description(self, *, end_phrase=None):
    method get_instruction_args (line 1318) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1321) | def get_instruction_args_keys(self):
    method check_following (line 1325) | def check_following(self, value):
  class TitleChecker (line 1332) | class TitleChecker(Instruction):
    method build_description (line 1335) | def build_description(self):
    method get_instruction_args (line 1343) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1346) | def get_instruction_args_keys(self):
    method check_following (line 1350) | def check_following(self, value):
  class LetterFrequencyChecker (line 1362) | class LetterFrequencyChecker(Instruction):
    method build_description (line 1365) | def build_description(self, *, letter=None, let_frequency=None, let_re...
    method get_instruction_args (line 1417) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1425) | def get_instruction_args_keys(self):
    method check_following (line 1429) | def check_following(self, value):
  class CapitalLettersEnglishChecker (line 1440) | class CapitalLettersEnglishChecker(Instruction):
    method build_description (line 1443) | def build_description(self):
    method get_instruction_args (line 1450) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1453) | def get_instruction_args_keys(self):
    method check_following (line 1457) | def check_following(self, value):
  class LowercaseLettersEnglishChecker (line 1471) | class LowercaseLettersEnglishChecker(Instruction):
    method build_description (line 1474) | def build_description(self):
    method get_instruction_args (line 1482) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1485) | def get_instruction_args_keys(self):
    method check_following (line 1489) | def check_following(self, value):
  class CommaChecker (line 1503) | class CommaChecker(Instruction):
    method build_description (line 1506) | def build_description(self):
    method get_instruction_args (line 1513) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1516) | def get_instruction_args_keys(self):
    method check_following (line 1520) | def check_following(self, value):
  class CapitalWordFrequencyChecker (line 1525) | class CapitalWordFrequencyChecker(Instruction):
    method build_description (line 1528) | def build_description(
    method get_instruction_args (line 1566) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1573) | def get_instruction_args_keys(self):
    method check_following (line 1577) | def check_following(self, value):
  class QuotationChecker (line 1591) | class QuotationChecker(Instruction):
    method build_description (line 1594) | def build_description(self):
    method get_instruction_args (line 1601) | def get_instruction_args(self):
    method get_instruction_args_keys (line 1605) | def get_instruction_args_keys(self):
    method check_following (line 1609) | def check_following(self, value):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/leaderboard/ifeval/instructions_registry.py
  function conflict_make (line 153) | def conflict_make(conflicts):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/leaderboard/ifeval/instructions_util.py
  function download_nltk_resources (line 34) | def download_nltk_resources():
  function split_into_sentences (line 1623) | def split_into_sentences(text):
  function count_words (line 1674) | def count_words(text):
  function _get_sentence_tokenizer (line 1683) | def _get_sentence_tokenizer():
  function count_sentences (line 1687) | def count_sentences(text):
  function generate_keywords (line 1694) | def generate_keywords(num_keywords):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/leaderboard/ifeval/utils.py
  class InputExample (line 8) | class InputExample:
  class OutputExample (line 16) | class OutputExample:
  function test_instruction_following_strict (line 24) | def test_instruction_following_strict(
  function test_instruction_following_loose (line 57) | def test_instruction_following_loose(
  function process_results (line 111) | def process_results(doc, results):
  function agg_inst_level_acc (line 131) | def agg_inst_level_acc(items):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/leaderboard/math/utils.py
  function doc_to_text (line 22) | def doc_to_text(doc: dict) -> str:
  function process_docs (line 26) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
  function list_fewshot_samples (line 42) | def list_fewshot_samples() -> list[dict]:
  function process_results (line 67) | def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
  function last_boxed_only_string (line 84) | def last_boxed_only_string(string: str) -> Optional[str]:
  function remove_boxed (line 114) | def remove_boxed(s: str) -> str:
  class timeout (line 128) | class timeout:
    method __init__ (line 129) | def __init__(self, seconds=1, error_message="Timeout"):
    method handle_timeout (line 133) | def handle_timeout(self, signum, frame):
    method __enter__ (line 136) | def __enter__(self):
    method __exit__ (line 140) | def __exit__(self, type, value, traceback):
  function is_equiv (line 144) | def is_equiv(x1: str, x2: str) -> bool:
  function get_unnormalized_answer (line 187) | def get_unnormalized_answer(text: str) -> str:
  function normalize_final_answer (line 259) | def normalize_final_answer(final_answer: str) -> str:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/leaderboard/mmlu_pro/utils.py
  function doc_to_text (line 4) | def doc_to_text(doc):
  function doc_to_choice (line 14) | def doc_to_choice(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/leaderboard/musr/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):
  function doc_to_text (line 14) | def doc_to_text(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/lingoly/script.py
  function clean_answer (line 6) | def clean_answer(answer: str):
  function safe_exact (line 31) | def safe_exact(references: list[str], predictions: list[str]):
  function parse_str_list_score (line 42) | def parse_str_list_score(model, correct, scoring_func):
  function exact_match (line 91) | def exact_match(references: list[str], predictions: list[str]):
  function aggregate_scores (line 124) | def aggregate_scores(input):
  function aggregate_metrics (line 128) | def aggregate_metrics(

FILE: eval/lm-evaluation-harness/lm_eval/tasks/lingoly/utils.py
  function load_questionsheet (line 6) | def load_questionsheet(qsheet: dict, no_context: bool = False):
  function format_answers (line 31) | def format_answers(questionpart_ns: list[str], answers: list[str]):
  function load_question (line 43) | def load_question(
  function load_all_questions (line 77) | def load_all_questions(

FILE: eval/lm-evaluation-harness/lm_eval/tasks/logiqa/utils_logiqa.py
  function doc_to_text (line 2) | def doc_to_text(doc) -> str:
  function doc_to_target (line 22) | def doc_to_target(doc) -> int:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/logiqa2/utils_logiqa2.py
  function doc_to_text (line 2) | def doc_to_text(doc) -> str:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/mathqa/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/med_concepts_qa/_generate_configs.py
  function generate_yaml_content (line 6) | def generate_yaml_content(vocab_name: str, level: str):
  function generate_yaml_files (line 17) | def generate_yaml_files(

FILE: eval/lm-evaluation-harness/lm_eval/tasks/medmcqa/utils_medmcqa.py
  function doc_to_text (line 2) | def doc_to_text(doc) -> str:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/medqa/preprocess_medqa.py
  function doc_to_text (line 1) | def doc_to_text(doc) -> str:
  function doc_to_target (line 12) | def doc_to_target(doc) -> int:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/metamathqa/utils.py
  function doc_to_text (line 29) | def doc_to_text(doc: dict) -> str:
  function process_docs (line 33) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
  function process_results (line 47) | def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
  function get_answer (line 87) | def get_answer(string: str):
  function last_boxed_only_string (line 96) | def last_boxed_only_string(string: str) -> Optional[str]:
  function remove_boxed (line 126) | def remove_boxed(s: str) -> str:
  class timeout (line 140) | class timeout:
    method __init__ (line 141) | def __init__(self, seconds=1, error_message="Timeout"):
    method handle_timeout (line 145) | def handle_timeout(self, signum, frame):
    method __enter__ (line 148) | def __enter__(self):
    method __exit__ (line 152) | def __exit__(self, type, value, traceback):
  function is_equiv (line 156) | def is_equiv(x1: str, x2: str) -> bool:
  function get_unnormalized_answer (line 199) | def get_unnormalized_answer(text: str) -> str:
  function normalize_final_answer (line 271) | def normalize_final_answer(final_answer: str) -> str:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/mgsm/utils.py
  function add_regex_pattern (line 97) | def add_regex_pattern(regex_pattern):
  function gen_lang_yamls (line 131) | def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
  function main (line 204) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/minerva_math/utils.py
  function doc_to_text (line 22) | def doc_to_text(doc: dict) -> str:
  function process_docs (line 26) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
  function list_fewshot_samples (line 42) | def list_fewshot_samples() -> list[dict]:
  function process_results (line 67) | def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
  function last_boxed_only_string (line 84) | def last_boxed_only_string(string: str) -> Optional[str]:
  function remove_boxed (line 114) | def remove_boxed(s: str) -> str:
  class timeout (line 128) | class timeout:
    method __init__ (line 129) | def __init__(self, seconds=1, error_message="Timeout"):
    method handle_timeout (line 133) | def handle_timeout(self, signum, frame):
    method __enter__ (line 136) | def __enter__(self):
    method __exit__ (line 140) | def __exit__(self, type, value, traceback):
  function is_equiv (line 144) | def is_equiv(x1: str, x2: str) -> bool:
  function get_unnormalized_answer (line 187) | def get_unnormalized_answer(text: str) -> str:
  function normalize_final_answer (line 259) | def normalize_final_answer(final_answer: str) -> str:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/mmlu/_generate_configs.py
  function parse_args (line 77) | def parse_args():

FILE: eval/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_cot_zeroshot/utils.py
  class MultiChoiceRegexFilter (line 8) | class MultiChoiceRegexFilter(RegexFilter):
    method __init__ (line 11) | def __init__(
    method apply (line 34) | def apply(self, resps, docs):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/generative/utils.py
  class MultiChoiceRegexFilter (line 8) | class MultiChoiceRegexFilter(RegexFilter):
    method __init__ (line 11) | def __init__(
    method apply (line 34) | def apply(self, resps, docs):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/mmlu_pro/utils.py
  function format_cot_example (line 24) | def format_cot_example(example, including_answer=True):
  function process_docs (line 46) | def process_docs(dataset, subject):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/mmlusr/answer_only/utils.py
  function process_docs (line 4) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/mmlusr/config.py
  function parse_args (line 79) | def parse_args():

FILE: eval/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer/utils.py
  function process_docs (line 4) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_only/utils.py
  function process_docs (line 4) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/mmmu/utils.py
  function doc_to_image (line 26) | def doc_to_image(doc):
  function doc_to_text (line 42) | def doc_to_text(doc):
  function _doc_to_text (line 54) | def _doc_to_text(doc):
  function process_results (line 76) | def process_results(doc, results):
  function parse_multi_choice_response (line 105) | def parse_multi_choice_response(response, all_choices, index2ans):
  function check_is_number (line 163) | def check_is_number(string):
  function normalize_str (line 175) | def normalize_str(string):
  function extract_numbers (line 200) | def extract_numbers(string):
  function parse_open_response (line 223) | def parse_open_response(response):
  function eval_multi_choice (line 299) | def eval_multi_choice(gold_i, pred_i):
  function eval_open (line 316) | def eval_open(gold_i, pred_i):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py
  function main (line 6) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/model_written_evals/persona/_generate_configs.py
  function main (line 6) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/mutual/utils.py
  function process_docs (line 4) | def process_docs(dataset):
  function process_results (line 30) | def process_results(doc, results):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/noticia/utils.py
  function clean_text (line 6) | def clean_text(text: str) -> str:
  function rouge1 (line 20) | def rouge1(items):
  function average_len (line 27) | def average_len(items):
  function rouge1_agg (line 34) | def rouge1_agg(items):
  function average_len_agg (line 48) | def average_len_agg(items):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/okapi/arc_multilingual/utils.py
  function preprocess (line 6) | def preprocess(text):
  function process_docs (line 14) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/okapi/hellaswag_multilingual/utils.py
  function preprocess (line 6) | def preprocess(text):
  function process_docs (line 15) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/okapi/mmlu_multilingual/_generate_configs.py
  function main (line 6) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/okapi/truthfulqa_multilingual/utils.py
  function preprocess (line 23) | def preprocess(text):
  function process_docs (line 33) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
  function process_results_mc2 (line 48) | def process_results_mc2(doc, results):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/openai_math/utils.py
  function extract_answer_idx (line 123) | def extract_answer_idx(sampler, options: List[str], attempt: str):
  class ChatCompletionSampler (line 134) | class ChatCompletionSampler:
    method __init__ (line 139) | def __init__(
    method _handle_image (line 155) | def _handle_image(
    method _handle_text (line 166) | def _handle_text(self, text: str):
    method _pack_message (line 169) | def _pack_message(self, role: str, content):
    method __call__ (line 172) | def __call__(self, message_list) -> str:
  function doc_to_text (line 199) | def doc_to_text(doc: dict) -> str:
  function process_docs (line 202) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
  function process_docs_openai_math_cot_quality_check (line 219) | def process_docs_openai_math_cot_quality_check(dataset: datasets.Dataset...
  function process_results (line 241) | def process_results(doc: dict, results: List[str]) -> Dict[str, int]:
  function last_boxed_only_string (line 321) | def last_boxed_only_string(string: str) -> Optional[str]:
  function remove_boxed (line 350) | def remove_boxed(s: str) -> str:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/paloma/paloma_utils.py
  function doc_to_target (line 1) | def doc_to_target(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/paws-x/_generate_config.py
  function gen_lang_yamls (line 49) | def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
  function main (line 91) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/portuguese_bench/flores_pt/create_yamls_flores_pt.py
  function doc_to_text (line 257) | def doc_to_text(src: str, tgt: str) -> str:
  function doc_to_target (line 265) | def doc_to_target(tgt: str) -> str:
  function gen_lang_yamls (line 272) | def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
  function main (line 315) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/pubmedqa/preprocess_pubmedqa.py
  function doc_to_text (line 1) | def doc_to_text(doc) -> str:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/qa4mre/preprocess_qa4mre.py
  function qa4mre_process (line 1) | def qa4mre_process(doc):
  function doc_to_target (line 5) | def doc_to_target(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/qasper/metrics.py
  function normalize_answer (line 6) | def normalize_answer(s):
  function f1_abstractive (line 28) | def f1_abstractive(predictions, references):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/qasper/utils.py
  function process_docs (line 6) | def process_docs(dataset, set_answer_type="bool"):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/race/preprocess_race.py
  function process_ast (line 4) | def process_ast(string):
  function last_problem (line 8) | def last_problem(doc):
  function get_answer_option (line 12) | def get_answer_option(problem):
  function doc_to_choice (line 18) | def doc_to_choice(doc):
  function doc_to_text (line 24) | def doc_to_text(doc):
  function doc_to_target (line 37) | def doc_to_target(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/realtoxicityprompts/metric.py
  function toxicity_perspective_api (line 10) | def toxicity_perspective_api(

FILE: eval/lm-evaluation-harness/lm_eval/tasks/scrolls/task.py
  function _download_metric (line 44) | def _download_metric():
  function _process_doc_prepended_question (line 62) | def _process_doc_prepended_question(doc):
  function _drop_duplicates_in_input (line 78) | def _drop_duplicates_in_input(untokenized_dataset):
  function _num_cpu_cores (line 99) | def _num_cpu_cores():
  class _SCROLLSTask (line 111) | class _SCROLLSTask(ConfigurableTask):
    method __init__ (line 119) | def __init__(self, config=None):
    method has_training_docs (line 124) | def has_training_docs(self):
    method has_validation_docs (line 127) | def has_validation_docs(self):
    method has_test_docs (line 130) | def has_test_docs(self):
    method training_docs (line 133) | def training_docs(self):
    method validation_docs (line 144) | def validation_docs(self):
    method should_decontaminate (line 155) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 158) | def doc_to_decontamination_query(self, doc):
    method download (line 161) | def download(self, *args, **kwargs):
    method _get_prune_text (line 169) | def _get_prune_text(self, sample):
    method prune (line 172) | def prune(self):
    method doc_to_target (line 198) | def doc_to_target(self, doc):
    method doc_to_text (line 201) | def doc_to_text(self, doc):
    method higher_is_better (line 204) | def higher_is_better(self):
    method _scrolls_metrics (line 208) | def _scrolls_metrics(self):
    method _make_compute_metrics (line 211) | def _make_compute_metrics(self, value):
    method aggregation (line 221) | def aggregation(self):
  class _SCROLLSMultipleChoiceTask (line 228) | class _SCROLLSMultipleChoiceTask(_SCROLLSTask):
    method __post_init__ (line 229) | def __post_init__(self):
    method _scrolls_metrics (line 232) | def _scrolls_metrics(self):
    method aggregation (line 235) | def aggregation(self):
    method higher_is_better (line 238) | def higher_is_better(self):
    method process_results (line 241) | def process_results(self, doc, results):
    method construct_requests (line 255) | def construct_requests(self, doc, ctx, **kwargs):
  class _SCROLLSSummaryTask (line 269) | class _SCROLLSSummaryTask(_SCROLLSTask):
    method _process_doc (line 270) | def _process_doc(self, doc):
    method _scrolls_metrics (line 273) | def _scrolls_metrics(self):
    method process_results (line 280) | def process_results(self, doc, results):
    method construct_requests (line 287) | def construct_requests(self, doc, ctx, **kwargs):
    method doc_to_text (line 296) | def doc_to_text(self, doc):
  class Qasper (line 300) | class Qasper(_SCROLLSTask):
    method _process_doc (line 307) | def _process_doc(self, doc):
    method _scrolls_metrics (line 317) | def _scrolls_metrics(self):
    method process_results (line 320) | def process_results(self, doc, results):
    method construct_requests (line 329) | def construct_requests(self, doc, ctx, **kwargs):
  class QuALITY (line 357) | class QuALITY(_SCROLLSMultipleChoiceTask):
    method _normalize_answer (line 366) | def _normalize_answer(text):
    method _process_doc (line 369) | def _process_doc(self, doc):
  class NarrativeQA (line 385) | class NarrativeQA(_SCROLLSTask):
    method _process_doc (line 392) | def _process_doc(self, doc):
    method _scrolls_metrics (line 395) | def _scrolls_metrics(self):
    method _get_prune_text (line 398) | def _get_prune_text(self, doc):
    method process_results (line 405) | def process_results(self, doc, results):
    method construct_requests (line 408) | def construct_requests(self, doc, ctx, **kwargs):
  class ContractNLI (line 418) | class ContractNLI(_SCROLLSMultipleChoiceTask):
    method _process_doc (line 426) | def _process_doc(self, doc):
    method doc_to_text (line 432) | def doc_to_text(self, doc):
  class GovReport (line 436) | class GovReport(_SCROLLSSummaryTask):
  class SummScreenFD (line 449) | class SummScreenFD(_SCROLLSSummaryTask):
  class QMSum (line 457) | class QMSum(_SCROLLSSummaryTask):
    method _process_doc (line 466) | def _process_doc(self, doc):
    method doc_to_text (line 469) | def doc_to_text(self, doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/spanish_bench/flores_es/create_yamls_flores_es.py
  function doc_to_text (line 257) | def doc_to_text(src: str, tgt: str) -> str:
  function doc_to_target (line 265) | def doc_to_target(tgt: str) -> str:
  function gen_lang_yamls (line 272) | def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
  function main (line 315) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/spanish_bench/utils.py
  function lowercase_first_letter (line 10) | def lowercase_first_letter(text):
  function process_doc_nli (line 14) | def process_doc_nli(dataset):
  function process_results_qa (line 38) | def process_results_qa(doc, results):
  function process_xlsum (line 47) | def process_xlsum(dataset):
  function process_docs_paraphrases (line 57) | def process_docs_paraphrases(dataset):
  function rouge1 (line 85) | def rouge1(items):
  function rouge1_agg (line 92) | def rouge1_agg(items):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/squad_completion/task.py
  class SQUADCompletion (line 10) | class SQUADCompletion(ConfigurableTask):
    method __init__ (line 15) | def __init__(self, **kwargs):
    method has_training_docs (line 18) | def has_training_docs(self):
    method has_validation_docs (line 21) | def has_validation_docs(self):
    method has_test_docs (line 24) | def has_test_docs(self):
    method validation_docs (line 27) | def validation_docs(self):
    method doc_to_text (line 30) | def doc_to_text(self, doc):
    method doc_to_target (line 33) | def doc_to_target(self, doc):
    method construct_requests (line 36) | def construct_requests(self, doc, ctx, **kwargs):
    method process_results (line 58) | def process_results(self, doc, results):
    method aggregation (line 73) | def aggregation(self):
    method higher_is_better (line 83) | def higher_is_better(self):
  function contains_score (line 94) | def contains_score(prediction: str, labels: List[str]):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/squadv2/task.py
  function _squad_metric (line 39) | def _squad_metric(predictions, references):
  function _squad_agg (line 46) | def _squad_agg(key, items):
  class SQuAD2 (line 52) | class SQuAD2(ConfigurableTask):
    method __init__ (line 57) | def __init__(self, config=None):
    method has_training_docs (line 65) | def has_training_docs(self):
    method has_validation_docs (line 68) | def has_validation_docs(self):
    method has_test_docs (line 71) | def has_test_docs(self):
    method training_docs (line 74) | def training_docs(self):
    method validation_docs (line 77) | def validation_docs(self):
    method doc_to_text (line 80) | def doc_to_text(self, doc):
    method should_decontaminate (line 94) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 97) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 100) | def doc_to_target(self, doc):
    method construct_requests (line 108) | def construct_requests(self, doc, ctx, **kwargs):
    method process_results (line 137) | def process_results(self, doc, results):
    method aggregation (line 195) | def aggregation(self):
    method higher_is_better (line 228) | def higher_is_better(self):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/super_glue/cb/aggregate.py
  function cb_multi_fi (line 4) | def cb_multi_fi(items):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/super_glue/cb/t5_utils.py
  function mean_3class_f1 (line 1) | def mean_3class_f1(predictions, references):  # This is a passthrough fu...
  function agg_mean_3class_f1 (line 11) | def agg_mean_3class_f1(items):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/super_glue/copa/utils.py
  function convert_choice (line 1) | def convert_choice(choice):
  function doc_to_text (line 5) | def doc_to_text(doc):
  function doc_to_target (line 14) | def doc_to_target(doc):
  function doc_to_choice (line 20) | def doc_to_choice(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/super_glue/multirc/t5_utils.py
  function f1 (line 6) | def f1(predictions, references):  # This is a passthrough function
  function agg_f1 (line 20) | def agg_f1(items):
  function em (line 29) | def em(predictions, references):  # This is a passthrough function
  function agg_em (line 43) | def agg_em(items):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/super_glue/record/t5_utils.py
  function doc_to_text (line 11) | def doc_to_text(doc):
  function process_docs (line 28) | def process_docs(dataset):
  function normalize_squad (line 50) | def normalize_squad(answer):
  function em (line 76) | def em(predictions, references):  # This is a passthrough function
  function f1 (line 80) | def f1(predictions, references):  # This is a passthrough function
  function squad_em_agg (line 84) | def squad_em_agg(items):
  function squad_f1_agg (line 104) | def squad_f1_agg(items):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/super_glue/record/util.py
  function doc_to_text (line 8) | def doc_to_text(doc):
  function format_answer (line 16) | def format_answer(query, entity):
  function doc_to_target (line 20) | def doc_to_target(doc):
  function doc_to_choice (line 25) | def doc_to_choice(doc):
  function process_docs (line 29) | def process_docs(dataset: datasets.Dataset):
  function process_results (line 41) | def process_results(doc, results):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/super_glue/wsc/preprocess_wsc.py
  function default_doc_to_text (line 4) | def default_doc_to_text(x):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/super_glue/wsc/t5_utils.py
  function doc_to_text (line 5) | def doc_to_text(x):
  function _wsc_inputs (line 10) | def _wsc_inputs(x):
  function clean (line 80) | def clean(s: str) -> str:
  function process_results (line 86) | def process_results(docs: dict, resps: List):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/swde/task.py
  class SWDE (line 10) | class SWDE(ConfigurableTask):
    method __init__ (line 15) | def __init__(self, **kwargs):
    method has_training_docs (line 18) | def has_training_docs(self):
    method has_validation_docs (line 21) | def has_validation_docs(self):
    method has_test_docs (line 24) | def has_test_docs(self):
    method validation_docs (line 27) | def validation_docs(self):
    method doc_to_text (line 30) | def doc_to_text(self, doc):
    method doc_to_target (line 33) | def doc_to_target(self, doc):
    method construct_requests (line 36) | def construct_requests(self, doc, ctx, **kwargs):
    method process_results (line 58) | def process_results(self, doc, results):
    method aggregation (line 73) | def aggregation(self):
    method higher_is_better (line 83) | def higher_is_better(self):
  function contains_score (line 94) | def contains_score(prediction: str, labels: List[str]):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/tinyBenchmarks/agg_functions.py
  function agg_pirt (line 15) | def agg_pirt(items: List[float], benchmark: str) -> float:
  function agg_gpirt_arc (line 21) | def agg_gpirt_arc(items: List[float], benchmark: str = "arc") -> float:
  function agg_gpirt_gsm8k (line 27) | def agg_gpirt_gsm8k(items: List[float], benchmark: str = "gsm8k") -> float:
  function agg_gpirt_hellaswag (line 33) | def agg_gpirt_hellaswag(items: List[float], benchmark: str = "hellaswag"...
  function agg_gpirt_mmlu (line 39) | def agg_gpirt_mmlu(items: List[float], benchmark: str = "mmlu") -> float:
  function agg_gpirt_truthfulqa (line 45) | def agg_gpirt_truthfulqa(items: List[float], benchmark: str = "truthfulq...
  function agg_gpirt_winogrande (line 51) | def agg_gpirt_winogrande(items: List[float], benchmark: str = "winogrand...

FILE: eval/lm-evaluation-harness/lm_eval/tasks/tinyBenchmarks/utils_hellaswag.py
  function preprocess (line 9) | def preprocess(text):
  function process_docs (line 18) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/tinyBenchmarks/utils_truthfulqa.py
  function process_results_mc2 (line 12) | def process_results_mc2(doc, results):
  function process_docs_gen (line 25) | def process_docs_gen(dataset: datasets.Dataset) -> datasets.Dataset:
  function preprocess_function (line 29) | def preprocess_function(examples):
  function process_results_gen (line 53) | def process_results_gen(doc, results):
  function bleu (line 124) | def bleu(refs, preds):
  function rouge (line 147) | def rouge(refs, preds):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/tinyBenchmarks/utils_winogrande.py
  function doc_to_text (line 4) | def doc_to_text(doc):
  function doc_to_target (line 9) | def doc_to_target(doc):
  function doc_to_choice (line 14) | def doc_to_choice(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/tmlu/default/_generate_configs.py
  function parse_args (line 78) | def parse_args():

FILE: eval/lm-evaluation-harness/lm_eval/tasks/tmlu/default/utils.py
  function process_docs (line 4) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/_generate_configs.py
  function parse_args (line 109) | def parse_args():

FILE: eval/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/utils.py
  function process_docs (line 4) | def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/toxigen/utils.py
  function doc_to_target (line 4) | def doc_to_target(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/translation/utils.py
  function code_to_language (line 35) | def code_to_language(code):
  function gen_lang_yamls (line 41) | def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
  function main (line 100) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/truthfulqa/utils.py
  function process_results_mc2 (line 10) | def process_results_mc2(doc, results):
  function process_docs_gen (line 23) | def process_docs_gen(dataset: datasets.Dataset) -> datasets.Dataset:
  function preprocess_function (line 27) | def preprocess_function(examples):
  function process_results_gen (line 51) | def process_results_gen(doc, results):
  function bleu (line 122) | def bleu(refs, preds):
  function rouge (line 145) | def rouge(refs, preds):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/unitxt/task.py
  function score (line 28) | def score(items, metric):
  class Unitxt (line 37) | class Unitxt(ConfigurableTask):
    method __init__ (line 40) | def __init__(
    method has_training_docs (line 55) | def has_training_docs(self):
    method has_validation_docs (line 58) | def has_validation_docs(self):
    method has_test_docs (line 61) | def has_test_docs(self):
    method training_docs (line 64) | def training_docs(self):
    method validation_docs (line 67) | def validation_docs(self):
    method test_docs (line 70) | def test_docs(self):
    method doc_to_text (line 73) | def doc_to_text(self, doc):
    method should_decontaminate (line 76) | def should_decontaminate(self):
    method doc_to_target (line 79) | def doc_to_target(self, doc):
    method construct_requests (line 82) | def construct_requests(self, doc, ctx, **kwargs):
    method process_results (line 104) | def process_results(self, doc, results):
    method aggregation (line 125) | def aggregation(self):
    method higher_is_better (line 136) | def higher_is_better(self):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/webqs/utils.py
  function doc_to_choice (line 4) | def doc_to_choice(doc: Dict) -> List[str]:
  function doc_to_target (line 9) | def doc_to_target(doc: Dict) -> List[int]:
  function _remove_prefixes (line 15) | def _remove_prefixes(aliases):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/wikitext/preprocess_wikitext.py
  function wikitext_detokenizer (line 4) | def wikitext_detokenizer(doc):
  function process_results (line 39) | def process_results(doc, results):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/winogrande/preprocess_winogrande.py
  function doc_to_text (line 1) | def doc_to_text(doc):
  function doc_to_target (line 6) | def doc_to_target(doc):
  function doc_to_choice (line 11) | def doc_to_choice(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/wmt2016/metrics.py
  function bleu (line 4) | def bleu(predictions, references):
  function agg_bleu (line 8) | def agg_bleu(items):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/wsc273/utils.py
  function process_doc (line 16) | def process_doc(dataset):
  function __normalize_option (line 27) | def __normalize_option(doc, option):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/xcopa/utils.py
  function convert_choice (line 4) | def convert_choice(choice):
  function doc_to_text (line 8) | def doc_to_text(doc, connector):
  function doc_to_choice (line 14) | def doc_to_choice(doc):

FILE: eval/lm-evaluation-harness/lm_eval/tasks/xnli/utils.py
  function gen_lang_yamls (line 104) | def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
  function main (line 148) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/tasks/xwinograd/utils.py
  function doc_to_text (line 13) | def doc_to_text(doc: Dict) -> int:
  function doc_to_target (line 25) | def doc_to_target(doc: Dict) -> str:
  function doc_to_choice (line 36) | def doc_to_choice(doc: Dict) -> List[str]:
  function gen_lang_yamls (line 43) | def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
  function main (line 76) | def main() -> None:

FILE: eval/lm-evaluation-harness/lm_eval/utils.py
  function hash_string (line 35) | def hash_string(string: str) -> str:
  function escaped_split (line 39) | def escaped_split(text, sep_char, maxsplit=-1):
  function handle_arg_string (line 62) | def handle_arg_string(arg):
  function handle_non_serializable (line 75) | def handle_non_serializable(o):
  function sanitize_list (line 84) | def sanitize_list(sub):
  function simple_parse_args_string (line 96) | def simple_parse_args_string(args_string):
  function join_iters (line 112) | def join_iters(iters):
  function group (line 117) | def group(arr, fn):
  function pattern_match (line 128) | def pattern_match(patterns, source_list):
  function softmax (line 139) | def softmax(x):
  function general_detokenize (line 145) | def general_detokenize(string):
  function get_file_task_name (line 155) | def get_file_task_name(filename: str) -> str:
  function get_file_datetime (line 162) | def get_file_datetime(filename: str) -> str:
  function sanitize_model_name (line 169) | def sanitize_model_name(model_name: str) -> str:
  function sanitize_task_name (line 176) | def sanitize_task_name(task_name: str) -> str:
  function get_latest_filename (line 183) | def get_latest_filename(filenames: List[str]) -> str:
  function get_results_filenames (line 190) | def get_results_filenames(filenames: List[str]) -> List[str]:
  function get_sample_results_filenames (line 197) | def get_sample_results_filenames(filenames: List[str]) -> List[str]:
  function get_rolling_token_windows (line 204) | def get_rolling_token_windows(token_list, prefix_token, max_seq_len, con...
  function make_disjoint_window (line 245) | def make_disjoint_window(pair):
  class EnhancedJSONEncoder (line 251) | class EnhancedJSONEncoder(json.JSONEncoder):
    method default (line 257) | def default(self, o):
  class Reorderer (line 263) | class Reorderer:
    method __init__ (line 264) | def __init__(self, arr: List[Any], fn: Callable) -> None:
    method get_reordered (line 281) | def get_reordered(self):
    method get_original (line 289) | def get_original(self, newarr):
  function make_table (line 311) | def make_table(result_dict, column: str = "results", sort_results: bool ...
  function positional_deprecated (line 383) | def positional_deprecated(fn):
  function ignore_constructor (line 402) | def ignore_constructor(loader, node):
  function import_function (line 406) | def import_function(loader, node):
  function load_yaml_config (line 423) | def load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None, mo...
  function regex_replace (line 469) | def regex_replace(string, pattern, repl, count: int = 0):
  function apply_template (line 480) | def apply_template(template: str, doc: dict) -> str:
  function create_iterator (line 485) | def create_iterator(raw_iterator, *, rank=0, world_size=1, limit=None):
  function weighted_f1_score (line 494) | def weighted_f1_score(items):

FILE: eval/lm-evaluation-harness/scripts/build_benchmark.py
  function parse_args (line 15) | def parse_args():

FILE: eval/lm-evaluation-harness/scripts/clean_training_data/compress_and_package.py
  function process_task (line 16) | def process_task(
  function compress_and_move (line 31) | def compress_and_move(working_directory, output_directory, process_count):

FILE: eval/lm-evaluation-harness/scripts/clean_training_data/generate_13_grams.py
  function handler (line 46) | def handler(signal_received, frame):
  function yield_pile (line 51) | def yield_pile(start_offsets=None, checkpoint_offset=None):
  class Buckets (line 86) | class Buckets:
    method __init__ (line 87) | def __init__(self, directory, num_buckets):
    method add_data (line 104) | def add_data(self, key, value):
    method save_checkpoint (line 109) | def save_checkpoint(self):
    method close_buckets (line 116) | def close_buckets(self):
  function do_ngrams_in_buckets (line 121) | def do_ngrams_in_buckets(n_value, working_directory, bucket_count):

FILE: eval/lm-evaluation-harness/scripts/clean_training_data/investigate_pile.py
  function get_file_stats (line 12) | def get_file_stats(file_path, tqdm_func, global_tqdm):
  function get_files (line 36) | def get_files():
  function get_stats (line 43) | def get_stats():

FILE: eval/lm-evaluation-harness/scripts/clean_training_data/janitor_util.cpp
  function is_whitespace (line 9) | bool is_whitespace(char ch) noexcept {
  function is_punctuation (line 15) | bool is_punctuation(char c) noexcept {
  function clean_ngram (line 24) | std::vector<std::string> clean_ngram(std::string const &input,
  function clean_ngram_with_indices (line 109) | std::vector<std::tuple<std::string, size_t, size_t>>
  function PYBIND11_MODULE (line 194) | PYBIND11_MODULE(janitor_util, m) {

FILE: eval/lm-evaluation-harness/scripts/clean_training_data/process_sorted_buckets.py
  function process_bucket (line 35) | def process_bucket(
  function process_sorted_buckets (line 97) | def process_sorted_buckets(working_directory, move_dir, process_count):

FILE: eval/lm-evaluation-harness/scripts/clean_training_data/sort_13_gram_buckets.py
  function handler (line 28) | def handler(signal_received, frame):
  function sort_13_gram_buckets (line 33) | def sort_13_gram_buckets(working_directory):

FILE: eval/lm-evaluation-harness/scripts/cost_estimate.py
  class DryrunLM (line 9) | class DryrunLM(LM):
    method __init__ (line 10) | def __init__(self):
    method create_from_arg_string (line 16) | def create_from_arg_string(cls, arg_string):
    method loglikelihood (line 19) | def loglikelihood(self, requests):
    method generate_until (line 28) | def generate_until(self, requests):
    method loglikelihood_rolling (line 39) | def loglikelihood_rolling(self, requests):
  function main (line 49) | def main():

FILE: eval/lm-evaluation-harness/scripts/make_table_results.py
  function make_table (line 17) | def make_table(result_dict):

FILE: eval/lm-evaluation-harness/scripts/make_table_tasks.py
  function check (line 18) | def check(tf):

FILE: eval/lm-evaluation-harness/scripts/model_comparator.py
  function memory_stats (line 18) | def memory_stats():
  function calculate_z_value (line 24) | def calculate_z_value(res1: Dict, res2: Dict) -> Tuple[float, float]:
  function print_results (line 35) | def print_results(
  function parse_args (line 63) | def parse_args():

FILE: eval/lm-evaluation-harness/scripts/regression.py
  function parse_args (line 29) | def parse_args():
  function eval_models (line 47) | def eval_models(args, branch=None):
  function extract_value (line 108) | def extract_value(args, results, model, task, err=False):
  function format_value (line 126) | def format_value(args, results, model, task):
  function format_diff (line 132) | def format_diff(args, results1, results2, model, task):
  function main (line 139) | def main():

FILE: eval/lm-evaluation-harness/scripts/requests_caching.py
  function run_model_for_task_caching (line 34) | def run_model_for_task_caching(tasks: List[str], cache_requests: str):

FILE: eval/lm-evaluation-harness/scripts/write_out.py
  function parse_args (line 16) | def parse_args():
  function main (line 39) | def main():

FILE: eval/lm-evaluation-harness/scripts/zeno_visualize.py
  function parse_args (line 18) | def parse_args():
  function main (line 35) | def main():
  function tasks_for_model (line 139) | def tasks_for_model(model: str, data_path: str):
  function generate_dataset (line 158) | def generate_dataset(
  function generate_system_df (line 201) | def generate_system_df(data, config):

FILE: eval/lm-evaluation-harness/tests/models/test_api.py
  function api (line 9) | def api():
  function api_tokenized (line 16) | def api_tokenized():
  function test_create_payload_generate (line 24) | def test_create_payload_generate(api):
  function test_create_payload_loglikelihood (line 45) | def test_create_payload_loglikelihood(api):
  function test_model_generate_call_usage (line 91) | def test_model_generate_call_usage(
  function test_model_tokenized_call_usage (line 131) | def test_model_tokenized_call_usage(

FILE: eval/lm-evaluation-harness/tests/models/test_gguf.py
  function gguf_completion_mock (line 15) | def gguf_completion_mock(base_url=None, **kwargs):
  class GGUFLMTest (line 92) | class GGUFLMTest(unittest.TestCase):
    method test_loglikelihood (line 96) | def test_loglikelihood(self, gguf_completion_mock):
    method test_generate_until (line 118) | def test_generate_until(self, gguf_completion_mock):

FILE: eval/lm-evaluation-harness/tests/models/test_huggingface.py
  class Test_HFLM (line 23) | class Test_HFLM:
    method test_logliklihood (line 107) | def test_logliklihood(self) -> None:
    method test_generate_until (line 126) | def test_generate_until(self) -> None:
    method test_logliklihood_rolling (line 130) | def test_logliklihood_rolling(self) -> None:
    method test_toc_encode (line 134) | def test_toc_encode(self) -> None:
    method test_toc_decode (line 138) | def test_toc_decode(self) -> None:
    method test_batch_encode (line 142) | def test_batch_encode(self) -> None:
    method test_model_generate (line 146) | def test_model_generate(self) -> None:

FILE: eval/lm-evaluation-harness/tests/models/test_neuralmagic.py
  function test_sparseml_eval (line 28) | def test_sparseml_eval(model_id, task):
  function test_deepsparse_eval (line 48) | def test_deepsparse_eval(model_id, task):

FILE: eval/lm-evaluation-harness/tests/models/test_openvino.py
  function test_evaluator (line 20) | def test_evaluator(model_id, task):
  function test_ov_config (line 77) | def test_ov_config():

FILE: eval/lm-evaluation-harness/tests/models/test_vllm.py
  class Test_VLLM (line 13) | class Test_VLLM:
    method test_logliklihood (line 35) | def test_logliklihood(self) -> None:
    method test_generate_until (line 41) | def test_generate_until(self) -> None:
    method test_logliklihood_rolling (line 47) | def test_logliklihood_rolling(self) -> None:

FILE: eval/lm-evaluation-harness/tests/test_cli.py
  function test_cli_parse_error (line 8) | def test_cli_parse_error():
  function test_cli_parse_no_error (line 27) | def test_cli_parse_no_error():

FILE: eval/lm-evaluation-harness/tests/test_evaluator.py
  function test_evaluator (line 38) | def test_evaluator(
  function test_printed_results (line 113) | def test_printed_results(task_name: List[str], limit: int, model: str, m...

FILE: eval/lm-evaluation-harness/tests/test_include_path.py
  function test_include_correctness (line 20) | def test_include_correctness(limit: int, model: str, model_args: str):
  function test_no_include_defaults (line 71) | def test_no_include_defaults():

FILE: eval/lm-evaluation-harness/tests/test_janitor.py
  function simple_ngram (line 34) | def simple_ngram(sequence, n):
  function test_form_ngrams (line 46) | def test_form_ngrams():
  function test_word_ngrams (line 57) | def test_word_ngrams():
  function test_split_indices (line 71) | def test_split_indices():
  function test_word_ngrams_indices (line 101) | def test_word_ngrams_indices():
  function test_janitor1 (line 137) | def test_janitor1():
  function test_janitor2 (line 180) | def test_janitor2():
  function test_janitor3 (line 217) | def test_janitor3():
  function test_janitor4 (line 251) | def test_janitor4():
  function test_janitor5 (line 293) | def test_janitor5():
  function test_janitor6 (line 337) | def test_janitor6():
  function test_janitor7 (line 389) | def test_janitor7():
  function test_janitor8 (line 444) | def test_janitor8():

FILE: eval/lm-evaluation-harness/tests/test_misc.py
  function test_bootstrapping (line 8) | def test_bootstrapping():

FILE: eval/lm-evaluation-harness/tests/test_prompt.py
  function test_mmlu_prompt_rendering (line 73) | def test_mmlu_prompt_rendering(

FILE: eval/lm-evaluation-harness/tests/test_requests_caching.py
  function setup_and_teardown (line 26) | def setup_and_teardown():
  function clear_cache (line 35) | def clear_cache():
  function get_cache_files (line 44) | def get_cache_files(tasks: Optional[List[str]] = None) -> Tuple[List[str...
  function assert_created (line 57) | def assert_created(tasks: List[str], file_task_names: List[str]):
  function requests_caching_true (line 65) | def requests_caching_true(tasks: List[str]):
  function requests_caching_refresh (line 74) | def requests_caching_refresh(tasks: List[str]):
  function requests_caching_delete (line 94) | def requests_caching_delete(tasks: List[str]):
  function run_tests (line 108) | def run_tests():

FILE: eval/lm-evaluation-harness/tests/test_task_manager.py
  function custom_task_name (line 10) | def custom_task_name():
  function custom_task_tag (line 15) | def custom_task_tag():
  function task_yaml (line 20) | def task_yaml(pytestconfig, custom_task_name, custom_task_tag):
  function task_code (line 30) | def task_code():
  function custom_task_files_dir (line 49) | def custom_task_files_dir(task_yaml, task_code, custom_task_name):
  function test_python_task_inclusion (line 60) | def test_python_task_inclusion(

FILE: eval/lm-evaluation-harness/tests/test_tasks.py
  function task_class (line 19) | def task_class():
  function limit (line 32) | def limit() -> int:
  class TestNewTasks (line 38) | class TestNewTasks:
    method test_download (line 39) | def test_download(self, task_class: ConfigurableTask):
    method test_has_training_docs (line 43) | def test_has_training_docs(self, task_class: ConfigurableTask):
    method test_check_training_docs (line 46) | def test_check_training_docs(self, task_class: ConfigurableTask):
    method test_has_validation_docs (line 50) | def test_has_validation_docs(self, task_class):
    method test_check_validation_docs (line 53) | def test_check_validation_docs(self, task_class):
    method test_has_test_docs (line 57) | def test_has_test_docs(self, task_class):
    method test_check_test_docs (line 60) | def test_check_test_docs(self, task_class):
    method test_should_decontaminate (line 65) | def test_should_decontaminate(self, task_class):
    method test_doc_to_text (line 71) | def test_doc_to_text(self, task_class, limit):
    method test_create_choices (line 85) | def test_create_choices(self, task_class, limit):
    method test_doc_to_target (line 97) | def test_doc_to_target(self, task_class, limit):
    method test_build_all_requests (line 112) | def test_build_all_requests(self, task_class, limit):
    method test_construct_requests (line 117) | def test_construct_requests(self, task_class, limit):

FILE: eval/lm-evaluation-harness/tests/test_utils.py
  function test_get_rolling_token_windows_v1 (line 21) | def test_get_rolling_token_windows_v1():
  function test_get_rolling_token_windows_v2 (line 51) | def test_get_rolling_token_windows_v2():
  function test_get_rolling_token_windows_v3 (line 80) | def test_get_rolling_token_windows_v3():
  function test_get_rolling_token_windows_v4 (line 125) | def test_get_rolling_token_windows_v4():
  function test_get_rolling_token_windows_v5 (line 166) | def test_get_rolling_token_windows_v5():
  function test_get_rolling_token_windows_v6 (line 195) | def test_get_rolling_token_windows_v6():
  function test_get_rolling_token_windows_empty (line 219) | def test_get_rolling_token_windows_empty():
  function test_make_disjoint_window (line 232) | def test_make_disjoint_window():
  class TestCollator (line 241) | class TestCollator:
    method make_generate_sample (line 242) | def make_generate_sample(self, end=10):
    method make_loglikelihood_sample (line 255) | def make_loglikelihood_sample(self, end=11):
    method make_loglikelihood_sample_group (line 262) | def make_loglikelihood_sample_group(self, end=11):
    method test_generations (line 271) | def test_generations(self, batch_size, end):
    method test_loglikelihood (line 301) | def test_loglikelihood(self, batch_size, end):
    method test_context_grouping (line 324) | def test_context_grouping(self, batch_size):
  function test_aggregate_mean (line 365) | def test_aggregate_mean():
  function test_aggregate_stderrs (line 384) | def test_aggregate_stderrs(samples):

FILE: eval/lm-evaluation-harness/tests/utils.py
  function load_changed_files (line 14) | def load_changed_files(file_path: str) -> List[str]:
  function parser (line 25) | def parser(full_path: List[str]) -> List[str]:
  function new_tasks (line 37) | def new_tasks() -> Union[List[str], None]:

FILE: eval/rebase/inference_scaling/evaluate/data_processing/answer_extraction.py
  function _fix_fracs (line 4) | def _fix_fracs(string):
  function _fix_a_slash_b (line 36) | def _fix_a_slash_b(string):
  function _fix_sqrt (line 53) | def _fix_sqrt(string):
  function _fix_tan (line 59) | def _fix_tan(string):
  function strip_string (line 65) | def strip_string(string):
  function extract_boxed_answers (line 177) | def extract_boxed_answers(text):
  function extract_program_output (line 194) | def extract_program_output(pred_str):
  function extract_answer (line 207) | def extract_answer(pred_str, exhaust=False):
  function extract_math_answer (line 245) | def extract_math_answer(question, reasoning, task):
  function extract_math_few_shot_cot_answer (line 256) | def extract_math_few_shot_cot_answer(question, reasoning, task):
  function extract_last_single_answer (line 261) | def extract_last_single_answer(question, reasoning, task):
  function extract_gsm_few_shot_cot_answer (line 264) | def extract_gsm_few_shot_cot_answer(question, reasoning, task):
  function extract_agieval_gaokao_mathcloze_few_shot_cot_test (line 273) | def extract_agieval_gaokao_mathcloze_few_shot_cot_test(question, reasoni...
  function extract_agieval_gaokao_mathqa_few_shot_cot_test (line 284) | def extract_agieval_gaokao_mathqa_few_shot_cot_test(question, reasoning,...
  function extract_sat_few_shot_answer (line 294) | def extract_sat_few_shot_answer(question, reasoning, task):
  function extract_ocwcourses_few_shot_answer (line 302) | def extract_ocwcourses_few_shot_answer(question, reasoning, task):
  function extract_mmlu_stem (line 313) | def extract_mmlu_stem(question, reasoning, task):
  function extract_minif2f_isabelle (line 318) | def extract_minif2f_isabelle(question, reasoning, task):
  function extract_cmath_few_shot_test (line 323) | def extract_cmath_few_shot_test(question, reasoning, task):

FILE: eval/rebase/inference_scaling/evaluate/data_processing/process_utils.py
  function process_gsm8k_test (line 5) | def process_gsm8k_test(item):
  function process_math_test (line 17) | def process_math_test(item):
  function process_math_sat (line 37) | def process_math_sat(item):
  function process_ocwcourses (line 58) | def process_ocwcourses(item):
  function process_mmlu_stem (line 72) | def process_mmlu_stem(item):
  function process_mgsm_zh (line 91) | def process_mgsm_zh(item):
  function process_cmath (line 95) | def process_cmath(item):
  function process_agieval_gaokao_math_cloze (line 109) | def process_agieval_gaokao_math_cloze(item):
  function process_agieval_gaokao_mathqa (line 121) | def process_agieval_gaokao_mathqa(item):
  function process_agieval_gaokao_mathqa_few_shot_cot_test (line 143) | def process_agieval_gaokao_mathqa_few_shot_cot_test(item):
  function process_minif2f_isabelle (line 158) | def process_minif2f_isabelle(item):

FILE: eval/rebase/inference_scaling/evaluate/evaluate_utils/grader.py
  function _sympy_parse (line 21) | def _sympy_parse(expr: str):
  function _parse_latex (line 33) | def _parse_latex(expr: str) -> str:
  function _is_float (line 51) | def _is_float(num: str) -> bool:
  function _is_int (line 59) | def _is_int(x: float) -> bool:
  function _is_frac (line 66) | def _is_frac(expr: str) -> bool:
  function _str_is_int (line 70) | def _str_is_int(x: str) -> bool:
  function _str_to_int (line 79) | def _str_to_int(x: str) -> bool:
  function _inject_implicit_mixed_number (line 85) | def _inject_implicit_mixed_number(step: str):
  function _strip_properly_formatted_commas (line 95) | def _strip_properly_formatted_commas(expr: str):
  function _normalize (line 106) | def _normalize(expr: str) -> str:
  function count_unknown_letters_in_expr (line 179) | def count_unknown_letters_in_expr(expr: str):
  function should_allow_eval (line 186) | def should_allow_eval(expr: str):
  function are_equal_under_sympy (line 202) | def are_equal_under_sympy(ground_truth_normalized: str, given_normalized...
  function split_tuple (line 216) | def split_tuple(expr: str):
  function grade_answer (line 235) | def grade_answer(given_answer: str, ground_truth: str) -> bool:

FILE: eval/rebase/inference_scaling/evaluate/evaluate_utils/math_normalize.py
  function normalize_answer (line 8) | def normalize_answer(answer: Optional[str]) -> Optional[str]:
  function _fix_fracs (line 22) | def _fix_fracs(string):
  function _fix_a_slash_b (line 54) | def _fix_a_slash_b(string):
  function _remove_right_units (line 69) | def _remove_right_units(string):
  function _fix_sqrt (line 79) | def _fix_sqrt(string):
  function _strip_string (line 94) | def _strip_string(string):

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/convert_checkpoint_to_hf.py
  function load_and_merge_models (line 19) | def load_and_merge_models(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/data_utils/common_utils.py
  function zip_ (line 37) | def zip_(*args: Sequence):
  function mean (line 45) | def mean(*seqs: Sequence[Numeric]) -> Union[Numeric, Sequence[Numeric]]:
  function alleq (line 51) | def alleq(l: Sequence, f: Optional[Callable] = lambda x, y: x == y):
  function flatten_dict (line 64) | def flatten_dict(nested, sep=".", postprocess_fn=lambda *args: args):
  function unpack_dict (line 80) | def unpack_dict(
  function merge_dict (line 91) | def merge_dict(dicts: Sequence[dict], merge_fn: Callable = lambda *args:...
  function prepare_inputs (line 98) | def prepare_inputs(
  function pad_inputs_on_batch (line 112) | def pad_inputs_on_batch(
  function pad (line 135) | def pad(
  function left_pad (line 150) | def left_pad(
  function right_pad (line 156) | def right_pad(
  function manual_seed (line 162) | def manual_seed(args_or_seed: Union[int, argparse.Namespace], fix_cudnn=...
  class InfiniteLoader (line 175) | class InfiniteLoader(object):
    method __init__ (line 178) | def __init__(self, loader: DataLoader):
    method __iter__ (line 184) | def __iter__(self):
    method __next__ (line 187) | def __next__(self):
  class DisableLogger (line 205) | class DisableLogger:
    method __enter__ (line 206) | def __enter__(self):
    method __exit__ (line 209) | def __exit__(self, exit_type, exit_value, exit_traceback):
  class DataCollatorForStackableDataset (line 214) | class DataCollatorForStackableDataset(object):
    method __call__ (line 215) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
  function local_dataset (line 222) | def local_dataset(dataset_name):
  function _get_generator (line 237) | def _get_generator(seed: int) -> torch.Generator:
  function split_train_into_train_and_eval (line 243) | def split_train_into_train_and_eval(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/data_utils/data_utils_pm_pairwise.py
  function format_prompt (line 56) | def format_prompt(
  function format_double_outputs (line 76) | def format_double_outputs(
  function _tokenize_fn (line 88) | def _tokenize_fn(
  function preprocess_for_preference_modeling (line 114) | def preprocess_for_preference_modeling(
  class PairwisePreferenceModelingDataset (line 159) | class PairwisePreferenceModelingDataset(Dataset):
    method __init__ (line 160) | def __init__(
    method __len__ (line 195) | def __len__(self):
    method __getitem__ (line 198) | def __getitem__(self, i) -> Dict[str, torch.Tensor]:
  function make_pairwise_preference_modeling_data_module (line 209) | def make_pairwise_preference_modeling_data_module(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/data_utils/data_utils_rm_pairwise.py
  function format_prompt (line 54) | def format_prompt(
  function format_output (line 74) | def format_output(
  function _tokenize_fn (line 81) | def _tokenize_fn(
  function preprocess_for_reward_modeling (line 112) | def preprocess_for_reward_modeling(
  class PairwiseRewardModelingDataset (line 165) | class PairwiseRewardModelingDataset(Dataset):
    method __init__ (line 166) | def __init__(
    method __len__ (line 190) | def __len__(self):
    method __getitem__ (line 193) | def __getitem__(self, i) -> Dict[str, torch.Tensor]:
  function make_pairwise_reward_modeling_data_module (line 200) | def make_pairwise_reward_modeling_data_module(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/data_utils/data_utils_rm_pointwise.py
  class DataCollatorForKNRewardModeling (line 30) | class DataCollatorForKNRewardModeling(object):
    method __call__ (line 37) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
  class DataCollatorForKNRewardModelingDenoise (line 81) | class DataCollatorForKNRewardModelingDenoise(object):
    method __call__ (line 88) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
  class DataCollatorForPointwiseRewardModeling (line 137) | class DataCollatorForPointwiseRewardModeling(object):
    method __call__ (line 144) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
  class DataCollatorForPointwiseRewardModelingV2 (line 244) | class DataCollatorForPointwiseRewardModelingV2(object):
    method __call__ (line 251) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
  function extract_prm_dataset (line 397) | def extract_prm_dataset(example):
  function extract_prm_v2_dataset (line 417) | def extract_prm_v2_dataset(example):
  function extract_prm_v3_dataset (line 437) | def extract_prm_v3_dataset(example):
  function extract_prm_v4_dataset (line 457) | def extract_prm_v4_dataset(example):
  function extract_reward_dataset (line 473) | def extract_reward_dataset(example):
  function extract_apps_dataset (line 480) | def extract_apps_dataset(example):
  function make_pointwise_reward_modeling_data_module (line 488) | def make_pointwise_reward_modeling_data_module(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/data_utils/data_utils_sft.py
  function preprocess_for_sft (line 54) | def preprocess_for_sft(
  class DataCollatorForCausalLM (line 160) | class DataCollatorForCausalLM(object):
    method __call__ (line 169) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
  function extract_alpaca_dataset (line 182) | def extract_alpaca_dataset(example):
  function extract_dromedary_dataset (line 190) | def extract_dromedary_dataset(example, meta_prompts):
  function extract_prm_dataset (line 206) | def extract_prm_dataset(example):
  function extract_prm_v2_dataset (line 227) | def extract_prm_v2_dataset(example):
  function extract_metamath_dataset (line 248) | def extract_metamath_dataset(example):
  function extract_metamath_mathshepherd_dataset (line 257) | def extract_metamath_mathshepherd_dataset(example):
  function extract_apps_dataset (line 267) | def extract_apps_dataset(example):
  function make_sft_data_module (line 287) | def make_sft_data_module(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/data_utils/tokenizer_utils.py
  function is_numpy_array (line 44) | def is_numpy_array(x):
  function is_torch_device (line 51) | def is_torch_device(x):
  class ExplicitEnum (line 55) | class ExplicitEnum(str, Enum):
    method _missing_ (line 61) | def _missing_(cls, value):
  class TensorType (line 67) | class TensorType(ExplicitEnum):
  class PaddingStrategy (line 77) | class PaddingStrategy(ExplicitEnum):
  class BatchEncoding (line 88) | class BatchEncoding(UserDict):
    method __init__ (line 108) | def __init__(
    method __getitem__ (line 119) | def __getitem__(self, item: Union[int, str]) -> Any:
    method __getattr__ (line 139) | def __getattr__(self, item: str):
    method keys (line 145) | def keys(self):
    method values (line 148) | def values(self):
    method items (line 151) | def items(self):
    method convert_to_tensors (line 154) | def convert_to_tensors(
    method to (line 226) | def to(self, device: Union[str, "torch.device"]) -> "BatchEncoding":
  class FakePreTrainedTokenizer (line 250) | class FakePreTrainedTokenizer:
    method __init__ (line 253) | def __init__(
    method __call__ (line 305) | def __call__(
    method encode (line 398) | def encode(self, s: str, bos: bool, eos: bool) -> List[int]:
    method decode (line 428) | def decode(self, t: List[int], skip_special_tokens: bool = True) -> str:
    method sp_convert_tokens_to_string (line 450) | def sp_convert_tokens_to_string(self, tokens, skip_special_tokens=False):
    method batch_encode (line 482) | def batch_encode(
    method batch_decode (line 521) | def batch_decode(
  function batch_encode_tokens (line 561) | def batch_encode_tokens(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/finetune.py
  function model_forward (line 70) | def model_forward(model, x, input_pos):
  function model_forward_with_loss (line 74) | def model_forward_with_loss(
  function encode_tokens (line 117) | def encode_tokens(tokenizer, string, bos=True, device="cuda"):
  function main (line 124) | def main(
  function prepare_batch (line 436) | def prepare_batch(input_ids, labels, tokenizer, use_tp, sync_group):

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/finetune_rm.py
  function model_forward (line 72) | def model_forward(model, x):
  function model_forward_with_loss (line 76) | def model_forward_with_loss(
  function model_forward_with_regression_loss (line 112) | def model_forward_with_regression_loss(
  function encode_tokens (line 144) | def encode_tokens(tokenizer, string, bos=True, device="cuda"):
  function main (line 151) | def main(
  function prepare_batch (line 483) | def prepare_batch(input_ids, labels, weights, tokenizer, use_tp, sync_gr...

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/finetune_rm_pairwise.py
  function model_forward (line 71) | def model_forward(model, x, eos_pos):
  function model_forward_with_loss (line 75) | def model_forward_with_loss(
  function compute_reward_modeling_metrics (line 126) | def compute_reward_modeling_metrics(logits, labels) -> Dict:
  function encode_tokens (line 145) | def encode_tokens(tokenizer, string, bos=True, device="cuda"):
  function main (line 152) | def main(
  function prepare_batch (line 473) | def prepare_batch(input_ids, choice, tokenizer, use_tp, sync_group):

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/models/GPTQ.py
  class InputRecorder (line 28) | class InputRecorder(lm_eval.base.BaseLM):
    method __init__ (line 41) | def __init__(
    method eot_token_id (line 73) | def eot_token_id(self):
    method max_length (line 77) | def max_length(self):
    method max_gen_toks (line 81) | def max_gen_toks(self):
    method batch_size (line 85) | def batch_size(self):
    method device (line 89) | def device(self):
    method tok_encode (line 92) | def tok_encode(self, string: str):
    method tok_decode (line 102) | def tok_decode(self, tokens):
    method add_input (line 106) | def add_input(self, args):
    method get_recorded_inputs (line 114) | def get_recorded_inputs(self):
    method _model_call (line 117) | def _model_call(self, inps):
    method _model_generate (line 154) | def _model_generate(self, context, max_length, eos_token_id):
  class MultiInput (line 161) | class MultiInput:
    method __init__ (line 162) | def __init__(self, inputs):
    method add_input (line 165) | def add_input(self, input):
    method __getitem__ (line 169) | def __getitem__(self, slice):
    method cuda (line 172) | def cuda(self):
  class GenericGPTQRunner (line 178) | class GenericGPTQRunner(fx.Interpreter):
    method __init__ (line 191) | def __init__(
    method configure_quantization_mode (line 212) | def configure_quantization_mode(
    method run (line 242) | def run(self):
    method get_quantized_state_dict (line 249) | def get_quantized_state_dict(self):
    method call_function (line 263) | def call_function(self, target, args, kwargs, skip_quant=False):
    method faster_quant (line 400) | def faster_quant(self, H, W):

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/models/model.py
  function find_multiple (line 29) | def find_multiple(n: int, k: int) -> int:
  class ModelArgs (line 36) | class ModelArgs:
    method __post_init__ (line 48) | def __post_init__(self):
    method from_name (line 58) | def from_name(cls, name: str):
  class KVCache (line 110) | class KVCache(nn.Module):
    method __init__ (line 111) | def __init__(
    method update (line 119) | def update(self, input_pos, k_val, v_val):
  class Transformer (line 130) | class Transformer(nn.Module):
    method __init__ (line 131) | def __init__(
    method setup_caches (line 170) | def setup_caches(self, max_batch_size, max_seq_length, kv_cache=True):
    method forward (line 210) | def forward(
    method from_name (line 249) | def from_name(cls, name: str, **kwargs):
  class TransformerBlock (line 253) | class TransformerBlock(nn.Module):
    method __init__ (line 254) | def __init__(self, config: ModelArgs, freeze_norm: bool = False) -> None:
    method forward (line 269) | def forward(
  class Attention (line 284) | class Attention(nn.Module):
    method __init__ (line 285) | def __init__(self, config: ModelArgs):
    method load_hook (line 301) | def load_hook(self, state_dict, prefix, *args):
    method forward (line 308) | def forward(
  class FeedForward (line 350) | class FeedForward(nn.Module):
    method __init__ (line 351) | def __init__(self, config: ModelArgs) -> None:
    method forward (line 357) | def forward(self, x: Tensor) -> Tensor:
  class RMSNorm (line 361) | class RMSNorm(nn.Module):
    method __init__ (line 362) | def __init__(self, dim: int, eps: float = 1e-5):
    method _norm (line 370) | def _norm(self, x):
    method forward (line 373) | def forward(self, x: Tensor) -> Tensor:
  function precompute_freqs_cis (line 389) | def precompute_freqs_cis(seq_len: int, n_elem: int, base: int = 10000) -...
  function apply_rotary_emb (line 400) | def apply_rotary_emb(x: Tensor, freqs_cis: Tensor) -> Tensor:
  class FrozenEmbedding (line 415) | class FrozenEmbedding(nn.Module):
    method __init__ (line 436) | def __init__(
    method forward (line 459) | def forward(self, input: Tensor) -> Tensor:
    method extra_repr (line 494) | def extra_repr(self) -> str:
  class FrozenRMSNorm (line 509) | class FrozenRMSNorm(nn.Module):
    method __init__ (line 510) | def __init__(self, dim: int, eps: float = 1e-5):
    method _norm (line 518) | def _norm(self, x):
    method forward (line 521) | def forward(self, x: Tensor) -> Tensor:
  class FrozenLinear (line 537) | class FrozenLinear(nn.Module):
    method __init__ (line 543) | def __init__(
    method forward (line 563) | def forward(self, input: Tensor) -> Tensor:
    method extra_repr (line 566) | def extra_repr(self) -> str:
  function set_global_compile_mode (line 570) | def set_global_compile_mode(mode: bool):

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/models/ppo_trainer.py
  class PPOTrainer (line 97) | class PPOTrainer(RLTrainer):
    method __init__ (line 98) | def __init__(
    method _shape_reward (line 139) | def _shape_reward(
    method _estimate_advantage (line 196) | def _estimate_advantage(
    method _calculate_outcome_accuracy (line 225) | def _calculate_outcome_accuracy(
    method rollout (line 309) | def rollout(self, queries_data) -> Dict[str, torch.Tensor]:
    method post_terminating_reward (line 756) | def post_terminating_reward(
    method _post_process_math_rollouts (line 779) | def _post_process_math_rollouts(self, text_responses, answer_gt_levels):
    method _stop_token_penalty (line 833) | def _stop_token_penalty(self, text_responses):
    method shape_process_rewards (line 888) | def shape_process_rewards(
    method compute_policy_loss (line 1004) | def compute_policy_loss(
    method compute_value_loss (line 1073) | def compute_value_loss(
    method record_step_stats (line 1140) | def record_step_stats(self, train_stats, rollouts, step_idx, **kwargs):
    method sub_batched_outputs (line 1219) | def sub_batched_outputs(
    method resume_training (line 1267) | def resume_training(self, checkpoint_dir: Path) -> int:
    method evaluate (line 1302) | def evaluate(self, step_idx: int, temperature: float = 0.0):
    method _evaluate (line 1344) | def _evaluate(
    method save_model (line 1575) | def save_model(
  function make_models (line 1610) | def make_models(
  function whiten (line 1879) | def whiten(
  function truncate_after_stop_token (line 1922) | def truncate_after_stop_token(
  function rank0_print (line 1940) | def rank0_print(*args):
  function remove_all_backward_hooks (line 1946) | def remove_all_backward_hooks(model: torch.nn.Module) -> Dict[str, Order...
  function recover_all_backward_hooks (line 1956) | def recover_all_backward_hooks(
  function post_process_newline_scores (line 1964) | def post_process_newline_scores(
  function last_boxed_only_string (line 2079) | def last_boxed_only_string(string):
  function remove_boxed (line 2105) | def remove_boxed(s):

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/models/quantize.py
  function dynamically_quantize_per_channel (line 31) | def dynamically_quantize_per_channel(x, quant_min, quant_max, target_dty...
  function get_group_qparams (line 66) | def get_group_qparams(w, n_bit=4, groupsize=128):
  function pack_scales_and_zeros (line 87) | def pack_scales_and_zeros(scales, zeros):
  function unpack_scales_and_zeros (line 104) | def unpack_scales_and_zeros(scales_and_zeros):
  function group_quantize_tensor_from_qparams (line 110) | def group_quantize_tensor_from_qparams(w, scales, zeros, n_bit=4, groups...
  function group_quantize_tensor (line 139) | def group_quantize_tensor(w, n_bit=4, groupsize=128):
  function group_dequantize_tensor_from_qparams (line 146) | def group_dequantize_tensor_from_qparams(
  function group_dequantize_tensor (line 166) | def group_dequantize_tensor(w_int32, scales_and_zeros, n_bit=4, groupsiz...
  class QuantHandler (line 173) | class QuantHandler:
    method __init__ (line 174) | def __init__(self, mod):
    method create_quantized_state_dict (line 177) | def create_quantized_state_dict(self) -> "StateDict":
    method convert_for_runtime (line 180) | def convert_for_runtime(self) -> "nn.Module":
    method convert_for_runtime_on_the_fly (line 183) | def convert_for_runtime_on_the_fly(self) -> "nn.Module":
  class GPTQQuantHandler (line 187) | class GPTQQuantHandler(QuantHandler):
    method __init__ (line 250) | def __init__(self):
    method get_inputs (line 259) | def get_inputs(
    method create_quantized_state_dict (line 285) | def create_quantized_state_dict(
    method convert_for_runtime (line 324) | def convert_for_runtime(self) -> "nn.Module":
    method convert_for_runtime_on_the_fly (line 327) | def convert_for_runtime_on_the_fly(self) -> "nn.Module":
  function replace_linear_weight_only_int8_per_channel (line 334) | def replace_linear_weight_only_int8_per_channel(module):
  function replace_linear_weight_only_int8_per_channel_on_the_fly (line 346) | def replace_linear_weight_only_int8_per_channel_on_the_fly(module):
  class WeightOnlyInt8QuantHandler (line 371) | class WeightOnlyInt8QuantHandler:
    method __init__ (line 372) | def __init__(self, mod):
    method create_quantized_state_dict (line 376) | def create_quantized_state_dict(self):
    method convert_for_runtime (line 388) | def convert_for_runtime(self):
    method convert_for_runtime_on_the_fly (line 392) | def convert_for_runtime_on_the_fly(self):
  class WeightOnlyInt8Linear (line 397) | class WeightOnlyInt8Linear(torch.nn.Module):
    method __init__ (line 403) | def __init__(
    method forward (line 420) | def forward(self, input: torch.Tensor) -> torch.Tensor:
  function prepare_int4_weight_and_scales_and_zeros (line 427) | def prepare_int4_weight_and_scales_and_zeros(weight_bf16, groupsize, inn...
  function linear_forward_int4 (line 437) | def linear_forward_int4(x, weight_int4pack, scales_and_zeros, out_featur...
  function _check_linear_int4_k (line 448) | def _check_linear_int4_k(k, groupsize=1, inner_k_tiles=1):
  function replace_linear_int4 (line 452) | def replace_linear_int4(module, groupsize, inner_k_tiles, padding):
  class WeightOnlyInt4QuantHandler (line 485) | class WeightOnlyInt4QuantHandler:
    method __init__ (line 486) | def __init__(self, mod, groupsize=128, inner_k_tiles=8, padding=True):
    method create_quantized_state_dict (line 495) | def create_quantized_state_dict(self):
    method convert_for_runtime (line 539) | def convert_for_runtime(self):
    method convert_for_runtime_on_the_fly (line 543) | def convert_for_runtime_on_the_fly(self):
  class WeightOnlyInt4GPTQQuantHandler (line 547) | class WeightOnlyInt4GPTQQuantHandler(GPTQQuantHandler):
    method __init__ (line 548) | def __init__(self, mod, groupsize=128, inner_k_tiles=8, padding=True):
    method convert_for_runtime (line 591) | def convert_for_runtime(self):
  class WeightOnlyInt4Linear (line 596) | class WeightOnlyInt4Linear(torch.nn.Module):
    method __init__ (line 602) | def __init__(
    method forward (line 650) | def forward(self, input: torch.Tensor) -> torch.Tensor:
  function quantize (line 661) | def quantize(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/models/reward_model.py
  function unpack_dict (line 28) | def unpack_dict(
  function batch_select (line 39) | def batch_select(input: Tensor, index: Tensor):
  class RewardArgs (line 60) | class RewardArgs:
    method from_name (line 64) | def from_name(cls, name: str):
  class RewardModel (line 68) | class RewardModel(nn.Module):
    method __init__ (line 69) | def __init__(self, config: RewardArgs, **kwargs) -> None:
    method forward (line 74) | def forward(
    method from_name (line 90) | def from_name(cls, name: str, **kwargs):
  function apply_reward_modeling_head (line 94) | def apply_reward_modeling_head(
  function compute_pairwise_reward_modeling_loss (line 135) | def compute_pairwise_reward_modeling_loss(model, inputs, return_outputs=...
  function compute_pairwise_reward_modeling_metrics (line 169) | def compute_pairwise_reward_modeling_metrics(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/models/rl_model.py
  class Policy (line 41) | class Policy(nn.Module, abc.ABC):
    method __init__ (line 42) | def __init__(
    method forward (line 64) | def forward(
    method respond (line 74) | def respond(
    method _respond (line 87) | def _respond(
    method _post_respond (line 96) | def _post_respond(self, respond_outputs: Dict[str, Tensor]) -> Dict[st...
  class AutoregressivePolicy (line 100) | class AutoregressivePolicy(Policy):
    method forward (line 101) | def forward(
    method _respond (line 161) | def _respond(
  class Value (line 232) | class Value(nn.Module, abc.ABC):
    method __init__ (line 233) | def __init__(
    method forward (line 246) | def forward(
  class AutoregressiveValue (line 252) | class AutoregressiveValue(Value):
    method forward (line 253) | def forward(
  function make_policy_with_base_model (line 282) | def make_policy_with_base_model(
  function make_value_with_base_model (line 291) | def make_value_with_base_model(
  function prepare_right_pad_sequences (line 301) | def prepare_right_pad_sequences(input_ids, attention_mask=None, pad_toke...
  function restore_from_right_pad_sequences (line 323) | def restore_from_right_pad_sequences(inputs, shifts):
  function prepare_left_pad_mask_pos (line 338) | def prepare_left_pad_mask_pos(input_ids, attention_mask=None, pad_token_...
  function multinomial_sample_one_no_sync (line 346) | def multinomial_sample_one_no_sync(
  function logits_to_probs (line 354) | def logits_to_probs(logits, temperature: float = 1.0, top_k: Optional[in...
  function sample (line 365) | def sample(
  function prefill (line 381) | def prefill(
  function _decode_one_token (line 393) | def _decode_one_token(
  function decode_n_tokens (line 409) | def decode_n_tokens(
  function compute_logprobs (line 446) | def compute_logprobs(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/models/rl_trainer.py
  class KLController (line 52) | class KLController(abc.ABC):
    method step (line 55) | def step(self, *args, **kwargs):
  class FixedKLController (line 59) | class FixedKLController(KLController):
    method __init__ (line 60) | def __init__(self, kl_coef):
  class RLTrainer (line 65) | class RLTrainer(object):
    method __init__ (line 66) | def __init__(
    method rollout (line 113) | def rollout(self, queries_data) -> Dict[str, torch.Tensor]:
    method compute_policy_loss (line 117) | def compute_policy_loss(
    method compute_value_loss (line 123) | def compute_value_loss(
    method record_step_stats (line 130) | def record_step_stats(self, train_stats, rollouts, step_idx, **kwargs):
    method policy_optimizable_params (line 134) | def policy_optimizable_params(self):
    method value_optimizable_params (line 142) | def value_optimizable_params(self):
    method optimizable_params (line 150) | def optimizable_params(self):
    method _compute_grad_norm_policy (line 157) | def _compute_grad_norm_policy(self):
    method _compute_grad_norm_value (line 164) | def _compute_grad_norm_value(self):
    method _compute_param_norm (line 171) | def _compute_param_norm(self):
    method step_with_rollouts (line 175) | def step_with_rollouts(self, rollouts):
    method step (line 285) | def step(self, train_dataloader, step_idx: int):
    method create_optimizer_and_scheduler (line 334) | def create_optimizer_and_scheduler(self, num_training_steps: int) -> N...
    method train (line 363) | def train(self, resume_training_ckpt: Optional[str] = None):
    method evaluate (line 402) | def evaluate(self, step_idx: int, temperature: float = 0.0):
    method save_model (line 407) | def save_model(self, step_idx: int, metrics: Optional[Dict] = None):
    method resume_training (line 412) | def resume_training(self, checkpoint_dir: str):
    method _log_batch_size (line 415) | def _log_batch_size(self, loader: DataLoader, loader_name):
    method get_train_dataloader (line 430) | def get_train_dataloader(self):
    method get_rollouts_dataloader (line 456) | def get_rollouts_dataloader(
    method is_main_process (line 485) | def is_main_process(self):
    method device (line 492) | def device(self):
    method prepare_tp_batch (line 502) | def prepare_tp_batch(
  function truncate_after_eos (line 566) | def truncate_after_eos(completions, eos_token_id):
  function truncate_after_eos_padded (line 578) | def truncate_after_eos_padded(completions, eos_token_id, pad_token_id):
  function no_op_context_manager (line 600) | def no_op_context_manager():

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/models/tp.py
  function _get_local_rank (line 41) | def _get_local_rank() -> int:
  function _get_rank (line 45) | def _get_rank() -> int:
  function is_local (line 49) | def is_local():
  function local_break (line 53) | def local_break():
  function _get_world_size (line 59) | def _get_world_size() -> int:
  function maybe_init_dist (line 63) | def maybe_init_dist() -> Optional[int]:
  function _apply_tp_embedding (line 83) | def _apply_tp_embedding(
  function _apply_tp_linear (line 120) | def _apply_tp_linear(
  function _apply_tp_ffn (line 196) | def _apply_tp_ffn(
  function _apply_tp_attn (line 216) | def _apply_tp_attn(
  function _apply_tp_Transformer (line 246) | def _apply_tp_Transformer(Transformer: Transformer) -> None:
  function apply_tp (line 254) | def apply_tp(
  function apply_reward_head_tp (line 287) | def apply_reward_head_tp(model: nn.Module, requires_grad: bool = False) ...
  function aggregate_forward_tensor_hook (line 295) | def aggregate_forward_tensor_hook(
  function aggregate_grad_hook (line 311) | def aggregate_grad_hook(
  function reduce_from_model_parallel_region_hook (line 332) | def reduce_from_model_parallel_region_hook(
  function scatter_to_model_parallel_region_pre_hook (line 345) | def scatter_to_model_parallel_region_pre_hook(
  function scatter_to_sequence_parallel_region_hook (line 361) | def scatter_to_sequence_parallel_region_hook(
  function gather_from_sequence_parallel_region_pre_hook (line 373) | def gather_from_sequence_parallel_region_pre_hook(
  function reduce_scatter_to_sequence_parallel_region_hook (line 394) | def reduce_scatter_to_sequence_parallel_region_hook(
  function ensure_divisibility (line 406) | def ensure_divisibility(numerator: int, denominator: int) -> None:
  function divide_and_check_no_remainder (line 413) | def divide_and_check_no_remainder(numerator: int, denominator: int) -> int:
  function initialize_model_parallel (line 420) | def initialize_model_parallel(
  function get_model_parallel_group (line 484) | def get_model_parallel_group() -> torch.distributed.ProcessGroup:
  function get_data_parallel_group (line 490) | def get_data_parallel_group() -> torch.distributed.ProcessGroup:
  function get_model_parallel_world_size (line 496) | def get_model_parallel_world_size() -> int:
  function get_model_parallel_rank (line 506) | def get_model_parallel_rank() -> int:
  function get_data_parallel_world_size (line 517) | def get_data_parallel_world_size() -> int:
  function get_data_parallel_rank (line 527) | def get_data_parallel_rank() -> int:
  function split_tensor_along_last_dim (line 538) | def split_tensor_along_last_dim(
  class _ReduceFromModelParallelRegion (line 567) | class _ReduceFromModelParallelRegion(torch.autograd.Function):
    method forward (line 571) | def forward(ctx, input_):  # type: ignore
    method backward (line 575) | def backward(ctx, grad_output):  # type: ignore
  class _ScatterToModelParallelRegion (line 579) | class _ScatterToModelParallelRegion(torch.autograd.Function):
    method forward (line 583) | def forward(ctx, input_):  # type: ignore
    method backward (line 587) | def backward(ctx, grad_output):  # type: ignore
  class _GatherFromSequenceParallelRegion (line 591) | class _GatherFromSequenceParallelRegion(torch.autograd.Function):
    method symbolic (line 595) | def symbolic(graph, input_, tensor_parallel_output_grad=True):
    method forward (line 599) | def forward(ctx, input_, tensor_parallel_output_grad=True):
    method backward (line 604) | def backward(ctx, grad_output):
  class _ReduceScatterToSequenceParallelRegion (line 617) | class _ReduceScatterToSequenceParallelRegion(torch.autograd.Function):
    method symbolic (line 621) | def symbolic(graph, input_):
    method forward (line 625) | def forward(ctx, input_):
    method backward (line 629) | def backward(ctx, grad_output):
  class _ScatterToSequenceParallelRegion (line 633) | class _ScatterToSequenceParallelRegion(torch.autograd.Function):
    method symbolic (line 637) | def symbolic(graph, input_):
    method forward (line 641) | def forward(ctx, input_):
    method backward (line 645) | def backward(ctx, grad_output):
  function _reduce (line 649) | def _reduce(ctx: Any, input_: torch.Tensor) -> torch.Tensor:
  function _split (line 667) | def _split(input_: torch.Tensor) -> torch.Tensor:
  function _gather (line 687) | def _gather(input_: torch.Tensor) -> torch.Tensor:
  function _split_along_first_dim (line 711) | def _split_along_first_dim(input_):
  function _gather_along_first_dim (line 734) | def _gather_along_first_dim(input_):
  function _reduce_scatter_along_first_dim (line 755) | def _reduce_scatter_along_first_dim(input_):
  function clip_grad_norm_ (line 778) | def clip_grad_norm_(
  class _VocabParallelCrossEntropy (line 886) | class _VocabParallelCrossEntropy(torch.autograd.Function):
    method forward (line 888) | def forward(ctx, vocab_parallel_logits, target):  # type: ignore
    method backward (line 949) | def backward(ctx, grad_output):  # type: ignore
  function vocab_parallel_cross_entropy (line 970) | def vocab_parallel_cross_entropy(
  function compute_vocab_parallel_logprobs (line 977) | def compute_vocab_parallel_logprobs(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/scripts/convert_hf_checkpoint.py
  function convert_hf_checkpoint (line 22) | def convert_hf_checkpoint(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/scripts/download.py
  function hf_download (line 13) | def hf_download(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/training_utils/checkpoint_hook.py
  function rank0_print (line 39) | def rank0_print(*args):
  function get_trainable_state_dict (line 45) | def get_trainable_state_dict(
  function save_checkpoint (line 140) | def save_checkpoint(
  function load_checkpoint (line 220) | def load_checkpoint(
  function load_inference_checkpoint (line 299) | def load_inference_checkpoint(
  function merge_tp_checkpoint (line 382) | def merge_tp_checkpoint(
  function split_tp_checkpoint (line 441) | def split_tp_checkpoint(
  function get_checkpoint_path (line 502) | def get_checkpoint_path(
  function get_latest_checkpoint_path (line 525) | def get_latest_checkpoint_path(
  function checkpoint_hook (line 575) | def checkpoint_hook(
  function load_model_from_from_ckpt (line 633) | def load_model_from_from_ckpt(
  function load_reward_model_from_ckpt (line 686) | def load_reward_model_from_ckpt(
  function load_reward_model_from_sft_ckpt (line 744) | def load_reward_model_from_sft_ckpt(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/training_utils/fsdp_utils.py
  function fixed_scatter_full_optim_state_dict (line 47) | def fixed_scatter_full_optim_state_dict(
  function fixed_optim_state_dict_to_load_impl (line 129) | def fixed_optim_state_dict_to_load_impl(
  function fixed_flatten_optim_state_dict (line 187) | def fixed_flatten_optim_state_dict(
  function fixed_broadcast_processed_state (line 334) | def fixed_broadcast_processed_state(
  function fixed_broadcast_state (line 357) | def fixed_broadcast_state(
  function fixed_full_optim_state_dict (line 384) | def fixed_full_optim_state_dict(
  function fixed_optim_state_dict_impl (line 457) | def fixed_optim_state_dict_impl(
  function fixed_optim_state_dict (line 509) | def fixed_optim_state_dict(
  function fixed_map_param_key_to_optim_keys (line 665) | def fixed_map_param_key_to_optim_keys(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/training_utils/hf_argparser.py
  function string_to_bool (line 46) | def string_to_bool(v):
  function make_choice_type_function (line 59) | def make_choice_type_function(choices: list) -> Callable[[str], Any]:
  function HfArg (line 74) | def HfArg(
  class HfArgumentParser (line 123) | class HfArgumentParser(ArgumentParser):
    method __init__ (line 134) | def __init__(
    method _parse_dataclass_field (line 155) | def _parse_dataclass_field(parser: ArgumentParser, field: dataclasses....
    method _add_dataclass_arguments (line 268) | def _add_dataclass_arguments(self, dtype: DataClassType):
    method parse_args_into_dataclasses (line 305) | def parse_args_into_dataclasses(
    method parse_dict (line 393) | def parse_dict(
    method parse_json_file (line 425) | def parse_json_file(
    method parse_yaml_file (line 449) | def parse_yaml_file(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/training_utils/memory_efficient_adam.py
  class MemoryEfficientAdamW (line 13) | class MemoryEfficientAdamW(Optimizer):
    method __init__ (line 33) | def __init__(
    method torch_adam_update_cpu (line 55) | def torch_adam_update_cpu(
    method step (line 101) | def step(self, closure=None):
    method load_state_dict (line 165) | def load_state_dict(self, state_dict: StateDict) -> None:

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/training_utils/trainer_utils.py
  function create_optimizer (line 37) | def create_optimizer(
  function create_fsdp_model_for_finetune (line 70) | def create_fsdp_model_for_finetune(
  function _get_cosine_schedule_with_warmup_lr_lambda (line 103) | def _get_cosine_schedule_with_warmup_lr_lambda(
  function get_cosine_schedule_with_warmup (line 124) | def get_cosine_schedule_with_warmup(

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/training_utils/training_args.py
  class TrainingArguments (line 15) | class TrainingArguments:
    method __post_init__ (line 559) | def __post_init__(self):

FILE: eval/rebase/inference_scaling/finetune/gpt-accelera/util.py
  function last_boxed_only (line 3) | def last_boxed_only(sample):
  function last_boxed_only_string (line 10) | def last_boxed_only_string(string):
  function only_until_first_boxed_from_tokens (line 37) | def only_until_first_boxed_from_tokens(string, tokens):
  function clean_numbers (line 54) | def clean_numbers(sample):
  function _clean_numbers (line 63) | def _clean_numbers(string):
  function fix_fracs (line 95) | def fix_fracs(string):
  function fix_a_slash_b (line 126) | def fix_a_slash_b(string):
  function remove_right_units (line 140) | def remove_right_units(string):
  function fix_sqrt (line 149) | def fix_sqrt(string):
  function strip_string (line 164) | def strip_string(string):
  function is_equiv (line 229) | def is_equiv(str1, str2, verbose=False):
  class NotEqual (line 246) | class NotEqual:
    method __eq__ (line 247) | def __eq__(self, other):

FILE: eval/rebase/inference_scaling/finetune/hf_finetune/hf_train.py
  class ModelArguments (line 28) | class ModelArguments:
  class DataArguments (line 33) | class DataArguments:
  class TrainingArguments (line 39) | class TrainingArguments(transformers.TrainingArguments):
  function smart_tokenizer_and_embedding_resize (line 48) | def smart_tokenizer_and_embedding_resize(
  function _tokenize_fn (line 71) | def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrai...
  function preprocess (line 95) | def preprocess(
  class SupervisedDataset (line 111) | class SupervisedDataset(Dataset):
    method __init__ (line 114) | def __init__(self, data_path: str, tokenizer: transformers.PreTrainedT...
    method __len__ (line 132) | def __len__(self):
    method __getitem__ (line 135) | def __getitem__(self, i) -> Dict[str, torch.Tensor]:
  class DataCollatorForSupervisedDataset (line 140) | class DataCollatorForSupervisedDataset(object):
    method __call__ (line 145) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
  function make_supervised_data_module (line 158) | def make_supervised_data_module(tokenizer: transformers.PreTrainedTokeni...
  function train (line 165) | def train():

FILE: eval/rebase/inference_scaling/hf_score.py
  function main (line 15) | def main(args):

FILE: eval/rebase/inference_scaling/math_evaluate.py
  function extract_gsm_answer (line 20) | def extract_gsm_answer(completion):
  function extract_shepherd_answer (line 30) | def extract_shepherd_answer(completion):
  function extract_metamath_answer (line 43) | def extract_metamath_answer(completion):
  function extract_ground_truth (line 53) | def extract_ground_truth(completion):
  function agg_min (line 61) | def agg_min(step_scores):
  function agg_mean (line 71) | def agg_mean(step_scores):
  function agg_prod (line 76) | def agg_prod(step_scores):
  function agg_last (line 84) | def agg_last(step_scores):
  function evaluate (line 91) | def evaluate(path, aggfunc, extract_function):
  function majority_vote (line 110) | def majority_vote(path, weighted, weight_func, extract_function):
  function main (line 149) | def main(args):

FILE: eval/rebase/inference_scaling/rebase.py
  function read_jsonl (line 21) | def read_jsonl(path: str):
  function get_prompts (line 26) | def get_prompts(args):
  class TreeNode (line 35) | class TreeNode:
    method __init__ (line 36) | def __init__(self, id, state, score, num_step_tokens=0, parent=None):
    method get_id (line 54) | def get_id(self):
    method get_parent (line 57) | def get_parent(self):
    method get_text (line 60) | def get_text(self):
    method get_state (line 63) | def get_state(self):
    method get_depth (line 66) | def get_depth(self):
    method get_score (line 69) | def get_score(self):
    method is_leaf (line 72) | def is_leaf(self):
    method get_cum_tokens (line 75) | def get_cum_tokens(self):
  class Tree (line 78) | class Tree:
    method __init__ (line 79) | def __init__(self, root_state, paras, reward_backend):
    method reset_running_list (line 92) | def reset_running_list(self):
    method get_running_list (line 95) | def get_running_list(self):
    method get_history_list (line 98) | def get_history_list(self):
    method get_nodes (line 101) | def get_nodes(self):
    method expand (line 104) | def expand(self, node, wid):
    method insert (line 118) | def insert(self, state, parent):
    method select_softmax (line 130) | def select_softmax(self, node_list, node_weights, width):
    method select_softmax_with_truncation (line 151) | def select_softmax_with_truncation(self, node_list, node_weights, width):
    method select_and_expand (line 177) | def select_and_expand(self, depth):
  function reward_guided_search (line 199) | def reward_guided_search(s, id, question, ground_truth_answer, paras, re...
  function search_worker (line 259) | def search_worker(search_dict, lock, prompts, test_examples, paras, poli...
  function main (line 276) | def main(args):

FILE: eval/rebase/inference_scaling/rebase_ori.py
  function read_jsonl (line 21) | def read_jsonl(path: str):
  function get_prompts (line 26) | def get_prompts(args):
  class TreeNode (line 35) | class TreeNode:
    method __init__ (line 36) | def __init__(self, id, state, score, num_step_tokens=0, parent=None):
    method get_id (line 54) | def get_id(self):
    method get_parent (line 57) | def get_parent(self):
    method get_text (line 60) | def get_text(self):
    method get_state (line 63) | def get_state(self):
    method get_depth (line 66) | def get_depth(self):
    method get_score (line 69) | def get_score(self):
    method is_leaf (line 72) | def is_leaf(self):
    method get_cum_tokens (line 75) | def get_cum_tokens(self):
  class Tree (line 78) | class Tree:
    method __init__ (line 79) | def __init__(self, root_state, paras, reward_backend):
    method reset_running_list (line 92) | def reset_running_list(self):
    method get_running_list (line 95) | def get_running_list(self):
    method get_history_list (line 98) | def get_history_list(self):
    method get_nodes (line 101) | def get_nodes(self):
    method expand (line 104) | def expand(self, node, wid):
    method insert (line 118) | def insert(self, state, parent):
    method select_softmax (line 130) | def select_softmax(self, node_list, node_weights, width):
    method select_softmax_with_truncation (line 151) | def select_softmax_with_truncation(self, node_list, node_weights, width):
    method select_and_expand (line 176) | def select_and_expand(self, depth):
  function reward_guided_search (line 198) | def reward_guided_search(s, id, question, ground_truth_answer, paras, re...
  function search_worker (line 261) | def search_worker(search_dict, lock, prompts, test_examples, paras, poli...
  function main (line 278) | def main(args):

FILE: eval/rebase/inference_scaling/sgl_baseline.py
  function read_jsonl (line 12) | def read_jsonl(path: str):
  function get_prompts_math (line 17) | def get_prompts_math(args):
  function get_prompts_gsm (line 24) | def get_prompts_gsm(args):
  function multiple_sampling (line 36) | def multiple_sampling(s, id, question, sampling_num, max_tokens, ground_...
  function main (line 58) | def main(args):

FILE: eval/rebase/sglang/benchmark/dspy/bench_dspy_intro.py
  class BasicQA (line 11) | class BasicQA(dspy.Signature):
  class GenerateAnswer (line 18) | class GenerateAnswer(dspy.Signature):
  class RAG (line 26) | class RAG(dspy.Module):
    method __init__ (line 27) | def __init__(self, num_passages=3):
    method forward (line 33) | def forward(self, question):
  function main (line 39) | def main(args):

FILE: eval/rebase/sglang/benchmark/generative_agents/agent_functions.py
  function poignancy_event (line 8) | def poignancy_event(s, persona_name, persona_iss, event):
  function poignancy_event_prompt (line 18) | def poignancy_event_prompt(persona_name, persona_iss, event):
  function generate_event_triple (line 31) | def generate_event_triple(s, persona_name, action):
  function generate_event_triple_prompt (line 56) | def generate_event_triple_prompt(persona_name, action):
  function generate_pronunciatio (line 83) | def generate_pronunciatio(s, action):
  function generate_pronunciatio_prompt (line 89) | def generate_pronunciatio_prompt(action):
  function action_location_sector (line 98) | def action_location_sector(
  function action_location_sector_prompt (line 138) | def action_location_sector_prompt(
  function action_location_object (line 179) | def action_location_object(s, persona_name, target_sector, target_sector...
  function action_location_object_prompt (line 205) | def action_location_object_prompt(persona_name, target_sector,

FILE: eval/rebase/sglang/benchmark/generative_agents/bench_other.py
  function main (line 25) | def main(args):

FILE: eval/rebase/sglang/benchmark/generative_agents/bench_sglang.py
  function main (line 21) | def main(args):

FILE: eval/rebase/sglang/benchmark/gsm8k/bench_other.py
  function get_one_example (line 19) | def get_one_example(lines, i, include_answer):
  function get_few_shot_examples (line 26) | def get_few_shot_examples(lines, k):
  function get_answer_value (line 33) | def get_answer_value(answer_str):
  function main (line 44) | def main(args):

FILE: eval/rebase/sglang/benchmark/gsm8k/bench_sglang.py
  function get_one_example (line 15) | def get_one_example(lines, i, include_answer):
  function get_few_shot_examples (line 22) | def get_few_shot_examples(lines, k):
  function get_answer_value (line 29) | def get_answer_value(answer_str):
  function main (line 40) | def main(args):

FILE: eval/rebase/sglang/benchmark/hellaswag/bench_other.py
  function get_one_example (line 13) | def get_one_example(lines, i, include_answer):
  function get_few_shot_examples (line 20) | def get_few_shot_examples(lines, k):
  function main (line 27) | def main(args):

FILE: eval/rebase/sglang/benchmark/hellaswag/bench_sglang.py
  function get_one_example (line 10) | def get_one_example(lines, i, include_answer):
  function get_few_shot_examples (line 17) | def get_few_shot_examples(lines, k):
  function main (line 24) | def main(args):

FILE: eval/rebase/sglang/benchmark/json_decode_regex/bench_other.py
  function json_decode (line 19) | def json_decode(document, generate):
  function main (line 40) | def main(args):

FILE: eval/rebase/sglang/benchmark/json_decode_regex/bench_sglang.py
  function json_warm_up (line 17) | def json_warm_up(s):
  function json_decode (line 32) | def json_decode(s, document):
  function main (line 47) | def main(args):

FILE: eval/rebase/sglang/benchmark/json_decode_regex/build_dataset.py
  function get_content (line 35) | def get_content(city_name):

FILE: eval/rebase/sglang/benchmark/json_jump_forward/bench_other.py
  function character_gen (line 46) | def character_gen(name, generate):
  function city_gen (line 53) | def city_gen(document, generate):
  function character_maker (line 63) | def character_maker(lm, name):
  function city_maker (line 88) | def city_maker(lm, document):
  function bench_character (line 111) | def bench_character(args):
  function bench_city_doc (line 159) | def bench_city_doc(args):
  function main (line 206) | def main(args):

FILE: eval/rebase/sglang/benchmark/json_jump_forward/bench_sglang.py
  function character_gen (line 44) | def character_gen(s, name):
  function city_gen (line 51) | def city_gen(s, document):
  function bench_city_doc (line 59) | def bench_city_doc(args):
  function bench_character (line 82) | def bench_character(args):
  function main (line 106) | def main(args):

FILE: eval/rebase/sglang/benchmark/json_jump_forward/build_dataset.py
  function get_content (line 35) | def get_content(city_name):

FILE: eval/rebase/sglang/benchmark/latency_throughput/bench_throughput.py
  function sample_requests (line 35) | def sample_requests(
  function get_request (line 83) | async def get_request(
  function send_request (line 100) | async def send_request(
  function benchmark (line 177) | async def benchmark(
  function main (line 195) | def main(args: argparse.Namespace):

FILE: eval/rebase/sglang/benchmark/line_retrieval/bench_sglang.py
  function line_retrieval (line 13) | def line_retrieval(s, prefix, suffix, body_0, body_1, body_2, body_3):
  function eval_model (line 26) | def eval_model(args, line_obj, num_hoops, src_indices, dst_percents):
  function main (line 115) | def main(args):

FILE: eval/rebase/sglang/benchmark/line_retrieval/gen_data.py
  function generate_lines (line 15) | def generate_lines(random_words, num_lines, redirect_ratio):

FILE: eval/rebase/sglang/benchmark/llava_bench/bench_sglang.py
  function image_qa (line 14) | def image_qa(s, image_file, question):
  function main (line 19) | def main(args):

FILE: eval/rebase/sglang/benchmark/llm_judge/bench_other.py
  function multi_dimension_judge (line 28) | def multi_dimension_judge(article, generate):
  function main (line 52) | def main(args):

FILE: eval/rebase/sglang/benchmark/llm_judge/bench_sglang.py
  function multi_dimension_judge (line 26) | def multi_dimension_judge(s, article):
  function main (line 47) | def main(args):

FILE: eval/rebase/sglang/benchmark/long_json_decode/bench_other.py
  function json_decode (line 14) | def json_decode(document, generate):
  function main (line 31) | def main(args):

FILE: eval/rebase/sglang/benchmark/long_json_decode/bench_sglang.py
  function json_decode (line 12) | def json_decode(s, document):
  function main (line 24) | def main(args):

FILE: eval/rebase/sglang/benchmark/mmlu/bench_other.py
  function format_subject (line 21) | def format_subject(subject):
  function format_example (line 28) | def format_example(df, idx, include_answer=True):
  function gen_prompt (line 38) | def gen_prompt(train_df, subject, k=-1):
  function evaluate (line 50) | def evaluate(args, subject, dev_df, test_df):
  function main (line 160) | def main(args):

FILE: eval/rebase/sglang/benchmark/mmlu/bench_sglang.py
  function format_subject (line 18) | def format_subject(subject):
  function format_example (line 25) | def format_example(df, idx, include_answer=True):
  function gen_prompt (line 35) | def gen_prompt(train_df, subject, k=-1):
  function evaluate (line 43) | def evaluate(args, subject, dev_df, test_df):
  function main (line 103) | def main(args):

FILE: eval/rebase/sglang/benchmark/mtbench/bench_other.py
  function load_questions (line 14) | def load_questions(filename):
  function write_answers (line 23) | def write_answers(filename, model_id, questions, answers):
  function main (line 39) | def main(args):

FILE: eval/rebase/sglang/benchmark/mtbench/bench_sglang.py
  function load_questions (line 11) | def load_questions(filename):
  function write_answers (line 20) | def write_answers(filename, model_id, questions, answers):
  function answer_mt_bench (line 37) | def answer_mt_bench(s, question_1, question_2):
  function main (line 45) | def main(args):

FILE: eval/rebase/sglang/benchmark/multi_chain_reasoning/bench_other.py
  function get_answer_value (line 18) | def get_answer_value(answer_str):
  function multi_chain_gsm8k (line 39) | def multi_chain_gsm8k(question, num_chains, call_generate):
  function main (line 59) | def main(args):

FILE: eval/rebase/sglang/benchmark/multi_chain_reasoning/bench_sglang.py
  function get_answer_value (line 15) | def get_answer_value(answer_str):
  function main (line 36) | def main(args):

FILE: eval/rebase/sglang/benchmark/multi_document_qa/bench_other.py
  function multi_document_qa (line 20) | def multi_document_qa(docs, question, generate):
  function main (line 35) | def main(args):

FILE: eval/rebase/sglang/benchmark/multi_document_qa/bench_sglang.py
  function multi_document_qa (line 12) | def multi_document_qa(s, docs, question):
  function main (line 27) | def main(args):

FILE: eval/rebase/sglang/benchmark/multi_turn_chat/bench_other.py
  function get_generate (line 15) | def get_generate(args):
  function multi_turns (line 60) | def multi_turns(generate, qas):
  function main (line 69) | def main(args):

FILE: eval/rebase/sglang/benchmark/multi_turn_chat/bench_sglang.py
  function multi_turns (line 17) | def multi_turns(s, qas):
  function main (line 23) | def main(args):

FILE: eval/rebase/sglang/benchmark/multi_turn_chat/data_gen.py
  function gen_prompt (line 7) | def gen_prompt(tokenizer, token_num):
  function gen_arguments (line 15) | def gen_arguments(args, tokenizer):

FILE: eval/rebase/sglang/benchmark/react/bench_other.py
  function get_prompt (line 18) | def get_prompt(question):
  function main (line 90) | def main(args):

FILE: eval/rebase/sglang/benchmark/react/bench_sglang.py
  function webthink (line 14) | def webthink(s, question, triplets):
  function main (line 98) | def main(args):

FILE: eval/rebase/sglang/benchmark/tip_suggestion/bench_other.py
  function expand_tip (line 17) | def expand_tip(topic, tip, generate):
  function suggest_tips (line 37) | def suggest_tips(topic, generate):
  function main (line 56) | def main(args):

FILE: eval/rebase/sglang/benchmark/tip_suggestion/bench_sglang.py
  function expand_tip (line 15) | def expand_tip(s, topic, tip):
  function suggest_tips (line 36) | def suggest_tips(s, topic):
  function main (line 50) | def main(args):

FILE: eval/rebase/sglang/benchmark/tree_of_thought_deep/bench_other.py
  function get_answer_value (line 20) | def get_answer_value(answer_str):
  function most_frequent_number (line 31) | def most_frequent_number(numbers):
  function propose_plan (line 49) | def propose_plan(s, question, num_branches, call_generate):
  function execute_plan (line 58) | def execute_plan(s, num_branches, call_generate):
  function reflect_solution (line 66) | def reflect_solution(s, num_branches, call_generate):
  function get_final_answer (line 74) | def get_final_answer(s, num_branches, call_generate):
  function tree_search (line 82) | def tree_search(question, num_branches, call_generate):
  function main (line 103) | def main(args):

FILE: eval/rebase/sglang/benchmark/tree_of_thought_deep/bench_sglang.py
  function get_answer_value (line 17) | def get_answer_value(answer_str):
  function most_frequent_number (line 28) | def most_frequent_number(numbers):
  function propose_plan (line 41) | def propose_plan(s, question, num_branches):
  function execute_plan (line 49) | def execute_plan(s, num_branches):
  function reflect_solution (line 57) | def reflect_solution(s, num_branches):
  function get_final_answer (line 65) | def get_final_answer(s, num_branches):
  function tree_search (line 75) | def tree_search(s, question, num_branches):
  function main (line 96) | def main(args):

FILE: eval/rebase/sglang/benchmark/tree_of_thought_v0/bench_other.py
  function get_answer_value (line 20) | def get_answer_value(answer_str):
  function most_frequent_number (line 31) | def most_frequent_number(numbers):
  function propose_plan (line 49) | def propose_plan(s, question, num_branches, call_generate):
  function execute_plan (line 58) | def execute_plan(s, num_branches, call_generate):
  function reflect_solution (line 66) | def reflect_solution(s, num_branches, call_generate):
  function tree_search (line 74) | def tree_search(question, num_branches, call_generate):
  function main (line 88) | def main(args):

FILE: eval/rebase/sglang/benchmark/tree_of_thought_v0/bench_sglang.py
  function get_answer_value (line 17) | def get_answer_value(answer_str):
  function most_frequent_number (line 28) | def most_frequent_number(numbers):
  function propose_plan (line 41) | def propose_plan(s, question, num_branches):
  function execute_plan (line 49) | def execute_plan(s, num_branches):
  function reflect_solution (line 57) | def reflect_solution(s, num_branches):
  function tree_search (line 66) | def tree_search(s, question, num_branches):
  function main (line 86) | def main(args):

FILE: eval/rebase/sglang/examples/quick_start/anthropic_example_chat.py
  function multi_turn_question (line 10) | def multi_turn_question(s, question_1, question_2):
  function single (line 17) | def single():
  function stream (line 29) | def stream():
  function batch (line 41) | def batch():

FILE: eval/rebase/sglang/examples/quick_start/anthropic_example_complete.py
  function few_shot_qa (line 11) | def few_shot_qa(s, question):
  function single (line 25) | def single():
  function stream (line 34) | def stream():
  function batch (line 44) | def batch():

FILE: eval/rebase/sglang/examples/quick_start/azure_openai_example_chat.py
  function multi_turn_question (line 11) | def multi_turn_question(s, question_1, question_2):
  function single (line 19) | def single():
  function stream (line 31) | def stream():
  function batch (line 43) | def batch():

FILE: eval/rebase/sglang/examples/quick_start/gemini_example_chat.py
  function multi_turn_question (line 10) | def multi_turn_question(s, question_1, question_2):
  function single (line 17) | def single():
  function stream (line 29) | def stream():
  function batch (line 41) | def batch():

FILE: eval/rebase/sglang/examples/quick_start/gemini_example_complete.py
  function few_shot_qa (line 11) | def few_shot_qa(s, question):
  function single (line 25) | def single():
  function stream (line 34) | def stream():
  function batch (line 44) | def batch():

FILE: eval/rebase/sglang/examples/quick_start/gemini_example_multimodal_chat.py
  function image_qa (line 10) | def image_qa(s, image_file1, image_file2, question):

FILE: eval/rebase/sglang/examples/quick_start/openai_example_chat.py
  function multi_turn_question (line 10) | def multi_turn_question(s, question_1, question_2):
  function single (line 18) | def single():
  function stream (line 30) | def stream():
  function batch (line 42) | def batch():

FILE: eval/rebase/sglang/examples/quick_start/openai_example_complete.py
  function few_shot_qa (line 11) | def few_shot_qa(s, question):
  function single (line 25) | def single():
  function stream (line 34) | def stream():
  function batch (line 44) | def batch():

FILE: eval/rebase/sglang/examples/quick_start/srt_example_chat.py
  function multi_turn_question (line 9) | def multi_turn_question(s, question_1, question_2):
  function single (line 16) | def single():
  function stream (line 28) | def stream():
  function batch (line 40) | def batch():

FILE: eval/rebase/sglang/examples/quick_start/srt_example_complete.py
  function few_shot_qa (line 9) | def few_shot_qa(s, question):
  function single (line 23) | def single():
  function stream (line 32) | def stream():
  function batch (line 42) | def batch():

FILE: eval/rebase/sglang/examples/quick_start/srt_example_llava.py
  function image_qa (line 8) | def image_qa(s, image_path, question):
  function single (line 13) | def single():
  function stream (line 21) | def stream():
  function batch (line 33) | def batch():

FILE: eval/rebase/sglang/examples/quick_start/srt_example_yi_vl.py
  function image_qa (line 8) | def image_qa(s, image_path, question):
  function single (line 13) | def single():
  function stream (line 22) | def stream():
  function batch (line 35) | def batch():

FILE: eval/rebase/sglang/examples/quick_start/together_example_chat.py
  function multi_turn_question (line 11) | def multi_turn_question(s, question_1, question_2):
  function single (line 19) | def single():
  function stream (line 31) | def stream():
  function batch (line 43) | def batch():

FILE: eval/rebase/sglang/examples/quick_start/together_example_complete.py
  function few_shot_qa (line 12) | def few_shot_qa(s, question):
  function single (line 26) | def single():
  function stream (line 35) | def stream():
  function batch (line 45) | def batch():

FILE: eval/rebase/sglang/examples/usage/async_io.py
  function generate (line 9) | async def generate(

FILE: eval/rebase/sglang/examples/usage/choices_logprob.py
  function tool_use (line 10) | def tool_use(s, question):
  function main (line 15) | def main():

FILE: eval/rebase/sglang/examples/usage/json_decode.py
  function character_gen (line 31) | def character_gen(s, name):
  function driver_character_gen (line 39) | def driver_character_gen():
  class Weapon (line 44) | class Weapon(str, Enum):
  class Wizard (line 53) | class Wizard(BaseModel):
  function pydantic_wizard_gen (line 60) | def pydantic_wizard_gen(s):
  function driver_pydantic_wizard_gen (line 70) | def driver_pydantic_wizard_gen():

FILE: eval/rebase/sglang/examples/usage/openai_speculative.py
  function gen_character_spec (line 9) | def gen_character_spec(s):

FILE: eval/rebase/sglang/examples/usage/parallel_sample.py
  function parallel_sample (line 9) | def parallel_sample(s, question, n):

FILE: eval/rebase/sglang/examples/usage/readme_examples.py
  function tool_use (line 10) | def tool_use(s, question):
  function tip_suggestion (line 21) | def tip_suggestion(s):
  function regular_expression_gen (line 38) | def regular_expression_gen(s):
  function text_qa (line 48) | def text_qa(s, question):
  function driver_tool_use (line 53) | def driver_tool_use():
  function driver_tip_suggestion (line 59) | def driver_tip_suggestion():
  function driver_regex (line 65) | def driver_regex():
  function driver_batching (line 71) | def driver_batching():
  function driver_stream (line 86) | def driver_stream():

FILE: eval/rebase/sglang/examples/usage/streaming.py
  function multi_turn_question (line 10) | def multi_turn_question(s, question_1, question_2):
  function stream_a_variable (line 21) | def stream_a_variable():
  function async_stream (line 33) | async def async_stream():

FILE: eval/rebase/sglang/examples/usage/triton/models/character_generation/1/model.py
  class Character (line 11) | class Character(BaseModel):
  function character_gen (line 17) | def character_gen(s, name):
  class TritonPythonModel (line 25) | class TritonPythonModel:
    method initialize (line 26) | def initialize(self, args):
    method execute (line 28) | def execute(self, requests):

FILE: eval/rebase/sglang/python/sglang/api.py
  function function (line 24) | def function(
  function Runtime (line 36) | def Runtime(*args, **kwargs):
  function set_default_backend (line 43) | def set_default_backend(backend: BaseBackend):
  function flush_cache (line 47) | def flush_cache(backend: BaseBackend = None):
  function get_server_args (line 54) | def get_server_args(backend: BaseBackend = None):
  function gen (line 61) | def gen(
  function gen_int (line 104) | def gen_int(
  function gen_string (line 130) | def gen_string(
  function image (line 156) | def image(expr: SglExpr):
  function select (line 160) | def select(
  function _role_common (line 169) | def _role_common(name: str, expr: Optional[SglExpr] = None):
  function system (line 176) | def system(expr: Optional[SglExpr] = None):
  function user (line 180) | def user(expr: Optional[SglExpr] = None):
  function assistant (line 184) | def assistant(expr: Optional[SglExpr] = None):
  function user_begin (line 188) | def user_begin():
  function user_end (line 192) | def user_end():
  function assistant_begin (line 196) | def assistant_begin():
  function assistant_end (line 200) | def assistant_end():

FILE: eval/rebase/sglang/python/sglang/backend/anthropic.py
  class Anthropic (line 15) | class Anthropic(BaseBackend):
    method __init__ (line 16) | def __init__(self, model_name):
    method get_chat_template (line 25) | def get_chat_template(self):
    method generate (line 28) | def generate(
    method generate_stream (line 47) | def generate_stream(

FILE: eval/rebase/sglang/python/sglang/backend/base_backend.py
  class BaseBackend (line 8) | class BaseBackend:
    method __init__ (line 9) | def __init__(self) -> None:
    method get_model_name (line 13) | def get_model_name(self):
    method get_chat_template (line 16) | def get_chat_template(self):
    method cache_prefix (line 19) | def cache_prefix(self, prefix_str: str):
    method uncache_prefix (line 22) | def uncache_prefix(self, rid: str):
    method end_request (line 25) | def end_request(self, rid: Union[str, List[str]]):
    method begin_program (line 28) | def begin_program(self, s: StreamExecutor):
    method end_program (line 31) | def end_program(self, s: Union[StreamExecutor, List[StreamExecutor]]):
    method commit_lazy_operations (line 34) | def commit_lazy_operations(self, s: StreamExecutor):
    method fork_program (line 37) | def fork_program(
    method fill_image (line 45) | def fill_image(self, s: StreamExecutor):
    method generate (line 48) | def generate(
    method generate_stream (line 55) | def generate_stream(
    method select (line 62) | def select(
    method concatenate_and_append (line 70) | def concatenate_and_append(self, src_rids: List[str], dst_rid: str):
    method shutdown (line 73) | def shutdown(self):
    method flush_cache (line 76) | def flush_cache(self):
    method get_server_args (line 79) | def get_server_args(self):

FILE: eval/rebase/sglang/python/sglang/backend/openai.py
  function create_logit_bias_int (line 21) | def create_logit_bias_int(tokenizer):
  class OpenAI (line 43) | class OpenAI(BaseBackend):
    method __init__ (line 44) | def __init__(
    method get_chat_template (line 84) | def get_chat_template(self):
    method generate (line 87) | def generate(
    method generate_stream (line 140) | def generate_stream(
    method select (line 168) | def select(
  function openai_completion (line 233) | def openai_completion(client, retries=3, is_chat=None, prompt=None, **kw...
  function openai_completion_stream (line 260) | def openai_completion_stream(client, retries=3, is_chat=None, prompt=Non...

FILE: eval/rebase/sglang/python/sglang/backend/runtime_endpoint.py
  class RuntimeEndpoint (line 14) | class RuntimeEndpoint(BaseBackend):
    method __init__ (line 15) | def __init__(
    method get_model_name (line 43) | def get_model_name(self):
    method flush_cache (line 46) | def flush_cache(self):
    method get_server_args (line 54) | def get_server_args(self):
    method get_chat_template (line 62) | def get_chat_template(self):
    method cache_prefix (line 65) | def cache_prefix(self, prefix_str: str):
    method commit_lazy_operations (line 75) | def commit_lazy_operations(self, s: StreamExecutor):
    method fill_image (line 85) | def fill_image(self, s: StreamExecutor):
    method generate (line 97) | def generate(
    method generate_stream (line 138) | def generate_stream(
    method select (line 192) | def select(
    method concatenate_and_append (line 238) | def concatenate_and_append(self, src_rids: List[str], dst_rid: str):
    method _add_images (line 248) | def _add_images(self, s: StreamExecutor, data):

FILE: eval/rebase/sglang/python/sglang/backend/vertexai.py
  class VertexAI (line 22) | class VertexAI(BaseBackend):
    method __init__ (line 23) | def __init__(self, model_name):
    method get_chat_template (line 36) | def get_chat_template(self):
    method generate (line 39) | def generate(
    method generate_stream (line 62) | def generate_stream(
    method text_to_vertexai_input (line 84) | def text_to_vertexai_input(self, text, images):
    method messages_to_vertexai_input (line 98) | def messages_to_vertexai_input(self, messages):

FILE: eval/rebase/sglang/python/sglang/global_config.py
  class GlobalConfig (line 4) | class GlobalConfig:
    method __init__ (line 5) | def __init__(self):

FILE: eval/rebase/sglang/python/sglang/lang/chat_template.py
  class ChatTemplateStyle (line 6) | class ChatTemplateStyle(Enum):
  class ChatTemplate (line 12) | class ChatTemplate:
    method get_prefix_and_suffix (line 20) | def get_prefix_and_suffix(
    method get_prompt (line 41) | def get_prompt(self, messages: List[Dict]) -> str:
  function register_chat_template (line 59) | def register_chat_template(template):
  function register_chat_template_matching_function (line 63) | def register_chat_template_matching_function(func):
  function get_chat_template (line 67) | def get_chat_template(name):
  function get_chat_template_by_model_path (line 71) | def get_chat_template_by_model_path(model_path):
  function match_vicuna (line 197) | def match_vicuna(model_path: str):
  function match_llama2_chat (line 205) | def match_llama2_chat(model_path: str):
  function match_chat_ml (line 218) | def match_chat_ml(model_path: str):
  function match_chat_yi (line 229) | def match_chat_yi(model_path: str):
  function match_gemma_it (line 236) | def match_gemma_it(model_path: str):

FILE: eval/rebase/sglang/python/sglang/lang/compiler.py
  function compile_func (line 17) | def compile_func(function, backend):
  class CompiledFunction (line 23) | class CompiledFunction:
    method __init__ (line 24) | def __init__(self, tracer, function):
    method build_graph (line 32) | def build_graph(self, tracer):
    method topological_sort (line 72) | def topological_sort(self):
    method print_graph (line 89) | def print_graph(
    method run_internal (line 95) | def run_internal(
    method run (line 125) | def run(
    method run_batch (line 154) | def run_batch(
  class CompGraphNode (line 215) | class CompGraphNode:
    method __init__ (line 216) | def __init__(
    method add_next_node (line 224) | def add_next_node(self, other):
    method __repr__ (line 227) | def __repr__(self):

FILE: eval/rebase/sglang/python/sglang/lang/interpreter.py
  function run_internal (line 33) | def run_internal(state, program, func_args, func_kwargs, sync):
  function run_program (line 48) | def run_program(
  function run_program_batch (line 76) | def run_program_batch(
  function pin_program (line 155) | def pin_program(program, backend):
  function unpin_program (line 168) | def unpin_program(program, backend):
  class StreamExecutor (line 172) | class StreamExecutor:
    method __init__ (line 175) | def __init__(
    method submit (line 240) | def submit(self, expr: SglExpr):
    method sync (line 248) | def sync(self):
    method get_var (line 252) | def get_var(self, name):
    method set_var (line 257) | def set_var(self, name, value):
    method get_meta_info (line 260) | def get_meta_info(self, name):
    method set_score_backend (line 266) | def set_score_backend(self, score_backend):
    method fork (line 269) | def fork(self, number: int, position_ids_offset: Optional[List[int]] =...
    method text (line 294) | def text(self):
    method scores (line 298) | def scores(self):
    method text_ready (line 302) | def text_ready(self):
    method score_ready (line 305) | def score_ready(self):
    method messages (line 308) | def messages(self):
    method end (line 312) | def end(self):
    method _thread_worker_func (line 318) | def _thread_worker_func(self):
    method _execute (line 335) | def _execute(self, other):
    method _execute_fill (line 375) | def _execute_fill(self, value: str):
    method _execute_image (line 383) | def _execute_image(self, expr: SglImage):
    method _execute_gen (line 395) | def _execute_gen(self, expr: SglGen):
    method _execute_select (line 502) | def _execute_select(self, expr: SglSelect):
    method _execute_variable (line 516) | def _execute_variable(self, expr: SglVariable):
    method _execute_role_begin (line 521) | def _execute_role_begin(self, expr: SglRoleBegin):
    method _execute_role_end (line 539) | def _execute_role_end(self, expr: SglRoleEnd):
    method _execute_var_scope_begin (line 568) | def _execute_var_scope_begin(self, expr: SglVarScopeBegin):
    method _execute_var_scope_end (line 571) | def _execute_var_scope_end(self, expr: SglVarScopeEnd):
    method _execute_commit_lazy_operations (line 575) | def _execute_commit_lazy_operations(self, expr: SglCommitLazy):
    method _execute_concatenate_and_append_text (line 578) | def _execute_concatenate_and_append_text(self, expr: SglConcateAndAppe...
    method _execute_concatenate_and_append_kv_cache (line 587) | def _execute_concatenate_and_append_kv_cache(self, expr: SglConcateAnd...
    method _init_var_event (line 603) | def _init_var_event(self, expr):
    method _resolve_sampling_params (line 612) | def _resolve_sampling_params(self, sampling_params):
    method __del__ (line 645) | def __del__(self):
  class ProgramState (line 649) | class ProgramState:
    method __init__ (line 652) | def __init__(self, stream_executor: StreamExecutor):
    method _role_common (line 655) | def _role_common(self, name: str, expr: Optional[SglExpr] = None):
    method system (line 670) | def system(self, expr: Optional[SglExpr] = None):
    method user (line 673) | def user(self, expr: Optional[SglExpr] = None):
    method assistant (line 676) | def assistant(self, expr: Optional[SglExpr] = None):
    method var_scope (line 680) | def var_scope(self, name: str):
    method fork (line 685) | def fork(self, number: int = 1, position_ids_offset: Optional[List[int...
    method copy (line 692) | def copy(self, position_ids_offset: Optional[List[int]] = None):
    method text (line 699) | def text(self):
    method text_processed (line 702) | def text_processed(self):
    method text_ready (line 705) | def text_ready(self):
    method score_ready (line 708) | def score_ready(self):
    method text_prepare (line 711) | def text_prepare(self):
    method score_prepare (line 714) | def score_prepare(self):
    method scores (line 717) | def scores(self):
    method set_score_backend (line 720) | def set_score_backend(self, score_backend):
    method messages (line 723) | def messages(self):
    method sync (line 726) | def sync(self):
    method text_iter (line 729) | def text_iter(self, var_name: Optional[str] = None):
    method text_async_iter (line 760) | async def text_async_iter(
    method get_var (line 798) | def get_var(self, name):
    method set_var (line 801) | def set_var(self, name, value):
    method get_meta_info (line 804) | def get_meta_info(self, name):
    method __iadd__ (line 807) | def __iadd__(self, other):
    method __getitem__ (line 811) | def __getitem__(self, name):
    method __setitem__ (line 814) | def __setitem__(self, name, value):
    method __del__ (line 817) | def __del__(self):
    method __repr__ (line 820) | def __repr__(self) -> str:
  class ProgramStateGroup (line 824) | class ProgramStateGroup:
    method __init__ (line 825) | def __init__(
    method join (line 831) | def join(self, mode: str = "gather_variable"):
    method __getitem__ (line 857) | def __getitem__(self, i: int):
    method __setitem__ (line 860) | def __setitem__(self, i: int, value):
    method __iadd__ (line 863) | def __iadd__(self, other):

FILE: eval/rebase/sglang/python/sglang/lang/ir.py
  class SglSamplingParams (line 17) | class SglSamplingParams:
    method clone (line 33) | def clone(self):
    method to_openai_kwargs (line 44) | def to_openai_kwargs(self):
    method to_vertexai_kwargs (line 57) | def to_vertexai_kwargs(self):
    method to_anthropic_kwargs (line 71) | def to_anthropic_kwargs(self):
    method to_srt_kwargs (line 87) | def to_srt_kwargs(self):
  class SglFunction (line 103) | class SglFunction:
    method __init__ (line 104) | def __init__(self, func, api_num_spec_tokens=None, bind_arguments=None):
    method bind (line 115) | def bind(self, **kwargs):
    method run (line 121) | def run(
    method run_batch (line 155) | def run_batch(
    method trace (line 202) | def trace(self, *, backend=None, **kwargs):
    method pin (line 208) | def pin(self, backend=None):
    method unpin (line 214) | def unpin(self, backend=None):
    method compile (line 220) | def compile(self, *, backend=None):
    method __call__ (line 225) | def __call__(self, *args, **kwargs):
  class SglExpr (line 236) | class SglExpr:
    method __init__ (line 239) | def __init__(self):
    method __add__ (line 245) | def __add__(self, other):
    method __radd__ (line 252) | def __radd__(self, other):
    method concatenate_ir (line 259) | def concatenate_ir(self, a, b):
    method print_graph_dfs (line 270) | def print_graph_dfs(self):
  class SglExprList (line 306) | class SglExprList(SglExpr):
    method __init__ (line 307) | def __init__(self, expr_list: List[SglExpr]):
    method __repr__ (line 311) | def __repr__(self):
  class SglArgument (line 315) | class SglArgument(SglExpr):
    method __init__ (line 316) | def __init__(self, name: str, value: str):
    method __repr__ (line 321) | def __repr__(self):
    method __len__ (line 324) | def __len__(self):
    method __getitem__ (line 327) | def __getitem__(self, i):
    method __int__ (line 330) | def __int__(self):
    method __bool__ (line 333) | def __bool__(self):
    method __format__ (line 336) | def __format__(self, *args):
  class SglImage (line 343) | class SglImage(SglExpr):
    method __init__ (line 344) | def __init__(self, path):
    method __repr__ (line 347) | def __repr__(self) -> str:
  class SglGen (line 351) | class SglGen(SglExpr):
    method __init__ (line 352) | def __init__(
    method __repr__ (line 385) | def __repr__(self):
  class SglConstantText (line 389) | class SglConstantText(SglExpr):
    method __init__ (line 390) | def __init__(self, value):
    method __repr__ (line 394) | def __repr__(self):
  class SglRoleBegin (line 398) | class SglRoleBegin(SglExpr):
    method __init__ (line 399) | def __init__(self, role):
    method __repr__ (line 403) | def __repr__(self):
  class SglRoleEnd (line 407) | class SglRoleEnd(SglExpr):
    method __init__ (line 408) | def __init__(self, role):
    method __repr__ (line 412) | def __repr__(self):
  class SglSelect (line 416) | class SglSelect(SglExpr):
    method __init__ (line 417) | def __init__(self, name, choices, temperature):
    method __repr__ (line 423) | def __repr__(self):
  class SglFork (line 427) | class SglFork(SglExpr):
    method __init__ (line 428) | def __init__(self, number, position_ids_offset=None):
    method __repr__ (line 433) | def __repr__(self):
  class SglGetForkItem (line 440) | class SglGetForkItem(SglExpr):
    method __init__ (line 441) | def __init__(self, index):
    method __repr__ (line 445) | def __repr__(self):
  class SglVariable (line 449) | class SglVariable(SglExpr):
    method __init__ (line 450) | def __init__(self, name, source):
    method __repr__ (line 455) | def __repr__(self):
  class SglVarScopeBegin (line 459) | class SglVarScopeBegin(SglExpr):
    method __init__ (line 460) | def __init__(self, name):
    method __repr__ (line 464) | def __repr__(self):
  class SglVarScopeEnd (line 468) | class SglVarScopeEnd(SglExpr):
    method __init__ (line 469) | def __init__(self, name):
    method __repr__ (line 473) | def __repr__(self):
  class SglConcateAndAppend (line 477) | class SglConcateAndAppend(SglExpr):
    method __init__ (line 478) | def __init__(self, states):
    method __repr__ (line 482) | def __repr__(self):
  class SglCommitLazy (line 486) | class SglCommitLazy(SglExpr):
    method __init__ (line 487) | def __init__(self):
    method __repr__ (line 490) | def __repr__(self):

FILE: eval/rebase/sglang/python/sglang/lang/tracer.py
  class StopTracing (line 29) | class StopTracing(Exception):
  function extract_prefix_by_tracing (line 33) | def extract_prefix_by_tracing(program, backend):
  function trace_program (line 58) | def trace_program(program, arguments, backend):
  class TracerProgramState (line 79) | class TracerProgramState(ProgramState):
    method __init__ (line 80) | def __init__(self, backend, arguments, only_trace_prefix):
    method fork (line 112) | def fork(self, number: int, position_ids_offset: Optional[List[int]] =...
    method _append_node (line 141) | def _append_node(self, other: SglExpr):
    method _execute (line 146) | def _execute(self, other: SglExpr):
    method __iadd__ (line 177) | def __iadd__(self, other):
    method _execute_fill (line 181) | def _execute_fill(self, expr: SglConstantText):
    method _execute_gen (line 186) | def _execute_gen(self, expr: SglGen):
    method _execute_select (line 192) | def _execute_select(self, expr: SglSelect):
    method _execute_role_begin (line 200) | def _execute_role_begin(self, expr: SglRoleBegin):
    method _execute_role_end (line 219) | def _execute_role_end(self, expr: SglRoleEnd):
    method _execute_var_scope_end (line 230) | def _execute_var_scope_end(self, expr: SglVarScopeEnd):
    method get_var (line 234) | def get_var(self, name):
    method flatten_nodes (line 242) | def flatten_nodes(self):
    method __del__ (line 255) | def __del__(self):
  class TracingScope (line 259) | class TracingScope:
    method __init__ (line 262) | def __init__(self, tracer_state: TracerProgramState):
    method __enter__ (line 266) | def __enter__(self):
    method __exit__ (line 270) | def __exit__(self, exc_type, exc_value, traceback):
    method get_current_scope (line 274) | def get_current_scope():
    method add_child_state (line 277) | def add_child_state(self, state: TracerProgramState):

FILE: eval/rebase/sglang/python/sglang/srt/backend_config.py
  class BackendConfig (line 9) | class BackendConfig:

FILE: eval/rebase/sglang/python/sglang/srt/constrained/__init__.py
  function build_regex_from_object (line 18) | def build_regex_from_object(

FILE: eval/rebase/sglang/python/sglang/srt/constrained/base_cache.py
  class BaseCache (line 6) | class BaseCache:
    method __init__ (line 7) | def __init__(self, enable=True):
    method reset (line 11) | def reset(self):
    method query (line 15) | def query(self, key):
    method init_value (line 41) | def init_value(self, key):
    method get_cache_hit_rate (line 44) | def get_cache_hit_rate(self):
    method get_avg_init_time (line 49) | def get_avg_init_time(self):

FILE: eval/rebase/sglang/python/sglang/srt/constrained/fsm_cache.py
  class FSMCache (line 5) | class FSMCache(BaseCache):
    method __init__ (line 6) | def __init__(self, tokenizer_path, tokenizer_args_dict, enable=True):
    method init_value (line 23) | def init_value(self, regex):

FILE: eval/rebase/sglang/python/sglang/srt/constrained/jump_forward.py
  class JumpForwardMap (line 8) | class JumpForwardMap:
    method __init__ (line 9) | def __init__(self, regex_string):
    method valid_states (line 43) | def valid_states(self):
    method jump_forward (line 46) | def jump_forward(self, state):
  class JumpForwardCache (line 59) | class JumpForwardCache(BaseCache):
    method __init__ (line 60) | def __init__(self):
    method init_value (line 63) | def init_value(self, regex):
  function test_main (line 67) | def test_main():

FILE: eval/rebase/sglang/python/sglang/srt/conversation.py
  class SeparatorStyle (line 10) | class SeparatorStyle(IntEnum):
  class Conversation (line 34) | class Conversation:
    method get_prompt (line 57) | def get_prompt(self) -> str:
    method set_system_message (line 247) | def set_system_message(self, system_message: str):
    method append_message (line 251) | def append_message(self, role: str, message: str):
    method append_image (line 255) | def append_image(self, image: str):
    method update_last_message (line 259) | def update_last_message(self, message: str):
    method to_gradio_chatbot (line 267) | def to_gradio_chatbot(self):
    method to_openai_api_messages (line 277) | def to_openai_api_messages(self):
    method copy (line 292) | def copy(self):
    method dict (line 306) | def dict(self):
  function register_conv_template (line 320) | def register_conv_template(template: Conversation, override: bool = False):
  function chat_template_exists (line 330) | def chat_template_exists(template_name: str) -> bool:
  function generate_chat_conv (line 334) | def generate_chat_conv(

FILE: eval/rebase/sglang/python/sglang/srt/hf_transformers_utils.py
  function download_from_hf (line 19) | def download_from_hf(model_path: str):
  function get_config_json (line 26) | def get_config_json(model_path: str):
  function get_config (line 32) | def get_config(model: str, trust_remote_code: bool, revision: Optional[s...
  function get_context_length (line 52) | def get_context_length(config):
  function get_tokenizer (line 71) | def get_tokenizer(
  function get_processor (line 149) | def get_processor(

FILE: eval/rebase/sglang/python/sglang/srt/layers/context_flashattention_nopad.py
  function _fwd_kernel (line 12) | def _fwd_kernel(
  function context_attention_fwd (line 124) | def context_attention_fwd(q, k, v, o, b_start_loc, b_seq_len, max_input_...

FILE: eval/rebase/sglang/python/sglang/srt/layers/extend_attention.py
  function _fwd_kernel (line 11) | def _fwd_kernel(
  function extend_attention_fwd (line 162) | def extend_attention_fwd(
  function redundant_attention (line 277) | def redundant_attention(
  function test (line 317) | def test():

FILE: eval/rebase/sglang/python/sglang/srt/layers/logits_processor.py
  class LogitsProcessor (line 10) | class LogitsProcessor(nn.Module):
    method __init__ (line 11) | def __init__(self, config):
    method forward (line 16) | def forward(self, input_ids, hidden_states, weight, input_metadata):

FILE: eval/rebase/sglang/python/sglang/srt/layers/radix_attention.py
  class RadixAttention (line 9) | class RadixAttention(nn.Module):
    method __init__ (line 10) | def __init__(self, num_heads, head_dim, scaling, num_kv_heads, layer_id):
    method prefill_forward_triton (line 29) | def prefill_forward_triton(self, q, k, v, input_metadata: InputMetadata):
    method extend_forward_triton (line 45) | def extend_forward_triton(self, q, k, v, input_metadata: InputMetadata):
    method decode_forward_triton (line 68) | def decode_forward_triton(self, q, k, v, input_metadata: InputMetadata):
    method prefill_forward_flashinfer (line 88) | def prefill_forward_flashinfer(self, q, k, v, input_metadata: InputMet...
    method decode_forward_flashinfer (line 98) | def decode_forward_flashinfer(self, q, k, v, input_metadata: InputMeta...
    method forward (line 108) | def forward(self, q, k, v, input_metadata: InputMetadata):
    method store_kv_cache (line 119) | def store_kv_cache(self, cache_k, cache_v, input_metadata: InputMetada...

FILE: eval/rebase/sglang/python/sglang/srt/layers/token_attention.py
  function _fwd_kernel_stage1 (line 19) | def _fwd_kernel_stage1(
  function _fwd_kernel_stage2 (line 84) | def _fwd_kernel_stage2(
  function _token_att_m_fwd (line 158) | def _token_att_m_fwd(
  function _token_softmax_reducev_fwd (line 231) | def _token_softmax_reducev_fwd(
  function token_attention_fwd (line 294) | def token_attention_fwd(

FILE: eval/rebase/sglang/python/sglang/srt/managers/detokenizer_manager.py
  class DetokenizerManager (line 14) | class DetokenizerManager:
    method __init__ (line 15) | def __init__(
    method handle_loop (line 33) | async def handle_loop(self):
  function start_detokenizer_process (line 85) | def start_detokenizer_process(

FILE: eval/rebase/sglang/python/sglang/srt/managers/io_struct.py
  class GenerateReqInput (line 9) | class GenerateReqInput:
    method post_init (line 31) | def post_init(self):
  class TokenizedGenerateReqInput (line 81) | class TokenizedGenerateReqInput:
  class BatchTokenIDOut (line 97) | class BatchTokenIDOut:
  class BatchStrOut (line 110) | class BatchStrOut:
  class FlushCacheReq (line 120) | class FlushCacheReq:
  class DetokenizeReqInput (line 125) | class DetokenizeReqInput:

FILE: eval/rebase/sglang/python/sglang/srt/managers/openai_protocol.py
  class LogProbs (line 8) | class LogProbs(BaseModel):
  class UsageInfo (line 15) | class UsageInfo(BaseModel):
  class CompletionRequest (line 21) | class CompletionRequest(BaseModel):
  class CompletionResponseChoice (line 43) | class CompletionResponseChoice(BaseModel):
  class CompletionResponse (line 50) | class CompletionResponse(BaseModel):
  class CompletionResponseStreamChoice (line 59) | class CompletionResponseStreamChoice(BaseModel):
  class CompletionStreamResponse (line 66) | class CompletionStreamResponse(BaseModel):
  class ChatCompletionMessageGenericParam (line 75) | class ChatCompletionMessageGenericParam(BaseModel):
  class ChatCompletionMessageContentTextPart (line 80) | class ChatCompletionMessageContentTextPart(BaseModel):
  class ChatCompletionMessageContentImageURL (line 85) | class ChatCompletionMessageContentImageURL(BaseModel):
  class ChatCompletionMessageContentImagePart (line 90) | class ChatCompletionMessageContentImagePart(BaseModel):
  class ChatCompletionMessageUserParam (line 100) | class ChatCompletionMessageUserParam(BaseModel):
  class ChatCompletionRequest (line 110) | class ChatCompletionRequest(BaseModel):
  class ChatMessage (line 129) | class ChatMessage(BaseModel):
  class ChatCompletionResponseChoice (line 134) | class ChatCompletionResponseChoice(BaseModel):
  class ChatCompletionResponse (line 140) | class ChatCompletionResponse(BaseModel):
  class DeltaMessage (line 149) | class DeltaMessage(BaseModel):
  class ChatCompletionResponseStreamChoice (line 154) | class ChatCompletionResponseStreamChoice(BaseModel):
  class ChatCompletionStreamResponse (line 160) | class ChatCompletionStreamResponse(BaseModel):

FILE: eval/rebase/sglang/python/sglang/srt/managers/router/infer_batch.py
  class ForwardMode (line 11) | class ForwardMode(Enum):
  class FinishReason (line 17) | class FinishReason(Enum):
  class Req (line 23) | class Req:
    method __init__ (line 24) | def __init__(self, rid, input_text, input_ids):
    method max_new_tokens (line 70) | def max_new_tokens(self):
    method jump_forward_and_retokenize (line 73) | def jump_forward_and_retokenize(self, jump_forward_str, next_state):
    method check_finished (line 117) | def check_finished(self):
    method __repr__ (line 148) | def __repr__(self):
  class Batch (line 153) | class Batch:
    method init_new (line 190) | def init_new(cls, reqs, req_to_token_pool, token_to_kv_pool, tree_cache):
    method is_empty (line 201) | def is_empty(self):
    method prepare_for_extend (line 204) | def prepare_for_extend(self, vocab_size: int, int_token_logit_bias: to...
    method check_decode_mem (line 301) | def check_decode_mem(self):
    method retract_decode (line 313) | def retract_decode(self):
    method check_for_jump_forward (line 346) | def check_for_jump_forward(self):
    method prepare_for_decode (line 386) | def prepare_for_decode(self, input_ids=None):
    method filter_batch (line 417) | def filter_batch(self, unfinished_indices: List[int]):
    method merge (line 438) | def merge(self, other):
    method sample (line 464) | def sample(self, logits: torch.Tensor):
  function _top_p_top_k (line 503) | def _top_p_top_k(probs: torch.Tensor, top_ps: torch.Tensor, top_ks: torc...

FILE: eval/rebase/sglang/python/sglang/srt/managers/router/manager.py
  class RouterManager (line 15) | class RouterManager:
    method __init__ (line 16) | def __init__(self, model_client: ModelRpcClient, port_args: PortArgs):
    method loop_for_forward (line 34) | async def loop_for_forward(self):
    method loop_for_recv_requests (line 51) | async def loop_for_recv_requests(self):
  function start_router_process (line 57) | def start_router_process(

FILE: eval/rebase/sglang/python/sglang/srt/managers/router/model_rpc.py
  class ModelRpcServer (line 39) | class ModelRpcServer(rpyc.Service):
    method exposed_init_model (line 40) | def exposed_init_model(
    method flush_cache (line 151) | def flush_cache(self):
    method exposed_step (line 169) | def exposed_step(self, recv_reqs):
    method forward_step (line 194) | def forward_step(self):
    method handle_generate_request (line 245) | def handle_generate_request(
    method get_new_fill_batch (line 287) | def get_new_fill_batch(self):
    method forward_fill_batch (line 405) | def forward_fill_batch(self, batch: Batch):
    method forward_decode_batch (line 472) | def forward_decode_batch(self, batch: Batch):
    method handle_finished_requests (line 543) | def handle_finished_requests(self, batch: Batch):
  class ModelRpcClient (line 635) | class ModelRpcClient:
    method __init__ (line 636) | def __init__(self, server_args: ServerArgs, port_args: PortArgs):
  function _init_service (line 679) | def _init_service(port):
  function start_model_process (line 688) | def start_model_process(port):

FILE: eval/rebase/sglang/python/sglang/srt/managers/router/model_runner.py
  function import_model_classes (line 36) | def import_model_classes():
  function get_model_cls_by_arch_name (line 48) | def get_model_cls_by_arch_name(model_arch_names):
  class InputMetadata (line 65) | class InputMetadata:
    method init_flashinfer_args (line 100) | def init_flashinfer_args(self, tp_size):
    method init_extend_args (line 168) | def init_extend_args(self):
    method create (line 175) | def create(
  class ModelRunner (line 250) | class ModelRunner:
    method __init__ (line 251) | def __init__(
    method load_model (line 295) | def load_model(self):
    method profile_max_num_token (line 339) | def profile_max_num_token(self, total_gpu_memory):
    method init_memory_pool (line 352) | def init_memory_pool(self, total_gpu_memory):
    method forward_prefill (line 373) | def forward_prefill(
    method forward_extend (line 397) | def forward_extend(
    method forward_decode (line 423) | def forward_decode(
    method forward_extend_multi_modal (line 451) | def forward_extend_multi_modal(
    method forward (line 484) | def forward(self, batch: Batch, forward_mode: ForwardMode, return_logp...

FILE: eval/rebase/sglang/python/sglang/srt/managers/router/radix_cache.py
  class TreeNode (line 10) | class TreeNode:
    method __init__ (line 11) | def __init__(self):
    method __lt__ (line 18) | def __lt__(self, other):
  function match (line 22) | def match(key, seq):
  class RadixCache (line 31) | class RadixCache:
    method __init__ (line 32) | def __init__(self, disable=False):
    method reset (line 38) | def reset(self):
    method match_prefix (line 44) | def match_prefix(self, key):
    method insert (line 55) | def insert(self, key, value=None):
    method pretty_print (line 63) | def pretty_print(self):
    method total_size (line 67) | def total_size(self):
    method evict (line 70) | def evict(self, num_tokens, evict_callback):
    method inc_ref_counter (line 92) | def inc_ref_counter(self, node):
    method dec_ref_counter (line 102) | def dec_ref_counter(self, node):
    method evictable_size (line 112) | def evictable_size(self):
    method _match_prefix_helper (line 116) | def _match_prefix_helper(self, node, key, value, last_node):
    method _split_node (line 132) | def _split_node(self, key, child, split_len):
    method _insert_helper (line 145) | def _insert_helper(self, node, key, value):
    method _print_helper (line 173) | def _print_helper(self, node, indent):
    method _delete_leaf (line 178) | def _delete_leaf(self, node):
    method _total_size_helper (line 185) | def _total_size_helper(self, node):
    method _collect_leaves (line 191) | def _collect_leaves(self):

FILE: eval/rebase/sglang/python/sglang/srt/managers/router/scheduler.py
  class Scheduler (line 5) | class Scheduler:
    method __init__ (line 6) | def __init__(
    method get_priority_queue (line 20) | def get_priority_queue(self, forward_queue):
    method _calc_weight_recursive (line 51) | def _calc_weight_recursive(self, cur_node, last_node_to_reqs, node_to_...
    method _get_weight_priority_recursive (line 59) | def _get_weight_priority_recursive(

FILE: eval/rebase/sglang/python/sglang/srt/managers/tokenizer_manager.py
  class ReqState (line 35) | class ReqState:
  function init_global_processor (line 45) | def init_global_processor(server_args: ServerArgs):
  function get_pixel_values (line 55) | def get_pixel_values(
  class TokenizerManager (line 79) | class TokenizerManager:
    method __init__ (line 80) | def __init__(
    method get_pixel_values (line 124) | async def get_pixel_values(self, image_data):
    method generate_request (line 143) | async def generate_request(self, obj: GenerateReqInput):
    method detokenize (line 244) | async def detokenize(self, obj: DetokenizeReqInput):
    method flush_cache (line 248) | async def flush_cache(self):
    method create_handle_loop (line 252) | async def create_handle_loop(self):
    method handle_loop (line 257) | async def handle_loop(self):

FILE: eval/rebase/sglang/python/sglang/srt/memory_pool.py
  class ReqToTokenPool (line 10) | class ReqToTokenPool:
    method __init__ (line 11) | def __init__(self, size, max_context_len):
    method alloc (line 18) | def alloc(self, need_size):
    method free (line 27) | def free(self, free_index):
    method clear (line 37) | def clear(self):
  class TokenToKVPool (line 42) | class TokenToKVPool:
    method __init__ (line 43) | def __init__(self, size, dtype, head_num, head_dim, layer_num):
    method get_key_buffer (line 53) | def get_key_buffer(self, layer_id):
    method get_value_buffer (line 56) | def get_value_buffer(self, layer_id):
    method alloc (line 59) | def alloc(self, need_size):
    method alloc_contiguous (line 67) | def alloc_contiguous(self, need_size):
    method free (line 86) | def free(self, free_index):
    method used_size (line 89) | def used_size(self):
    method available_size (line 92) | def available_size(self):
    method add_refs (line 95) | def add_refs(self, token_index: torch.Tensor):
    method decrease_refs (line 99) | def decrease_refs(self, token_index: torch.Tensor):
    method clear (line 110) | def clear(self):

FILE: eval/rebase/sglang/python/sglang/srt/mm_utils.py
  function select_best_resolution (line 11) | def select_best_resolution(original_size, possible_resolutions):
  function resize_and_pad_image (line 48) | def resize_and_pad_image(image, target_resolution):
  function divide_to_patches (line 83) | def divide_to_patches(image, patch_size):
  function get_anyres_image_grid_shape (line 105) | def get_anyres_image_grid_shape(image_size, grid_pinpoints, patch_size):
  function process_anyres_image (line 125) | def process_anyres_image(image, processor, grid_pinpoints):
  function load_image_from_base64 (line 158) | def load_image_from_base64(image):
  function expand2square (line 162) | def expand2square(pil_img, background_color):
  function unpad_image (line 178) | def unpad_image(tensor, original_size):
  function unpad_image_shape (line 209) | def unpad_image_shape(current_height, current_width, original_size):
  function process_images (line 233) | def process_images(images, image_processor, model_cfg):

FILE: eval/rebase/sglang/python/sglang/srt/model_config.py
  class ModelConfig (line 6) | class ModelConfig:
    method __init__ (line 7) | def __init__(

FILE: eval/rebase/sglang/python/sglang/srt/models/gemma.py
  class GemmaMLP (line 32) | class GemmaMLP(nn.Module):
    method __init__ (line 33) | def __init__(
    method forward (line 51) | def forward(self, x):
  class GemmaAttention (line 58) | class GemmaAttention(nn.Module):
    method __init__ (line 59) | def __init__(
    method forward (line 122) | def forward(
  class GemmaDecoderLayer (line 136) | class GemmaDecoderLayer(nn.Module):
    method __init__ (line 137) | def __init__(
    method forward (line 165) | def forward(
  class GemmaModel (line 190) | class GemmaModel(nn.Module):
    method __init__ (line 191) | def __init__(
    method forward (line 211) | def forward(
  class GemmaForCausalLM (line 239) | class GemmaForCausalLM(nn.Module):
    method __init__ (line 263) | def __init__(
    method forward (line 277) | def forward(
    method load_weights (line 289) | def load_weights(

FILE: eval/rebase/sglang/python/sglang/srt/models/llama2.py
  class LlamaMLP (line 34) | class LlamaMLP(nn.Module):
    method __init__ (line 35) | def __init__(
    method forward (line 59) | def forward(self, x):
  class LlamaAttention (line 66) | class LlamaAttention(nn.Module):
    method __init__ (line 67) | def __init__(
    method forward (line 131) | def forward(
  class LlamaDecoderLayer (line 145) | class LlamaDecoderLayer(nn.Module):
    method __init__ (line 146) | def __init__(
    method forward (line 178) | def forward(
  class LlamaModel (line 203) | class LlamaModel(nn.Module):
    method __init__ (line 204) | def __init__(
    method forward (line 225) | def forward(
  class LlamaForCausalLM (line 249) | class LlamaForCausalLM(nn.Module):
    method __init__ (line 250) | def __init__(
    method forward (line 262) | def forward(
    method load_weights (line 274) | def load_weights(

FILE: eval/rebase/sglang/python/sglang/srt/models/mistral.py
  class MistralForCausalLM (line 6) | class MistralForCausalLM(LlamaForCausalLM):
    method __init__ (line 7) | def __init__(self, *args, **kwargs):

FILE: eval/rebase/sglang/python/sglang/srt/models/qwen.py
  class QWenMLP (line 31) | class QWenMLP(nn.Module):
    method __init__ (line 32) | def __init__(
    method forward (line 61) | def forward(self, x):
  class QWenAttention (line 68) | class QWenAttention(nn.Module):
    method __init__ (line 69) | def __init__(
    method forward (line 118) | def forward(
  class QWenBlock (line 132) | class QWenBlock(nn.Module):
    method __init__ (line 133) | def __init__(self, config: PretrainedConfig, layer_id, linear_method=N...
    method forward (line 157) | def forward(
  class QWenModel (line 181) | class QWenModel(nn.Module):
    method __init__ (line 182) | def __init__(self, config: PretrainedConfig, linear_method=None):
    method forward (line 200) | def forward(
  class QWenLMHeadModel (line 218) | class QWenLMHeadModel(nn.Module):
    method __init__ (line 219) | def __init__(self, config: PretrainedConfig, linear_method=None):
    method forward (line 227) | def forward(
    method load_weights (line 239) | def load_weights(

FILE: eval/rebase/sglang/python/sglang/srt/models/qwen2.py
  class Qwen2MLP (line 35) | class Qwen2MLP(nn.Module):
    method __init__ (line 36) | def __init__(
    method forward (line 60) | def forward(self, x):
  class Qwen2Attention (line 67) | class Qwen2Attention(nn.Module):
    method __init__ (line 68) | def __init__(
    method forward (line 132) | def forward(
  class Qwen2DecoderLayer (line 146) | class Qwen2DecoderLayer(nn.Module):
    method __init__ (line 147) | def __init__(
    method forward (line 179) | def forward(
  class Qwen2Model (line 204) | class Qwen2Model(nn.Module):
    method __init__ (line 205) | def __init__(
    method forward (line 226) | def forward(
  class Qwen2ForCausalLM (line 250) | class Qwen2ForCausalLM(nn.Module):
    method __init__ (line 251) | def __init__(
    method forward (line 263) | def forward(
    method load_weights (line 275) | def load_weights(

FILE: eval/rebase/sglang/python/sglang/srt/models/stablelm.py
  class StablelmMLP (line 35) | class StablelmMLP(nn.Module):
    method __init__ (line 36) | def __init__(
    method forward (line 54) | def forward(self, x: torch.Tensor) -> torch.Tensor:
  class StablelmAttention (line 61) | class StablelmAttention(nn.Module):
    method __init__ (line 62) | def __init__(
    method forward (line 130) | def forward(
  class StablelmDecoderLayer (line 144) | class StablelmDecoderLayer(nn.Module):
    method __init__ (line 145) | def __init__(
    method forward (line 158) | def forward(
  class StableLMEpochModel (line 183) | class StableLMEpochModel(nn.Module):
    method __init__ (line 184) | def __init__(
    method forward (line 201) | def forward(
  class StableLmForCausalLM (line 223) | class StableLmForCausalLM(nn.Module):
    method __init__ (line 224) | def __init__(
    method forward (line 236) | def forward(
    method load_weights (line 248) | def load_weights(

FILE: eval/rebase/sglang/python/sglang/srt/sampling_params.py
  class SamplingParams (line 8) | class SamplingParams:
    method __init__ (line 9) | def __init__(
    method verify (line 48) | def verify(self):
    method normalize (line 73) | def normalize(self, tokenizer):

FILE: eval/rebase/sglang/python/sglang/srt/server.py
  class APIKeyValidatorMiddleware (line 65) | class APIKeyValidatorMiddleware(BaseHTTPMiddleware):
    method __init__ (line 66) | def __init__(self, app, api_key: str):
    method dispatch (line 70) | async def dispatch(self, request: Request, call_next):
  function jsonify_pydantic_model (line 91) | def jsonify_pydantic_model(obj: BaseModel):
  function health (line 98) | async def health() -> Response:
  function get_model_info (line 104) | async def get_model_info():
  function get_server_args (line 112) | async def get_server_args():
  function flush_cache (line 117) | async def flush_cache():
  function detokenize_logprob_tokens (line 126) | async def detokenize_logprob_tokens(token_logprobs):
  function stream_generator (line 132) | async def stream_generator(obj: GenerateReqInput):
  function make_openai_style_logprobs (line 141) | async def make_openai_style_logprobs(token_logprobs):
  function generate_request (line 155) | async def generate_request(obj: GenerateReqInput):
  function v1_completions (line 177) | async def v1_completions(raw_request: Request):
  function v1_chat_completions (line 293) | async def v1_chat_completions(raw_request: Request):
  function launch_server (line 415) | def launch_server(server_args, pipe_finish_writer):
  class Runtime (line 574) | class Runtime:
    method __init__ (line 575) | def __init__(
    method shutdown (line 647) | def shutdown(self):
    method get_tokenizer (line 661) | def get_tokenizer(self):
    method add_request (line 668) | async def add_request(
    method __del__ (line 695) | def __del__(self):

FILE: eval/rebase/sglang/python/sglang/srt/server_args.py
  class ServerArgs (line 7) | class ServerArgs:
    method __post_init__ (line 37) | def __post_init__(self):
    method add_cli_args (line 55) | def add_cli_args(parser: argparse.ArgumentParser):
    method from_cli_args (line 213) | def from_cli_args(cls, args: argparse.Namespace):
    method url (line 217) | def url(self):
    method get_optional_modes_logging (line 220) | def get_optional_modes_logging(self):
  class PortArgs (line 231) | class PortArgs:

FILE: eval/rebase/sglang/python/sglang/srt/utils.py
  function mark_cost_time (line 19) | def mark_cost_time(func_name):
  function mark_start (line 43) | def mark_start(key):
  function mark_end (line 50) | def mark_end(key, print_min_cost=0.0):
  function calculate_time (line 58) | def calculate_time(show=False, min_cost_ms=0.0):
  function set_random_seed (line 77) | def set_random_seed(seed: int) -> None:
  function alloc_usable_network_port (line 85) | def alloc_usable_network_port(num, used_list=()):
  function check_port (line 103) | def check_port(port):
  function handle_port_init (line 113) | def handle_port_init(
  function get_exception_traceback (line 145) | def get_exception_traceback():
  function get_int_token_logit_bias (line 151) | def get_int_token_logit_bias(tokenizer, vocab_size):
  function wrap_kernel_launcher (line 166) | def wrap_kernel_launcher(kernel):
  function is_multimodal_model (line 234) | def is_multimodal_model(model):
  function load_image (line 245) | def load_image(image_file):

FILE: eval/rebase/sglang/python/sglang/test/test_conversation.py
  function test_chat_completion_to_conv_image (line 12) | def test_chat_completion_to_conv_image():

FILE: eval/rebase/sglang/python/sglang/test/test_openai_protocol.py
  function test_chat_completion_request_image (line 11) | def test_chat_completion_request_image():

FILE: eval/rebase/sglang/python/sglang/test/test_programs.py
  function test_few_shot_qa (line 11) | def test_few_shot_qa():
  function test_mt_bench (line 39) | def test_mt_bench():
  function test_select (line 60) | def test_select(check_answer):
  function test_decode_int (line 94) | def test_decode_int():
  function test_decode_json_regex (line 105) | def test_decode_json_regex():
  function test_decode_json (line 128) | def test_decode_json():
  function test_expert_answer (line 148) | def test_expert_answer():
  function test_tool_use (line 169) | def test_tool_use():
  function test_react (line 191) | def test_react():
  function test_parallel_decoding (line 227) | def test_parallel_decoding():
  function test_parallel_encoding (line 264) | def test_parallel_encoding(check_answer=True):
  function test_image_qa (line 296) | def test_image_qa():
  function test_stream (line 313) | def test_stream():
  function test_regex (line 336) | def test_regex():

FILE: eval/rebase/sglang/python/sglang/test/test_utils.py
  function call_generate_lightllm (line 10) | def call_generate_lightllm(prompt, temperature, max_tokens, stop, url):
  function call_generate_vllm (line 25) | def call_generate_vllm(prompt, temperature, max_tokens, stop, url, n=1):
  function call_generate_outlines (line 42) | def call_generate_outlines(
  function call_generate_srt_raw (line 62) | def call_generate_srt_raw(prompt, temperature, max_tokens, stop, url):
  function call_select_lightllm (line 78) | def call_select_lightllm(context, choices, url):
  function call_select_vllm (line 93) | def call_select_vllm(context, choices, url):
  function add_common_other_args_and_parse (line 115) | def add_common_other_args_and_parse(parser):
  function add_common_sglang_args_and_parse (line 142) | def add_common_sglang_args_and_parse(parser):
  function select_sglang_backend (line 152) | def select_sglang_backend(args):

FILE: eval/rebase/sglang/python/sglang/utils.py
  function get_available_gpu_memory (line 13) | def get_available_gpu_memory(gpu_id, distributed=True):
  function is_same_type (line 41) | def is_same_type(values):
  function read_jsonl (line 50) | def read_jsonl(filename: str):
  function dump_state_text (line 61) | def dump_state_text(filename, states, mode="w"):
  class HttpResponse (line 79) | class HttpResponse:
    method __init__ (line 80) | def __init__(self, resp):
    method json (line 83) | def json(self):
    method status_code (line 87) | def status_code(self):
  function http_request (line 91) | def http_request(
  function encode_image_base64 (line 117) | def encode_image_base64(image_path):
  function _is_chinese_char (line 133) | def _is_chinese_char(cp):
  function find_printable_text (line 158) | def find_printable_text(text):
  function run_with_timeout (line 177) | def run_with_timeout(func, args=(), kwargs=None, timeout=None):

FILE: eval/rebase/sglang/scripts/convert_yi_vl.py
  function add_image_token (line 13) | def add_image_token(model_path: str):
  function edit_model_config (line 23) | def edit_model_config(model_path):

FILE: eval/rebase/sglang/test/lang/run_all.py
  function run_unittest_files (line 11) | def run_unittest_files(files, args):

FILE: eval/rebase/sglang/test/lang/test_anthropic_backend.py
  class TestAnthropicBackend (line 9) | class TestAnthropicBackend(unittest.TestCase):
    method setUp (line 13) | def setUp(self):
    method test_mt_bench (line 20) | def test_mt_bench(self):
    method test_stream (line 23) | def test_stream(self):

FILE: eval/rebase/sglang/test/lang/test_bind_pin.py
  class TestBind (line 8) | class TestBind(unittest.TestCase):
    method setUp (line 11) | def setUp(self):
    method test_bind (line 17) | def test_bind(self):
    method test_pin (line 33) | def test_pin(self):

FILE: eval/rebase/sglang/test/lang/test_openai_backend.py
  class TestOpenAIBackend (line 21) | class TestOpenAIBackend(unittest.TestCase):
    method setUp (line 26) | def setUp(self):
    method test_few_shot_qa (line 34) | def test_few_shot_qa(self):
    method test_mt_bench (line 38) | def test_mt_bench(self):
    method test_select (line 42) | def test_select(self):
    method test_decode_int (line 46) | def test_decode_int(self):
    method test_decode_json (line 50) | def test_decode_json(self):
    method test_expert_answer (line 54) | def test_expert_answer(self):
    method test_tool_use (line 58) | def test_tool_use(self):
    method test_react (line 62) | def test_react(self):
    method test_parallel_decoding (line 66) | def test_parallel_decoding(self):
    method test_parallel_encoding (line 70) | def test_parallel_encoding(self):
    method test_image_qa (line 74) | def test_image_qa(self):
    method test_stream (line 78) | def test_stream(self):

FILE: eval/rebase/sglang/test/lang/test_openai_spec.py
  function gen_character_default (line 5) | def gen_character_default(s):
  function gen_character_spec (line 14) | def gen_character_spec(s):
  function gen_character_no_stop (line 23) | def gen_character_no_stop(s):
  function gen_character_multi_stop (line 32) | def gen_character_multi_stop(s):

FILE: eval/rebase/sglang/test/lang/test_srt_backend.py
  class TestSRTBackend (line 26) | class TestSRTBackend(unittest.TestCase):
    method setUp (line 29) | def setUp(self):
    method test_few_shot_qa (line 36) | def test_few_shot_qa(self):
    method test_mt_bench (line 39) | def test_mt_bench(self):
    method test_select (line 42) | def test_select(self):
    method test_decode_int (line 45) | def test_decode_int(self):
    method test_decode_json_regex (line 48) | def test_decode_json_regex(self):
    method test_expert_answer (line 51) | def test_expert_answer(self):
    method test_tool_use (line 54) | def test_tool_use(self):
    method test_parallel_decoding (line 57) | def test_parallel_decoding(self):
    method test_stream (line 60) | def test_stream(self):
    method test_regex (line 63) | def test_regex(self):

FILE: eval/rebase/sglang/test/lang/test_tracing.py
  class TestTracing (line 9) | class TestTracing(unittest.TestCase):
    method test_few_shot_qa (line 10) | def test_few_shot_qa(self):
    method test_select (line 22) | def test_select(self):
    method test_raise_warning (line 32) | def test_raise_warning(self):
    method test_multi_function (line 45) | def test_multi_function(self):
    method test_role (line 87) | def test_role(self):
    method test_fork (line 101) | def test_fork(self):

FILE: eval/rebase/sglang/test/lang/test_vertexai_backend.py
  class TestVertexAIBackend (line 16) | class TestVertexAIBackend(unittest.TestCase):
    method setUp (line 21) | def setUp(self):
    method test_few_shot_qa (line 29) | def test_few_shot_qa(self):
    method test_mt_bench (line 33) | def test_mt_bench(self):
    method test_expert_answer (line 37) | def test_expert_answer(self):
    method test_parallel_decoding (line 41) | def test_parallel_decoding(self):
    method test_parallel_encoding (line 45) | def test_parallel_encoding(self):
    method test_image_qa (line 49) | def test_image_qa(self):
    method test_stream (line 53) | def test_stream(self):

FILE: eval/rebase/sglang/test/srt/model/bench_llama_low_api.py
  class BenchBatch (line 12) | class BenchBatch:
    method __init__ (line 25) | def __init__(self, model_runner: ModelRunner):
    method init_prefill_batch (line 29) | def init_prefill_batch(self, input_ids, batch_size, seq_len):
    method update_extend (line 47) | def update_extend(
    method update_decode (line 73) | def update_decode(self, predict_ids, batch_size):
  function prefill (line 90) | def prefill(model_runner: ModelRunner, batch: BenchBatch):
  function extend (line 108) | def extend(model_runner: ModelRunner, batch: BenchBatch):
  function decode (line 126) | def decode(model_runner: ModelRunner, batch: BenchBatch):
  function bench_generate_worker (line 145) | def bench_generate_worker(
  function bench_generate (line 222) | def bench_generate(

FILE: eval/rebase/sglang/test/srt/model/reference_hf.py
  function normal_text (line 9) | def normal_text(args):
  function synthetic_tokens (line 42) | def synthetic_tokens(args):

FILE: eval/rebase/sglang/test/srt/model/test_llama_extend.py
  function test_generate_worker (line 15) | def test_generate_worker(model_path, tp_rank, tp_size):
  function test_generate (line 73) | def test_generate(model_path, tp_size):

FILE: eval/rebase/sglang/test/srt/model/test_llama_low_api.py
  function test_generate_worker (line 11) | def test_generate_worker(
  function test_generate (line 147) | def test_generate(model_path, tp_size, batch_size, input_len, output_len):

FILE: eval/rebase/sglang/test/srt/model/test_llava_low_api.py
  function init_batch_data (line 14) | def init_batch_data(model, batch_size, input_len):
  function prefill (line 35) | def prefill(model, tp_rank, params, print_logits):
  function decode (line 50) | def decode(step, model, tp_rank, batch_size, predict_ids, params, print_...
  function test_generate_worker (line 85) | def test_generate_worker(
  function test_generate (line 144) | def test_generate(model_path, tp_size):

FILE: eval/rebase/sglang/test/srt/test_flashinfer.py
  function test_batch_prefill_with_paged_kv_cache (line 15) | def test_batch_prefill_with_paged_kv_cache(
  function test_batch_decode_with_paged_kv_cache (line 98) | def test_batch_decode_with_paged_kv_cache(

FILE: eval/rebase/sglang/test/srt/test_httpserver_concurrent.py
  function send_request (line 19) | async def send_request(url, data, delay=0):
  function main (line 27) | async def main(args):

FILE: eval/rebase/sglang/test/srt/test_httpserver_decode.py
  function test_decode (line 16) | def test_decode(url, return_logprob):

FILE: eval/rebase/sglang/test/srt/test_httpserver_decode_stream.py
  function test_decode_stream (line 16) | def test_decode_stream(url, return_logprob):

FILE: eval/rebase/sglang/test/srt/test_httpserver_llava.py
  function send_request (line 19) | async def send_request(url, data, delay=0):
  function test_concurrent (line 27) | async def test_concurrent(args):
  function test_streaming (line 51) | def test_streaming(args):

FILE: eval/rebase/sglang/test/srt/test_jump_forward.py
  function regex_gen (line 29) | def regex_gen(s):
  function json_gen (line 52) | def json_gen(s):
  class Weapon (line 62) | class Weapon(str, Enum):
  class Armor (line 71) | class Armor(str, Enum):
  class Character (line 77) | class Character(BaseModel):
  function character_gen (line 86) | def character_gen(s):
  function main (line 96) | def main(args):

FILE: eval/rebase/sglang/test/srt/test_openai_server.py
  function test_completion (line 22) | def test_completion(args, echo, logprobs):
  function test_completion_stream (line 49) | def test_completion_stream(args, echo, logprobs):
  function test_chat_completion (line 80) | def test_chat_completion(args):
  function test_chat_completion_image (line 99) | def test_chat_completion_image(args):
  function test_chat_completion_stream (line 129) | def test_chat_completion_stream(args):
  function test_regex (line 155) | def test_regex(args):

FILE: eval/rebase/sglang/test/srt/test_robust.py
  function gen_prompt (line 18) | def gen_prompt(token_num):
  function robust_test_dfs (line 30) | def robust_test_dfs(s, d, args, leaf_states):
  function robust_test_bfs (line 54) | def robust_test_bfs(s, args, leaf_states):
  function robust_test (line 83) | def robust_test(s, args):
  function main (line 92) | def main(args):

FILE: eval/rebase/tot/metric_utils.py
  function _aggregate_scores (line 5) | def _aggregate_scores(step_scores, method="min"):
  function _extract_shepherd_answer (line 26) | def _extract_shepherd_answer(completion):
  function _majority_vote (line 45) | def _majority_vote(model_answers, weighted=False, weight_func="min"):

FILE: eval/rebase/tot/o1_rebase.py
  class o1 (line 16) | class o1:
    method __init__ (line 17) | def __init__(
    method generate (line 43) | def generate(self, input_ids, dir_name="tot/outputs/math_o1", temperat...
    method _generate_base (line 104) | def _generate_base(self, input_ids, question, thinking_trajectory_so_f...
    method _generate_rebase (line 130) | def _generate_rebase(self, input_ids, question, thinking_trajectory_so...
    method _generate_step (line 187) | def _generate_step(self, state, temperature, max_length, do_sample, **...
    method _calculate_scores (line 200) | def _calculate_scores(self, outputs, question, thinking_trajectory_so_...
    method reward_fn_unified (line 205) | def reward_fn_unified(self, outputs, question, thinking_trajectory_so_...
    method _extract_question (line 244) | def _extract_question(self, input_ids):
    method _should_generate_multiple_outputs (line 248) | def _should_generate_multiple_outputs(self, chosen_step):
    method _generate_multiple_outputs (line 251) | def _generate_multiple_outputs(self, input_ids, temperature, max_lengt...
    method _generate_single_output (line 257) | def _generate_single_output(self, input_ids, temperature, max_length, ...
    method _remove_extra_eos (line 262) | def _remove_extra_eos(self, o):
    method _process_ranked_output (line 270) | def _process_ranked_output(self, ranked_output, thinking_trajectory_so...
    method _should_break (line 281) | def _should_break(self, chosen_step, input_ids, max_length):
    method _update_input_ids (line 284) | def _update_input_ids(self, input_ids, chosen_step):
    method _write_final_output (line 292) | def _write_final_output(self, input_ids, f):
    method _write_problem_dict (line 297) | def _write_problem_dict(self, file_name, question, thinking_trajectory...
    method _write_price (line 303) | def _write_price(self, file_name, question, all_price):
  function prepare_input_ids (line 308) | def prepare_input_ids(prompts, tokenizer):

FILE: eval/rebase/tot/o1_rebase_text.py
  class o1 (line 21) | class o1:
    method __init__ (line 22) | def __init__(
    method generate (line 64) | def generate(self, input_ids, dir_name="/home/weijias/o1/output_swj/op...
    method _generate_rebase (line 110) | def _generate_rebase(self, prompts, params, file_name):
    method _generate_base (line 172) | def _generate_base(self, input_ids, question, thinking_trajectory_so_f...
    method _generate_step (line 198) | def _generate_step(self, state, temperature, max_length, do_sample, **...
    method _calculate_scores (line 211) | def _calculate_scores(self, outputs, question, thinking_trajectory_so_...
    method reward_fn_unified (line 216) | def reward_fn_unified(self, outputs, question, thinking_trajectory_so_...
    method _extract_question (line 255) | def _extract_question(self, input_ids):
    method _should_generate_multiple_outputs (line 259) | def _should_generate_multiple_outputs(self, chosen_step):
    method _generate_multiple_outputs (line 262) | def _generate_multiple_outputs(self, input_ids, temperature, max_lengt...
    method _generate_single_output (line 268) | def _generate_single_output(self, input_ids, temperature, max_length, ...
    method _remove_extra_eos (line 273) | def _remove_extra_eos(self, o):
    method _process_ranked_output (line 281) | def _process_ranked_output(self, ranked_output, thinking_trajectory_so...
    method _should_break (line 292) | def _should_break(self, chosen_step, input_ids, max_length):
    method _update_input_ids (line 295) | def _update_input_ids(self, input_ids, chosen_step):
    method _write_final_output (line 303) | def _write_final_output(self, input_ids, f):
    method _write_problem_dict (line 308) | def _write_problem_dict(self, file_name, question, thinking_trajectory...
    method _write_price (line 314) | def _write_price(self, file_name, question, all_price):
  function prepare_input_ids (line 319) | def prepare_input_ids(prompts, tokenizer):

FILE: eval/rebase/tot/rebase_utils.py
  function convert_text_to_reward_format_llama (line 20) | def convert_text_to_reward_format_llama(text):
  function convert_text_to_reward_format_qwen (line 38) | def convert_text_to_reward_format_qwen(text):
  function convert_text_to_reward_format_qwen_steps_left (line 69) | def convert_text_to_reward_format_qwen_steps_left(text):
  class TreeNode (line 102) | class TreeNode:
    method __init__ (line 103) | def __init__(self, id, state, score, num_step_tokens=0, parent=None):
    method get_id (line 125) | def get_id(self):
    method get_parent (line 128) | def get_parent(self):
    method get_text (line 131) | def get_text(self):
    method get_state (line 134) | def get_state(self):
    method get_depth (line 137) | def get_depth(self):
    method get_score (line 140) | def get_score(self):
    method is_leaf (line 143) | def is_leaf(self):
    method get_cum_tokens (line 146) | def get_cum_tokens(self):
  class Tree (line 149) | class Tree:
    method __init__ (line 150) | def __init__(self, root_state, paras, reward_backend):
    method reset_running_list (line 163) | def reset_running_list(self):
    method get_running_list (line 166) | def get_running_list(self):
    method get_history_list (line 169) | def get_history_list(self):
    method get_nodes (line 172) | def get_nodes(self):
    method expand (line 175) | def expand(self, node, wid):
    method insert (line 282) | def insert(self, state, parent):
    method select_softmax (line 295) | def select_softmax(self, node_list, node_weights, width):
    method select_softmax_with_truncation (line 316) | def select_softmax_with_truncation(self, node_list, node_weights, width):
    method select_and_expand (line 342) | def select_and_expand(self, depth):
  function reward_guided_search (line 377) | def reward_guided_search(s, id, question, paras, reward_host):

FILE: train/sft.py
  class TrainingConfig (line 13) | class TrainingConfig:
    method __post_init__ (line 21) | def __post_init__(self):
  function train (line 25) | def train():