SYMBOL INDEX (1636 symbols across 97 files)

FILE: datautils.py
  function set_seed (line 9) | def set_seed(seed):
  function get_pile (line 16) | def get_pile(nsamples, seed, seqlen, model):
  function get_wikitext2 (line 35) | def get_wikitext2(nsamples, seed, seqlen, model):
  function get_ptb (line 56) | def get_ptb(nsamples, seed, seqlen, model):
  function get_c4 (line 78) | def get_c4(nsamples, seed, seqlen, model):
  function get_ptb_new (line 120) | def get_ptb_new(nsamples, seed, seqlen, model):
  function get_c4_new (line 143) | def get_c4_new(nsamples, seed, seqlen, model):
  function get_loaders (line 174) | def get_loaders(

FILE: gptq.py
  class Observer (line 15) | class Observer:
    method __init__ (line 17) | def __init__(self, topk=32):
    method submit (line 21) | def submit(self, name: str, layerid: int, gptq, error: float):
    method print (line 39) | def print(self):
    method items (line 52) | def items(self):
  class GPTQ (line 56) | class GPTQ:
    method __init__ (line 58) | def __init__(self, layer, observe=False):
    method add_batch (line 73) | def add_batch(self, inp, out):
    method print_loss (line 101) | def print_loss(self, name, q_weight, weight_error, timecost):
    method fasterquant (line 128) | def fasterquant(self, blocksize=128, percdamp=.01, groupsize=-1, actor...
    method free (line 233) | def free(self):

FILE: irqlora.py
  function replace_to_qlora_model (line 18) | def replace_to_qlora_model(model, model_fp, blocksize2=256, tau_range=0....
  function prod (line 22) | def prod(iterable):
  function quantize_tensor (line 26) | def quantize_tensor(X, L, idx=False):
  function dequantize_tensor (line 36) | def dequantize_tensor(X, L):
  function nf4_quant (line 41) | def nf4_quant(weight, weight_shape, tau, compress_statistics, quant_type...
  function evaluate_entropy (line 51) | def evaluate_entropy(weight_int8, blocksize):
  function search (line 64) | def search(fp_weight: Tensor, fp_weight_shape, compress_statistics, quan...
  class IRQLoraLinear4bit (line 86) | class IRQLoraLinear4bit(bnb.nn.Linear4bit, LoraLayer):
    method __init__ (line 87) | def __init__(
    method forward (line 118) | def forward(self, x: torch.Tensor):
  function _replace_with_ours_lora_4bit_linear (line 159) | def _replace_with_ours_lora_4bit_linear(

FILE: llama.py
  function get_llama (line 13) | def get_llama(model):
  function llama_sequential (line 28) | def llama_sequential(model, dataloader, dev):
  function llama_eval (line 174) | def llama_eval(model, testenc, dev):
  function llama_pack (line 265) | def llama_pack(model, quantizers, wbits, groupsize):
  function load_quant (line 279) | def load_quant(model, checkpoint, wbits, groupsize=-1, fused_mlp=True, e...
  function llama_multigpu (line 328) | def llama_multigpu(model, gpus, gpu_dist):
  function benchmark (line 385) | def benchmark(model, input_ids, check=False):

FILE: lm_eval/base.py
  class LM (line 20) | class LM(abc.ABC):
    method __init__ (line 21) | def __init__(self):
    method loglikelihood (line 25) | def loglikelihood(self, requests):
    method loglikelihood_rolling (line 49) | def loglikelihood_rolling(self, requests):
    method greedy_until (line 92) | def greedy_until(self, requests):
    method create_from_arg_string (line 110) | def create_from_arg_string(cls, arg_string, additional_config=None):
    method set_cache_hook (line 116) | def set_cache_hook(self, cache_hook):
  class BaseLM (line 120) | class BaseLM(LM):
    method eot_token_id (line 123) | def eot_token_id(self):
    method max_length (line 128) | def max_length(self):
    method max_gen_toks (line 133) | def max_gen_toks(self):
    method batch_size (line 138) | def batch_size(self):
    method device (line 143) | def device(self):
    method tok_encode (line 147) | def tok_encode(self, string: str):
    method tok_decode (line 151) | def tok_decode(self, tokens: Iterable[int]):
    method _model_generate (line 155) | def _model_generate(self, context, max_length, eos_token_id):
    method _model_call (line 159) | def _model_call(self, inps):
    method loglikelihood (line 172) | def loglikelihood(self, requests):
    method loglikelihood_rolling (line 187) | def loglikelihood_rolling(self, requests):
    method _loglikelihood_tokens (line 221) | def _loglikelihood_tokens(self, requests, disable_tqdm=False):
    method greedy_until (line 332) | def greedy_until(self, requests):
  class Task (line 372) | class Task(abc.ABC):
    method __init__ (line 389) | def __init__(self, data_dir=None, cache_dir=None, download_mode=None):
    method download (line 416) | def download(self, data_dir=None, cache_dir=None, download_mode=None):
    method should_decontaminate (line 449) | def should_decontaminate(self):
    method has_training_docs (line 454) | def has_training_docs(self):
    method has_validation_docs (line 459) | def has_validation_docs(self):
    method has_test_docs (line 464) | def has_test_docs(self):
    method training_docs (line 468) | def training_docs(self):
    method validation_docs (line 475) | def validation_docs(self):
    method test_docs (line 482) | def test_docs(self):
    method _process_doc (line 489) | def _process_doc(self, doc):
    method fewshot_examples (line 500) | def fewshot_examples(self, k, rnd):
    method doc_to_decontamination_query (line 506) | def doc_to_decontamination_query(self, doc):
    method doc_to_text (line 513) | def doc_to_text(self, doc):
    method doc_to_target (line 517) | def doc_to_target(self, doc):
    method construct_requests (line 521) | def construct_requests(self, doc, ctx):
    method process_results (line 535) | def process_results(self, doc, results):
    method aggregation (line 548) | def aggregation(self):
    method higher_is_better (line 557) | def higher_is_better(self):
    method fewshot_description (line 565) | def fewshot_description(self):
    method fewshot_context (line 576) | def fewshot_context(
  class MultipleChoiceTask (line 645) | class MultipleChoiceTask(Task):
    method doc_to_target (line 646) | def doc_to_target(self, doc):
    method construct_requests (line 649) | def construct_requests(self, doc, ctx):
    method process_results (line 656) | def process_results(self, doc, results):
    method higher_is_better (line 668) | def higher_is_better(self):
    method aggregation (line 674) | def aggregation(self):
  class PerplexityTask (line 681) | class PerplexityTask(Task, abc.ABC):
    method should_decontaminate (line 682) | def should_decontaminate(self):
    method has_training_docs (line 686) | def has_training_docs(self):
    method fewshot_examples (line 689) | def fewshot_examples(self, k, rnd):
    method fewshot_context (line 693) | def fewshot_context(
    method higher_is_better (line 715) | def higher_is_better(self):
    method doc_to_decontamination_query (line 722) | def doc_to_decontamination_query(self, doc):
    method doc_to_text (line 725) | def doc_to_text(self, doc):
    method doc_to_target (line 728) | def doc_to_target(self, doc):
    method construct_requests (line 731) | def construct_requests(self, doc, ctx):
    method process_results (line 736) | def process_results(self, doc, results):
    method aggregation (line 746) | def aggregation(self):
    method count_bytes (line 754) | def count_bytes(cls, doc):
    method count_words (line 758) | def count_words(cls, doc):
  function hash_args (line 763) | def hash_args(attr, args):
  class CacheHook (line 768) | class CacheHook:
    method __init__ (line 769) | def __init__(self, cachinglm):
    method add_partial (line 776) | def add_partial(self, attr, req, res):
  class CachingLM (line 783) | class CachingLM:
    method __init__ (line 784) | def __init__(self, lm, cache_db):
    method __getattr__ (line 801) | def __getattr__(self, attr):
    method get_cache_hook (line 839) | def get_cache_hook(self):
  class Request (line 850) | class Request:
    method __init__ (line 851) | def __init__(self, request_type, args, index=None):
    method __iter__ (line 861) | def __iter__(self):
    method __getitem__ (line 867) | def __getitem__(self, i):
    method __eq__ (line 872) | def __eq__(self, other):
    method __repr__ (line 879) | def __repr__(self):
  class RequestFactory (line 883) | class RequestFactory:
    method __getattr__ (line 884) | def __getattr__(self, attr):

FILE: lm_eval/datasets/asdiv/asdiv.py
  class ASDiv (line 52) | class ASDiv(datasets.GeneratorBasedBuilder):
    method _info (line 65) | def _info(self):
    method _split_generators (line 83) | def _split_generators(self, dl_manager):
    method _generate_examples (line 101) | def _generate_examples(self, filepath, split):

FILE: lm_eval/datasets/coqa/coqa.py
  class Coqa (line 88) | class Coqa(datasets.GeneratorBasedBuilder):
    method _info (line 99) | def _info(self):
    method _split_generators (line 159) | def _split_generators(self, dl_manager):
    method _generate_examples (line 182) | def _generate_examples(self, filepath, split):

FILE: lm_eval/datasets/drop/drop.py
  class Drop (line 68) | class Drop(datasets.GeneratorBasedBuilder):
    method _info (line 79) | def _info(self):
    method _split_generators (line 120) | def _split_generators(self, dl_manager):
    method _generate_examples (line 147) | def _generate_examples(self, filepath, split):

FILE: lm_eval/datasets/headqa/headqa.py
  class HeadQA (line 61) | class HeadQA(datasets.GeneratorBasedBuilder):
    method _info (line 77) | def _info(self):
    method _split_generators (line 102) | def _split_generators(self, dl_manager):
    method _generate_examples (line 133) | def _generate_examples(self, data_dir, filepath):

FILE: lm_eval/datasets/hendrycks_ethics/hendrycks_ethics.py
  class EthicsConfig (line 50) | class EthicsConfig(datasets.BuilderConfig):
    method __init__ (line 53) | def __init__(self, prefix, features, **kwargs):
  class HendrycksEthics (line 67) | class HendrycksEthics(datasets.GeneratorBasedBuilder):
    method _info (line 136) | def _info(self):
    method _split_generators (line 145) | def _split_generators(self, dl_manager):
    method _generate_examples (line 178) | def _generate_examples(self, filepath, split):

FILE: lm_eval/datasets/hendrycks_math/hendrycks_math.py
  class HendrycksMath (line 57) | class HendrycksMath(datasets.GeneratorBasedBuilder):
    method _info (line 67) | def _info(self):
    method _split_generators (line 84) | def _split_generators(self, dl_manager):
    method _generate_examples (line 111) | def _generate_examples(self, basepath, split):

FILE: lm_eval/datasets/logiqa/logiqa.py
  class Logiqa (line 51) | class Logiqa(datasets.GeneratorBasedBuilder):
    method _info (line 62) | def _info(self):
    method _split_generators (line 79) | def _split_generators(self, dl_manager):
    method _generate_examples (line 111) | def _generate_examples(self, filepath, split):

FILE: lm_eval/datasets/mutual/mutual.py
  class Mutual (line 47) | class Mutual(datasets.GeneratorBasedBuilder):
    method _info (line 63) | def _info(self):
    method _split_generators (line 80) | def _split_generators(self, dl_manager):
    method _generate_examples (line 117) | def _generate_examples(self, basepath, split):

FILE: lm_eval/datasets/pile/pile.py
  class Pile (line 75) | class Pile(datasets.GeneratorBasedBuilder):
    method _info (line 85) | def _info(self):
    method _split_generators (line 99) | def _split_generators(self, dl_manager):
    method _generate_examples (line 119) | def _generate_examples(self, filepath, split):

FILE: lm_eval/datasets/quac/quac.py
  class Quac (line 51) | class Quac(datasets.GeneratorBasedBuilder):
    method _info (line 62) | def _info(self):
    method _split_generators (line 80) | def _split_generators(self, dl_manager):
    method _generate_examples (line 100) | def _generate_examples(self, filepath, split):

FILE: lm_eval/datasets/sat_analogies/sat_analogies.py
  class SatAnalogies (line 46) | class SatAnalogies(datasets.GeneratorBasedBuilder):
    method manual_download_instructions (line 60) | def manual_download_instructions(self):
    method _info (line 69) | def _info(self):
    method _split_generators (line 86) | def _split_generators(self, dl_manager):
    method _generate_examples (line 103) | def _generate_examples(self, filepath):

FILE: lm_eval/datasets/triviaqa/triviaqa.py
  class Triviaqa (line 52) | class Triviaqa(datasets.GeneratorBasedBuilder):
    method _info (line 63) | def _info(self):
    method _split_generators (line 95) | def _split_generators(self, dl_manager):
    method _generate_examples (line 120) | def _generate_examples(self, filepath):

FILE: lm_eval/datasets/unscramble/unscramble.py
  class Unscramble (line 61) | class Unscramble(datasets.GeneratorBasedBuilder):
    method _info (line 73) | def _info(self):
    method _split_generators (line 88) | def _split_generators(self, dl_manager):
    method _generate_examples (line 103) | def _generate_examples(self, filepath, split):

FILE: lm_eval/decontamination/archiver.py
  function json_serial (line 12) | def json_serial(obj):
  class Archive (line 21) | class Archive:
    method __init__ (line 22) | def __init__(self, file_path, compression_level=3):
    method add_data (line 31) | def add_data(self, data, meta={}):
    method commit (line 39) | def commit(self):
  class Reader (line 46) | class Reader:
    method __init__ (line 47) | def __init__(self):
    method read (line 50) | def read(self, file, get_meta=False, autojoin_paragraphs=True, para_jo...
  class TextArchive (line 74) | class TextArchive:
    method __init__ (line 75) | def __init__(self, file_path, mode="rb+"):
    method add_data (line 86) | def add_data(self, data):
    method commit (line 89) | def commit(self):
  class TextReader (line 94) | class TextReader:
    method __init__ (line 95) | def __init__(self, file_path):
    method read_tqdm (line 100) | def read_tqdm(self, update_frequency=10000):
    method read_and_tell (line 121) | def read_and_tell(self):
    method read (line 132) | def read(self):
    method read_slow (line 139) | def read_slow(self):
  class ZStdTextReader (line 151) | class ZStdTextReader:
    method __init__ (line 152) | def __init__(self, file):
    method read_tqdm (line 155) | def read_tqdm(self):

FILE: lm_eval/decontamination/decontaminate.py
  function get_train_overlap_stub (line 14) | def get_train_overlap_stub(docs, ngrams_path, ngrams_n_size):
  function get_train_overlap (line 36) | def get_train_overlap(docs_by_task_set, ngrams_path, limit):

FILE: lm_eval/decontamination/janitor.py
  function form_ngrams (line 22) | def form_ngrams(sequence, n):
  function word_ngrams (line 39) | def word_ngrams(s, n):
  function split_indices (line 71) | def split_indices(s):
  function word_ngrams_indices (line 78) | def word_ngrams_indices(s, n):
  class Janitor (line 106) | class Janitor:
    method __init__ (line 109) | def __init__(
    method save_contamination_ngrams (line 138) | def save_contamination_ngrams(self, filename):
    method load_contamination_ngrams (line 142) | def load_contamination_ngrams(self, filename):
    method register_contaminant (line 150) | def register_contaminant(self, dirt_string):
    method clean (line 159) | def clean(self, dirty_string):
    method _split_chunks (line 169) | def _split_chunks(self, dirty_string, dirty_parts):
    method register_contaminant_cpp (line 192) | def register_contaminant_cpp(self, dirt_string):
    method clean_cpp (line 197) | def clean_cpp(self, dirty_string):
    method normalize_string (line 207) | def normalize_string(self, s):
    method register_contaminant_python (line 210) | def register_contaminant_python(self, dirt_string):
    method clean_python (line 215) | def clean_python(self, dirty_string):

FILE: lm_eval/evaluator copy.py
  function simple_evaluate (line 17) | def simple_evaluate(
  function evaluate (line 127) | def evaluate(
  function make_table (line 311) | def make_table(result_dict):

FILE: lm_eval/evaluator.py
  function pattern_match (line 11) | def pattern_match(patterns, source_list):
  function simple_evaluate (line 18) | def simple_evaluate(
  function evaluate (line 96) | def evaluate(
  function make_table (line 284) | def make_table(result_dict):

FILE: lm_eval/metrics.py
  function mean (line 10) | def mean(arr):
  function pop_stddev (line 14) | def pop_stddev(arr):
  function sample_stddev (line 19) | def sample_stddev(arr):
  function mean_stderr (line 24) | def mean_stderr(arr):
  function median (line 28) | def median(arr):
  function matthews_corrcoef (line 32) | def matthews_corrcoef(items):
  function f1_score (line 39) | def f1_score(items):
  function acc_all (line 48) | def acc_all(items):
  function acc_all_stderr (line 67) | def acc_all_stderr(items):
  function metric_max_over_ground_truths (line 85) | def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
  function perplexity (line 94) | def perplexity(items):
  function weighted_mean (line 98) | def weighted_mean(items):
  function weighted_perplexity (line 103) | def weighted_perplexity(items):
  function bits_per_byte (line 107) | def bits_per_byte(items):
  function bleu (line 111) | def bleu(items):
  function chrf (line 128) | def chrf(items):
  function ter (line 142) | def ter(items):
  function is_non_str_iterable (line 157) | def is_non_str_iterable(obj):
  function _sacreformat (line 161) | def _sacreformat(refs, preds):
  class _bootstrap_internal (line 192) | class _bootstrap_internal:
    method __init__ (line 193) | def __init__(self, f, n):
    method __call__ (line 197) | def __call__(self, v):
  function bootstrap_stderr (line 207) | def bootstrap_stderr(f, xs, iters):
  function stderr_for_metric (line 236) | def stderr_for_metric(metric, bootstrap_iters):
  function yesno (line 255) | def yesno(x):

FILE: lm_eval/models/__init__.py
  function get_model (line 18) | def get_model(model_name):

FILE: lm_eval/models/dummy.py
  class DummyLM (line 5) | class DummyLM(LM):
    method __init__ (line 6) | def __init__(self):
    method create_from_arg_string (line 10) | def create_from_arg_string(cls, arg_string, additional_config=None):
    method loglikelihood (line 13) | def loglikelihood(self, requests):
    method greedy_until (line 21) | def greedy_until(self, requests):
    method loglikelihood_rolling (line 30) | def loglikelihood_rolling(self, requests):

FILE: lm_eval/models/gpt2.py
  class HFLM (line 6) | class HFLM(BaseLM):
    method __init__ (line 7) | def __init__(
    method eot_token_id (line 81) | def eot_token_id(self):
    method max_length (line 86) | def max_length(self):
    method max_gen_toks (line 94) | def max_gen_toks(self):
    method batch_size (line 98) | def batch_size(self):
    method device (line 103) | def device(self):
    method tok_encode (line 107) | def tok_encode(self, string: str):
    method tok_decode (line 110) | def tok_decode(self, tokens):
    method _model_call (line 113) | def _model_call(self, inps):
    method _model_generate (line 124) | def _model_generate(self, context, max_length, eos_token_id):

FILE: lm_eval/models/gpt3.py
  function get_result (line 10) | def get_result(response, ctxlen):
  function oa_completion (line 38) | def oa_completion(**kwargs):
  class GPT3LM (line 57) | class GPT3LM(BaseLM):
    method __init__ (line 60) | def __init__(self, engine, truncate=False):
    method eot_token_id (line 89) | def eot_token_id(self):
    method max_length (line 93) | def max_length(self):
    method max_gen_toks (line 98) | def max_gen_toks(self):
    method batch_size (line 102) | def batch_size(self):
    method device (line 107) | def device(self):
    method tok_encode (line 111) | def tok_encode(self, string: str):
    method tok_decode (line 114) | def tok_decode(self, tokens):
    method _loglikelihood_tokens (line 117) | def _loglikelihood_tokens(self, requests, disable_tqdm=False):
    method greedy_until (line 168) | def greedy_until(self, requests):
    method _model_call (line 224) | def _model_call(self, inps):
    method _model_generate (line 228) | def _model_generate(self, context, max_length, eos_token_id):

FILE: lm_eval/models/huggingface.py
  function _get_accelerate_args (line 18) | def _get_accelerate_args(
  function _get_dtype (line 43) | def _get_dtype(
  class HuggingFaceAutoLM (line 57) | class HuggingFaceAutoLM(BaseLM):
    method __init__ (line 66) | def __init__(
    method _create_auto_model (line 190) | def _create_auto_model(
    method _create_auto_tokenizer (line 212) | def _create_auto_tokenizer(
    method add_special_tokens (line 229) | def add_special_tokens(self) -> bool:
    method eot_token (line 249) | def eot_token(self) -> str:
    method eot_token_id (line 253) | def eot_token_id(self) -> int:
    method max_gen_toks (line 257) | def max_gen_toks(self) -> int:
    method max_length (line 261) | def max_length(self) -> int:
    method batch_size (line 283) | def batch_size(self) -> int:
    method device (line 288) | def device(self) -> Union[int, str, torch.device]:
    method tok_encode (line 291) | def tok_encode(self, string: str) -> TokenSequence:
    method tok_encode_batch (line 295) | def tok_encode_batch(self, strings: List[str]) -> TokenSequence:
    method tok_decode (line 303) | def tok_decode(self, tokens: torch.LongTensor) -> List[str]:
    method greedy_until (line 306) | def greedy_until(self, requests: List[Tuple[str, dict]]) -> List[str]:
  class AutoCausalLM (line 358) | class AutoCausalLM(HuggingFaceAutoLM):
    method _create_auto_tokenizer (line 366) | def _create_auto_tokenizer(
    method _model_call (line 383) | def _model_call(
    method _model_generate (line 388) | def _model_generate(
  class AutoSeq2SeqLM (line 422) | class AutoSeq2SeqLM(HuggingFaceAutoLM):
    method max_length (line 431) | def max_length(self) -> int:
    method loglikelihood (line 439) | def loglikelihood(
    method loglikelihood_rolling (line 467) | def loglikelihood_rolling(self, requests: List[Tuple[str, str]]) -> Li...
    method _loglikelihood_tokens (line 515) | def _loglikelihood_tokens(
    method _model_call (line 551) | def _model_call(
    method _model_generate (line 556) | def _model_generate(
  class MultiTokenEOSCriteria (line 589) | class MultiTokenEOSCriteria(transformers.StoppingCriteria):
    method __init__ (line 592) | def __init__(
    method __call__ (line 606) | def __call__(self, input_ids, scores, **kwargs) -> bool:
  function stop_sequences_criteria (line 620) | def stop_sequences_criteria(

FILE: lm_eval/models/textsynth.py
  function textsynth_completion (line 25) | def textsynth_completion(**kwargs):
  class TextSynthLM (line 41) | class TextSynthLM(BaseLM):
    method __init__ (line 42) | def __init__(self, engine, truncate=False):
    method eot_token_id (line 58) | def eot_token_id(self):
    method max_length (line 63) | def max_length(self):
    method max_gen_toks (line 68) | def max_gen_toks(self):
    method batch_size (line 72) | def batch_size(self):
    method device (line 77) | def device(self):
    method tok_encode (line 81) | def tok_encode(self, string: str):
    method tok_decode (line 85) | def tok_decode(self, tokens):
    method loglikelihood (line 89) | def loglikelihood(self, requests):
    method loglikelihood_rolling (line 109) | def loglikelihood_rolling(self, requests):
    method greedy_until (line 119) | def greedy_until(self, requests):
    method _model_call (line 149) | def _model_call(self, inps):
    method _model_generate (line 153) | def _model_generate(self, context, max_length, eos_token_id):

FILE: lm_eval/quantizer/irqlora.py
  function replace_to_qlora_model (line 18) | def replace_to_qlora_model(model, model_fp, blocksize2=256, tau_range=0....
  function prod (line 22) | def prod(iterable):
  function quantize_tensor (line 26) | def quantize_tensor(X, L, idx=False):
  function dequantize_tensor (line 36) | def dequantize_tensor(X, L):
  function nf4_quant (line 41) | def nf4_quant(weight, weight_shape, tau, compress_statistics, quant_type...
  function evaluate_entropy (line 51) | def evaluate_entropy(weight_int8, blocksize):
  function search (line 64) | def search(fp_weight: Tensor, fp_weight_shape, compress_statistics, quan...
  class IRQLoraLinear4bit (line 86) | class IRQLoraLinear4bit(bnb.nn.Linear4bit, LoraLayer):
    method __init__ (line 87) | def __init__(
    method forward (line 118) | def forward(self, x: torch.Tensor):
  function _replace_with_ours_lora_4bit_linear (line 159) | def _replace_with_ours_lora_4bit_linear(

FILE: lm_eval/tasks/__init__.py
  function get_task (line 319) | def get_task(task_name):
  function get_task_name_from_object (line 328) | def get_task_name_from_object(task_object):
  function get_task_dict (line 341) | def get_task_dict(task_name_list: List[Union[str, lm_eval.base.Task]]):

FILE: lm_eval/tasks/anli.py
  class ANLIBase (line 33) | class ANLIBase(Task):
    method has_training_docs (line 39) | def has_training_docs(self):
    method has_validation_docs (line 42) | def has_validation_docs(self):
    method has_test_docs (line 45) | def has_test_docs(self):
    method training_docs (line 48) | def training_docs(self):
    method validation_docs (line 54) | def validation_docs(self):
    method test_docs (line 58) | def test_docs(self):
    method doc_to_text (line 62) | def doc_to_text(self, doc):
    method should_decontaminate (line 74) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 77) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 80) | def doc_to_target(self, doc):
    method construct_requests (line 86) | def construct_requests(self, doc, ctx):
    method process_results (line 102) | def process_results(self, doc, results):
    method aggregation (line 116) | def aggregation(self):
    method higher_is_better (line 124) | def higher_is_better(self):
  class ANLIRound1 (line 133) | class ANLIRound1(ANLIBase):
  class ANLIRound2 (line 137) | class ANLIRound2(ANLIBase):
  class ANLIRound3 (line 141) | class ANLIRound3(ANLIBase):

FILE: lm_eval/tasks/arc.py
  class ARCEasy (line 29) | class ARCEasy(MultipleChoiceTask):
    method has_training_docs (line 34) | def has_training_docs(self):
    method has_validation_docs (line 37) | def has_validation_docs(self):
    method has_test_docs (line 40) | def has_test_docs(self):
    method training_docs (line 43) | def training_docs(self):
    method validation_docs (line 48) | def validation_docs(self):
    method test_docs (line 51) | def test_docs(self):
    method _process_doc (line 54) | def _process_doc(self, doc):
    method doc_to_text (line 67) | def doc_to_text(self, doc):
    method should_decontaminate (line 70) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 73) | def doc_to_decontamination_query(self, doc):
  class ARCChallenge (line 77) | class ARCChallenge(ARCEasy):

FILE: lm_eval/tasks/arithmetic.py
  class Arithmetic (line 29) | class Arithmetic(Task):
    method has_training_docs (line 33) | def has_training_docs(self):
    method has_validation_docs (line 36) | def has_validation_docs(self):
    method has_test_docs (line 39) | def has_test_docs(self):
    method training_docs (line 42) | def training_docs(self):
    method validation_docs (line 45) | def validation_docs(self):
    method test_docs (line 48) | def test_docs(self):
    method doc_to_text (line 51) | def doc_to_text(self, doc):
    method should_decontaminate (line 54) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 57) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 60) | def doc_to_target(self, doc):
    method construct_requests (line 63) | def construct_requests(self, doc, ctx):
    method process_results (line 67) | def process_results(self, doc, results):
    method aggregation (line 71) | def aggregation(self):
    method higher_is_better (line 76) | def higher_is_better(self):
  class Arithmetic2DPlus (line 80) | class Arithmetic2DPlus(Arithmetic):
  class Arithmetic2DMinus (line 84) | class Arithmetic2DMinus(Arithmetic):
  class Arithmetic3DPlus (line 88) | class Arithmetic3DPlus(Arithmetic):
  class Arithmetic3DMinus (line 92) | class Arithmetic3DMinus(Arithmetic):
  class Arithmetic4DPlus (line 96) | class Arithmetic4DPlus(Arithmetic):
  class Arithmetic4DMinus (line 100) | class Arithmetic4DMinus(Arithmetic):
  class Arithmetic5DPlus (line 104) | class Arithmetic5DPlus(Arithmetic):
  class Arithmetic5DMinus (line 108) | class Arithmetic5DMinus(Arithmetic):
  class Arithmetic2DMultiplication (line 112) | class Arithmetic2DMultiplication(Arithmetic):
  class Arithmetic1DComposite (line 116) | class Arithmetic1DComposite(Arithmetic):

FILE: lm_eval/tasks/asdiv.py
  class Asdiv (line 35) | class Asdiv(Task):
    method has_training_docs (line 39) | def has_training_docs(self):
    method has_validation_docs (line 42) | def has_validation_docs(self):
    method has_test_docs (line 45) | def has_test_docs(self):
    method training_docs (line 48) | def training_docs(self):
    method validation_docs (line 51) | def validation_docs(self):
    method test_docs (line 54) | def test_docs(self):
    method fewshot_context (line 57) | def fewshot_context(
    method doc_to_text (line 65) | def doc_to_text(self, doc):
    method should_decontaminate (line 69) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 72) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 75) | def doc_to_target(self, doc):
    method construct_requests (line 81) | def construct_requests(self, doc, ctx):
    method process_results (line 85) | def process_results(self, doc, results):
    method aggregation (line 90) | def aggregation(self):
    method higher_is_better (line 93) | def higher_is_better(self):

FILE: lm_eval/tasks/blimp.py
  class BlimpTask (line 34) | class BlimpTask(Task):
    method has_training_docs (line 38) | def has_training_docs(self):
    method has_validation_docs (line 41) | def has_validation_docs(self):
    method has_test_docs (line 44) | def has_test_docs(self):
    method validation_docs (line 47) | def validation_docs(self):
    method fewshot_context (line 53) | def fewshot_context(
    method doc_to_text (line 73) | def doc_to_text(self, doc):
    method should_decontaminate (line 77) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 80) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 83) | def doc_to_target(self, doc):
    method construct_requests (line 87) | def construct_requests(self, doc, ctx):
    method process_results (line 97) | def process_results(self, doc, results):
    method higher_is_better (line 107) | def higher_is_better(self):
    method aggregation (line 112) | def aggregation(self):
  class BlimpAdjunctIsland (line 118) | class BlimpAdjunctIsland(BlimpTask):
  class BlimpAnaphorGenderAgreement (line 122) | class BlimpAnaphorGenderAgreement(BlimpTask):
  class BlimpAnaphorNumberAgreement (line 126) | class BlimpAnaphorNumberAgreement(BlimpTask):
  class BlimpAnimateSubjectPassive (line 130) | class BlimpAnimateSubjectPassive(BlimpTask):
  class BlimpAnimateSubjectTrans (line 134) | class BlimpAnimateSubjectTrans(BlimpTask):
  class BlimpCausative (line 138) | class BlimpCausative(BlimpTask):
  class BlimpComplex_NPIsland (line 142) | class BlimpComplex_NPIsland(BlimpTask):
  class BlimpCoordinateStructureConstraintComplexLeftBranch (line 146) | class BlimpCoordinateStructureConstraintComplexLeftBranch(BlimpTask):
  class BlimpCoordinateStructureConstraintObjectExtraction (line 150) | class BlimpCoordinateStructureConstraintObjectExtraction(BlimpTask):
  class BlimpDeterminerNounAgreement_1 (line 154) | class BlimpDeterminerNounAgreement_1(BlimpTask):
  class BlimpDeterminerNounAgreement_2 (line 158) | class BlimpDeterminerNounAgreement_2(BlimpTask):
  class BlimpDeterminerNounAgreementIrregular_1 (line 162) | class BlimpDeterminerNounAgreementIrregular_1(BlimpTask):
  class BlimpDeterminerNounAgreementIrregular_2 (line 166) | class BlimpDeterminerNounAgreementIrregular_2(BlimpTask):
  class BlimpDeterminerNounAgreementWithAdj_2 (line 170) | class BlimpDeterminerNounAgreementWithAdj_2(BlimpTask):
  class BlimpDeterminerNounAgreementWithAdjIrregular_1 (line 174) | class BlimpDeterminerNounAgreementWithAdjIrregular_1(BlimpTask):
  class BlimpDeterminerNounAgreementWithAdjIrregular_2 (line 178) | class BlimpDeterminerNounAgreementWithAdjIrregular_2(BlimpTask):
  class BlimpDeterminerNounAgreementWithAdjective_1 (line 182) | class BlimpDeterminerNounAgreementWithAdjective_1(BlimpTask):
  class BlimpDistractorAgreementRelationalNoun (line 186) | class BlimpDistractorAgreementRelationalNoun(BlimpTask):
  class BlimpDistractorAgreementRelativeClause (line 190) | class BlimpDistractorAgreementRelativeClause(BlimpTask):
  class BlimpDropArgument (line 194) | class BlimpDropArgument(BlimpTask):
  class BlimpEllipsisNBar_1 (line 198) | class BlimpEllipsisNBar_1(BlimpTask):
  class BlimpEllipsisNBar_2 (line 202) | class BlimpEllipsisNBar_2(BlimpTask):
  class BlimpExistentialThereObjectRaising (line 206) | class BlimpExistentialThereObjectRaising(BlimpTask):
  class BlimpExistentialThereQuantifiers_1 (line 210) | class BlimpExistentialThereQuantifiers_1(BlimpTask):
  class BlimpExistentialThereQuantifiers_2 (line 214) | class BlimpExistentialThereQuantifiers_2(BlimpTask):
  class BlimpExistentialThereSubjectRaising (line 218) | class BlimpExistentialThereSubjectRaising(BlimpTask):
  class BlimpExpletiveItObjectRaising (line 222) | class BlimpExpletiveItObjectRaising(BlimpTask):
  class BlimpInchoative (line 226) | class BlimpInchoative(BlimpTask):
  class BlimpIntransitive (line 230) | class BlimpIntransitive(BlimpTask):
  class BlimpIrregularPastParticipleAdjectives (line 234) | class BlimpIrregularPastParticipleAdjectives(BlimpTask):
  class BlimpIrregularPastParticipleVerbs (line 238) | class BlimpIrregularPastParticipleVerbs(BlimpTask):
  class BlimpIrregularPluralSubjectVerbAgreement_1 (line 242) | class BlimpIrregularPluralSubjectVerbAgreement_1(BlimpTask):
  class BlimpIrregularPluralSubjectVerbAgreement_2 (line 246) | class BlimpIrregularPluralSubjectVerbAgreement_2(BlimpTask):
  class BlimpLeftBranchIslandEchoQuestion (line 250) | class BlimpLeftBranchIslandEchoQuestion(BlimpTask):
  class BlimpLeftBranchIslandSimpleQuestion (line 254) | class BlimpLeftBranchIslandSimpleQuestion(BlimpTask):
  class BlimpMatrixQuestionNpiLicensorPresent (line 258) | class BlimpMatrixQuestionNpiLicensorPresent(BlimpTask):
  class BlimpNpiPresent_1 (line 262) | class BlimpNpiPresent_1(BlimpTask):
  class BlimpNpiPresent_2 (line 266) | class BlimpNpiPresent_2(BlimpTask):
  class BlimpOnlyNpiLicensorPresent (line 270) | class BlimpOnlyNpiLicensorPresent(BlimpTask):
  class BlimpOnlyNpiScope (line 274) | class BlimpOnlyNpiScope(BlimpTask):
  class BlimpPassive_1 (line 278) | class BlimpPassive_1(BlimpTask):
  class BlimpPassive_2 (line 282) | class BlimpPassive_2(BlimpTask):
  class BlimpPrinciple_ACCommand (line 286) | class BlimpPrinciple_ACCommand(BlimpTask):
  class BlimpPrinciple_ACase_1 (line 290) | class BlimpPrinciple_ACase_1(BlimpTask):
  class BlimpPrinciple_ACase_2 (line 294) | class BlimpPrinciple_ACase_2(BlimpTask):
  class BlimpPrinciple_ADomain_1 (line 298) | class BlimpPrinciple_ADomain_1(BlimpTask):
  class BlimpPrinciple_ADomain_2 (line 302) | class BlimpPrinciple_ADomain_2(BlimpTask):
  class BlimpPrinciple_ADomain_3 (line 306) | class BlimpPrinciple_ADomain_3(BlimpTask):
  class BlimpPrinciple_AReconstruction (line 310) | class BlimpPrinciple_AReconstruction(BlimpTask):
  class BlimpRegularPluralSubjectVerbAgreement_1 (line 314) | class BlimpRegularPluralSubjectVerbAgreement_1(BlimpTask):
  class BlimpRegularPluralSubjectVerbAgreement_2 (line 318) | class BlimpRegularPluralSubjectVerbAgreement_2(BlimpTask):
  class BlimpSententialNegationNpiLicensorPresent (line 322) | class BlimpSententialNegationNpiLicensorPresent(BlimpTask):
  class BlimpSententialNegationNpiScope (line 326) | class BlimpSententialNegationNpiScope(BlimpTask):
  class BlimpSententialSubjectIsland (line 330) | class BlimpSententialSubjectIsland(BlimpTask):
  class BlimpSuperlativeQuantifiers_1 (line 334) | class BlimpSuperlativeQuantifiers_1(BlimpTask):
  class BlimpSuperlativeQuantifiers_2 (line 338) | class BlimpSuperlativeQuantifiers_2(BlimpTask):
  class BlimpToughVsRaising_1 (line 342) | class BlimpToughVsRaising_1(BlimpTask):
  class BlimpToughVsRaising_2 (line 346) | class BlimpToughVsRaising_2(BlimpTask):
  class BlimpTransitive (line 350) | class BlimpTransitive(BlimpTask):
  class BlimpWhIsland (line 354) | class BlimpWhIsland(BlimpTask):
  class BlimpWhQuestionsObjectGap (line 358) | class BlimpWhQuestionsObjectGap(BlimpTask):
  class BlimpWhQuestionsSubjectGap (line 362) | class BlimpWhQuestionsSubjectGap(BlimpTask):
  class BlimpWhQuestionsSubjectGapLongDistance (line 366) | class BlimpWhQuestionsSubjectGapLongDistance(BlimpTask):
  class BlimpWhVsThatNoGap (line 370) | class BlimpWhVsThatNoGap(BlimpTask):
  class BlimpWhVsThatNoGapLongDistance (line 374) | class BlimpWhVsThatNoGapLongDistance(BlimpTask):
  class BlimpWhVsThatWithGap (line 378) | class BlimpWhVsThatWithGap(BlimpTask):
  class BlimpWhVsThatWithGapLongDistance (line 382) | class BlimpWhVsThatWithGapLongDistance(BlimpTask):

FILE: lm_eval/tasks/cbt.py
  class CBTBase (line 32) | class CBTBase(Task):
    method has_training_docs (line 37) | def has_training_docs(self):
    method has_validation_docs (line 40) | def has_validation_docs(self):
    method has_test_docs (line 43) | def has_test_docs(self):
    method training_docs (line 46) | def training_docs(self):
    method validation_docs (line 51) | def validation_docs(self):
    method test_docs (line 54) | def test_docs(self):
    method detokenize (line 57) | def detokenize(self, text):
    method doc_to_text (line 73) | def doc_to_text(self, doc):
    method should_decontaminate (line 78) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 81) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 85) | def doc_to_target(self, doc):
    method fewshot_examples (line 88) | def fewshot_examples(self, k, rnd):
    method construct_requests (line 94) | def construct_requests(self, doc, ctx):
    method process_results (line 113) | def process_results(self, doc, results):
    method aggregation (line 127) | def aggregation(self):
    method higher_is_better (line 135) | def higher_is_better(self):
  class CBTCN (line 144) | class CBTCN(CBTBase):
  class CBTNE (line 148) | class CBTNE(CBTBase):

FILE: lm_eval/tasks/coqa.py
  class CoQA (line 31) | class CoQA(Task):
    method has_training_docs (line 36) | def has_training_docs(self):
    method has_validation_docs (line 39) | def has_validation_docs(self):
    method has_test_docs (line 42) | def has_test_docs(self):
    method training_docs (line 45) | def training_docs(self):
    method validation_docs (line 48) | def validation_docs(self):
    method test_docs (line 51) | def test_docs(self):
    method doc_to_text (line 54) | def doc_to_text(self, doc):
    method should_decontaminate (line 66) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 69) | def doc_to_decontamination_query(self, doc):
    method get_answers (line 73) | def get_answers(cls, doc, turn_id):
    method get_answer_choice (line 90) | def get_answer_choice(self, raw_text):
    method compute_scores (line 104) | def compute_scores(gold_list, pred):
    method doc_to_target (line 126) | def doc_to_target(self, doc, turnid=None):
    method construct_requests (line 133) | def construct_requests(self, doc, ctx):
    method process_results (line 147) | def process_results(self, doc, results):
    method higher_is_better (line 168) | def higher_is_better(self):
    method aggregation (line 174) | def aggregation(self):

FILE: lm_eval/tasks/crowspairs.py
  class CrowsPairsMutilingual (line 55) | class CrowsPairsMutilingual(Task):
    method has_training_docs (line 60) | def has_training_docs(self):
    method has_validation_docs (line 63) | def has_validation_docs(self):
    method has_test_docs (line 66) | def has_test_docs(self):
    method validation_docs (line 69) | def validation_docs(self):
    method fewshot_context (line 77) | def fewshot_context(
    method doc_to_text (line 97) | def doc_to_text(self, doc):
    method should_decontaminate (line 101) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 104) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 107) | def doc_to_target(self, doc):
    method construct_requests (line 111) | def construct_requests(self, doc, ctx):
    method process_results (line 121) | def process_results(self, doc, results):
    method higher_is_better (line 133) | def higher_is_better(self):
    method aggregation (line 137) | def aggregation(self):
  class CrowsPairsEnglish (line 141) | class CrowsPairsEnglish(CrowsPairsMutilingual):
  class CrowsPairsFrench (line 145) | class CrowsPairsFrench(CrowsPairsMutilingual):
  class CrowsPairsEnglishRaceColor (line 149) | class CrowsPairsEnglishRaceColor(CrowsPairsMutilingual):
  class CrowsPairsEnglishSocioeconomic (line 154) | class CrowsPairsEnglishSocioeconomic(CrowsPairsMutilingual):
  class CrowsPairsEnglishGender (line 159) | class CrowsPairsEnglishGender(CrowsPairsMutilingual):
  class CrowsPairsEnglishAge (line 164) | class CrowsPairsEnglishAge(CrowsPairsMutilingual):
  class CrowsPairsEnglishReligion (line 169) | class CrowsPairsEnglishReligion(CrowsPairsMutilingual):
  class CrowsPairsEnglishDisability (line 174) | class CrowsPairsEnglishDisability(CrowsPairsMutilingual):
  class CrowsPairsEnglishSexualOrientation (line 179) | class CrowsPairsEnglishSexualOrientation(CrowsPairsMutilingual):
  class CrowsPairsEnglishNationality (line 184) | class CrowsPairsEnglishNationality(CrowsPairsMutilingual):
  class CrowsPairsEnglishPhysicalAppearance (line 189) | class CrowsPairsEnglishPhysicalAppearance(CrowsPairsMutilingual):
  class CrowsPairsEnglishAutre (line 194) | class CrowsPairsEnglishAutre(CrowsPairsMutilingual):
  class CrowsPairsFrenchRaceColor (line 199) | class CrowsPairsFrenchRaceColor(CrowsPairsMutilingual):
  class CrowsPairsFrenchSocioeconomic (line 204) | class CrowsPairsFrenchSocioeconomic(CrowsPairsMutilingual):
  class CrowsPairsFrenchGender (line 209) | class CrowsPairsFrenchGender(CrowsPairsMutilingual):
  class CrowsPairsFrenchAge (line 214) | class CrowsPairsFrenchAge(CrowsPairsMutilingual):
  class CrowsPairsFrenchReligion (line 219) | class CrowsPairsFrenchReligion(CrowsPairsMutilingual):
  class CrowsPairsFrenchDisability (line 224) | class CrowsPairsFrenchDisability(CrowsPairsMutilingual):
  class CrowsPairsFrenchSexualOrientation (line 229) | class CrowsPairsFrenchSexualOrientation(CrowsPairsMutilingual):
  class CrowsPairsFrenchNationality (line 234) | class CrowsPairsFrenchNationality(CrowsPairsMutilingual):
  class CrowsPairsFrenchPhysicalAppearance (line 239) | class CrowsPairsFrenchPhysicalAppearance(CrowsPairsMutilingual):
  class CrowsPairsFrenchAutre (line 244) | class CrowsPairsFrenchAutre(CrowsPairsMutilingual):

FILE: lm_eval/tasks/drop.py
  class DROP (line 40) | class DROP(Task):
    method has_training_docs (line 45) | def has_training_docs(self):
    method has_validation_docs (line 48) | def has_validation_docs(self):
    method has_test_docs (line 51) | def has_test_docs(self):
    method training_docs (line 54) | def training_docs(self):
    method validation_docs (line 59) | def validation_docs(self):
    method _process_doc (line 62) | def _process_doc(self, doc):
    method get_answers (line 71) | def get_answers(cls, qa):
    method parse_answer (line 102) | def parse_answer(cls, answer):
    method doc_to_text (line 114) | def doc_to_text(self, doc):
    method should_decontaminate (line 117) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 120) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 123) | def doc_to_target(self, doc):
    method construct_requests (line 126) | def construct_requests(self, doc, ctx):
    method process_results (line 140) | def process_results(self, doc, results):
    method get_metrics (line 160) | def get_metrics(self, predicted, gold):
    method _answer_to_bags (line 183) | def _answer_to_bags(self, answer):
    method _align_bags (line 196) | def _align_bags(self, predicted, gold):
    method _compute_f1 (line 215) | def _compute_f1(self, predicted_bag, gold_bag):
    method _match_numbers_if_present (line 232) | def _match_numbers_if_present(self, gold_bag, predicted_bag):
    method _is_number (line 245) | def _is_number(self, text):
    method _remove_articles (line 252) | def _remove_articles(self, text):
    method _white_space_fix (line 255) | def _white_space_fix(self, text):
    method _remove_punc (line 258) | def _remove_punc(self, text):
    method _fix_number (line 265) | def _fix_number(self, text):
    method _tokenize (line 268) | def _tokenize(self, text):
    method _normalize (line 271) | def _normalize(self, answer):
    method aggregation (line 284) | def aggregation(self):
    method higher_is_better (line 292) | def higher_is_better(self):

FILE: lm_eval/tasks/glue.py
  class CoLA (line 48) | class CoLA(Task):
    method has_training_docs (line 53) | def has_training_docs(self):
    method has_validation_docs (line 56) | def has_validation_docs(self):
    method has_test_docs (line 59) | def has_test_docs(self):
    method training_docs (line 62) | def training_docs(self):
    method validation_docs (line 67) | def validation_docs(self):
    method doc_to_text (line 70) | def doc_to_text(self, doc):
    method should_decontaminate (line 75) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 78) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 81) | def doc_to_target(self, doc):
    method construct_requests (line 84) | def construct_requests(self, doc, ctx):
    method process_results (line 89) | def process_results(self, doc, results):
    method higher_is_better (line 95) | def higher_is_better(self):
    method aggregation (line 98) | def aggregation(self):
  class SST (line 102) | class SST(Task):
    method has_training_docs (line 107) | def has_training_docs(self):
    method has_validation_docs (line 110) | def has_validation_docs(self):
    method has_test_docs (line 113) | def has_test_docs(self):
    method training_docs (line 116) | def training_docs(self):
    method validation_docs (line 121) | def validation_docs(self):
    method doc_to_text (line 124) | def doc_to_text(self, doc):
    method doc_to_target (line 129) | def doc_to_target(self, doc):
    method construct_requests (line 132) | def construct_requests(self, doc, ctx):
    method process_results (line 137) | def process_results(self, doc, results):
    method higher_is_better (line 143) | def higher_is_better(self):
    method aggregation (line 146) | def aggregation(self):
  class MNLI (line 153) | class MNLI(Task):
    method has_training_docs (line 158) | def has_training_docs(self):
    method has_validation_docs (line 161) | def has_validation_docs(self):
    method has_test_docs (line 164) | def has_test_docs(self):
    method training_docs (line 167) | def training_docs(self):
    method validation_docs (line 172) | def validation_docs(self):
    method test_docs (line 176) | def test_docs(self):
    method doc_to_text (line 180) | def doc_to_text(self, doc):
    method doc_to_target (line 187) | def doc_to_target(self, doc):
    method construct_requests (line 193) | def construct_requests(self, doc, ctx):
    method process_results (line 199) | def process_results(self, doc, results):
    method higher_is_better (line 204) | def higher_is_better(self):
    method aggregation (line 207) | def aggregation(self):
  class MNLIMismatched (line 211) | class MNLIMismatched(MNLI):
    method validation_docs (line 214) | def validation_docs(self):
    method test_docs (line 218) | def test_docs(self):
  class QNLI (line 223) | class QNLI(Task):
    method has_training_docs (line 228) | def has_training_docs(self):
    method has_validation_docs (line 231) | def has_validation_docs(self):
    method has_test_docs (line 234) | def has_test_docs(self):
    method training_docs (line 237) | def training_docs(self):
    method validation_docs (line 242) | def validation_docs(self):
    method doc_to_text (line 245) | def doc_to_text(self, doc):
    method doc_to_target (line 253) | def doc_to_target(self, doc):
    method construct_requests (line 258) | def construct_requests(self, doc, ctx):
    method process_results (line 263) | def process_results(self, doc, results):
    method higher_is_better (line 269) | def higher_is_better(self):
    method aggregation (line 272) | def aggregation(self):
  class WNLI (line 276) | class WNLI(Task):
    method has_training_docs (line 281) | def has_training_docs(self):
    method has_validation_docs (line 284) | def has_validation_docs(self):
    method has_test_docs (line 287) | def has_test_docs(self):
    method training_docs (line 290) | def training_docs(self):
    method validation_docs (line 295) | def validation_docs(self):
    method doc_to_text (line 298) | def doc_to_text(self, doc):
    method doc_to_target (line 304) | def doc_to_target(self, doc):
    method construct_requests (line 309) | def construct_requests(self, doc, ctx):
    method process_results (line 314) | def process_results(self, doc, results):
    method higher_is_better (line 320) | def higher_is_better(self):
    method aggregation (line 323) | def aggregation(self):
  class RTE (line 327) | class RTE(Task):
    method has_training_docs (line 332) | def has_training_docs(self):
    method has_validation_docs (line 335) | def has_validation_docs(self):
    method has_test_docs (line 338) | def has_test_docs(self):
    method training_docs (line 341) | def training_docs(self):
    method validation_docs (line 346) | def validation_docs(self):
    method doc_to_text (line 349) | def doc_to_text(self, doc):
    method doc_to_target (line 355) | def doc_to_target(self, doc):
    method construct_requests (line 360) | def construct_requests(self, doc, ctx):
    method process_results (line 365) | def process_results(self, doc, results):
    method higher_is_better (line 371) | def higher_is_better(self):
    method aggregation (line 374) | def aggregation(self):
  class MRPC (line 381) | class MRPC(Task):
    method has_training_docs (line 386) | def has_training_docs(self):
    method has_validation_docs (line 389) | def has_validation_docs(self):
    method has_test_docs (line 392) | def has_test_docs(self):
    method training_docs (line 395) | def training_docs(self):
    method validation_docs (line 400) | def validation_docs(self):
    method doc_to_text (line 403) | def doc_to_text(self, doc):
    method doc_to_target (line 409) | def doc_to_target(self, doc):
    method construct_requests (line 412) | def construct_requests(self, doc, ctx):
    method process_results (line 417) | def process_results(self, doc, results):
    method higher_is_better (line 426) | def higher_is_better(self):
    method aggregation (line 429) | def aggregation(self):
  class QQP (line 433) | class QQP(Task):
    method has_training_docs (line 438) | def has_training_docs(self):
    method has_validation_docs (line 441) | def has_validation_docs(self):
    method has_test_docs (line 444) | def has_test_docs(self):
    method training_docs (line 447) | def training_docs(self):
    method validation_docs (line 452) | def validation_docs(self):
    method doc_to_text (line 455) | def doc_to_text(self, doc):
    method doc_to_target (line 461) | def doc_to_target(self, doc):
    method construct_requests (line 464) | def construct_requests(self, doc, ctx):
    method process_results (line 469) | def process_results(self, doc, results):
    method higher_is_better (line 478) | def higher_is_better(self):
    method aggregation (line 481) | def aggregation(self):
  class STSB (line 485) | class STSB(Task):
    method has_training_docs (line 490) | def has_training_docs(self):
    method has_validation_docs (line 493) | def has_validation_docs(self):
    method has_test_docs (line 496) | def has_test_docs(self):
    method training_docs (line 499) | def training_docs(self):
    method validation_docs (line 504) | def validation_docs(self):
    method test_docs (line 507) | def test_docs(self):
    method doc_to_text (line 510) | def doc_to_text(self, doc):
    method doc_to_target (line 516) | def doc_to_target(self, doc):
    method construct_requests (line 519) | def construct_requests(self, doc, ctx):
    method process_results (line 533) | def process_results(self, doc, results):
    method aggregation (line 546) | def aggregation(self):
    method higher_is_better (line 555) | def higher_is_better(self):

FILE: lm_eval/tasks/gsm8k.py
  class GradeSchoolMath8K (line 40) | class GradeSchoolMath8K(Task):
    method has_training_docs (line 45) | def has_training_docs(self):
    method has_validation_docs (line 48) | def has_validation_docs(self):
    method has_test_docs (line 51) | def has_test_docs(self):
    method training_docs (line 54) | def training_docs(self):
    method validation_docs (line 57) | def validation_docs(self):
    method test_docs (line 60) | def test_docs(self):
    method doc_to_text (line 63) | def doc_to_text(self, doc):
    method doc_to_target (line 66) | def doc_to_target(self, doc):
    method construct_requests (line 69) | def construct_requests(self, doc, ctx):
    method _extract_answer (line 85) | def _extract_answer(self, completion):
    method _is_correct (line 94) | def _is_correct(self, completion, answer):
    method process_results (line 99) | def process_results(self, doc, results):
    method aggregation (line 113) | def aggregation(self):
    method higher_is_better (line 121) | def higher_is_better(self):

FILE: lm_eval/tasks/headqa.py
  class HeadQABase (line 28) | class HeadQABase(MultipleChoiceTask):
    method has_training_docs (line 32) | def has_training_docs(self):
    method has_validation_docs (line 35) | def has_validation_docs(self):
    method has_test_docs (line 38) | def has_test_docs(self):
    method training_docs (line 41) | def training_docs(self):
    method validation_docs (line 46) | def validation_docs(self):
    method test_docs (line 49) | def test_docs(self):
    method _process_doc (line 52) | def _process_doc(self, doc):
    method doc_to_text (line 61) | def doc_to_text(self, doc):
    method should_decontaminate (line 64) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 67) | def doc_to_decontamination_query(self, doc):
  class HeadQAEn (line 71) | class HeadQAEn(HeadQABase):
  class HeadQAEs (line 75) | class HeadQAEs(HeadQABase):
  class HeadQAEsDeprecated (line 80) | class HeadQAEsDeprecated(HeadQABase):
    method __init__ (line 83) | def __init__(self):

FILE: lm_eval/tasks/hellaswag.py
  class HellaSwag (line 30) | class HellaSwag(MultipleChoiceTask):
    method has_training_docs (line 35) | def has_training_docs(self):
    method has_validation_docs (line 38) | def has_validation_docs(self):
    method has_test_docs (line 41) | def has_test_docs(self):
    method training_docs (line 44) | def training_docs(self):
    method validation_docs (line 49) | def validation_docs(self):
    method _process_doc (line 52) | def _process_doc(self, doc):
    method preprocess (line 62) | def preprocess(cls, text):
    method doc_to_text (line 70) | def doc_to_text(self, doc):
    method should_decontaminate (line 73) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 76) | def doc_to_decontamination_query(self, doc):

FILE: lm_eval/tasks/hendrycks_ethics.py
  class Ethics (line 37) | class Ethics(Task):
    method has_training_docs (line 41) | def has_training_docs(self):
    method has_validation_docs (line 44) | def has_validation_docs(self):
    method has_test_docs (line 47) | def has_test_docs(self):
    method training_docs (line 52) | def training_docs(self):
    method validation_docs (line 55) | def validation_docs(self):
    method test_docs (line 58) | def test_docs(self):
    method doc_to_text (line 62) | def doc_to_text(self, doc):
    method doc_to_target (line 66) | def doc_to_target(self, doc):
    method construct_requests (line 70) | def construct_requests(self, doc, ctx):
    method process_results (line 74) | def process_results(self, doc, results):
    method aggregation (line 78) | def aggregation(self):
    method higher_is_better (line 82) | def higher_is_better(self):
  class EthicsCM (line 86) | class EthicsCM(Ethics):
    method doc_to_text (line 90) | def doc_to_text(self, doc):
    method should_decontaminate (line 93) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 96) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 99) | def doc_to_target(self, doc):
    method construct_requests (line 102) | def construct_requests(self, doc, ctx):
    method process_results (line 107) | def process_results(self, doc, results):
    method aggregation (line 113) | def aggregation(self):
    method higher_is_better (line 116) | def higher_is_better(self):
  class EthicsDeontology (line 120) | class EthicsDeontology(Ethics):
    method doc_to_text (line 124) | def doc_to_text(self, doc):
    method should_decontaminate (line 130) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 133) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 136) | def doc_to_target(self, doc):
    method construct_requests (line 140) | def construct_requests(self, doc, ctx):
    method process_results (line 145) | def process_results(self, doc, results):
    method calc_em (line 150) | def calc_em(self, items):
    method aggregation (line 164) | def aggregation(self):
    method higher_is_better (line 167) | def higher_is_better(self):
  class EthicsJustice (line 171) | class EthicsJustice(Ethics):
    method doc_to_text (line 175) | def doc_to_text(self, doc):
    method should_decontaminate (line 180) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 183) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 186) | def doc_to_target(self, doc):
    method construct_requests (line 190) | def construct_requests(self, doc, ctx):
    method process_results (line 195) | def process_results(self, doc, results):
    method calc_em (line 200) | def calc_em(self, items):
    method aggregation (line 214) | def aggregation(self):
    method higher_is_better (line 217) | def higher_is_better(self):
  class EthicsUtilitarianismOriginal (line 221) | class EthicsUtilitarianismOriginal(Ethics):
    method has_training_docs (line 225) | def has_training_docs(self):
    method fewshot_examples (line 229) | def fewshot_examples(self, k, rnd):
    method doc_to_text (line 253) | def doc_to_text(self, doc):
    method should_decontaminate (line 256) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 259) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 262) | def doc_to_target(self, doc):
    method construct_requests (line 265) | def construct_requests(self, doc, ctx):
    method process_results (line 273) | def process_results(self, doc, results):
    method aggregation (line 287) | def aggregation(self):
    method higher_is_better (line 290) | def higher_is_better(self):
  class EthicsUtilitarianism (line 294) | class EthicsUtilitarianism(Ethics):
    method training_docs (line 303) | def training_docs(self):
    method validation_docs (line 307) | def validation_docs(self):
    method test_docs (line 310) | def test_docs(self):
    method _process_doc (line 314) | def _process_doc(self, doc):
    method doc_to_text (line 325) | def doc_to_text(self, doc):
    method doc_to_target (line 330) | def doc_to_target(self, doc):
    method construct_requests (line 333) | def construct_requests(self, doc, ctx):
    method process_results (line 338) | def process_results(self, doc, results):
    method aggregation (line 344) | def aggregation(self):
    method higher_is_better (line 347) | def higher_is_better(self):
  class EthicsVirtue (line 351) | class EthicsVirtue(Ethics):
    method _process_doc (line 355) | def _process_doc(self, doc):
    method doc_to_text (line 358) | def doc_to_text(self, doc):
    method doc_to_target (line 363) | def doc_to_target(self, doc):
    method construct_requests (line 366) | def construct_requests(self, doc, ctx):
    method process_results (line 371) | def process_results(self, doc, results):
    method calc_em (line 377) | def calc_em(self, items):
    method aggregation (line 392) | def aggregation(self):
    method higher_is_better (line 395) | def higher_is_better(self):

FILE: lm_eval/tasks/hendrycks_math.py
  class Math (line 27) | class Math(Task):
    method has_training_docs (line 31) | def has_training_docs(self):
    method has_validation_docs (line 34) | def has_validation_docs(self):
    method has_test_docs (line 37) | def has_test_docs(self):
    method training_docs (line 40) | def training_docs(self):
    method validation_docs (line 43) | def validation_docs(self):
    method test_docs (line 46) | def test_docs(self):
    method _process_doc (line 49) | def _process_doc(self, doc):
    method doc_to_text (line 53) | def doc_to_text(self, doc):
    method should_decontaminate (line 56) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 59) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 62) | def doc_to_target(self, doc):
    method construct_requests (line 65) | def construct_requests(self, doc, ctx):
    method process_results (line 68) | def process_results(self, doc, results):
    method aggregation (line 82) | def aggregation(self):
    method higher_is_better (line 85) | def higher_is_better(self):
    method is_equiv (line 88) | def is_equiv(self, str1, str2, verbose=False):
    method remove_boxed (line 104) | def remove_boxed(self, s):
    method last_boxed_only_string (line 117) | def last_boxed_only_string(self, string):
    method fix_fracs (line 147) | def fix_fracs(self, string):
    method fix_a_slash_b (line 178) | def fix_a_slash_b(self, string):
    method remove_right_units (line 192) | def remove_right_units(self, string):
    method fix_sqrt (line 201) | def fix_sqrt(self, string):
    class NotEqual (line 215) | class NotEqual:
      method __eq__ (line 216) | def __eq__(self, other):
    method strip_string (line 219) | def strip_string(self, string):
  class MathAlgebra (line 284) | class MathAlgebra(Math):
  class MathCountingAndProbability (line 289) | class MathCountingAndProbability(Math):
  class MathGeometry (line 294) | class MathGeometry(Math):
  class MathIntermediateAlgebra (line 299) | class MathIntermediateAlgebra(Math):
  class MathNumberTheory (line 304) | class MathNumberTheory(Math):
  class MathPrealgebra (line 309) | class MathPrealgebra(Math):
  class MathPrecalculus (line 314) | class MathPrecalculus(Math):

FILE: lm_eval/tasks/hendrycks_test.py
  function create_all_tasks (line 89) | def create_all_tasks():
  function create_task (line 97) | def create_task(subject):
  class GeneralHendrycksTest (line 105) | class GeneralHendrycksTest(MultipleChoiceTask):
    method __init__ (line 110) | def __init__(self, subject):
    method has_training_docs (line 114) | def has_training_docs(self):
    method has_validation_docs (line 117) | def has_validation_docs(self):
    method has_test_docs (line 120) | def has_test_docs(self):
    method validation_docs (line 123) | def validation_docs(self):
    method test_docs (line 126) | def test_docs(self):
    method _process_doc (line 129) | def _process_doc(self, doc):
    method fewshot_examples (line 156) | def fewshot_examples(self, k, rnd):
    method doc_to_text (line 165) | def doc_to_text(self, doc):
    method should_decontaminate (line 168) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 171) | def doc_to_decontamination_query(self, doc):

FILE: lm_eval/tasks/lambada.py
  class LambadaBase (line 31) | class LambadaBase(Task):
    method training_docs (line 34) | def training_docs(self):
    method validation_docs (line 38) | def validation_docs(self):
    method test_docs (line 42) | def test_docs(self):
    method doc_to_text (line 46) | def doc_to_text(self, doc):
    method should_decontaminate (line 49) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 52) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 55) | def doc_to_target(self, doc):
    method construct_requests (line 58) | def construct_requests(self, doc, ctx):
    method process_results (line 63) | def process_results(self, doc, results):
    method aggregation (line 68) | def aggregation(self):
    method higher_is_better (line 71) | def higher_is_better(self):
  class LambadaStandard (line 75) | class LambadaStandard(LambadaBase):
    method has_training_docs (line 81) | def has_training_docs(self):
    method has_validation_docs (line 84) | def has_validation_docs(self):
    method has_test_docs (line 87) | def has_test_docs(self):
  class LambadaOpenAI (line 91) | class LambadaOpenAI(LambadaBase):
    method has_training_docs (line 101) | def has_training_docs(self):
    method has_validation_docs (line 104) | def has_validation_docs(self):
    method has_test_docs (line 107) | def has_test_docs(self):

FILE: lm_eval/tasks/lambada_cloze.py
  class LambadaStandardCloze (line 31) | class LambadaStandardCloze(LambadaStandard):
    method doc_to_text (line 36) | def doc_to_text(self, doc):
    method should_decontaminate (line 39) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 42) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 45) | def doc_to_target(self, doc):
  class LambadaOpenAICloze (line 49) | class LambadaOpenAICloze(LambadaOpenAI):
    method doc_to_text (line 54) | def doc_to_text(self, doc):
    method should_decontaminate (line 57) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 60) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 63) | def doc_to_target(self, doc):

FILE: lm_eval/tasks/lambada_multilingual.py
  class LambadaOpenAIMultilingualEnglish (line 33) | class LambadaOpenAIMultilingualEnglish(LambadaOpenAI):
  class LambadaOpenAIMultilingualFrench (line 38) | class LambadaOpenAIMultilingualFrench(LambadaOpenAI):
  class LambadaOpenAIMultilingualGerman (line 43) | class LambadaOpenAIMultilingualGerman(LambadaOpenAI):
  class LambadaOpenAIMultilingualItalian (line 48) | class LambadaOpenAIMultilingualItalian(LambadaOpenAI):
  class LambadaOpenAIMultilingualSpanish (line 53) | class LambadaOpenAIMultilingualSpanish(LambadaOpenAI):
  function construct_tasks (line 67) | def construct_tasks():

FILE: lm_eval/tasks/logiqa.py
  class LogiQA (line 30) | class LogiQA(MultipleChoiceTask):
    method has_training_docs (line 35) | def has_training_docs(self):
    method has_validation_docs (line 38) | def has_validation_docs(self):
    method has_test_docs (line 41) | def has_test_docs(self):
    method training_docs (line 44) | def training_docs(self):
    method validation_docs (line 49) | def validation_docs(self):
    method test_docs (line 52) | def test_docs(self):
    method _process_doc (line 55) | def _process_doc(self, doc):
    method doc_to_text (line 82) | def doc_to_text(self, doc):
    method should_decontaminate (line 85) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 88) | def doc_to_decontamination_query(self, doc):

FILE: lm_eval/tasks/mathqa.py
  class MathQA (line 27) | class MathQA(MultipleChoiceTask):
    method has_training_docs (line 32) | def has_training_docs(self):
    method has_validation_docs (line 35) | def has_validation_docs(self):
    method has_test_docs (line 38) | def has_test_docs(self):
    method training_docs (line 41) | def training_docs(self):
    method validation_docs (line 46) | def validation_docs(self):
    method test_docs (line 49) | def test_docs(self):
    method _process_doc (line 52) | def _process_doc(self, doc):
    method doc_to_text (line 66) | def doc_to_text(self, doc):
    method should_decontaminate (line 69) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 72) | def doc_to_decontamination_query(self, doc):

FILE: lm_eval/tasks/mc_taco.py
  class MCTACO (line 37) | class MCTACO(Task):
    method has_training_docs (line 42) | def has_training_docs(self):
    method has_validation_docs (line 45) | def has_validation_docs(self):
    method has_test_docs (line 48) | def has_test_docs(self):
    method validation_docs (line 51) | def validation_docs(self):
    method test_docs (line 54) | def test_docs(self):
    method doc_to_text (line 57) | def doc_to_text(self, doc):
    method should_decontaminate (line 63) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 66) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 69) | def doc_to_target(self, doc):
    method construct_requests (line 72) | def construct_requests(self, doc, ctx):
    method process_results (line 87) | def process_results(self, doc, results):
    method _question2id (line 104) | def _question2id(self, doc):
    method aggregation (line 108) | def aggregation(self):
    method higher_is_better (line 114) | def higher_is_better(self):
  function exact_match (line 121) | def exact_match(items):
  function f1 (line 133) | def f1(items):

FILE: lm_eval/tasks/mutual.py
  class MuTualBase (line 28) | class MuTualBase(Task):
    method has_training_docs (line 34) | def has_training_docs(self):
    method has_validation_docs (line 37) | def has_validation_docs(self):
    method has_test_docs (line 40) | def has_test_docs(self):
    method training_docs (line 43) | def training_docs(self):
    method validation_docs (line 46) | def validation_docs(self):
    method test_docs (line 49) | def test_docs(self):
    method doc_to_text (line 52) | def doc_to_text(self, doc):
    method should_decontaminate (line 55) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 58) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 61) | def doc_to_target(self, doc):
    method construct_requests (line 64) | def construct_requests(self, doc, ctx):
    method detokenize (line 70) | def detokenize(self, text):
    method process_results (line 86) | def process_results(self, doc, results):
    method aggregation (line 94) | def aggregation(self):
    method higher_is_better (line 97) | def higher_is_better(self):
  class MuTual (line 101) | class MuTual(MuTualBase):
  class MuTualPlus (line 105) | class MuTualPlus(MuTualBase):

FILE: lm_eval/tasks/naturalqs.py
  class NaturalQs (line 32) | class NaturalQs(Task):
    method has_training_docs (line 37) | def has_training_docs(self):
    method has_validation_docs (line 40) | def has_validation_docs(self):
    method has_test_docs (line 43) | def has_test_docs(self):
    method training_docs (line 46) | def training_docs(self):
    method validation_docs (line 53) | def validation_docs(self):
    method fewshot_examples (line 56) | def fewshot_examples(self, k, rnd):
    method doc_to_text (line 63) | def doc_to_text(self, doc):
    method should_decontaminate (line 66) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 69) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 72) | def doc_to_target(self, doc):
    method construct_requests (line 91) | def construct_requests(self, doc, ctx):
    method process_results (line 105) | def process_results(self, doc, results):
    method aggregation (line 118) | def aggregation(self):
    method higher_is_better (line 127) | def higher_is_better(self):

FILE: lm_eval/tasks/openbookqa.py
  class OpenBookQA (line 30) | class OpenBookQA(MultipleChoiceTask):
    method has_training_docs (line 35) | def has_training_docs(self):
    method has_validation_docs (line 38) | def has_validation_docs(self):
    method has_test_docs (line 41) | def has_test_docs(self):
    method training_docs (line 44) | def training_docs(self):
    method validation_docs (line 49) | def validation_docs(self):
    method test_docs (line 52) | def test_docs(self):
    method _process_doc (line 55) | def _process_doc(self, doc):
    method doc_to_text (line 64) | def doc_to_text(self, doc):
    method should_decontaminate (line 67) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 70) | def doc_to_decontamination_query(self, doc):

FILE: lm_eval/tasks/pile.py
  class PilePerplexityTask (line 28) | class PilePerplexityTask(PerplexityTask):
    method has_validation_docs (line 33) | def has_validation_docs(self):
    method has_test_docs (line 36) | def has_test_docs(self):
    method validation_docs (line 39) | def validation_docs(self):
    method test_docs (line 43) | def test_docs(self):
  class PileArxiv (line 48) | class PileArxiv(PilePerplexityTask):
  class PileBooks3 (line 52) | class PileBooks3(PilePerplexityTask):
  class PileBookCorpus2 (line 56) | class PileBookCorpus2(PilePerplexityTask):
  class PileDmMathematics (line 60) | class PileDmMathematics(PilePerplexityTask):
  class PileEnron (line 64) | class PileEnron(PilePerplexityTask):
  class PileEuroparl (line 68) | class PileEuroparl(PilePerplexityTask):
  class PileFreeLaw (line 72) | class PileFreeLaw(PilePerplexityTask):
  class PileGithub (line 76) | class PileGithub(PilePerplexityTask):
  class PileGutenberg (line 80) | class PileGutenberg(PilePerplexityTask):
  class PileHackernews (line 84) | class PileHackernews(PilePerplexityTask):
  class PileNIHExporter (line 88) | class PileNIHExporter(PilePerplexityTask):
  class PileOpenSubtitles (line 92) | class PileOpenSubtitles(PilePerplexityTask):
  class PileOpenWebText2 (line 96) | class PileOpenWebText2(PilePerplexityTask):
  class PilePhilPapers (line 100) | class PilePhilPapers(PilePerplexityTask):
  class PilePileCc (line 104) | class PilePileCc(PilePerplexityTask):
  class PilePubmedAbstracts (line 108) | class PilePubmedAbstracts(PilePerplexityTask):
  class PilePubmedCentral (line 112) | class PilePubmedCentral(PilePerplexityTask):
  class PileStackExchange (line 116) | class PileStackExchange(PilePerplexityTask):
  class PileUspto (line 120) | class PileUspto(PilePerplexityTask):
  class PileUbuntuIrc (line 124) | class PileUbuntuIrc(PilePerplexityTask):
  class PileWikipedia (line 128) | class PileWikipedia(PilePerplexityTask):
  class PileYoutubeSubtitles (line 132) | class PileYoutubeSubtitles(PilePerplexityTask):

FILE: lm_eval/tasks/piqa.py
  class PiQA (line 29) | class PiQA(MultipleChoiceTask):
    method has_training_docs (line 34) | def has_training_docs(self):
    method has_validation_docs (line 37) | def has_validation_docs(self):
    method has_test_docs (line 40) | def has_test_docs(self):
    method training_docs (line 43) | def training_docs(self):
    method validation_docs (line 48) | def validation_docs(self):
    method _process_doc (line 51) | def _process_doc(self, doc):
    method doc_to_text (line 59) | def doc_to_text(self, doc):
    method should_decontaminate (line 62) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 65) | def doc_to_decontamination_query(self, doc):

FILE: lm_eval/tasks/prost.py
  class PROST (line 38) | class PROST(MultipleChoiceTask):
    method has_training_docs (line 43) | def has_training_docs(self):
    method has_validation_docs (line 46) | def has_validation_docs(self):
    method has_test_docs (line 49) | def has_test_docs(self):
    method test_docs (line 52) | def test_docs(self):
    method fewshot_context (line 55) | def fewshot_context(
    method _process_doc (line 65) | def _process_doc(self, doc):
    method doc_to_text (line 73) | def doc_to_text(self, doc):
    method should_decontaminate (line 76) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 79) | def doc_to_decontamination_query(self, doc):

FILE: lm_eval/tasks/pubmedqa.py
  class Pubmed_QA (line 34) | class Pubmed_QA(Task):
    method has_training_docs (line 39) | def has_training_docs(self):
    method has_validation_docs (line 42) | def has_validation_docs(self):
    method has_test_docs (line 45) | def has_test_docs(self):
    method test_docs (line 48) | def test_docs(self):
    method doc_to_text (line 53) | def doc_to_text(self, doc):
    method should_decontaminate (line 59) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 62) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 65) | def doc_to_target(self, doc):
    method construct_requests (line 68) | def construct_requests(self, doc, ctx):
    method process_results (line 77) | def process_results(self, doc, results):
    method aggregation (line 85) | def aggregation(self):
    method higher_is_better (line 88) | def higher_is_better(self):

FILE: lm_eval/tasks/qa4mre.py
  class QA4MRE (line 29) | class QA4MRE(MultipleChoiceTask):
    method has_training_docs (line 34) | def has_training_docs(self):
    method has_validation_docs (line 37) | def has_validation_docs(self):
    method has_test_docs (line 40) | def has_test_docs(self):
    method test_docs (line 43) | def test_docs(self):
    method _process_doc (line 47) | def _process_doc(self, doc):
    method doc_to_text (line 57) | def doc_to_text(self, doc):
    method should_decontaminate (line 60) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 63) | def doc_to_decontamination_query(self, doc):
  class QA4MRE_2011 (line 67) | class QA4MRE_2011(QA4MRE):
  class QA4MRE_2012 (line 71) | class QA4MRE_2012(QA4MRE):
  class QA4MRE_2013 (line 75) | class QA4MRE_2013(QA4MRE):

FILE: lm_eval/tasks/qasper.py
  function normalize_answer (line 43) | def normalize_answer(s):
  function categorise_answer (line 65) | def categorise_answer(answer_blob):
  function token_f1_score (line 88) | def token_f1_score(prediction, ground_truth):
  class QASPER (line 104) | class QASPER(Task):
    method has_training_docs (line 109) | def has_training_docs(self):
    method has_validation_docs (line 112) | def has_validation_docs(self):
    method has_test_docs (line 115) | def has_test_docs(self):
    method doc_to_text (line 118) | def doc_to_text(self, doc):
    method doc_to_target (line 132) | def doc_to_target(self, doc):
    method training_docs (line 138) | def training_docs(self):
    method validation_docs (line 142) | def validation_docs(self):
    method _process_doc (line 146) | def _process_doc(self, doc):
    method process_results (line 167) | def process_results(self, doc, results):
    method aggregation (line 198) | def aggregation(self):
    method construct_requests (line 204) | def construct_requests(self, doc, ctx):
    method higher_is_better (line 225) | def higher_is_better(self):

FILE: lm_eval/tasks/quac.py
  class QuAC (line 28) | class QuAC(Task):
    method has_training_docs (line 33) | def has_training_docs(self):
    method has_validation_docs (line 36) | def has_validation_docs(self):
    method has_test_docs (line 39) | def has_test_docs(self):
    method training_docs (line 42) | def training_docs(self):
    method validation_docs (line 47) | def validation_docs(self):
    method test_docs (line 50) | def test_docs(self):
    method _process_doc (line 53) | def _process_doc(self, doc):
    method doc_to_text (line 57) | def doc_to_text(self, doc):
    method should_decontaminate (line 71) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 74) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 77) | def doc_to_target(self, doc):
    method construct_requests (line 80) | def construct_requests(self, doc, ctx):
    method process_results (line 94) | def process_results(self, doc, results):
    method aggregation (line 107) | def aggregation(self):
    method higher_is_better (line 116) | def higher_is_better(self):

FILE: lm_eval/tasks/race.py
  class each (line 29) | class each:
    method __init__ (line 30) | def __init__(self, f):
    method __rrshift__ (line 33) | def __rrshift__(self, other):
  class RACE (line 37) | class RACE(Task):
    method has_training_docs (line 45) | def has_training_docs(self):
    method has_validation_docs (line 48) | def has_validation_docs(self):
    method has_test_docs (line 51) | def has_test_docs(self):
    method _collate_data (line 54) | def _collate_data(self, set):
    method training_docs (line 87) | def training_docs(self):
    method validation_docs (line 90) | def validation_docs(self):
    method test_docs (line 93) | def test_docs(self):
    method get_answer_option (line 97) | def get_answer_option(cls, problem):
    method last_problem (line 102) | def last_problem(cls, doc):
    method doc_to_text (line 105) | def doc_to_text(self, doc):
    method should_decontaminate (line 119) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 122) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 125) | def doc_to_target(self, doc):
    method construct_requests (line 128) | def construct_requests(self, doc, ctx):
    method process_results (line 145) | def process_results(self, doc, results):
    method aggregation (line 159) | def aggregation(self):
    method higher_is_better (line 167) | def higher_is_better(self):

FILE: lm_eval/tasks/sat.py
  class SATAnalogies (line 29) | class SATAnalogies(MultipleChoiceTask):
    method __init__ (line 34) | def __init__(self, data_dir: str):
    method has_training_docs (line 42) | def has_training_docs(self):
    method has_validation_docs (line 45) | def has_validation_docs(self):
    method has_test_docs (line 48) | def has_test_docs(self):
    method training_docs (line 51) | def training_docs(self):
    method validation_docs (line 54) | def validation_docs(self):
    method test_docs (line 57) | def test_docs(self):
    method _process_doc (line 60) | def _process_doc(self, doc):
    method doc_to_text (line 70) | def doc_to_text(self, doc):
    method should_decontaminate (line 73) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 76) | def doc_to_decontamination_query(self, doc):

FILE: lm_eval/tasks/sciq.py
  class SciQ (line 25) | class SciQ(MultipleChoiceTask):
    method has_training_docs (line 30) | def has_training_docs(self):
    method has_validation_docs (line 33) | def has_validation_docs(self):
    method has_test_docs (line 36) | def has_test_docs(self):
    method training_docs (line 39) | def training_docs(self):
    method validation_docs (line 44) | def validation_docs(self):
    method test_docs (line 47) | def test_docs(self):
    method _process_doc (line 50) | def _process_doc(self, doc):
    method doc_to_text (line 66) | def doc_to_text(self, doc):
    method should_decontaminate (line 69) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 72) | def doc_to_decontamination_query(self, doc):

FILE: lm_eval/tasks/squad.py
  function _squad_metric (line 35) | def _squad_metric(predictions, references):
  function _squad_agg (line 40) | def _squad_agg(key, items):
  class SQuAD2 (line 46) | class SQuAD2(Task):
    method has_training_docs (line 56) | def has_training_docs(self):
    method has_validation_docs (line 59) | def has_validation_docs(self):
    method has_test_docs (line 62) | def has_test_docs(self):
    method training_docs (line 65) | def training_docs(self):
    method validation_docs (line 68) | def validation_docs(self):
    method doc_to_text (line 71) | def doc_to_text(self, doc):
    method should_decontaminate (line 85) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 88) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 91) | def doc_to_target(self, doc):
    method construct_requests (line 99) | def construct_requests(self, doc, ctx):
    method process_results (line 114) | def process_results(self, doc, results):
    method aggregation (line 171) | def aggregation(self):
    method higher_is_better (line 204) | def higher_is_better(self):

FILE: lm_eval/tasks/storycloze.py
  class StoryCloze (line 36) | class StoryCloze(Task):
    method __init__ (line 41) | def __init__(self, data_dir: str):
    method has_training_docs (line 49) | def has_training_docs(self):
    method has_validation_docs (line 52) | def has_validation_docs(self):
    method has_test_docs (line 55) | def has_test_docs(self):
    method training_docs (line 58) | def training_docs(self):
    method validation_docs (line 61) | def validation_docs(self):
    method test_docs (line 64) | def test_docs(self):
    method doc_to_text (line 67) | def doc_to_text(self, doc):
    method should_decontaminate (line 77) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 80) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 90) | def doc_to_target(self, doc):
    method construct_requests (line 95) | def construct_requests(self, doc, ctx):
    method process_results (line 110) | def process_results(self, doc, results):
    method aggregation (line 124) | def aggregation(self):
    method higher_is_better (line 132) | def higher_is_better(self):
  class StoryCloze2016 (line 141) | class StoryCloze2016(StoryCloze):
  class StoryCloze2018 (line 145) | class StoryCloze2018(StoryCloze):

FILE: lm_eval/tasks/superglue.py
  class BoolQ (line 35) | class BoolQ(Task):
    method has_training_docs (line 40) | def has_training_docs(self):
    method has_validation_docs (line 43) | def has_validation_docs(self):
    method has_test_docs (line 46) | def has_test_docs(self):
    method training_docs (line 49) | def training_docs(self):
    method validation_docs (line 54) | def validation_docs(self):
    method doc_to_text (line 57) | def doc_to_text(self, doc):
    method should_decontaminate (line 60) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 63) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 66) | def doc_to_target(self, doc):
    method construct_requests (line 69) | def construct_requests(self, doc, ctx):
    method process_results (line 76) | def process_results(self, doc, results):
    method higher_is_better (line 84) | def higher_is_better(self):
    method aggregation (line 87) | def aggregation(self):
  class CommitmentBank (line 91) | class CommitmentBank(Task):
    method has_training_docs (line 96) | def has_training_docs(self):
    method has_validation_docs (line 99) | def has_validation_docs(self):
    method has_test_docs (line 102) | def has_test_docs(self):
    method training_docs (line 105) | def training_docs(self):
    method validation_docs (line 110) | def validation_docs(self):
    method doc_to_text (line 113) | def doc_to_text(self, doc):
    method doc_to_target (line 119) | def doc_to_target(self, doc):
    method construct_requests (line 125) | def construct_requests(self, doc, ctx):
    method process_results (line 132) | def process_results(self, doc, results):
    method higher_is_better (line 139) | def higher_is_better(self):
    method cb_multi_fi (line 143) | def cb_multi_fi(cls, items):
    method aggregation (line 153) | def aggregation(self):
  class Copa (line 160) | class Copa(Task):
    method has_training_docs (line 165) | def has_training_docs(self):
    method has_validation_docs (line 168) | def has_validation_docs(self):
    method has_test_docs (line 171) | def has_test_docs(self):
    method training_docs (line 174) | def training_docs(self):
    method validation_docs (line 179) | def validation_docs(self):
    method doc_to_text (line 182) | def doc_to_text(self, doc):
    method doc_to_target (line 190) | def doc_to_target(self, doc):
    method construct_requests (line 195) | def construct_requests(self, doc, ctx):
    method process_results (line 204) | def process_results(self, doc, results):
    method higher_is_better (line 211) | def higher_is_better(self):
    method aggregation (line 214) | def aggregation(self):
    method convert_choice (line 218) | def convert_choice(choice):
  class MultiRC (line 222) | class MultiRC(Task):
    method has_training_docs (line 227) | def has_training_docs(self):
    method has_validation_docs (line 230) | def has_validation_docs(self):
    method has_test_docs (line 233) | def has_test_docs(self):
    method training_docs (line 236) | def training_docs(self):
    method validation_docs (line 241) | def validation_docs(self):
    method doc_to_text (line 244) | def doc_to_text(self, doc):
    method doc_to_target (line 247) | def doc_to_target(self, doc):
    method format_answer (line 251) | def format_answer(answer, label):
    method construct_requests (line 255) | def construct_requests(self, doc, ctx):
    method process_results (line 264) | def process_results(self, doc, results):
    method higher_is_better (line 269) | def higher_is_better(self):
    method aggregation (line 272) | def aggregation(self):
  class ReCoRD (line 276) | class ReCoRD(Task):
    method has_training_docs (line 281) | def has_training_docs(self):
    method has_validation_docs (line 284) | def has_validation_docs(self):
    method has_test_docs (line 287) | def has_test_docs(self):
    method training_docs (line 290) | def training_docs(self):
    method validation_docs (line 299) | def validation_docs(self):
    method _process_doc (line 305) | def _process_doc(cls, doc):
    method doc_to_text (line 313) | def doc_to_text(self, doc):
    method format_answer (line 321) | def format_answer(cls, query, entity):
    method doc_to_target (line 324) | def doc_to_target(self, doc):
    method construct_requests (line 328) | def construct_requests(self, doc, ctx):
    method process_results (line 335) | def process_results(self, doc, results):
    method higher_is_better (line 356) | def higher_is_better(self):
    method aggregation (line 362) | def aggregation(self):
  class WordsInContext (line 369) | class WordsInContext(Task):
    method has_training_docs (line 374) | def has_training_docs(self):
    method has_validation_docs (line 377) | def has_validation_docs(self):
    method has_test_docs (line 380) | def has_test_docs(self):
    method training_docs (line 383) | def training_docs(self):
    method validation_docs (line 388) | def validation_docs(self):
    method doc_to_text (line 391) | def doc_to_text(self, doc):
    method doc_to_target (line 401) | def doc_to_target(self, doc):
    method construct_requests (line 404) | def construct_requests(self, doc, ctx):
    method process_results (line 410) | def process_results(self, doc, results):
    method higher_is_better (line 418) | def higher_is_better(self):
    method aggregation (line 421) | def aggregation(self):
  class SGWinogradSchemaChallenge (line 425) | class SGWinogradSchemaChallenge(Task):
    method has_training_docs (line 432) | def has_training_docs(self):
    method has_validation_docs (line 435) | def has_validation_docs(self):
    method has_test_docs (line 438) | def has_test_docs(self):
    method training_docs (line 441) | def training_docs(self):
    method validation_docs (line 450) | def validation_docs(self):
    method doc_to_text (line 453) | def doc_to_text(self, doc):
    method doc_to_target (line 468) | def doc_to_target(self, doc):
    method construct_requests (line 471) | def construct_requests(self, doc, ctx):
    method process_results (line 478) | def process_results(self, doc, results):
    method higher_is_better (line 486) | def higher_is_better(self):
    method aggregation (line 489) | def aggregation(self):

FILE: lm_eval/tasks/swag.py
  class SWAG (line 28) | class SWAG(MultipleChoiceTask):
    method has_training_docs (line 33) | def has_training_docs(self):
    method has_validation_docs (line 36) | def has_validation_docs(self):
    method has_test_docs (line 39) | def has_test_docs(self):
    method training_docs (line 42) | def training_docs(self):
    method validation_docs (line 47) | def validation_docs(self):
    method _process_doc (line 50) | def _process_doc(self, doc):
    method doc_to_text (line 58) | def doc_to_text(self, doc):

FILE: lm_eval/tasks/toxigen.py
  class ToxiGen (line 24) | class ToxiGen(MultipleChoiceTask):
    method has_training_docs (line 29) | def has_training_docs(self):
    method has_validation_docs (line 32) | def has_validation_docs(self):
    method has_test_docs (line 35) | def has_test_docs(self):
    method training_docs (line 38) | def training_docs(self):
    method test_docs (line 48) | def test_docs(self):
    method _preprocess_dataset (line 53) | def _preprocess_dataset(self, split: str):
    method _process_doc (line 62) | def _process_doc(self, doc):
    method doc_to_text (line 69) | def doc_to_text(self, doc):

FILE: lm_eval/tasks/translation.py
  function create_tasks_from_benchmarks (line 52) | def create_tasks_from_benchmarks(benchmark_dict):
  function zh_split (line 78) | def zh_split(zh_text: List[str]) -> List[str]:
  function ja_split (line 89) | def ja_split(ja_text: List[str]) -> List[str]:
  function create_translation_task (line 107) | def create_translation_task(dataset, language_pair, version=0):
  class GeneralTranslationTask (line 117) | class GeneralTranslationTask(Task):
    method __init__ (line 121) | def __init__(self, sacrebleu_dataset, sacrebleu_language_pair=None):
    method download (line 128) | def download(self, data_dir=None, cache_dir=None, download_mode=None):
    method has_training_docs (line 138) | def has_training_docs(self):
    method has_validation_docs (line 143) | def has_validation_docs(self):
    method has_test_docs (line 147) | def has_test_docs(self):
    method test_docs (line 151) | def test_docs(self):
    method doc_to_text (line 160) | def doc_to_text(self, doc):
    method should_decontaminate (line 166) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 169) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 172) | def doc_to_target(self, doc):
    method construct_requests (line 176) | def construct_requests(self, doc, ctx):
    method process_results (line 189) | def process_results(self, doc, results):
    method aggregation (line 205) | def aggregation(self):
    method higher_is_better (line 217) | def higher_is_better(self):
    method __str__ (line 229) | def __str__(self):
  function code_to_language (line 241) | def code_to_language(code):

FILE: lm_eval/tasks/triviaqa.py
  class TriviaQA (line 31) | class TriviaQA(Task):
    method has_training_docs (line 36) | def has_training_docs(self):
    method has_validation_docs (line 39) | def has_validation_docs(self):
    method has_test_docs (line 42) | def has_test_docs(self):
    method training_docs (line 45) | def training_docs(self):
    method validation_docs (line 48) | def validation_docs(self):
    method test_docs (line 51) | def test_docs(self):
    method doc_to_text (line 54) | def doc_to_text(self, doc):
    method should_decontaminate (line 57) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 60) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 63) | def doc_to_target(self, doc):
    method _remove_prefixes (line 66) | def _remove_prefixes(self, aliases):
    method construct_requests (line 76) | def construct_requests(self, doc, ctx):
    method process_results (line 83) | def process_results(self, doc, results):
    method aggregation (line 86) | def aggregation(self):
    method higher_is_better (line 91) | def higher_is_better(self):

FILE: lm_eval/tasks/truthfulqa.py
  class TruthfulQAMultipleChoice (line 67) | class TruthfulQAMultipleChoice(Task):
    method has_training_docs (line 72) | def has_training_docs(self):
    method has_validation_docs (line 75) | def has_validation_docs(self):
    method has_test_docs (line 78) | def has_test_docs(self):
    method training_docs (line 81) | def training_docs(self):
    method validation_docs (line 84) | def validation_docs(self):
    method test_docs (line 87) | def test_docs(self):
    method doc_to_text (line 90) | def doc_to_text(self, doc):
    method should_decontaminate (line 93) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 96) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 99) | def doc_to_target(self, doc):
    method fewshot_context (line 102) | def fewshot_context(
    method construct_requests (line 112) | def construct_requests(self, doc, ctx):
    method process_results (line 133) | def process_results(self, doc, results):
    method aggregation (line 161) | def aggregation(self):
    method higher_is_better (line 164) | def higher_is_better(self):
  class TruthfulQAGeneration (line 168) | class TruthfulQAGeneration(Task):
    method __init__ (line 173) | def __init__(self):
    method has_training_docs (line 183) | def has_training_docs(self):
    method has_validation_docs (line 186) | def has_validation_docs(self):
    method has_test_docs (line 189) | def has_test_docs(self):
    method training_docs (line 192) | def training_docs(self):
    method _format_answers (line 195) | def _format_answers(self, answers):
    method validation_docs (line 207) | def validation_docs(self):
    method test_docs (line 219) | def test_docs(self):
    method doc_to_text (line 222) | def doc_to_text(self, doc):
    method doc_to_target (line 225) | def doc_to_target(self, doc):
    method fewshot_context (line 228) | def fewshot_context(
    method construct_requests (line 238) | def construct_requests(self, doc, ctx):
    method process_results (line 253) | def process_results(self, doc, results):
    method aggregation (line 332) | def aggregation(self):
    method higher_is_better (line 351) | def higher_is_better(self):
    method bleu (line 370) | def bleu(self, refs, preds):
    method rouge (line 392) | def rouge(self, refs, preds):

FILE: lm_eval/tasks/unscramble.py
  class WordUnscrambleTask (line 32) | class WordUnscrambleTask(Task):
    method has_training_docs (line 37) | def has_training_docs(self):
    method has_validation_docs (line 40) | def has_validation_docs(self):
    method has_test_docs (line 43) | def has_test_docs(self):
    method validation_docs (line 46) | def validation_docs(self):
    method doc_to_text (line 49) | def doc_to_text(self, doc):
    method should_decontaminate (line 52) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 55) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 58) | def doc_to_target(self, doc):
    method construct_requests (line 61) | def construct_requests(self, doc, ctx):
    method process_results (line 65) | def process_results(self, doc, results):
    method aggregation (line 70) | def aggregation(self):
    method higher_is_better (line 73) | def higher_is_better(self):
  class Anagrams1 (line 77) | class Anagrams1(WordUnscrambleTask):
  class Anagrams2 (line 81) | class Anagrams2(WordUnscrambleTask):
  class CycleLetters (line 85) | class CycleLetters(WordUnscrambleTask):
  class RandomInsertion (line 89) | class RandomInsertion(WordUnscrambleTask):
  class ReversedWords (line 93) | class ReversedWords(WordUnscrambleTask):

FILE: lm_eval/tasks/webqs.py
  class WebQs (line 34) | class WebQs(Task):
    method has_training_docs (line 39) | def has_training_docs(self):
    method has_validation_docs (line 42) | def has_validation_docs(self):
    method has_test_docs (line 45) | def has_test_docs(self):
    method training_docs (line 48) | def training_docs(self):
    method test_docs (line 53) | def test_docs(self):
    method doc_to_text (line 56) | def doc_to_text(self, doc):
    method should_decontaminate (line 59) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 62) | def doc_to_decontamination_query(self, doc):
    method doc_to_target (line 65) | def doc_to_target(self, doc):
    method _remove_prefixes (line 71) | def _remove_prefixes(self, aliases):
    method construct_requests (line 82) | def construct_requests(self, doc, ctx):
    method process_results (line 89) | def process_results(self, doc, results):
    method aggregation (line 92) | def aggregation(self):
    method higher_is_better (line 97) | def higher_is_better(self):

FILE: lm_eval/tasks/wikitext.py
  function wikitext_detokenizer (line 28) | def wikitext_detokenizer(string):
  class WikiText (line 62) | class WikiText(PerplexityTask):
    method has_training_docs (line 67) | def has_training_docs(self):
    method has_validation_docs (line 70) | def has_validation_docs(self):
    method has_test_docs (line 73) | def has_test_docs(self):
    method training_docs (line 76) | def training_docs(self):
    method validation_docs (line 79) | def validation_docs(self):
    method test_docs (line 82) | def test_docs(self):
    method _process_doc (line 85) | def _process_doc(self, doc):
    method doc_to_target (line 88) | def doc_to_target(self, doc):
    method should_decontaminate (line 91) | def should_decontaminate(self):
    method count_words (line 94) | def count_words(self, doc):

FILE: lm_eval/tasks/winogrande.py
  class Winogrande (line 32) | class Winogrande(Task):
    method has_training_docs (line 39) | def has_training_docs(self):
    method has_validation_docs (line 42) | def has_validation_docs(self):
    method has_test_docs (line 45) | def has_test_docs(self):
    method training_docs (line 48) | def training_docs(self):
    method validation_docs (line 53) | def validation_docs(self):
    method doc_to_text (line 56) | def doc_to_text(self, doc):
    method should_decontaminate (line 59) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 62) | def doc_to_decontamination_query(self, doc):
    method partial_context (line 66) | def partial_context(cls, doc, option):
    method doc_to_target (line 72) | def doc_to_target(self, doc):
    method partial_target (line 76) | def partial_target(cls, doc):
    method construct_requests (line 81) | def construct_requests(self, doc, ctx):
    method append_context (line 101) | def append_context(cls, ctx, partial_ctx):
    method process_results (line 106) | def process_results(self, doc, results):
    method aggregation (line 118) | def aggregation(self):
    method higher_is_better (line 126) | def higher_is_better(self):

FILE: lm_eval/tasks/wsc273.py
  class WinogradSchemaChallenge273 (line 38) | class WinogradSchemaChallenge273(Task):
    method has_training_docs (line 57) | def has_training_docs(self):
    method has_validation_docs (line 60) | def has_validation_docs(self):
    method has_test_docs (line 63) | def has_test_docs(self):
    method test_docs (line 66) | def test_docs(self):
    method _process_doc (line 69) | def _process_doc(self, doc):
    method __normalize_option (line 76) | def __normalize_option(self, doc, option):
    method fewshot_examples (line 87) | def fewshot_examples(self, k, rnd):
    method doc_to_text (line 96) | def doc_to_text(self, doc):
    method should_decontaminate (line 99) | def should_decontaminate(self):
    method doc_to_decontamination_query (line 102) | def doc_to_decontamination_query(self, doc):
    method partial_context (line 106) | def partial_context(cls, doc, option):
    method doc_to_target (line 111) | def doc_to_target(self, doc):
    method partial_target (line 115) | def partial_target(cls, doc):
    method construct_requests (line 120) | def construct_requests(self, doc, ctx):
    method append_context (line 140) | def append_context(cls, ctx, partial_ctx):
    method process_results (line 145) | def process_results(self, doc, results):
    method aggregation (line 157) | def aggregation(self):
    method higher_is_better (line 165) | def higher_is_better(self):

FILE: lm_eval/utils.py
  class ExitCodeError (line 13) | class ExitCodeError(Exception):
  function sh (line 17) | def sh(x):
  function simple_parse_args_string (line 22) | def simple_parse_args_string(args_string):
  function join_iters (line 36) | def join_iters(iters):
  function chunks (line 41) | def chunks(iter, n):
  function group (line 53) | def group(arr, fn):
  function general_detokenize (line 62) | def general_detokenize(string):
  function get_rolling_token_windows (line 72) | def get_rolling_token_windows(token_list, prefix_token, max_seq_len, con...
  function make_disjoint_window (line 113) | def make_disjoint_window(pair):
  class Reorderer (line 119) | class Reorderer:
    method __init__ (line 120) | def __init__(self, arr, fn):
    method get_reordered (line 129) | def get_reordered(self):
    method get_original (line 132) | def get_original(self, newarr):
  function positional_deprecated (line 146) | def positional_deprecated(fn):
  function find_test_root (line 166) | def find_test_root(start_path: pathlib.Path) -> pathlib.Path:
  function run_task_tests (line 184) | def run_task_tests(task_list: List[str]):

FILE: main.py
  function evaluate (line 54) | def evaluate(lm, args, logger):
  function main (line 189) | def main():

FILE: models/IRQLoRALMClass.py
  class IRQLoRALMClass (line 23) | class IRQLoRALMClass(BaseLM):
    method __init__ (line 24) | def __init__(self, args):
    method eot_token (line 62) | def eot_token(self) -> str:
    method eot_token_id (line 66) | def eot_token_id(self):
    method max_length (line 71) | def max_length(self):
    method max_gen_toks (line 79) | def max_gen_toks(self):
    method batch_size (line 84) | def batch_size(self):
    method device (line 89) | def device(self):
    method tok_encode (line 93) | def tok_encode(self, string: str):
    method tok_encode_batch (line 96) | def tok_encode_batch(self, strings):
    method tok_decode (line 104) | def tok_decode(self, tokens):
    method _model_call (line 107) | def _model_call(self, inps):
    method model_batched_set (line 118) | def model_batched_set(self, inps):
    method _model_generate (line 127) | def _model_generate(self, context, max_length, eos_token_id):

FILE: models/LMClass.py
  class LMClass (line 12) | class LMClass(BaseLM):
    method __init__ (line 13) | def __init__(self, args):
    method eot_token (line 36) | def eot_token(self) -> str:
    method eot_token_id (line 40) | def eot_token_id(self):
    method max_length (line 45) | def max_length(self):
    method max_gen_toks (line 53) | def max_gen_toks(self):
    method batch_size (line 58) | def batch_size(self):
    method device (line 63) | def device(self):
    method tok_encode (line 67) | def tok_encode(self, string: str):
    method tok_encode_batch (line 70) | def tok_encode_batch(self, strings):
    method tok_decode (line 78) | def tok_decode(self, tokens):
    method _model_call (line 81) | def _model_call(self, inps):
    method model_batched_set (line 92) | def model_batched_set(self, inps):
    method _model_generate (line 101) | def _model_generate(self, context, max_length, eos_token_id):

FILE: models/int_falcon_layer.py
  class QuantFalconMLP (line 20) | class QuantFalconMLP(nn.Module):
    method __init__ (line 21) | def __init__(self, org_module: nn.Module,args=None):
    method forward (line 28) | def forward(self, x: torch.Tensor) -> torch.Tensor:
  class QuantFalconAttention (line 35) | class QuantFalconAttention(nn.Module):
    method __init__ (line 36) | def __init__(self,  config: FalconConfig, org_module: nn.Module, args=...
    method _split_heads (line 65) | def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor,...
    method _merge_heads (line 97) | def _merge_heads(self, x: torch.Tensor) -> torch.Tensor:
    method forward (line 122) | def forward(
  class QuantFalconDecoderLayer (line 239) | class QuantFalconDecoderLayer(nn.Module):
    method __init__ (line 240) | def __init__(self, config: FalconConfig,
    method forward (line 261) | def forward(
    method set_quant_state (line 318) | def set_quant_state(self, weight_quant: bool = False, act_quant: bool ...
    method smooth_and_quant_inplace (line 329) | def smooth_and_quant_inplace(self):
    method clear_temp_variable (line 338) | def clear_temp_variable(self):
    method smooth_and_quant_temporary (line 344) | def smooth_and_quant_temporary(self):
    method let_parameters (line 363) | def let_parameters(self, use_shift=True):
    method lwc_parameters (line 371) | def lwc_parameters(self):
    method omni_parameters (line 378) | def omni_parameters(self, use_shift=True):
    method omni_state_dict (line 386) | def omni_state_dict(self, destination=None, prefix='', keep_vars=False):
    method register_scales_and_zeros (line 394) | def register_scales_and_zeros(self):

FILE: models/int_llama_layer.py
  class QuantLlamaMLP (line 20) | class QuantLlamaMLP(nn.Module):
    method __init__ (line 21) | def __init__(
    method forward (line 44) | def forward(self, x):
  class QuantLlamaAttention (line 48) | class QuantLlamaAttention(nn.Module):
    method __init__ (line 51) | def __init__(self,
    method _shape (line 100) | def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
    method forward (line 103) | def forward(
    method set_quant_state (line 181) | def set_quant_state(self, weight_quant: bool = False, act_quant: bool ...
  class QuantLlamaDecoderLayer (line 191) | class QuantLlamaDecoderLayer(nn.Module):
    method __init__ (line 192) | def __init__(self,
    method forward (line 213) | def forward(
    method set_quant_state (line 269) | def set_quant_state(self, weight_quant: bool = False, act_quant: bool ...
    method smooth_and_quant_temporary (line 279) | def smooth_and_quant_temporary(self):
    method clear_temp_variable (line 309) | def clear_temp_variable(self):
    method smooth_and_quant_inplace (line 316) | def smooth_and_quant_inplace(self):
    method let_parameters (line 334) | def let_parameters(self, use_shift=True):
    method lwc_parameters (line 342) | def lwc_parameters(self):
    method omni_parameters (line 349) | def omni_parameters(self, use_shift=True):
    method omni_state_dict (line 357) | def omni_state_dict(self, destination=None, prefix='', keep_vars=False):
    method register_scales_and_zeros (line 365) | def register_scales_and_zeros(self):

FILE: models/int_opt_layer.py
  class QuantOPTAttention (line 16) | class QuantOPTAttention(nn.Module):
    method __init__ (line 19) | def __init__(
    method _shape (line 74) | def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
    method forward (line 81) | def forward(
    method set_quant_state (line 215) | def set_quant_state(self, weight_quant: bool = False, act_quant: bool ...
  class QuantOPTDecoderLayer (line 230) | class QuantOPTDecoderLayer(nn.Module):
    method __init__ (line 231) | def __init__(
    method forward (line 268) | def forward(
    method set_quant_state (line 348) | def set_quant_state(self, weight_quant: bool = False, act_quant: bool ...
    method smooth_and_quant_inplace (line 359) | def smooth_and_quant_inplace(self):
    method clear_temp_variable (line 379) | def clear_temp_variable(self):
    method smooth_and_quant_temporary (line 385) | def smooth_and_quant_temporary(self):
    method let_parameters (line 416) | def let_parameters(self, use_shift=True):
    method lwc_parameters (line 424) | def lwc_parameters(self):
    method omni_parameters (line 431) | def omni_parameters(self, use_shift=True):
    method omni_state_dict (line 439) | def omni_state_dict(self, destination=None, prefix='', keep_vars=False):
    method register_scales_and_zeros (line 448) | def register_scales_and_zeros(self):

FILE: models/models_utils.py
  class TruncateFunction (line 13) | class TruncateFunction(torch.autograd.Function):
    method forward (line 15) | def forward(ctx, input, threshold):
    method backward (line 22) | def backward(ctx, grad_output):
  function truncate_number (line 26) | def truncate_number(number, threshold=1e-3):
  function find_layers (line 30) | def find_layers(module, layers=[nn.Conv2d, nn.Linear, transformers.Conv1...
  class CacheHook (line 43) | class CacheHook:
    method __init__ (line 44) | def __init__(self, cachinglm):
    method add_partial (line 51) | def add_partial(self, attr, req, res):
  class LM (line 58) | class LM(abc.ABC):
    method __init__ (line 59) | def __init__(self):
    method loglikelihood (line 63) | def loglikelihood(self, requests):
    method loglikelihood_rolling (line 87) | def loglikelihood_rolling(self, requests):
    method greedy_until (line 130) | def greedy_until(self, requests):
    method create_from_arg_string (line 148) | def create_from_arg_string(cls, additional_config=None):
    method set_cache_hook (line 153) | def set_cache_hook(self, cache_hook):
  class BaseLM (line 157) | class BaseLM(LM):
    method eot_token_id (line 160) | def eot_token_id(self):
    method max_length (line 165) | def max_length(self):
    method max_gen_toks (line 170) | def max_gen_toks(self):
    method batch_size (line 175) | def batch_size(self):
    method device (line 180) | def device(self):
    method tok_encode (line 184) | def tok_encode(self, string: str):
    method tok_decode (line 188) | def tok_decode(self, tokens: Iterable[int]):
    method _model_generate (line 192) | def _model_generate(self, context, max_length, eos_token_id):
    method _model_call (line 196) | def _model_call(self, inps):
    method loglikelihood (line 209) | def loglikelihood(self, requests):
    method loglikelihood_rolling (line 223) | def loglikelihood_rolling(self, requests):
    method _loglikelihood_tokens (line 257) | def _loglikelihood_tokens(self, requests, disable_tqdm=False):
    method greedy_until (line 434) | def greedy_until(self, requests):
  function make_disjoint_window (line 475) | def make_disjoint_window(pair):
  function hash_args (line 481) | def hash_args(attr, args):
  function simple_parse_args_string (line 486) | def simple_parse_args_string(args_string):
  function get_rolling_token_windows (line 503) | def get_rolling_token_windows(token_list, prefix_token, max_seq_len, con...
  class Reorderer (line 544) | class Reorderer:
    method __init__ (line 545) | def __init__(self, arr, fn):
    method get_reordered (line 554) | def get_reordered(self):
    method get_original (line 557) | def get_original(self, newarr):
  function join_iters (line 571) | def join_iters(iters):
  function chunks (line 576) | def chunks(iter, n):
  function group (line 588) | def group(arr, fn):

FILE: models/transformation.py
  class TruncateFunction (line 5) | class TruncateFunction(torch.autograd.Function):
    method forward (line 7) | def forward(ctx, input, threshold):
    method backward (line 14) | def backward(ctx, grad_output):
  function truncate_number (line 18) | def truncate_number(number, threshold=1e-2):
  function smooth_ln_fcs_temporary (line 24) | def smooth_ln_fcs_temporary(ln, fcs, scales,shifts):
  function smooth_fc_fc_temporary (line 44) | def smooth_fc_fc_temporary(fc1, fc2, scales,shifts=None):
  function smooth_q_k_temporary (line 63) | def smooth_q_k_temporary(q_proj, k_proj, scales):
  function smooth_ln_fcs_inplace (line 71) | def smooth_ln_fcs_inplace(ln, fcs, scales,shifts):
  function smooth_fc_fc_inplace (line 93) | def smooth_fc_fc_inplace(fc1, fc2, scales,shifts=None):
  function smooth_q_k_inplace (line 108) | def smooth_q_k_inplace(q_proj, k_proj, scales,):

FILE: parallel_utils.py
  function nvidia_smi_memory_info (line 12) | def nvidia_smi_memory_info():
  function get_gpu_memory (line 42) | def get_gpu_memory():
  function get_lowest_occupied_gpu (line 59) | def get_lowest_occupied_gpu(wait_memory=1000):
  function sort_layers_by_params (line 74) | def sort_layers_by_params(layers: List[nn.Module]):
  function get_all_gpu_free_memory (line 80) | def get_all_gpu_free_memory():
  function assign_layers_to_gpus (line 89) | def assign_layers_to_gpus(layers: List[nn.Module]):
  function forward_hook_wrapper (line 135) | def forward_hook_wrapper(gpu_id):
  function add_forward_hooks (line 148) | def add_forward_hooks(layer_gpu_map):
  function map_layers_to_multi_gpus (line 159) | def map_layers_to_multi_gpus(layers):

FILE: quant/int_linear.py
  class QuantLinear (line 11) | class QuantLinear(nn.Module):
    method __init__ (line 16) | def __init__(
    method forward (line 48) | def forward(self, input: torch.Tensor):
    method set_quant_state (line 67) | def set_quant_state(self, weight_quant: bool = False, act_quant: bool ...

FILE: quant/int_matmul.py
  class QuantMatMul (line 7) | class QuantMatMul(nn.Module):
    method __init__ (line 8) | def __init__(
    method set_quant_state (line 27) | def set_quant_state(self, weight_quant: bool = False, act_quant: bool ...
    method quant_x1 (line 31) | def quant_x1(self, x1):
    method quant_x2 (line 36) | def quant_x2(self, x2):
    method forward (line 41) | def forward(self, x1, x2):

FILE: quant/omni_norm.py
  class OmniLayerNorm (line 11) | class OmniLayerNorm(nn.Module):
    method __init__ (line 12) | def __init__(self, ori_layer_norm) -> None:
    method forward (line 26) | def forward(self, x):
    method set_quant_state (line 36) | def set_quant_state(self, use_weight_quant, use_act_quant):
  class OmniLlamaRMSNorm (line 40) | class OmniLlamaRMSNorm(nn.Module):
    method __init__ (line 41) | def __init__(self, ori_norm, eps=1e-6):
    method forward (line 52) | def forward(self, hidden_states):

FILE: quant/omniquant.py
  function get_named_linears (line 25) | def get_named_linears(module):
  function add_new_module (line 29) | def add_new_module(name, original_module, added_module):
  function omniquant (line 42) | def omniquant(

FILE: quant/quantizer.py
  function round_ste (line 15) | def round_ste(x: torch.Tensor):
  class UniformAffineQuantizer (line 23) | class UniformAffineQuantizer(nn.Module):
    method __init__ (line 24) | def __init__(
    method change_n_bits (line 85) | def change_n_bits(self, n_bits):
    method fake_quant (line 94) | def fake_quant(self, x, scale, round_zero_point):
    method forward (line 118) | def forward(self, x: torch.Tensor):
    method per_token_dynamic_calibration (line 132) | def per_token_dynamic_calibration(self, x):
    method register_scales_and_zeros (line 161) | def register_scales_and_zeros(self):

FILE: quant/utils.py
  function let_parameters (line 8) | def let_parameters(model, use_shift=True):
  function lwc_parameters (line 16) | def lwc_parameters(model):
  function get_omni_parameters (line 23) | def get_omni_parameters(model, use_shift=True):
  function omni_state_dict (line 31) | def omni_state_dict(model, destination=None, prefix='', keep_vars=False):
  function register_scales_and_zeros (line 39) | def register_scales_and_zeros(model):
  class TruncateFunction (line 44) | class TruncateFunction(torch.autograd.Function):
    method forward (line 46) | def forward(ctx, input, threshold):
    method backward (line 53) | def backward(ctx, grad_output):
  function truncate_number (line 58) | def truncate_number(number, threshold=1e-2):
  function smooth_and_quant_temporary (line 62) | def smooth_and_quant_temporary(model, args, isllama):
  function clear_temp_variable (line 103) | def clear_temp_variable(model):
  function smooth_and_quant_inplace (line 112) | def smooth_and_quant_inplace(model, args, isllama):
  function set_quant_state (line 138) | def set_quant_state(self, weight_quant: bool = False, act_quant: bool = ...

FILE: utils.py
  function ampscaler_get_grad_norm (line 12) | def ampscaler_get_grad_norm(parameters, norm_type: float = 2.0) -> torch...
  class NativeScalerWithGradNormCount (line 27) | class NativeScalerWithGradNormCount:
    method __init__ (line 30) | def __init__(self):
    method __call__ (line 33) | def __call__(self, loss, optimizer, clip_grad=None, parameters=None, c...
    method state_dict (line 49) | def state_dict(self):
    method load_state_dict (line 52) | def load_state_dict(self, state_dict):
  function create_logger (line 56) | def create_logger(output_dir, dist_rank=0, name=''):