SYMBOL INDEX (141 symbols across 21 files)

FILE: paq/download.py
  function untar (line 340) | def untar(tar_filename: str) -> List[str]:
  function unpack (line 349) | def unpack(gzip_file: str, out_file: str):
  function _get_root_dir (line 360) | def _get_root_dir(out_dir):
  function download_resource (line 371) | def download_resource(
  function download_file (line 414) | def download_file(s3_url: str, out_dir: str, file_name: str):
  function download (line 426) | def download(resource_key: str, out_dir: str = None):
  function main (line 484) | def main():

FILE: paq/evaluation/eval_reranker.py
  function evaluate_exact_match (line 12) | def evaluate_exact_match(preds, refs):

FILE: paq/evaluation/eval_retriever.py
  function eval_retriever (line 12) | def eval_retriever(refs, preds, hits_at_k):

FILE: paq/evaluation/eval_utils.py
  function normalize_answer (line 12) | def normalize_answer(s):
  function exact_match_score (line 31) | def exact_match_score(prediction, ground_truth):
  function metric_max_over_ground_truths (line 35) | def metric_max_over_ground_truths(metric_fn, predictions: Union[str, Lis...

FILE: paq/generation/answer_extractor/extract_answers.py
  function load_passages (line 16) | def load_passages(path):
  function extract_answers (line 23) | def extract_answers(config, input_file, verbose):
  function extract_answers_and_write_to_file (line 31) | def extract_answers_and_write_to_file(config, input_path, output_path, v...

FILE: paq/generation/answer_extractor/extractors.py
  function get_output_format (line 17) | def get_output_format(all_passages, all_answers):
  class SpacyNERExtractor (line 31) | class SpacyNERExtractor:
    method __init__ (line 37) | def __init__(self, model="en_core_web_sm"):
    method extract_from_passage (line 42) | def extract_from_passage(self, passage: str) -> List[Dict]:
    method extract_answers_from_passages (line 54) | def extract_answers_from_passages(self, passages_to_label, disable_tqd...
  class Span2DAnswerExtractor (line 72) | class Span2DAnswerExtractor:
    method __init__ (line 78) | def __init__(
    method _tokenize (line 112) | def _tokenize(self, passage: str):
    method extract_from_passage (line 127) | def extract_from_passage(self, passage: str):
    method extract_answers_from_passages (line 135) | def extract_answers_from_passages(self, passages_to_label, disable_tqd...
  function load_answer_extractor (line 148) | def load_answer_extractor(config):

FILE: paq/generation/answer_extractor/span2D_model.py
  class AnswerSpanExtractor2DModelOutput (line 21) | class AnswerSpanExtractor2DModelOutput(ModelOutput):
  class AnswerSpanExtractor2DModel (line 50) | class AnswerSpanExtractor2DModel(BertPreTrainedModel):
    method __init__ (line 53) | def __init__(self, config):
    method forward (line 78) | def forward(
  function sigmoid (line 171) | def sigmoid(x):
  function postprocess_span2d_output (line 175) | def postprocess_span2d_output(span2D_output: AnswerSpanExtractor2DModelO...

FILE: paq/generation/filtering/filter_questions.py
  function retrieve_documents_for_generated_questions (line 16) | def retrieve_documents_for_generated_questions(config, input_file, verbo...
  function generate_answers_for_generated_questions_with_retrieved_docs (line 24) | def generate_answers_for_generated_questions_with_retrieved_docs(config,...
  function filter_generated_questions_and_write_to_file (line 33) | def filter_generated_questions_and_write_to_file(config, input_path, out...

FILE: paq/generation/filtering/filterer.py
  function _load_corpus (line 32) | def _load_corpus(path):
  class DummyFilteringRetriever (line 41) | class DummyFilteringRetriever:
    method retrieve_documents (line 45) | def retrieve_documents(self, data):
  class LocalFilteringRetriever (line 49) | class LocalFilteringRetriever:
    method __init__ (line 54) | def __init__(self, corpus_path):
    method retrieve_documents (line 57) | def retrieve_documents(self, data):
  class DPRQuestionEncoder (line 70) | class DPRQuestionEncoder(nn.Module):
    method __init__ (line 73) | def __init__(self, model):
    method forward (line 77) | def forward(self, *args, **kwargs):
  class GlobalFilteringRetriever (line 82) | class GlobalFilteringRetriever:
    method __init__ (line 89) | def __init__(self,
    method _load_corpus (line 116) | def _load_corpus(self):
    method retrieve_documents (line 125) | def retrieve_documents(self, qa_pairs):
  class CompatableEncoderWrapper (line 138) | class CompatableEncoderWrapper(torch.nn.Module):
    method __init__ (line 141) | def __init__(self, encoder, use_checkpoint=False):
    method forward (line 145) | def forward(self, input_ids=None, attention_mask=None, **kwargs, ):
  class FIDReader (line 156) | class FIDReader:
    method __init__ (line 160) | def __init__(self,
    method _get_dataloader_for_examples (line 179) | def _get_dataloader_for_examples(self, examples):
    method generate_answers (line 197) | def generate_answers(self, examples):
  class DummyReader (line 227) | class DummyReader:
    method generate_answers (line 231) | def generate_answers(self, examples):
  function _get_reader_output_format (line 239) | def _get_reader_output_format(dataset):
  function load_reader (line 250) | def load_reader(config):
  function load_retriever (line 256) | def load_retriever(config):

FILE: paq/generation/generate_qa_pairs.py
  function touch (line 28) | def touch(path):
  function _run_pipeline_step (line 34) | def _run_pipeline_step(config, input_file, output_file, done_indicator, ...
  function run_passage_scoring (line 41) | def run_passage_scoring(config, input_file, output_dir, verbose=False):
  function run_answer_extraction (line 48) | def run_answer_extraction(config, input_file, output_dir, verbose=False):
  function run_question_generation (line 55) | def run_question_generation(config, input_file, output_dir, verbose=False):
  function run_filtering (line 62) | def run_filtering(config, input_file, output_dir, verbose=False):
  function combine_generated_files (line 69) | def combine_generated_files(document_ranker_file,
  function run_paq_generation_pipeline (line 107) | def run_paq_generation_pipeline(config: dict, input_file: str, output_di...
  function _is_job_finished (line 133) | def _is_job_finished(job_number, output_dir):

FILE: paq/generation/passage_scorer/score_passages.py
  function load_passages (line 16) | def load_passages(path):
  function score_passages (line 23) | def score_passages(config, input_file, verbose):
  function score_passages_and_write_to_file (line 31) | def score_passages_and_write_to_file(config, input_path, output_path, ve...

FILE: paq/generation/passage_scorer/scorer.py
  class DummyPassageScorer (line 13) | class DummyPassageScorer:
    method __init__ (line 19) | def __init__(self, default_score=0.0):
    method score_passage (line 22) | def score_passage(self, passage: Dict) -> float:
    method score_passages (line 25) | def score_passages(self, passages_to_label, disable_tqdm=False):
  class LookupPassageScorer (line 32) | class LookupPassageScorer:
    method __init__ (line 38) | def __init__(self, scores_file, default_score=-10000.0):
    method _load_passage_scores (line 42) | def _load_passage_scores(self, scores_file):
    method score_passage (line 48) | def score_passage(self, passage: Dict) -> float:
    method score_passages (line 51) | def score_passages(self, passages_to_label, disable_tqdm=False):
  class LearntPassageScorer (line 58) | class LearntPassageScorer:
    method __init__ (line 62) | def __init__(self,
    method _tokenize (line 80) | def _tokenize(self, texts):
    method score_passages (line 87) | def score_passages(self, passages_to_label, disable_tqdm=False):
  function load_passage_scorer (line 113) | def load_passage_scorer(config):

FILE: paq/generation/question_generator/generate_questions.py
  function generate_questions (line 16) | def generate_questions(config, input_file, verbose):
  function generate_questions_and_write_to_file (line 24) | def generate_questions_and_write_to_file(config, input_path, output_path...

FILE: paq/generation/question_generator/generator.py
  function _batch_iterator (line 24) | def _batch_iterator(context_answer_pairs,
  class QuestionGenerator (line 62) | class QuestionGenerator:
    method __init__ (line 65) | def __init__(
    method generate_question (line 104) | def generate_question(self, data: Union[str, List[str]]):
    method generate_questions_from_passage_answer_pairs (line 139) | def generate_questions_from_passage_answer_pairs(self, passage_answer_...
  function load_question_generator (line 171) | def load_question_generator(config):

FILE: paq/paq_utils.py
  function is_spacy_available (line 38) | def is_spacy_available():
  function is_submitit_available (line 42) | def is_submitit_available():
  function is_apex_available (line 46) | def is_apex_available():
  function to_fp16 (line 50) | def to_fp16(model):
  function load_jsonl_memory_friendly (line 58) | def load_jsonl_memory_friendly(fi):
  function load_jsonl_fast (line 70) | def load_jsonl_fast(fi):
  function load_jsonl (line 88) | def load_jsonl(fi, memory_friendly=False):
  function dump_jsonl (line 95) | def dump_jsonl(items, fi):
  function load_dpr_tsv (line 107) | def load_dpr_tsv(fi):
  function get_vectors_file_paths_in_vector_directory (line 117) | def get_vectors_file_paths_in_vector_directory(embeddings_dir):
  function parse_vectors_from_directory_chunks (line 124) | def parse_vectors_from_directory_chunks(embeddings_dir, half):
  function parse_vectors_from_directory_fast (line 138) | def parse_vectors_from_directory_fast(embeddings_dir):
  function parse_vectors_from_directory_memory_friendly (line 148) | def parse_vectors_from_directory_memory_friendly(embeddings_dir, size=No...
  function parse_vectors_from_directory (line 174) | def parse_vectors_from_directory(fi, memory_friendly=False, size=None, a...
  function get_submitit_executor (line 192) | def get_submitit_executor(n_jobs=10, comment="", partition='learnfair'):

FILE: paq/rerankers/rerank.py
  function load_reranker (line 23) | def load_reranker(model_name_or_path):
  function get_output_format (line 36) | def get_output_format(qas, prediction_indices, prediction_scores):
  function tokenize (line 48) | def tokenize(tokenizer, batch_qas, cuda, top_k):
  function predict (line 65) | def predict(model, tokenizer, qas, cuda=CUDA, bsz=16, fp16=False, top_k=...
  function run_predictions (line 111) | def run_predictions(qas_to_rerank_file, output_file, model_name_or_path,...
  function parse_files (line 126) | def parse_files(args):

FILE: paq/retrievers/build_index.py
  function get_vector_sample (line 19) | def get_vector_sample(cached_embeddings_path, sample_fraction):
  function get_vectors_dim (line 39) | def get_vectors_dim(cached_embeddings_path):
  function augment_vectors (line 46) | def augment_vectors(vectors, max_phi):
  function build_index_streaming (line 53) | def build_index_streaming(cached_embeddings_path,

FILE: paq/retrievers/embed.py
  function embed (line 21) | def embed(model, tokenizer, qas, bsz=256, cuda=CUDA, fp16=False):
  function embed_job (line 60) | def embed_job(qas_to_embed_path, model_name_or_path, output_file_name, n...

FILE: paq/retrievers/retrieve.py
  function get_output_format (line 23) | def get_output_format(qas_to_answer, qas_to_retrieve_from, top_indices, ...
  function _torch_mips (line 37) | def _torch_mips(index, query_batch, top_k):
  function _flat_index_mips (line 42) | def _flat_index_mips(index, query_batch, top_k):
  function _aux_dim_index_mips (line 46) | def _aux_dim_index_mips(index, query_batch, top_k):
  function _get_mips_function (line 53) | def _get_mips_function(index):
  function mips (line 62) | def mips(index, queries, top_k, n_queries_to_parallelize=256):
  function run_queries (line 88) | def run_queries(model, tokenizer, qas_to_retrieve_from, qas_to_answer, t...
  function _load_index_if_exists (line 102) | def _load_index_if_exists(faiss_index_path, precomputed_embeddings_dir, ...

FILE: paq/retrievers/retriever_utils.py
  function _get_proj_keys_from_state_dict (line 17) | def _get_proj_keys_from_state_dict(state_dict):
  function _get_proj_dim_from_model_path (line 25) | def _get_proj_dim_from_model_path(model_name_or_path):
  function load_retriever (line 34) | def load_retriever(model_name_or_path):
  class RetrieverEncoder (line 42) | class RetrieverEncoder(nn.Module):
    method __init__ (line 45) | def __init__(self, config, proj_dim):
    method from_pretrained (line 53) | def from_pretrained(cls, model_name_or_path):
    method forward (line 67) | def forward(self, *args, **kwargs):

FILE: paq/server/server.py
  class http_server (line 16) | class http_server:
    method __init__ (line 17) | def __init__(self, index, model, tokenizer, qas_to_retrieve_from, fp16):
  class WebServerHandler (line 28) | class WebServerHandler(BaseHTTPRequestHandler):
    method do_POST (line 31) | def do_POST(self):
  function main (line 62) | def main(args):