SYMBOL INDEX (1617 symbols across 153 files) FILE: docs/source/conf.py function setup (line 211) | def setup(app): FILE: kp_data_converter.py function heuristic_filter (line 24) | def heuristic_filter(src_token, tgts_token, tgts_str, opt): FILE: kp_evaluate.py function evaluate (line 20) | def evaluate(src_list, tgt_list, pred_list, function kp_results_to_str (line 211) | def kp_results_to_str(results_dict): function baseline_pred_loader (line 227) | def baseline_pred_loader(pred_path, model_name): function keyphrase_eval (line 247) | def keyphrase_eval(datasplit_name, src_path, tgt_path, pred_path, function summarize_scores (line 307) | def summarize_scores(score_dict, ckpt_name, function gather_eval_results (line 399) | def gather_eval_results(eval_root_dir, report_csv_dir=None, tokenizer=No... function init_opt (line 487) | def init_opt(): FILE: kp_gen_eval.py function scan_new_checkpoints (line 22) | def scan_new_checkpoints(ckpt_dir): function _get_parser (line 33) | def _get_parser(): FILE: kp_gen_eval_transfer.py function scan_new_checkpoints (line 29) | def scan_new_checkpoints(ckpt_dir): function scan_predictions (line 52) | def scan_predictions(exp_root_dir): function _get_parser (line 74) | def _get_parser(): FILE: kp_gen_magkp_transfer_labelling.py function scan_new_checkpoints (line 23) | def scan_new_checkpoints(ckpt_dir): function scan_predictions (line 46) | def scan_predictions(exp_root_dir): function _get_parser (line 68) | def _get_parser(): FILE: kp_generate.py function _get_parser (line 15) | def _get_parser(): FILE: onmt/bin/average_models.py function average_models (line 6) | def average_models(model_files, fp32=False): function main (line 39) | def main(): FILE: onmt/bin/build_vocab.py function build_vocab_main (line 11) | def build_vocab_main(opts): function _get_parser (line 57) | def _get_parser(): function main (line 63) | def main(): FILE: onmt/bin/release_model.py function get_ctranslate2_model_spec (line 6) | def get_ctranslate2_model_spec(opt): function main (line 25) | def main(): FILE: onmt/bin/server.py function start (line 14) | def start(config_file, function _get_parser (line 136) | def _get_parser(): function main (line 149) | def main(): FILE: onmt/bin/train.py function prepare_fields_transforms (line 32) | def prepare_fields_transforms(opt): function _init_train (line 60) | def _init_train(opt): function train (line 99) | def train(opt): function _get_parser (line 200) | def _get_parser(): function main (line 206) | def main(): FILE: onmt/bin/translate.py function translate (line 17) | def translate(opt): function _get_parser (line 41) | def _get_parser(): function main (line 49) | def main(): FILE: onmt/constants.py class DefaultTokens (line 4) | class DefaultTokens(object): class CorpusName (line 17) | class CorpusName(object): class SubwordMarker (line 23) | class SubwordMarker(object): class ModelTask (line 28) | class ModelTask(object): FILE: onmt/decoders/bart_decoder.py class BARTDecoder (line 15) | class BARTDecoder(DecoderBase): method __init__ (line 45) | def __init__(self, opt, embeddings, bart_model=None, prev_checkpoint=N... method from_opt (line 82) | def from_opt(cls, opt, embeddings, **kwargs): method init_state (line 88) | def init_state(self, src, memory_bank, enc_hidden): method map_state (line 92) | def map_state(self, fn): method forward (line 95) | def forward(self, tgt, function forward_bart_decoder (line 157) | def forward_bart_decoder( function extract_features (line 203) | def extract_features( FILE: onmt/decoders/cnn_decoder.py class CNNDecoder (line 14) | class CNNDecoder(DecoderBase): method __init__ (line 21) | def __init__(self, num_layers, hidden_size, attn_type, method from_opt (line 52) | def from_opt(cls, opt, embeddings): method init_state (line 64) | def init_state(self, _, memory_bank, enc_hidden): method map_state (line 69) | def map_state(self, fn): method detach_state (line 74) | def detach_state(self): method forward (line 77) | def forward(self, tgt, memory_bank, step=None, **kwargs): method update_dropout (line 131) | def update_dropout(self, dropout): FILE: onmt/decoders/decoder.py class DecoderBase (line 11) | class DecoderBase(nn.Module): method __init__ (line 18) | def __init__(self, attentional=True): method from_opt (line 23) | def from_opt(cls, opt, embeddings): class RNNDecoderBase (line 32) | class RNNDecoderBase(DecoderBase): method __init__ (line 85) | def __init__(self, rnn_type, bidirectional_encoder, num_layers, method from_opt (line 168) | def from_opt(cls, opt, embeddings, **kwargs): method init_state (line 189) | def init_state(self, src, memory_bank, encoder_final): method map_state (line 218) | def map_state(self, fn): method detach_state (line 228) | def detach_state(self): method forward (line 232) | def forward(self, tgt, memory_bank, memory_lengths=None, step=None, method update_dropout (line 277) | def update_dropout(self, dropout): class StdRNNDecoder (line 282) | class StdRNNDecoder(RNNDecoderBase): method _run_forward_pass (line 298) | def _run_forward_pass(self, tgt, memory_bank, memory_lengths=None): method _build_rnn (line 361) | def _build_rnn(self, rnn_type, **kwargs): method _input_size (line 366) | def _input_size(self): class InputFeedRNNDecoder (line 370) | class InputFeedRNNDecoder(RNNDecoderBase): method _run_forward_pass (line 398) | def _run_forward_pass(self, tgt, memory_bank, memory_lengths=None): method _build_rnn (line 490) | def _build_rnn(self, rnn_type, input_size, method _input_size (line 498) | def _input_size(self): method update_dropout (line 502) | def update_dropout(self, dropout): FILE: onmt/decoders/ensemble.py class EnsembleDecoderOutput (line 17) | class EnsembleDecoderOutput(object): method __init__ (line 19) | def __init__(self, model_dec_outs): method squeeze (line 22) | def squeeze(self, dim=None): method __getitem__ (line 29) | def __getitem__(self, index): class EnsembleEncoder (line 33) | class EnsembleEncoder(EncoderBase): method __init__ (line 35) | def __init__(self, model_encoders): method forward (line 39) | def forward(self, src, lengths=None): class EnsembleDecoder (line 46) | class EnsembleDecoder(DecoderBase): method __init__ (line 48) | def __init__(self, model_decoders): method forward (line 54) | def forward(self, tgt, memory_bank, memory_lengths=None, step=None, method combine_attns (line 69) | def combine_attns(self, attns): method init_state (line 76) | def init_state(self, src, memory_bank, enc_hidden): method map_state (line 81) | def map_state(self, fn): class EnsembleGenerator (line 86) | class EnsembleGenerator(nn.Module): method __init__ (line 91) | def __init__(self, model_generators, raw_probs=False): method forward (line 96) | def forward(self, hidden, attn=None, src_map=None): class EnsembleModel (line 112) | class EnsembleModel(NMTModel): method __init__ (line 114) | def __init__(self, models, raw_probs=False): function load_test_model (line 123) | def load_test_model(opt): FILE: onmt/decoders/transformer.py class TransformerDecoderLayerBase (line 15) | class TransformerDecoderLayerBase(nn.Module): method __init__ (line 16) | def __init__(self): method forward (line 19) | def forward(self, *args, **kwargs): method _forward (line 54) | def _forward(self, *args, **kwargs): method update_dropout (line 57) | def update_dropout(self, dropout, attention_dropout): class TransformerDecoderLayer (line 61) | class TransformerDecoderLayer(TransformerDecoderLayerBase): method __init__ (line 98) | def __init__( method update_dropout (line 135) | def update_dropout(self, dropout, attention_dropout): method _forward (line 141) | def _forward( class TransformerDecoderBase (line 223) | class TransformerDecoderBase(DecoderBase): method __init__ (line 224) | def __init__(self, d_model, copy_attn, embeddings, alignment_layer): method from_opt (line 241) | def from_opt(cls, opt, embeddings): method init_state (line 264) | def init_state(self, src, memory_bank, enc_hidden): method map_state (line 269) | def map_state(self, fn): method detach_state (line 283) | def detach_state(self): method forward (line 286) | def forward(self, *args, **kwargs): method update_dropout (line 289) | def update_dropout(self, dropout, attention_dropout): class TransformerDecoder (line 295) | class TransformerDecoder(TransformerDecoderBase): method __init__ (line 334) | def __init__( method detach_state (line 401) | def detach_state(self): method forward (line 404) | def forward(self, tgt, memory_bank=None, step=None, **kwargs): method _init_cache (line 511) | def _init_cache(self, memory_bank): class TransformerLMDecoderLayer (line 539) | class TransformerLMDecoderLayer(TransformerDecoderLayerBase): method __init__ (line 572) | def __init__( method _forward (line 606) | def _forward( method update_dropout (line 670) | def update_dropout(self, dropout, attention_dropout): class TransformerLMDecoder (line 676) | class TransformerLMDecoder(TransformerDecoderBase): method __init__ (line 707) | def __init__( method init_state (line 745) | def init_state(self, src=None, memory_bank=None, enc_hidden=None): method detach_state (line 748) | def detach_state(self): method forward (line 751) | def forward(self, tgt, memory_bank=None, step=None, **kwargs): method _init_cache (line 794) | def _init_cache(self, memory_bank=None): FILE: onmt/encoders/bart_encoder.py class BARTEncoder (line 17) | class BARTEncoder(EncoderBase): method __init__ (line 21) | def __init__(self, model_name, embeddings, cache_dir, max_src_length, ... method from_opt (line 82) | def from_opt(cls, opt, embeddings, **kwargs): method forward (line 96) | def forward(self, src, src_lengths): function forward_embedding (line 128) | def forward_embedding(self, src_tokens, src_labels, token_embedding: Opt... function forward_bart_encoder (line 158) | def forward_bart_encoder(self, FILE: onmt/encoders/cnn_encoder.py class CNNEncoder (line 12) | class CNNEncoder(EncoderBase): method __init__ (line 17) | def __init__(self, num_layers, hidden_size, method from_opt (line 28) | def from_opt(cls, opt, embeddings): method forward (line 37) | def forward(self, input, lengths=None, hidden=None): method update_dropout (line 54) | def update_dropout(self, dropout): FILE: onmt/encoders/encoder.py class EncoderBase (line 8) | class EncoderBase(nn.Module): method from_opt (line 34) | def from_opt(cls, opt, embeddings=None): method _check_args (line 37) | def _check_args(self, src, lengths=None, hidden=None): method forward (line 43) | def forward(self, src, lengths=None): FILE: onmt/encoders/ggnn_encoder.py class GGNNAttrProxy (line 10) | class GGNNAttrProxy(object): method __init__ (line 16) | def __init__(self, module, prefix): method __getitem__ (line 20) | def __getitem__(self, i): class GGNNPropogator (line 24) | class GGNNPropogator(nn.Module): method __init__ (line 29) | def __init__(self, state_dim, n_node, n_edge_types): method forward (line 48) | def forward(self, state_in, state_out, state_cur, edges, nodes): class GGNNEncoder (line 66) | class GGNNEncoder(EncoderBase): method __init__ (line 87) | def __init__(self, rnn_type, state_dim, bidir_edges, method from_opt (line 136) | def from_opt(cls, opt, embeddings): method _initialization (line 148) | def _initialization(self): method forward (line 154) | def forward(self, src, lengths=None): method _initialize_bridge (line 247) | def _initialize_bridge(self, rnn_type, method _bridge (line 262) | def _bridge(self, hidden): FILE: onmt/encoders/mean_encoder.py class MeanEncoder (line 7) | class MeanEncoder(EncoderBase): method __init__ (line 15) | def __init__(self, num_layers, embeddings): method from_opt (line 21) | def from_opt(cls, opt, embeddings): method forward (line 27) | def forward(self, src, lengths=None): FILE: onmt/encoders/pretrained_encoder.py class ExtClassifier (line 12) | class ExtClassifier(nn.Module): method __init__ (line 13) | def __init__(self, hidden_size): method forward (line 18) | def forward(self, x, mask=None): class PretrainedEncoder (line 28) | class PretrainedEncoder(EncoderBase): method __init__ (line 33) | def __init__(self, model_name, cache_dir, src_length, vocab_size, opt): method from_opt (line 76) | def from_opt(cls, opt, embeddings): method forward (line 87) | def forward(self, src, mask): FILE: onmt/encoders/rnn_encoder.py class RNNEncoder (line 12) | class RNNEncoder(EncoderBase): method __init__ (line 25) | def __init__(self, rnn_type, bidirectional, num_layers, method from_opt (line 52) | def from_opt(cls, opt, embeddings): method forward (line 63) | def forward(self, src, lengths=None): method _initialize_bridge (line 85) | def _initialize_bridge(self, rnn_type, method _bridge (line 100) | def _bridge(self, hidden): method update_dropout (line 117) | def update_dropout(self, dropout): FILE: onmt/encoders/transformer.py class TransformerEncoderLayer (line 13) | class TransformerEncoderLayer(nn.Module): method __init__ (line 26) | def __init__(self, d_model, heads, d_ff, dropout, attention_dropout, method forward (line 37) | def forward(self, inputs, mask): method update_dropout (line 54) | def update_dropout(self, dropout, attention_dropout): class TransformerEncoder (line 60) | class TransformerEncoder(EncoderBase): method __init__ (line 91) | def __init__(self, num_layers, d_model, heads, d_ff, dropout, method from_opt (line 104) | def from_opt(cls, opt, embeddings): method forward (line 117) | def forward(self, src, lengths=None): method update_dropout (line 132) | def update_dropout(self, dropout, attention_dropout): FILE: onmt/inputters/corpus.py function exfile_open (line 19) | def exfile_open(filename, *args, **kwargs): class DatasetAdapter (line 46) | class DatasetAdapter(object): method __init__ (line 53) | def __init__(self, fields, is_train): method _valid_fields (line 58) | def _valid_fields(cls, fields): method _process (line 66) | def _process(item, is_train): method _maybe_add_dynamic_dict (line 85) | def _maybe_add_dynamic_dict(self, example, fields): method _to_examples (line 94) | def _to_examples(self, bucket, is_train=False): method __call__ (line 107) | def __call__(self, bucket): class ParallelCorpus (line 113) | class ParallelCorpus(object): method __init__ (line 116) | def __init__(self, name, src, tgt, align=None, extra_label_paths=None,... method load (line 126) | def load(self, offset=0, stride=1): method __repr__ (line 175) | def __repr__(self): function get_corpora (line 181) | def get_corpora(opts, is_train=False): class ParallelCorpusIterator (line 219) | class ParallelCorpusIterator(object): method __init__ (line 230) | def __init__(self, corpus, transform, infinitely=False, method _tokenize (line 243) | def _tokenize(self, stream): method _transform (line 252) | def _transform(self, stream): method _add_index (line 265) | def _add_index(self, stream): method _iter_corpus (line 285) | def _iter_corpus(self): method __iter__ (line 302) | def __iter__(self): function build_corpora_iters (line 311) | def build_corpora_iters(corpora, transforms, corpora_info, is_train=False, function write_files_from_queues (line 327) | def write_files_from_queues(sample_path, queues): function build_sub_vocab (line 352) | def build_sub_vocab(corpora, transforms, opts, n_sample, stride, offset): function init_pool (line 380) | def init_pool(queues): function build_vocab (line 385) | def build_vocab(opts, transforms, n_sample=3): function save_transformed_sample (line 426) | def save_transformed_sample(opts, transforms, n_sample=3): FILE: onmt/inputters/datareader_base.py class MissingDependencyException (line 6) | class MissingDependencyException(Exception): class DataReaderBase (line 10) | class DataReaderBase(object): method from_opt (line 20) | def from_opt(cls, opt): method _read_file (line 30) | def _read_file(cls, path): method _raise_missing_dep (line 37) | def _raise_missing_dep(*missing_deps): method read (line 43) | def read(self, data, side): FILE: onmt/inputters/dataset_base.py function _join_dicts (line 13) | def _join_dicts(*args): function _dynamic_dict (line 25) | def _dynamic_dict(example, src_field, tgt_field): class Dataset (line 103) | class Dataset(TorchtextDataset): method __init__ (line 148) | def __init__(self, fields, readers, data, sort_key, filter_pred=None): method __getattr__ (line 178) | def __getattr__(self, attr): method save (line 187) | def save(self, path, remove_fields=True): method config (line 193) | def config(fields): FILE: onmt/inputters/dynamic_iterator.py class MixingStrategy (line 13) | class MixingStrategy(object): method __init__ (line 16) | def __init__(self, iterables, weights): method _valid_iterable (line 22) | def _valid_iterable(self, iterables, weights): method __iter__ (line 29) | def __iter__(self): class SequentialMixer (line 33) | class SequentialMixer(MixingStrategy): method _iter_datasets (line 36) | def _iter_datasets(self): method __iter__ (line 41) | def __iter__(self): class WeightedMixer (line 47) | class WeightedMixer(MixingStrategy): method __init__ (line 50) | def __init__(self, iterables, weights): method _reset_iter (line 57) | def _reset_iter(self, ds_name): method _iter_datasets (line 60) | def _iter_datasets(self): method __iter__ (line 65) | def __iter__(self): class SimpleInfiniteMixer (line 81) | class SimpleInfiniteMixer(MixingStrategy): method __init__ (line 82) | def __init__(self, iterables, weights): method _reset_iter (line 89) | def _reset_iter(self, ds_name): method __iter__ (line 92) | def __iter__(self): class DynamicDatasetIter (line 101) | class DynamicDatasetIter(object): method __init__ (line 127) | def __init__(self, corpora, corpora_info, transforms, fields, is_train, method from_opts (line 154) | def from_opts(cls, corpora, transforms, fields, opts, is_train, method _init_datasets (line 171) | def _init_datasets(self): method _bucketing (line 189) | def _bucketing(self): method __iter__ (line 196) | def __iter__(self): function build_dynamic_dataset_iter (line 218) | def build_dynamic_dataset_iter(fields, transforms_cls, opts, is_train=True, function build_dynamic_dataset_iter_given_examples (line 231) | def build_dynamic_dataset_iter_given_examples(examples, fields, transfor... FILE: onmt/inputters/fields.py function _get_dynamic_fields (line 10) | def _get_dynamic_fields(opts): function build_dynamic_fields (line 25) | def build_dynamic_fields(opts, src_specials=None, tgt_specials=None): function get_vocabs (line 65) | def get_vocabs(fields): function save_fields (line 73) | def save_fields(fields, save_data, overwrite=True): function load_fields (line 81) | def load_fields(save_data, checkpoint=None): FILE: onmt/inputters/inputter.py function _getstate (line 33) | def _getstate(self): function _setstate (line 37) | def _setstate(self, state): function make_src (line 46) | def make_src(data, vocab): function preprocessing_tokenize (line 56) | def preprocessing_tokenize(data, tokenizer, max_length=None): function postprocessing_length_cap (line 65) | def postprocessing_length_cap(data, vocab, max_length): function make_tgt (line 69) | def make_tgt(data, vocab, pad_idx=0): class AlignField (line 83) | class AlignField(LabelField): method __init__ (line 88) | def __init__(self, **kwargs): method process (line 93) | def process(self, batch, device=None): function parse_align_idx (line 107) | def parse_align_idx(align_pharaoh): function make_align (line 124) | def make_align(data, vocab): function get_task_spec_tokens (line 139) | def get_task_spec_tokens(data_task, pad, bos, eos): function get_fields (line 157) | def get_fields( function load_roberta_kp_tokenizer (line 269) | def load_roberta_kp_tokenizer(vocab_path, bpe_dropout): function reload_keyphrase_fields (line 324) | def reload_keyphrase_fields(fields, opt, tokenizer=None): function deprecated_reload_keyphrase_fields (line 342) | def deprecated_reload_keyphrase_fields(fields, opt, tokenizer=None): class IterOnDevice (line 438) | class IterOnDevice(object): method __init__ (line 441) | def __init__(self, iterable, device_id): method batch_to_device (line 446) | def batch_to_device(batch, device_id): method __iter__ (line 465) | def __iter__(self): function filter_example (line 471) | def filter_example(ex, use_src_len=True, use_tgt_len=True, function _pad_vocab_to_multiple (line 499) | def _pad_vocab_to_multiple(vocab, multiple): function _build_field_vocab (line 510) | def _build_field_vocab(field, counter, size_multiple=1, **kwargs): function _load_vocab (line 527) | def _load_vocab(vocab_path, name, counters, min_freq=0): function _build_fv_from_multifield (line 550) | def _build_fv_from_multifield(multifield, counters, build_fv_kwargs, function _build_fields_vocab (line 561) | def _build_fields_vocab(fields, counters, data_type, share_vocab, function build_vocab (line 606) | def build_vocab(train_dataset_files, fields, data_type, share_vocab, function _merge_field_vocabs (line 707) | def _merge_field_vocabs(src_field, tgt_field, vocab_size, min_freq, function _read_vocab_file (line 731) | def _read_vocab_file(vocab_path, tag): FILE: onmt/inputters/iterator.py function batch_iter (line 7) | def batch_iter(data, batch_size, batch_size_fn=None, batch_size_multiple... function _pool (line 47) | def _pool(data, batch_size, batch_size_fn, batch_size_multiple, class OrderedIterator (line 61) | class OrderedIterator(torchtext.data.Iterator): method __init__ (line 63) | def __init__(self, method create_batches (line 76) | def create_batches(self): method __iter__ (line 102) | def __iter__(self): function max_tok_len (line 135) | def max_tok_len(new, count, sofar): FILE: onmt/inputters/keyphrase_dataset.py function infer_dataset_type (line 43) | def infer_dataset_type(filepath): function parse_src_fn (line 66) | def parse_src_fn(ex_dict, title_field, text_field): function kpdict_parse_fn (line 71) | def kpdict_parse_fn(example, tokenizer, tgt_concat_type, dataset_type='s... class KeyphraseDataset (line 101) | class KeyphraseDataset(TorchtextDataset): method __init__ (line 148) | def __init__(self, fields, readers, data, dirs, sort_key, method __getattr__ (line 214) | def __getattr__(self, attr): method save (line 223) | def save(self, path, remove_fields=True): method load_config (line 228) | def load_config(self, opt): class KeyphraseDataReader (line 232) | class KeyphraseDataReader(DataReaderBase): method read (line 233) | def read(self, sequences, side, _dir=None): function obtain_sorted_indices (line 285) | def obtain_sorted_indices(src_seq, tgt_seqs, sort_by): function process_multiple_tgts (line 333) | def process_multiple_tgts(big_batch, tgt_type): function kp_sort_key (line 432) | def kp_sort_key(ex): function max_tok_len (line 439) | def max_tok_len(new, count, sofar): function copyseq_tokenize (line 462) | def copyseq_tokenize(text): function _feature_tokenize (line 469) | def _feature_tokenize( class KeyphraseField (line 500) | class KeyphraseField(RawField): method __init__ (line 517) | def __init__(self, base_name, base_field): method base_field (line 523) | def base_field(self): method pad_seqs (line 526) | def pad_seqs(self, batch, max_seq_num, max_seq_len, pad_token): method process (line 541) | def process(self, batch, device=None): method preprocess (line 607) | def preprocess(self, x): method __getitem__ (line 624) | def __getitem__(self, item): function keyphrase_fields (line 628) | def keyphrase_fields(**kwargs): FILE: onmt/inputters/news_dataset.py class Token (line 26) | class Token(): method __init__ (line 27) | def __init__(self, token, field): function process_news_example (line 31) | def process_news_example(ex_dict, tgt_fields, tokenizer=None, tgt_weight... function process_tokenized_news_example (line 161) | def process_tokenized_news_example(ex_dict, tokenizer_name, tokenizer, function get_meta_tokens (line 411) | def get_meta_tokens(doc, dataset_name, tgt_field, cls_token=None, token_... function process_news_examples_parallel (line 443) | def process_news_examples_parallel(news_examples, tgt_fields, tgt_weights, function load_tokenized_news_examples (line 499) | def load_tokenized_news_examples(news_examples, tokenizer_name, tokenizer, function build_dynamic_dict_and_masks_parallel (line 528) | def build_dynamic_dict_and_masks_parallel(read_iters, fields, boseos_add... class NewsDataset (line 577) | class NewsDataset(TorchtextDataset): method __init__ (line 623) | def __init__(self, fields, readers, data, dirs, sort_key, method reload_fields (line 704) | def reload_fields(self): method __getattr__ (line 707) | def __getattr__(self, attr): method save (line 716) | def save(self, path, remove_fields=True): method load_config (line 721) | def load_config(self, opt): function load_dataset_from_jsonl (line 724) | def load_dataset_from_jsonl(fields, paths, tokenizer, opt): class NewsDataReader (line 737) | class NewsDataReader(DataReaderBase): method read (line 738) | def read(self, sequences, side, _dir=None): method read_jsonl (line 763) | def read_jsonl(self, sequences, _dir=None): function news_sort_key (line 799) | def news_sort_key(ex): function _feature_tokenize (line 811) | def _feature_tokenize( class NewsMultiField (line 840) | class NewsMultiField(RawField): method __init__ (line 859) | def __init__(self, base_name, base_field, feats_fields): method base_field (line 871) | def base_field(self): method process (line 874) | def process(self, batch, device=None): method preprocess (line 910) | def preprocess(self, x): method __getitem__ (line 923) | def __getitem__(self, item): function news_fields (line 927) | def news_fields(**kwargs): function update_field_vocab (line 973) | def update_field_vocab(field, tokenizer): function load_pretrained_tokenizer (line 1007) | def load_pretrained_tokenizer(tokenizer_name, cache_dir, special_vocab_p... FILE: onmt/inputters/text_dataset.py class TextDataReader (line 12) | class TextDataReader(DataReaderBase): method read (line 13) | def read(self, sequences, side): function text_sort_key (line 35) | def text_sort_key(ex): function _feature_tokenize (line 43) | def _feature_tokenize( class TextMultiField (line 69) | class TextMultiField(RawField): method __init__ (line 88) | def __init__(self, base_name, base_field, feats_fields): method base_field (line 95) | def base_field(self): method process (line 98) | def process(self, batch, device=None): method preprocess (line 134) | def preprocess(self, x): method __getitem__ (line 148) | def __getitem__(self, item): function text_fields (line 152) | def text_fields(**kwargs): FILE: onmt/keyphrase/baseline/export_dataset.py class Document (line 18) | class Document(object): method __init__ (line 19) | def __init__(self): method __str__ (line 26) | def __str__(self): method to_dict (line 29) | def to_dict(self): class Dataset (line 39) | class Dataset(object): method __init__ (line 40) | def __init__(self, **kwargs): method load_dataset (line 52) | def load_dataset(self): method load_dataset_as_dicts (line 57) | def load_dataset_as_dicts(self): method _convert_docs_to_dicts (line 61) | def _convert_docs_to_dicts(self, docs): method load_train_test_dataset (line 72) | def load_train_test_dataset(self): method dump_train_test_to_json (line 101) | def dump_train_test_to_json(self): method load_text (line 114) | def load_text(self, textdir): method load_keyphrase (line 194) | def load_keyphrase(self, keyphrasedir): class INSPEC (line 209) | class INSPEC(Dataset): method __init__ (line 210) | def __init__(self, **kwargs): class NUS (line 215) | class NUS(Dataset): method __init__ (line 216) | def __init__(self, **kwargs): class SemEval (line 221) | class SemEval(Dataset): method __init__ (line 222) | def __init__(self, **kwargs): class KRAPIVIN (line 228) | class KRAPIVIN(Dataset): method __init__ (line 229) | def __init__(self, **kwargs): class DUC (line 234) | class DUC(Dataset): method __init__ (line 235) | def __init__(self, **kwargs): function get_from_module (line 247) | def get_from_module(identifier, module_params, module_name, instantiate=... function initialize_test_data_loader (line 261) | def initialize_test_data_loader(identifier, kwargs=None): function copyseq_tokenize (line 279) | def copyseq_tokenize(text): function load_pos_tagger (line 300) | def load_pos_tagger(stanford_base_dir): function export_extra_dataset_to_json (line 317) | def export_extra_dataset_to_json(): function add_testset_postag (line 341) | def add_testset_postag(test_dataset_names, dataset_base_dir, stanford_ba... function export_to_UTD_format (line 398) | def export_to_UTD_format(dataset_name, dataset_base_dir, dump_path): function export_to_maui_format (line 438) | def export_to_maui_format(dataset_name, dataset_base_dir, dump_path, mod... FILE: onmt/keyphrase/best_scores_in_report.py function main (line 34) | def main(): FILE: onmt/keyphrase/bleu.py function bleu (line 20) | def bleu(candidate, references, weights): function _modified_precision (line 81) | def _modified_precision(candidate, references, n): function _brevity_penalty (line 201) | def _brevity_penalty(candidate, references): FILE: onmt/keyphrase/eval.py function compute_match_scores (line 28) | def compute_match_scores(tgt_seqs, pred_seqs, do_lower=True, do_stem=Tru... function run_classic_metrics (line 113) | def run_classic_metrics(match_list, pred_list, tgt_list, score_names, to... function run_advanced_metrics (line 196) | def run_advanced_metrics(match_scores, pred_list, tgt_list): function compute_PRF1 (line 237) | def compute_PRF1(match_scores, preds, tgts): function compute_MRR (line 253) | def compute_MRR(match_scores): function compute_AP (line 273) | def compute_AP(match_scores, precisions, tgts): function compute_PR_AUC (line 291) | def compute_PR_AUC(precisions, recalls): function compute_SizeAdjustedDiscountedRecall (line 300) | def compute_SizeAdjustedDiscountedRecall(match_scores, tgts): function compute_NormalizedDiscountedCumulativeGain (line 323) | def compute_NormalizedDiscountedCumulativeGain(match_scores, tgts): function compute_alphaNormalizedDiscountedCumulativeGain (line 348) | def compute_alphaNormalizedDiscountedCumulativeGain(preds, tgts, k=5, al... function f1_score (line 403) | def f1_score(prediction, ground_truth): function macro_averaged_score (line 415) | def macro_averaged_score(precisionlist, recalllist): function self_redundancy (line 424) | def self_redundancy(_input): function eval_and_print (line 445) | def eval_and_print(src_text, tgt_kps, pred_kps, pred_scores, unk_token='... function print_predeval_result (line 518) | def print_predeval_result(src_text, FILE: onmt/keyphrase/extract_np.py function stem_word_list (line 29) | def stem_word_list(word_list): function if_present_duplicate_phrases (line 33) | def if_present_duplicate_phrases(src_seq, tgt_seqs, stemming=True, lower... function if_present_phrase (line 81) | def if_present_phrase(src_str_tokens, phrase_str_tokens): function meng17_tokenize (line 105) | def meng17_tokenize(text): function spacy_innate_noun_chunks (line 123) | def spacy_innate_noun_chunks(doc, remove_duplicate=True): function all_nested_NPs (line 163) | def all_nested_NPs(span): function spacy_noun_chunks_all_nested (line 182) | def spacy_noun_chunks_all_nested(doc, remove_duplicate=True): function noun_chunks_by_pos_regex (line 228) | def noun_chunks_by_pos_regex(text, min_len, max_len): function spacy_noun_chunks_wrapper (line 261) | def spacy_noun_chunks_wrapper(text, trim_punct=True, remove_stopword=True): function get_all_np (line 281) | def get_all_np(text, stem=True, return_set=True): function test_np (line 324) | def test_np(): function spacy_tokenize (line 369) | def spacy_tokenize(text): function check_NP_recallM (line 376) | def check_NP_recallM(): function check_model_recallM (line 470) | def check_model_recallM(): function extract_np_mag (line 571) | def extract_np_mag(): FILE: onmt/keyphrase/json_plus_process.py function BertTokenizer (line 18) | def BertTokenizer(tokens): function if_present_phrase (line 26) | def if_present_phrase(src_str_tokens, phrase_str_tokens): function recognise_nounchunks (line 55) | def recognise_nounchunks(tagged): function pos (line 98) | def pos(text, model="stanfordnlp", lowercase=False): function listToStr (line 108) | def listToStr(tokens): function meng17_tokenize (line 114) | def meng17_tokenize(text): function keyword_stemmer (line 131) | def keyword_stemmer(keywords): function nounchunk_stemmer (line 139) | def nounchunk_stemmer(nouns): function macth_word (line 148) | def macth_word(sentence_stemmer, word_stemmer): FILE: onmt/keyphrase/kpg_example.py function extract_bartkp (line 41) | def extract_bartkp(ex_dict): function extract_deepkp_deprecated (line 89) | def extract_deepkp_deprecated(text_to_extract): function extract_pke (line 116) | def extract_pke(text, method, dataset_path=None, df_path=None, top_k=10): FILE: onmt/keyphrase/mag/export_mag_cs.py function mag2kp (line 29) | def mag2kp(mag_ex, id_field, title_field, text_field, keyword_field): function extract_papers (line 54) | def extract_papers(input_dir, output_dir, chunk_size, lang, must_have_kp... function main (line 111) | def main(): FILE: onmt/keyphrase/mag/post_clean.py function example_iterator_from_json (line 14) | def example_iterator_from_json(path, dataset_name, id_field, title_field... FILE: onmt/keyphrase/migrated/post_evaluate.py function check_if_present (line 13) | def check_if_present(source_tokens, targets_tokens): function get_match_flags (line 33) | def get_match_flags(targets, predictions): function evaluate_ (line 47) | def evaluate_(source_str_list, targets_str_list, prediction_str_list, function init_logging (line 410) | def init_logging(logfile): function load_predictions_from_file (line 428) | def load_predictions_from_file(prediction_dir, file_suffix='.txt'): function load_plain_text (line 447) | def load_plain_text(text_dir): function load_postag_text (line 467) | def load_postag_text(postag_text_dir): function evaluate_baselines (line 488) | def evaluate_baselines(models, test_sets, output_dir, filter_criteria, p... FILE: onmt/keyphrase/move_ckpt_by_devscore.py function main (line 42) | def main(): FILE: onmt/keyphrase/pke/base.py class LoadFile (line 32) | class LoadFile(object): method __init__ (line 35) | def __init__(self): method load_document (line 65) | def load_document(self, input, **kwargs): method apply_stemming (line 157) | def apply_stemming(self): method normalize_pos_tags (line 172) | def normalize_pos_tags(self): method unescape_punctuation_marks (line 181) | def unescape_punctuation_marks(self): method is_redundant (line 190) | def is_redundant(self, candidate, prev, minimum_length=1): method get_n_best (line 220) | def get_n_best(self, n=10, redundancy_removal=False, stemming=False): method add_candidate (line 274) | def add_candidate(self, words, stems, pos, offset, sentence_id): method ngram_selection (line 303) | def ngram_selection(self, n=3): method longest_pos_sequence_selection (line 329) | def longest_pos_sequence_selection(self, valid_pos=None): method longest_keyword_sequence_selection (line 333) | def longest_keyword_sequence_selection(self, keywords): method longest_sequence_selection (line 337) | def longest_sequence_selection(self, key, valid_values): method grammar_selection (line 376) | def grammar_selection(self, grammar=None): method _is_alphanum (line 427) | def _is_alphanum(word, valid_punctuation_marks='-'): method candidate_filtering (line 440) | def candidate_filtering(self, FILE: onmt/keyphrase/pke/data_structures.py class Sentence (line 6) | class Sentence(object): method __init__ (line 9) | def __init__(self, words): method __eq__ (line 26) | def __eq__(self, other): class Candidate (line 57) | class Candidate(object): method __init__ (line 60) | def __init__(self): class Document (line 78) | class Document(object): method __init__ (line 81) | def __init__(self): method from_sentences (line 90) | def from_sentences(sentences, **kwargs): method __eq__ (line 126) | def __eq__(self, other): FILE: onmt/keyphrase/pke/readers.py class Reader (line 12) | class Reader(object): method read (line 13) | def read(self, path): class MinimalCoreNLPReader (line 17) | class MinimalCoreNLPReader(Reader): method __init__ (line 20) | def __init__(self): method read (line 23) | def read(self, path, **kwargs): class RawTextReader (line 48) | class RawTextReader(Reader): method __init__ (line 51) | def __init__(self, language=None): method read (line 63) | def read(self, text, **kwargs): FILE: onmt/keyphrase/pke/supervised/api.py class SupervisedLoadFile (line 16) | class SupervisedLoadFile(LoadFile): method __init__ (line 20) | def __init__(self): method feature_scaling (line 28) | def feature_scaling(self): method feature_extraction (line 37) | def feature_extraction(self): method classify_candidates (line 41) | def classify_candidates(self, model=None): method candidate_weighting (line 75) | def candidate_weighting(self): FILE: onmt/keyphrase/pke/supervised/feature_based/kea.py class Kea (line 33) | class Kea(SupervisedLoadFile): method __init__ (line 67) | def __init__(self): method candidate_selection (line 72) | def candidate_selection(self, stoplist=None, **kwargs): method feature_extraction (line 104) | def feature_extraction(self, df=None, training=False): method candidate_weighting (line 149) | def candidate_weighting(self, model_file=None, df=None): method train (line 162) | def train(training_instances, training_classes, model_file): FILE: onmt/keyphrase/pke/supervised/feature_based/topiccorank.py class TopicCoRank (line 32) | class TopicCoRank(TopicRank): method __init__ (line 64) | def __init__(self): method build_topic_graph (line 73) | def build_topic_graph(self): method unify_with_domain_graph (line 102) | def unify_with_domain_graph(self, input_file, excluded_file=None): method candidate_weighting (line 173) | def candidate_weighting(self, FILE: onmt/keyphrase/pke/supervised/feature_based/wingnus.py class WINGNUS (line 30) | class WINGNUS(SupervisedLoadFile): method __init__ (line 56) | def __init__(self): method candidate_selection (line 61) | def candidate_selection(self, grammar=None): method feature_extraction (line 83) | def feature_extraction(self, df=None, training=False, features_set=None): method candidate_weighting (line 232) | def candidate_weighting(self, model_file=None, df=None): method train (line 245) | def train(training_instances, training_classes, model_file): FILE: onmt/keyphrase/pke/supervised/neural_based/seq2seq.py class Seq2Seq (line 15) | class Seq2Seq(SupervisedLoadFile): method __init__ (line 17) | def __init__(self): method document_to_ix (line 28) | def document_to_ix(self): method candidate_selection (line 40) | def candidate_selection(self): method candidate_weighting (line 43) | def candidate_weighting(self): FILE: onmt/keyphrase/pke/unsupervised/graph_based/expandrank.py class ExpandRank (line 28) | class ExpandRank(SingleRank): method __init__ (line 68) | def __init__(self): method expand_word_graph (line 73) | def expand_word_graph(self, method candidate_weighting (line 120) | def candidate_weighting(self, FILE: onmt/keyphrase/pke/unsupervised/graph_based/multipartiterank.py class MultipartiteRank (line 30) | class MultipartiteRank(TopicRank): method __init__ (line 65) | def __init__(self): method topic_clustering (line 77) | def topic_clustering(self, method build_topic_graph (line 118) | def build_topic_graph(self): method weight_adjustment (line 153) | def weight_adjustment(self, alpha=1.1): method candidate_weighting (line 200) | def candidate_weighting(self, FILE: onmt/keyphrase/pke/unsupervised/graph_based/positionrank.py class PositionRank (line 27) | class PositionRank(SingleRank): method __init__ (line 65) | def __init__(self): method candidate_selection (line 73) | def candidate_selection(self, method build_word_graph (line 102) | def build_word_graph(self, window=10, pos=None): method candidate_weighting (line 147) | def candidate_weighting(self, window=10, pos=None, normalized=False): FILE: onmt/keyphrase/pke/unsupervised/graph_based/single_tpr.py class TopicalPageRank (line 35) | class TopicalPageRank(SingleRank): method __init__ (line 72) | def __init__(self): method candidate_selection (line 77) | def candidate_selection(self, grammar=None, **kwargs): method candidate_weighting (line 103) | def candidate_weighting(self, FILE: onmt/keyphrase/pke/unsupervised/graph_based/singlerank.py class SingleRank (line 24) | class SingleRank(TextRank): method __init__ (line 60) | def __init__(self): method build_word_graph (line 65) | def build_word_graph(self, window=10, pos=None): method candidate_weighting (line 107) | def candidate_weighting(self, window=10, pos=None, normalized=False): FILE: onmt/keyphrase/pke/unsupervised/graph_based/textrank.py class TextRank (line 27) | class TextRank(LoadFile): method __init__ (line 66) | def __init__(self): method candidate_selection (line 74) | def candidate_selection(self, pos=None): method build_word_graph (line 88) | def build_word_graph(self, window=2, pos=None): method candidate_weighting (line 130) | def candidate_weighting(self, FILE: onmt/keyphrase/pke/unsupervised/graph_based/topicrank.py class TopicRank (line 30) | class TopicRank(LoadFile): method __init__ (line 63) | def __init__(self): method candidate_selection (line 75) | def candidate_selection(self, pos=None, stoplist=None): method vectorize_candidates (line 104) | def vectorize_candidates(self): method topic_clustering (line 133) | def topic_clustering(self, threshold=0.74, method='average'): method build_topic_graph (line 165) | def build_topic_graph(self): method candidate_weighting (line 185) | def candidate_weighting(self, FILE: onmt/keyphrase/pke/unsupervised/statistical/firstphrases.py class FirstPhrases (line 14) | class FirstPhrases(LoadFile): method candidate_selection (line 43) | def candidate_selection(self, pos=None): method candidate_weighting (line 57) | def candidate_weighting(self): FILE: onmt/keyphrase/pke/unsupervised/statistical/kpminer.py class KPMiner (line 27) | class KPMiner(LoadFile): method candidate_selection (line 61) | def candidate_selection(self, lasf=3, cutoff=400, stoplist=None, **kwa... method candidate_weighting (line 98) | def candidate_weighting(self, df=None, sigma=3.0, alpha=2.3): FILE: onmt/keyphrase/pke/unsupervised/statistical/tfidf.py class TfIdf (line 19) | class TfIdf(LoadFile): method candidate_selection (line 47) | def candidate_selection(self, n=3, stoplist=None, **kwargs): method candidate_weighting (line 68) | def candidate_weighting(self, df=None): FILE: onmt/keyphrase/pke/unsupervised/statistical/yake.py class YAKE (line 31) | class YAKE(LoadFile): method __init__ (line 68) | def __init__(self): method candidate_selection (line 86) | def candidate_selection(self, n=3, stoplist=None, **kwargs): method _vocabulary_building (line 121) | def _vocabulary_building(self, use_stems=False): method _contexts_building (line 151) | def _contexts_building(self, use_stems=False, window=2): method _feature_extraction (line 197) | def _feature_extraction(self, stoplist=None): method candidate_weighting (line 324) | def candidate_weighting(self, window=2, stoplist=None, use_stems=False): method is_redundant (line 392) | def is_redundant(self, candidate, prev, threshold=0.8): method get_n_best (line 413) | def get_n_best(self, FILE: onmt/keyphrase/pke/utils.py function load_document_frequency_file (line 32) | def load_document_frequency_file(input_file, function compute_document_frequency (line 65) | def compute_document_frequency(input_dir, function train_supervised_model (line 146) | def train_supervised_model(input_dir, function load_references (line 254) | def load_references(input_file, function compute_lda_model (line 320) | def compute_lda_model(input_dir, function load_document_as_bos (line 405) | def load_document_as_bos(input_file, function compute_pairwise_similarity_matrix (line 446) | def compute_pairwise_similarity_matrix(input_dir, FILE: onmt/keyphrase/preprocess.py function init_opt (line 27) | def init_opt(): function meng17_tokenize (line 45) | def meng17_tokenize(text): function start_end_re (line 62) | def start_end_re(match_position_idxs_key, match_pos_ends_key, keywords_e... function start_end (line 82) | def start_end(match_position_idxs_key, match_pos_ends_key, keywords_exis... function prepend_space_to_words (line 98) | def prepend_space_to_words(words): function position_dict (line 111) | def position_dict(position, subwords, length): function words_to_subwords (line 115) | def words_to_subwords(tokenizer, words, pos = None): function if_present_phrase (line 156) | def if_present_phrase(src_str_tokens, phrase_str_tokens): function macth_word (line 182) | def macth_word(sentence_stemmer, word_stemmer, word_origin, poss): function keyword_stemmer (line 201) | def keyword_stemmer(keywords): function nounchunk_stemmer (line 208) | def nounchunk_stemmer(nouns): function pos_judge (line 215) | def pos_judge(text, model="stanfordnlp", lowercase=False): function listToStr (line 219) | def listToStr(tokens): function tokenize_doc (line 225) | def tokenize_doc(doc): function label (line 244) | def label(sentence, starts, ends): function recognise_nounchunks (line 252) | def recognise_nounchunks(tagged): FILE: onmt/keyphrase/run_infer_hfkpg.py class ModelArguments (line 70) | class ModelArguments: class DataTrainingArguments (line 113) | class DataTrainingArguments: method __post_init__ (line 235) | def __post_init__(self): class CheckOutputCallback (line 264) | class CheckOutputCallback(TrainerCallback): method on_evaluate (line 265) | def on_evaluate(self, args: TrainingArguments, state: TrainerState, co... method on_prediction_step (line 268) | def on_prediction_step(self, args: TrainingArguments, state: TrainerSt... function main (line 271) | def main(): function _mp_fn (line 658) | def _mp_fn(index): FILE: onmt/keyphrase/utils.py function stem_word_list (line 20) | def stem_word_list(word_list): function validate_phrases (line 24) | def validate_phrases(pred_seqs, unk_token): function if_present_duplicate_phrases (line 52) | def if_present_duplicate_phrases(src_seq, tgt_seqs, stemming=True, lower... function if_present_phrase (line 100) | def if_present_phrase(src_str_tokens, phrase_str_tokens): function gather_scores (line 124) | def gather_scores(gathered_scores, results_names, results_dicts): function print_predeval_result (line 142) | def print_predeval_result(i, src_text, tgt_seqs, present_tgt_flags, function meng17_tokenize (line 258) | def meng17_tokenize(text): function retain_punc_tokenize (line 276) | def retain_punc_tokenize(raw_text): function replace_numbers_to_DIGIT (line 306) | def replace_numbers_to_DIGIT(tokens, k=2): function time_usage (line 313) | def time_usage(func): function current_milli_time (line 328) | def current_milli_time(): class LoggerWriter (line 331) | class LoggerWriter: method __init__ (line 332) | def __init__(self, level): method write (line 337) | def write(self, message): method flush (line 343) | def flush(self): function tally_parameters (line 350) | def tally_parameters(model): function _print_progress (line 369) | def _print_progress(epoch_i, batch_i, num_batches): class Progbar (line 375) | class Progbar(object): method __init__ (line 376) | def __init__(self, logger, title, target, width=30, batch_size = None,... method update (line 399) | def update(self, current_epoch, current, values=[]): method add (line 493) | def add(self, n, values=[]): method clear (line 496) | def clear(self): function numpy_seed (line 569) | def numpy_seed(seed, *addl_seeds): FILE: onmt/model_builder.py function build_embeddings (line 26) | def build_embeddings(opt, text_field, for_encoder=True): function build_encoder (line 61) | def build_encoder(opt, embeddings, **kwargs): function build_decoder (line 74) | def build_decoder(opt, embeddings, **kwargs): function load_test_model (line 86) | def load_test_model(opt, model_path=None): function build_src_emb (line 137) | def build_src_emb(model_opt, fields): function build_encoder_with_embeddings (line 147) | def build_encoder_with_embeddings(model_opt, fields): function build_decoder_with_embeddings (line 154) | def build_decoder_with_embeddings( function build_task_specific_model (line 169) | def build_task_specific_model(model_opt, fields): function build_base_model (line 196) | def build_base_model(model_opt, fields, gpu, checkpoint=None, gpu_id=None): function build_model (line 326) | def build_model(model_opt, opt, fields, checkpoint): FILE: onmt/models/model.py class BaseModel (line 5) | class BaseModel(nn.Module): method __init__ (line 11) | def __init__(self, encoder, decoder): method forward (line 14) | def forward(self, src, tgt, lengths, bptt=False, with_align=False): method update_dropout (line 39) | def update_dropout(self, dropout): method count_parameters (line 42) | def count_parameters(self, log=print): class NMTModel (line 46) | class NMTModel(BaseModel): method __init__ (line 55) | def __init__(self, encoder, decoder): method forward (line 60) | def forward(self, src, tgt, lengths, bptt=False, with_align=False): method update_dropout (line 75) | def update_dropout(self, dropout): method count_parameters (line 79) | def count_parameters(self, log=print): class LanguageModel (line 101) | class LanguageModel(BaseModel): method __init__ (line 110) | def __init__(self, encoder=None, decoder=None): method forward (line 117) | def forward(self, src, tgt, lengths, bptt=False, with_align=False): method update_dropout (line 146) | def update_dropout(self, dropout): method count_parameters (line 149) | def count_parameters(self, log=print): FILE: onmt/models/model_saver.py function build_model_saver (line 10) | def build_model_saver(model_opt, opt, model, fields, optim): function load_checkpoint (line 24) | def load_checkpoint(ckpt_path): class ModelSaverBase (line 34) | class ModelSaverBase(object): method __init__ (line 42) | def __init__(self, base_path, model, model_opt, fields, optim, method save (line 54) | def save(self, step, moving_average=None): method _save (line 85) | def _save(self, step, model): method _rm_checkpoint (line 101) | def _rm_checkpoint(self, name): class ModelSaver (line 112) | class ModelSaver(ModelSaverBase): method _save (line 115) | def _save(self, step, model): method _rm_checkpoint (line 148) | def _rm_checkpoint(self, name): FILE: onmt/models/sru.py class CheckSRU (line 17) | class CheckSRU(configargparse.Action): method __init__ (line 18) | def __init__(self, option_strings, dest, **kwargs): method __call__ (line 21) | def __call__(self, parser, namespace, values, option_string=None): function check_sru_requirement (line 33) | def check_sru_requirement(abort=False): function load_sru_mod (line 354) | def load_sru_mod(): class SRU_Compute (line 380) | class SRU_Compute(Function): method __init__ (line 382) | def __init__(self, activation_type, d_out, bidirectional=False): method maybe_load_sru_mod (line 390) | def maybe_load_sru_mod(): method forward (line 397) | def forward(self, u, x, bias, init=None, mask_h=None): method backward (line 443) | def backward(self, grad_h, grad_last): class SRUCell (line 496) | class SRUCell(nn.Module): method __init__ (line 497) | def __init__(self, n_in, n_out, dropout=0, rnn_dropout=0, method init_weight (line 519) | def init_weight(self): method set_bias (line 524) | def set_bias(self, bias_val=0): method forward (line 531) | def forward(self, input, c0=None): method get_dropout_mask_ (line 565) | def get_dropout_mask_(self, size, p): class SRU (line 570) | class SRU(nn.Module): method __init__ (line 591) | def __init__(self, input_size, hidden_size, method set_bias (line 619) | def set_bias(self, bias_val=0): method forward (line 623) | def forward(self, input, c0=None, return_hidden=True): FILE: onmt/models/stacked_rnn.py class StackedLSTM (line 6) | class StackedLSTM(nn.Module): method __init__ (line 12) | def __init__(self, num_layers, input_size, rnn_size, dropout): method forward (line 22) | def forward(self, input_feed, hidden): class StackedGRU (line 39) | class StackedGRU(nn.Module): method __init__ (line 45) | def __init__(self, num_layers, input_size, rnn_size, dropout): method forward (line 55) | def forward(self, input_feed, hidden): FILE: onmt/modules/average_attn.py class AverageAttention (line 10) | class AverageAttention(nn.Module): method __init__ (line 22) | def __init__(self, model_dim, dropout=0.1, aan_useffn=False): method cumulative_average_mask (line 31) | def cumulative_average_mask(self, batch_size, inputs_len, device): method cumulative_average (line 54) | def cumulative_average(self, inputs, mask_or_step, method forward (line 84) | def forward(self, inputs, mask=None, layer_cache=None, step=None): FILE: onmt/modules/conv_multi_step_attention.py function seq_linear (line 11) | def seq_linear(linear, x): class ConvMultiStepAttention (line 19) | class ConvMultiStepAttention(nn.Module): method __init__ (line 27) | def __init__(self, input_size): method apply_mask (line 32) | def apply_mask(self, mask): method forward (line 36) | def forward(self, base_target_emb, input_from_dec, encoder_out_top, FILE: onmt/modules/copy_generator.py function collapse_copy_scores (line 8) | def collapse_copy_scores(scores, batch, tgt_vocab, src_vocabs=None, class CopyGenerator (line 50) | class CopyGenerator(nn.Module): method __init__ (line 97) | def __init__(self, input_size, output_size, pad_idx): method forward (line 103) | def forward(self, hidden, attn, src_map): class CopyGeneratorLoss (line 143) | class CopyGeneratorLoss(nn.Module): method __init__ (line 145) | def __init__(self, vocab_size, force_copy, unk_index=0, method forward (line 154) | def forward(self, scores, align, target): class CommonCopyGeneratorLossCompute (line 188) | class CommonCopyGeneratorLossCompute(CommonLossCompute): method __init__ (line 190) | def __init__(self, criterion, generator, tgt_vocab, normalize_by_length, method _compute_loss (line 210) | def _compute_loss(self, batch, output, target, copy_attn, align, method _make_shard_state (line 300) | def _make_shard_state(self, batch, output, range_, attns): class CopyGeneratorLossCompute (line 315) | class CopyGeneratorLossCompute(CommonCopyGeneratorLossCompute): method __init__ (line 317) | def __init__(self, criterion, generator, tgt_vocab, normalize_by_lengt... class CopyGeneratorLMLossCompute (line 327) | class CopyGeneratorLMLossCompute(CommonCopyGeneratorLossCompute): method __init__ (line 329) | def __init__(self, criterion, generator, tgt_vocab, normalize_by_length, FILE: onmt/modules/embeddings.py class SequenceTooLongError (line 13) | class SequenceTooLongError(Exception): class PositionalEncoding (line 17) | class PositionalEncoding(nn.Module): method __init__ (line 28) | def __init__(self, dropout, dim, max_len=5000): method forward (line 44) | def forward(self, emb, step=None): class Embeddings (line 68) | class Embeddings(nn.Module): method __init__ (line 109) | def __init__(self, word_vec_size, method _validate_args (line 185) | def _validate_args(self, feat_merge, feat_vocab_sizes, feat_vec_exponent, method word_lut (line 213) | def word_lut(self): method emb_luts (line 218) | def emb_luts(self): method load_pretrained_vectors (line 222) | def load_pretrained_vectors(self, emb_file): method forward (line 240) | def forward(self, source, step=None): method update_dropout (line 261) | def update_dropout(self, dropout): function read_embeddings (line 268) | def read_embeddings(path, skip_lines=0, filter_set=None): function calc_vocab_load_stats (line 294) | def calc_vocab_load_stats(vocab, loaded_embed_dict): function convert_to_torch_tensor (line 302) | def convert_to_torch_tensor(word_to_float_list_dict, vocab): function prepare_pretrained_embeddings (line 310) | def prepare_pretrained_embeddings(opt, fields): FILE: onmt/modules/gate.py function context_gate_factory (line 6) | def context_gate_factory(gate_type, embeddings_size, decoder_size, class ContextGate (line 20) | class ContextGate(nn.Module): method __init__ (line 29) | def __init__(self, embeddings_size, decoder_size, method forward (line 39) | def forward(self, prev_emb, dec_state, attn_state): class SourceContextGate (line 48) | class SourceContextGate(nn.Module): method __init__ (line 51) | def __init__(self, embeddings_size, decoder_size, method forward (line 58) | def forward(self, prev_emb, dec_state, attn_state): class TargetContextGate (line 64) | class TargetContextGate(nn.Module): method __init__ (line 67) | def __init__(self, embeddings_size, decoder_size, method forward (line 74) | def forward(self, prev_emb, dec_state, attn_state): class BothContextGate (line 79) | class BothContextGate(nn.Module): method __init__ (line 82) | def __init__(self, embeddings_size, decoder_size, method forward (line 89) | def forward(self, prev_emb, dec_state, attn_state): FILE: onmt/modules/global_attention.py class GlobalAttention (line 15) | class GlobalAttention(nn.Module): method __init__ (line 71) | def __init__(self, dim, coverage=False, attn_type="dot", method score (line 97) | def score(self, h_t, h_s): method forward (line 138) | def forward(self, source, memory_bank, memory_lengths=None, coverage=N... FILE: onmt/modules/multi_headed_attn.py class MultiHeadedAttention (line 11) | class MultiHeadedAttention(nn.Module): method __init__ (line 51) | def __init__(self, head_count, model_dim, dropout=0.1, method forward (line 77) | def forward(self, key, value, query, mask=None, method update_dropout (line 229) | def update_dropout(self, dropout): FILE: onmt/modules/position_ffn.py class PositionwiseFeedForward (line 6) | class PositionwiseFeedForward(nn.Module): method __init__ (line 16) | def __init__(self, d_model, d_ff, dropout=0.1): method forward (line 25) | def forward(self, x): method update_dropout (line 39) | def update_dropout(self, dropout): FILE: onmt/modules/sparse_activations.py function _make_ix_like (line 14) | def _make_ix_like(input, dim=0): function _threshold_and_support (line 22) | def _threshold_and_support(input, dim=0): class SparsemaxFunction (line 44) | class SparsemaxFunction(Function): method forward (line 48) | def forward(ctx, input, dim=0): method backward (line 68) | def backward(ctx, grad_output): class Sparsemax (line 83) | class Sparsemax(nn.Module): method __init__ (line 85) | def __init__(self, dim=0): method forward (line 89) | def forward(self, input): class LogSparsemax (line 93) | class LogSparsemax(nn.Module): method __init__ (line 95) | def __init__(self, dim=0): method forward (line 99) | def forward(self, input): FILE: onmt/modules/sparse_losses.py class SparsemaxLossFunction (line 9) | class SparsemaxLossFunction(Function): method forward (line 13) | def forward(ctx, input, target): method backward (line 37) | def backward(ctx, grad_output): class SparsemaxLoss (line 48) | class SparsemaxLoss(nn.Module): method __init__ (line 59) | def __init__(self, weight=None, ignore_index=-100, method forward (line 67) | def forward(self, input, target): FILE: onmt/modules/structured_attention.py class MatrixTree (line 6) | class MatrixTree(nn.Module): method __init__ (line 13) | def __init__(self, eps=1e-5): method forward (line 17) | def forward(self, input): FILE: onmt/modules/util_class.py class Elementwise (line 7) | class Elementwise(nn.ModuleList): method __init__ (line 18) | def __init__(self, merge=None, *args): method forward (line 23) | def forward(self, inputs): class Cast (line 37) | class Cast(nn.Module): method __init__ (line 43) | def __init__(self, dtype): method forward (line 47) | def forward(self, x): FILE: onmt/modules/weight_norm.py function get_var_maybe_avg (line 8) | def get_var_maybe_avg(namespace, var_name, training, polyak_decay): function get_vars_maybe_avg (line 22) | def get_vars_maybe_avg(namespace, var_names, training, polyak_decay): class WeightNormLinear (line 31) | class WeightNormLinear(nn.Linear): method __init__ (line 44) | def __init__(self, in_features, out_features, method reset_parameters (line 62) | def reset_parameters(self): method forward (line 65) | def forward(self, x, init=False): class WeightNormConv2d (line 101) | class WeightNormConv2d(nn.Conv2d): method __init__ (line 102) | def __init__(self, in_channels, out_channels, kernel_size, stride=1, method reset_parameters (line 121) | def reset_parameters(self): method forward (line 124) | def forward(self, x, init=False): class WeightNormConvTranspose2d (line 174) | class WeightNormConvTranspose2d(nn.ConvTranspose2d): method __init__ (line 175) | def __init__(self, in_channels, out_channels, kernel_size, stride=1, method reset_parameters (line 196) | def reset_parameters(self): method forward (line 199) | def forward(self, x, init=False): FILE: onmt/newssum/bart/example.py function run_model (line 7) | def run_model(model, tokens, return_all_hiddens=False, features_only=Fal... FILE: onmt/newssum/cnndm/docid_url_mapping.py function hashhex (line 26) | def hashhex(s): function get_url_hashes (line 32) | def get_url_hashes(url_list): FILE: onmt/newssum/cnndm/resort_examples.py function load_groundtruth_data (line 9) | def load_groundtruth_data(groundtruth_jsonl_path): function hashhex (line 39) | def hashhex(s): FILE: onmt/newssum/docutils.py function word_tokenize (line 33) | def word_tokenize(text, model="spacy", lowercase=False): function get_noun_chunks (line 54) | def get_noun_chunks(text, trim_punct=True, remove_stopword=True): function get_NERs (line 75) | def get_NERs(text): function get_verbs (line 91) | def get_verbs(text): function build_matching_mask (line 98) | def build_matching_mask(text_tokens, match_tokens, function build_bottomup_mask (line 126) | def build_bottomup_mask(src_sents, tgt_tokens): function build_oracle_sentence_mask (line 206) | def build_oracle_sentence_mask(src_sents, tgt_sents, metric='avg'): function mask_to_text (line 251) | def mask_to_text(sents_tokens, token_masks): function summary_sents_to_text (line 261) | def summary_sents_to_text(tgt_sents): function fragments_to_mask (line 271) | def fragments_to_mask(flat_fragments, src_sents): function fragment_to_text (line 293) | def fragment_to_text(top_fragments_list, src_sents): function build_oracle_fragment_mask (line 313) | def build_oracle_fragment_mask(src_sents, tgt_sents, function prepend_space_to_words (line 365) | def prepend_space_to_words(words): function words_to_subwords (line 378) | def words_to_subwords(tokenizer, words): function wordmasks_to_subwords (line 393) | def wordmasks_to_subwords(tokenizer, title_sents_words, oracle_masks): function sentence_split (line 438) | def sentence_split(text, model="spacy"): function calc_rouge (line 448) | def calc_rouge(pred_sent, gt_sent, stopwords_removal=False, stemming=Tru... function eval_rouge (line 469) | def eval_rouge(doc, extract_sent_idx, number_to_cutoff=3, stopwords_remo... function extract_entities (line 492) | def extract_entities(sentence, return_text=False): function calc_entity_coverage (line 502) | def calc_entity_coverage(pred_entities, gt_entities): function eval_entity_coverage (line 508) | def eval_entity_coverage(doc, extract_sent_idx, number_to_cutoff=3, logg... function print_hypothesis (line 536) | def print_hypothesis(doc, extract_sent_idx, extract_sent_scores=None, lo... function update_score (line 545) | def update_score(score_dict, new_score_dict): function tokenize (line 550) | def tokenize(text): function normalize (line 559) | def normalize(tokens, case=False): function clean_text (line 572) | def clean_text(text): function summary_sentence_segment (line 576) | def summary_sentence_segment(summary, dataset='cnndm'): function tokenize_source_summary (line 612) | def tokenize_source_summary(source, summary, title, dataset): function preprocess_sents (line 628) | def preprocess_sents(sents, case=False, stemming=True, function concat_sents (line 642) | def concat_sents(sents): FILE: onmt/newssum/fragutils.py function extend_fragment_to_boundary (line 17) | def extend_fragment_to_boundary(tokenized_sent, lend, rend): function obtain_matching_fragments (line 27) | def obtain_matching_fragments(summary_sent, src_sents, function extract_fragments_iterative (line 62) | def extract_fragments_iterative(summary_sent_id, summary_sent, src_sents, function extract_multiple_fragments (line 120) | def extract_multiple_fragments(tgt_sents, src_sents, function extract_singleton_fragments (line 159) | def extract_singleton_fragments(sum_sents, src_sents, function locate_sum_fragment (line 205) | def locate_sum_fragment(sum_sent, src_sent, src_start, src_end): function smooth_fragments (line 217) | def smooth_fragments(fragments, smoothing_window): function _lcs (line 249) | def _lcs(sum_sent, src_sent, src_sent_id, stemming): function extract_fragments_by_LCS (line 318) | def extract_fragments_by_LCS(sum_sent, src_sents, stemming): function extract_fragments_by_wordmatch (line 346) | def extract_fragments_by_wordmatch(sum_sent, src_sents, function get_top_results (line 409) | def get_top_results(fragments): function match_matrix (line 416) | def match_matrix(sum_tokens, src_tokens): function shrink_matrix (line 437) | def shrink_matrix(matrix, src_tokens, ngram_size=2): function merge_fragments (line 486) | def merge_fragments(src_tokens, fragments, window_size=3): function insert_values_to_matrix (line 504) | def insert_values_to_matrix(matrix, indices, value): function locate_fragments_in_matrix (line 510) | def locate_fragments_in_matrix(summary_sents, source_sents, matrix, frag... FILE: onmt/newssum/json_to_shards.py function init_opt (line 6) | def init_opt(): FILE: onmt/newssum/news_preprocess.py function init_opt (line 43) | def init_opt(): class Token (line 67) | class Token(): method __init__ (line 68) | def __init__(self, token, field): function tokenize_doc (line 73) | def tokenize_doc(doc, tokenizer_fn): FILE: onmt/newssum/rouge_eval/rouge.py class FilesRouge (line 9) | class FilesRouge: method __init__ (line 10) | def __init__(self, hyp_path, ref_path, metrics=None, stats=None, method get_scores (line 34) | def get_scores(self, avg=False, ignore_empty=False): class Rouge (line 53) | class Rouge: method __init__ (line 63) | def __init__(self, metrics=None, stats=None, stopwords_removal=False, ... method get_scores (line 86) | def get_scores(self, hyps, refs, avg=False, ignore_empty=False): method _get_scores (line 103) | def _get_scores(self, hyps, refs): method _get_avg_scores (line 119) | def _get_avg_scores(self, hyps, refs): FILE: onmt/newssum/rouge_eval/rouge_score.py function _get_ngrams (line 41) | def _get_ngrams(n, text): function _split_into_words (line 59) | def _split_into_words(sentences, stopwords_removal=True, stemming=True, ... function _get_word_ngrams (line 78) | def _get_word_ngrams(n, sentences, stopwords_removal=True, stemming=True... function _len_lcs (line 91) | def _len_lcs(x, y): function _lcs (line 109) | def _lcs(x, y): function _recon_lcs (line 136) | def _recon_lcs(x, y): function multi_rouge_n (line 166) | def multi_rouge_n(sequences, scores_ids, n=2): function rouge_n (line 203) | def rouge_n(evaluated_sentences, reference_sentences, n=2, stopwords_rem... function f_r_p_rouge_n (line 244) | def f_r_p_rouge_n(evaluated_count, reference_count, overlapping_count): function _union_lcs (line 261) | def _union_lcs(evaluated_sentences, reference_sentence, prev_union=None,... function rouge_l_summary_level (line 305) | def rouge_l_summary_level(evaluated_sentences, reference_sentences, stop... FILE: onmt/newssum/wandb/example_mnist.py class Net (line 11) | class Net(nn.Module): method __init__ (line 12) | def __init__(self): method forward (line 20) | def forward(self, x): function train (line 30) | def train(args, model, device, train_loader, optimizer, epoch): function test (line 45) | def test(args, model, device, test_loader): function main (line 73) | def main(): FILE: onmt/opts.py function config_opts (line 12) | def config_opts(parser): function _add_logging_opts (line 22) | def _add_logging_opts(parser, is_train=True): function _add_reproducibility_opts (line 74) | def _add_reproducibility_opts(parser): function _add_dynamic_corpus_opts (line 81) | def _add_dynamic_corpus_opts(parser, build_vocab_only=False): function _add_dynamic_fields_opts (line 132) | def _add_dynamic_fields_opts(parser, build_vocab_only=False): function _add_dynamic_transform_opts (line 209) | def _add_dynamic_transform_opts(parser): function dynamic_prepare_opts (line 219) | def dynamic_prepare_opts(parser, build_vocab_only=False): function model_opts (line 236) | def model_opts(parser): function _add_train_general_opts (line 471) | def _add_train_general_opts(parser): function _add_train_dynamic_data (line 676) | def _add_train_dynamic_data(parser): function train_opts (line 682) | def train_opts(parser): function _add_decoding_opts (line 692) | def _add_decoding_opts(parser): function translate_opts (line 767) | def translate_opts(parser): class StoreLoggingLevelAction (line 837) | class StoreLoggingLevelAction(configargparse.Action): method __init__ (line 851) | def __init__(self, option_strings, dest, help=None, **kwargs): method __call__ (line 855) | def __call__(self, parser, namespace, value, option_string=None): class DeprecateAction (line 861) | class DeprecateAction(configargparse.Action): method __init__ (line 864) | def __init__(self, option_strings, dest, help=None, **kwargs): method __call__ (line 868) | def __call__(self, parser, namespace, values, flag_name): FILE: onmt/tests/test_attention.py class TestAttention (line 11) | class TestAttention(unittest.TestCase): method test_masked_global_attention (line 13) | def test_masked_global_attention(self): FILE: onmt/tests/test_beam.py class GlobalScorerStub (line 7) | class GlobalScorerStub(object): method update_global_state (line 8) | def update_global_state(self, beam): method score (line 11) | def score(self, beam, scores): class TestBeam (line 15) | class TestBeam(unittest.TestCase): method test_advance_with_all_repeats_gets_blocked (line 18) | def test_advance_with_all_repeats_gets_blocked(self): method test_advance_with_some_repeats_gets_blocked (line 44) | def test_advance_with_some_repeats_gets_blocked(self): method test_repeating_excluded_index_does_not_die (line 80) | def test_repeating_excluded_index_does_not_die(self): method test_doesnt_predict_eos_if_shorter_than_min_len (line 125) | def test_doesnt_predict_eos_if_shorter_than_min_len(self): method test_beam_is_done_when_n_best_beams_eos_using_min_length (line 177) | def test_beam_is_done_when_n_best_beams_eos_using_min_length(self): class TestBeamAgainstReferenceCase (line 235) | class TestBeamAgainstReferenceCase(unittest.TestCase): method init_step (line 243) | def init_step(self, beam): method first_step (line 255) | def first_step(self, beam, expected_beam_scores, expected_len_pen): method second_step (line 287) | def second_step(self, beam, expected_beam_scores, expected_len_pen): method third_step (line 322) | def third_step(self, beam, expected_beam_scores, expected_len_pen): method test_beam_advance_against_known_reference (line 356) | def test_beam_advance_against_known_reference(self): class TestBeamWithLengthPenalty (line 369) | class TestBeamWithLengthPenalty(TestBeamAgainstReferenceCase): method test_beam_advance_against_known_reference (line 373) | def test_beam_advance_against_known_reference(self): FILE: onmt/tests/test_beam_search.py class GlobalScorerStub (line 10) | class GlobalScorerStub(object): method __init__ (line 14) | def __init__(self): method update_global_state (line 21) | def update_global_state(self, beam): method score (line 24) | def score(self, beam, scores): class TestBeamSearch (line 28) | class TestBeamSearch(unittest.TestCase): method test_advance_with_all_repeats_gets_blocked (line 31) | def test_advance_with_all_repeats_gets_blocked(self): method test_advance_with_some_repeats_gets_blocked (line 78) | def test_advance_with_some_repeats_gets_blocked(self): method test_repeating_excluded_index_does_not_die (line 143) | def test_repeating_excluded_index_does_not_die(self): method test_doesnt_predict_eos_if_shorter_than_min_len (line 198) | def test_doesnt_predict_eos_if_shorter_than_min_len(self): method test_beam_is_done_when_n_best_beams_eos_using_min_length (line 255) | def test_beam_is_done_when_n_best_beams_eos_using_min_length(self): method test_beam_returns_attn_with_correct_length (line 316) | def test_beam_returns_attn_with_correct_length(self): class TestBeamSearchAgainstReferenceCase (line 399) | class TestBeamSearchAgainstReferenceCase(unittest.TestCase): method random_attn (line 410) | def random_attn(self): method init_step (line 413) | def init_step(self, beam, expected_len_pen): method first_step (line 430) | def first_step(self, beam, expected_beam_scores, expected_len_pen): method second_step (line 463) | def second_step(self, beam, expected_beam_scores, expected_len_pen): method third_step (line 502) | def third_step(self, beam, expected_beam_scores, expected_len_pen): method test_beam_advance_against_known_reference (line 539) | def test_beam_advance_against_known_reference(self): class TestBeamWithLengthPenalty (line 553) | class TestBeamWithLengthPenalty(TestBeamSearchAgainstReferenceCase): method test_beam_advance_against_known_reference (line 557) | def test_beam_advance_against_known_reference(self): class TestBeamSearchLM (line 572) | class TestBeamSearchLM(TestBeamSearchAgainstReferenceCase): method finish_first_beam_step (line 573) | def finish_first_beam_step(self, beam): method test_beam_lm_increase_memory_length (line 589) | def test_beam_lm_increase_memory_length(self): method test_beam_lm_update_memory_length_when_finished (line 607) | def test_beam_lm_update_memory_length_when_finished(self): FILE: onmt/tests/test_copy_generator.py class TestCopyGenerator (line 13) | class TestCopyGenerator(unittest.TestCase): method dummy_inputs (line 27) | def dummy_inputs(cls, params, init_case): method expected_shape (line 37) | def expected_shape(cls, params, init_case): method test_copy_gen_forward_shape (line 41) | def test_copy_gen_forward_shape(self): method test_copy_gen_outp_has_no_prob_of_pad (line 50) | def test_copy_gen_outp_has_no_prob_of_pad(self): method test_copy_gen_trainable_params_update (line 59) | def test_copy_gen_trainable_params_update(self): class TestCopyGeneratorLoss (line 80) | class TestCopyGeneratorLoss(unittest.TestCase): method dummy_inputs (line 94) | def dummy_inputs(cls, params, init_case): method expected_shape (line 108) | def expected_shape(cls, params, init_case): method test_copy_loss_forward_shape (line 111) | def test_copy_loss_forward_shape(self): method test_copy_loss_ignore_index_is_ignored (line 120) | def test_copy_loss_ignore_index_is_ignored(self): method test_copy_loss_output_range_is_positive (line 131) | def test_copy_loss_output_range_is_positive(self): FILE: onmt/tests/test_data_prepare.py function get_default_opts (line 19) | def get_default_opts(): class TestData (line 39) | class TestData(unittest.TestCase): method __init__ (line 40) | def __init__(self, *args, **kwargs): method dataset_build (line 44) | def dataset_build(self, opt): function _add_test (line 69) | def _add_test(param_setting, methodname): FILE: onmt/tests/test_embeddings.py class TestEmbeddings (line 12) | class TestEmbeddings(unittest.TestCase): method case_is_degenerate (line 32) | def case_is_degenerate(cls, case): method cases (line 56) | def cases(cls): method dummy_inputs (line 62) | def dummy_inputs(cls, params, init_case): method expected_shape (line 81) | def expected_shape(cls, params, init_case): method test_embeddings_forward_shape (line 90) | def test_embeddings_forward_shape(self): method test_embeddings_trainable_params (line 98) | def test_embeddings_trainable_params(self): method test_embeddings_trainable_params_update (line 131) | def test_embeddings_trainable_params_update(self): FILE: onmt/tests/test_greedy_search.py class TestGreedySearch (line 7) | class TestGreedySearch(unittest.TestCase): method test_doesnt_predict_eos_if_shorter_than_min_len (line 14) | def test_doesnt_predict_eos_if_shorter_than_min_len(self): method test_returns_correct_scores_deterministic (line 55) | def test_returns_correct_scores_deterministic(self): method test_returns_correct_scores_non_deterministic (line 127) | def test_returns_correct_scores_non_deterministic(self): FILE: onmt/tests/test_models.py class TestModel (line 21) | class TestModel(unittest.TestCase): method __init__ (line 23) | def __init__(self, *args, **kwargs): method get_field (line 27) | def get_field(self): method get_batch (line 32) | def get_batch(self, source_l=3, bsize=1): method embeddings_forward (line 39) | def embeddings_forward(self, opt, source_l=3, bsize=1): method encoder_forward (line 62) | def encoder_forward(self, opt, source_l=3, bsize=1): method nmtmodel_forward (line 93) | def nmtmodel_forward(self, opt, source_l=3, bsize=1): function _add_test (line 125) | def _add_test(param_setting, methodname): FILE: onmt/tests/test_random_sampling.py class TestRandomSampling (line 7) | class TestRandomSampling(unittest.TestCase): method test_advance_with_repeats_gets_blocked (line 14) | def test_advance_with_repeats_gets_blocked(self): method test_advance_with_some_repeats_gets_blocked (line 38) | def test_advance_with_some_repeats_gets_blocked(self): method test_repeating_excluded_index_does_not_die (line 79) | def test_repeating_excluded_index_does_not_die(self): method test_doesnt_predict_eos_if_shorter_than_min_len (line 112) | def test_doesnt_predict_eos_if_shorter_than_min_len(self): method test_returns_correct_scores_deterministic (line 152) | def test_returns_correct_scores_deterministic(self): method test_returns_correct_scores_non_deterministic (line 224) | def test_returns_correct_scores_non_deterministic(self): FILE: onmt/tests/test_simple.py function test_load (line 4) | def test_load(): FILE: onmt/tests/test_structured_attention.py class TestStructuredAttention (line 7) | class TestStructuredAttention(unittest.TestCase): method test_matrix_tree_marg_pdfs_sum_to_1 (line 8) | def test_matrix_tree_marg_pdfs_sum_to_1(self): FILE: onmt/tests/test_text_dataset.py class TestTextMultiField (line 13) | class TestTextMultiField(unittest.TestCase): method initialize_case (line 26) | def initialize_case(cls, init_case, params): method test_process_shape (line 36) | def test_process_shape(self): method test_preprocess_shape (line 77) | def test_preprocess_shape(self): method test_base_field (line 86) | def test_base_field(self): method test_correct_n_fields (line 93) | def test_correct_n_fields(self): method test_fields_order_correct (line 101) | def test_fields_order_correct(self): method test_getitem_0_returns_correct_field (line 110) | def test_getitem_0_returns_correct_field(self): method test_getitem_nonzero_returns_correct_field (line 118) | def test_getitem_nonzero_returns_correct_field(self): method test_getitem_has_correct_number_of_indexes (line 131) | def test_getitem_has_correct_number_of_indexes(self): class TestTextDataReader (line 141) | class TestTextDataReader(unittest.TestCase): method test_read (line 142) | def test_read(self): class TestTextDataReaderFromFS (line 153) | class TestTextDataReaderFromFS(unittest.TestCase): method setUpClass (line 164) | def setUpClass(cls): method tearDownClass (line 171) | def tearDownClass(cls): method test_read (line 174) | def test_read(self): FILE: onmt/tests/test_translation_server.py class TestServerModel (line 16) | class TestServerModel(unittest.TestCase): method test_deferred_loading_model_and_unload (line 17) | def test_deferred_loading_model_and_unload(self): method test_load_model_on_init_and_unload (line 29) | def test_load_model_on_init_and_unload(self): method test_tokenizing_with_no_tokenizer_fails (line 39) | def test_tokenizing_with_no_tokenizer_fails(self): method test_detokenizing_with_no_tokenizer_fails (line 47) | def test_detokenizing_with_no_tokenizer_fails(self): method test_moving_to_gpu_and_back (line 56) | def test_moving_to_gpu_and_back(self): method test_initialize_on_gpu_and_move_back (line 72) | def test_initialize_on_gpu_and_move_back(self): method test_initialize_on_nonzero_gpu_and_back (line 90) | def test_initialize_on_nonzero_gpu_and_back(self): method test_run (line 108) | def test_run(self): class TestTranslationServer (line 134) | class TestTranslationServer(unittest.TestCase): method tearDown (line 141) | def tearDown(self): method write (line 145) | def write(self, cfg): method test_start_without_initial_loading (line 167) | def test_start_without_initial_loading(self): method test_start_with_initial_loading (line 192) | def test_start_with_initial_loading(self): method test_start_with_two_models (line 227) | def test_start_with_two_models(self): FILE: onmt/tests/utils_for_tests.py function product_dict (line 4) | def product_dict(**kwargs): FILE: onmt/train_single.py function configure_process (line 17) | def configure_process(opt, device_id): function _get_model_opts (line 23) | def _get_model_opts(opt, checkpoint=None): function _build_iter_given_examples (line 39) | def _build_iter_given_examples(examples, opt, fields, transforms_cls, is... function _build_valid_iter (line 46) | def _build_valid_iter(opt, fields, transforms_cls): function _build_train_iter (line 53) | def _build_train_iter(opt, fields, transforms_cls, stride=1, offset=0): function main (line 61) | def main(opt, fields, transforms_cls, checkpoint, device_id, FILE: onmt/trainer.py function build_trainer (line 20) | def build_trainer(opt, device_id, model, fields, optim, model_saver=None): class Trainer (line 101) | class Trainer(object): method __init__ (line 127) | def __init__(self, model, train_loss, valid_loss, optim, method _accum_count (line 170) | def _accum_count(self, step): method _maybe_update_dropout (line 176) | def _maybe_update_dropout(self, step): method _accum_batches (line 183) | def _accum_batches(self, iterator): method _update_average (line 203) | def _update_average(self, step): method train (line 217) | def train(self, method validate (line 312) | def validate(self, valid_iter, moving_average=None): method _gradient_accumulation (line 360) | def _gradient_accumulation(self, true_batches, normalization, total_st... method _start_report_manager (line 460) | def _start_report_manager(self, start_time=None): method _maybe_gather_stats (line 470) | def _maybe_gather_stats(self, stat): method _maybe_report_training (line 485) | def _maybe_report_training(self, step, num_steps, learning_rate, method _report_step (line 501) | def _report_step(self, learning_rate, step, train_stats=None, FILE: onmt/transforms/__init__.py function get_transforms_cls (line 13) | def get_transforms_cls(transform_names): function register_transform (line 27) | def register_transform(name): FILE: onmt/transforms/bart.py function word_start (line 11) | def word_start(x, ignore_subword=False, is_joiner=False): class BARTNoising (line 22) | class BARTNoising(object): method __init__ (line 25) | def __init__(self, vocab, mask_tok=DefaultTokens.MASK, mask_ratio=0.0, method _make_poisson (line 66) | def _make_poisson(self, poisson_lambda): method _is_full_stop (line 80) | def _is_full_stop(self, token): method permute_sentences (line 83) | def permute_sentences(self, tokens, p=1.0): method _is_word_start (line 114) | def _is_word_start(self, token): method whole_word_mask (line 117) | def whole_word_mask(self, tokens, p=1.0): # text span mask/infilling method insertion_noise (line 242) | def insertion_noise(self, tokens, p=1.0): method rolling_noise (line 264) | def rolling_noise(self, tokens, p=1.0): method apply (line 270) | def apply(self, tokens): method __repr__ (line 287) | def __repr__(self): class BARTNoiseTransform (line 310) | class BARTNoiseTransform(Transform): method __init__ (line 311) | def __init__(self, opts): method _set_seed (line 314) | def _set_seed(self, seed): method add_options (line 320) | def add_options(cls, parser): method warm_up (line 350) | def warm_up(self, vocabs): method apply (line 378) | def apply(self, example, is_train=False, stats=None, **kwargs): method _repr_args (line 385) | def _repr_args(self): FILE: onmt/transforms/keyphrase.py class KeyphraseTransform (line 23) | class KeyphraseTransform(Transform): method __init__ (line 35) | def __init__(self, opts): method _parse_opts (line 40) | def _parse_opts(self): method _set_seed (line 50) | def _set_seed(self, seed): method add_options (line 56) | def add_options(cls, parser): method get_specials (line 85) | def get_specials(cls, opts): method warm_up (line 88) | def warm_up(self, vocabs): method kpdict_parse_fn (line 94) | def kpdict_parse_fn(self, ex_dict, kp_concat_type, dataset_type='scipa... method infer_dataset_type (line 163) | def infer_dataset_type(self, example): method apply (line 180) | def apply(self, example, is_train=False, stats=None, **kwargs): class KeyphraseRandomSpanTargetTransform (line 206) | class KeyphraseRandomSpanTargetTransform(Transform): method __init__ (line 207) | def __init__(self, opts): method _parse_opts (line 212) | def _parse_opts(self): method add_options (line 226) | def add_options(cls, parser): method _set_seed (line 232) | def _set_seed(self, seed): method warm_up (line 237) | def warm_up(self, vocabs): method random_span_parse_fn (line 240) | def random_span_parse_fn(self, ex, sep_token, method apply (line 320) | def apply(self, example, is_train=False, stats=None, **kwargs): class KeyphraseReplaceTargetTransform (line 342) | class KeyphraseReplaceTargetTransform(KeyphraseRandomSpanTargetTransform): method __init__ (line 343) | def __init__(self, opts): method _parse_opts (line 348) | def _parse_opts(self): method _set_seed (line 358) | def _set_seed(self, seed): method add_options (line 364) | def add_options(cls, parser): method warm_up (line 369) | def warm_up(self, vocabs): method maybe_replace_target (line 372) | def maybe_replace_target(self, example, method apply (line 463) | def apply(self, example, is_train=False, stats=None, **kwargs): class ControlPrefixTransform (line 489) | class ControlPrefixTransform(Transform): method __init__ (line 491) | def __init__(self, opts): method _parse_opts (line 496) | def _parse_opts(self): method add_options (line 503) | def add_options(cls, parser): method apply (line 511) | def apply(self, example, is_train=False, stats=None, **kwargs): function findBalanced (line 540) | def findBalanced(text, openDelim=['[['], closeDelim=[']]']): function replaceInternalLinks (line 582) | def replaceInternalLinks(text, return_anchor_text=False): function extract_phrases (line 637) | def extract_phrases(text): class WikiPhraseTransform (line 664) | class WikiPhraseTransform(Transform): method __init__ (line 665) | def __init__(self, opts): method _parse_opts (line 670) | def _parse_opts(self): method _set_seed (line 687) | def _set_seed(self, seed): method add_options (line 693) | def add_options(cls, parser): method wiki_ex_parse_fn (line 701) | def wiki_ex_parse_fn(self, method apply (line 835) | def apply(self, example, is_train=False, stats=None, **kwargs): FILE: onmt/transforms/misc.py class FilterTooLongTransform (line 7) | class FilterTooLongTransform(Transform): method __init__ (line 10) | def __init__(self, opts): method add_options (line 14) | def add_options(cls, parser): method _parse_opts (line 22) | def _parse_opts(self): method apply (line 26) | def apply(self, example, is_train=False, stats=None, **kwargs): method _repr_args (line 36) | def _repr_args(self): class PrefixTransform (line 45) | class PrefixTransform(Transform): method __init__ (line 48) | def __init__(self, opts): method _get_prefix (line 52) | def _get_prefix(corpus): method get_prefix_dict (line 64) | def get_prefix_dict(cls, opts): method get_specials (line 75) | def get_specials(cls, opts): method warm_up (line 84) | def warm_up(self, vocabs=None): method _prepend (line 89) | def _prepend(self, example, prefix): method apply (line 95) | def apply(self, example, is_train=False, stats=None, **kwargs): method _repr_args (line 108) | def _repr_args(self): FILE: onmt/transforms/sampling.py class HammingDistanceSampling (line 10) | class HammingDistanceSampling(object): method _softmax (line 13) | def _softmax(self, x): method _sample_replace (line 17) | def _sample_replace(self, vocab, reject): method _sample_distance (line 24) | def _sample_distance(self, tokens, temperature): method _sample_position (line 33) | def _sample_position(self, tokens, distance): class HammingDistanceSamplingTransform (line 39) | class HammingDistanceSamplingTransform(Transform, HammingDistanceSampling): method _set_seed (line 42) | def _set_seed(self, seed): class SwitchOutTransform (line 49) | class SwitchOutTransform(HammingDistanceSamplingTransform): method __init__ (line 55) | def __init__(self, opts): method warm_up (line 58) | def warm_up(self, vocabs): method add_options (line 66) | def add_options(cls, parser): method _parse_opts (line 75) | def _parse_opts(self): method _switchout (line 78) | def _switchout(self, tokens, vocab, stats=None): method apply (line 96) | def apply(self, example, is_train=False, stats=None, **kwargs): method _repr_args (line 106) | def _repr_args(self): class TokenDropTransform (line 112) | class TokenDropTransform(HammingDistanceSamplingTransform): method __init__ (line 115) | def __init__(self, opts): method add_options (line 119) | def add_options(cls, parser): method _parse_opts (line 126) | def _parse_opts(self): method _token_drop (line 129) | def _token_drop(self, tokens, stats=None): method apply (line 141) | def apply(self, example, is_train=False, stats=None, **kwargs): method _repr_args (line 149) | def _repr_args(self): class TokenMaskTransform (line 155) | class TokenMaskTransform(HammingDistanceSamplingTransform): method __init__ (line 159) | def __init__(self, opts): method add_options (line 163) | def add_options(cls, parser): method _parse_opts (line 170) | def _parse_opts(self): method get_specials (line 174) | def get_specials(cls, opts): method _token_mask (line 178) | def _token_mask(self, tokens, stats=None): method apply (line 192) | def apply(self, example, is_train=False, stats=None, **kwargs): method _repr_args (line 200) | def _repr_args(self): FILE: onmt/transforms/tokenize.py class TokenizerTransform (line 12) | class TokenizerTransform(Transform): method __init__ (line 15) | def __init__(self, opts): method add_options (line 20) | def add_options(cls, parser): method _validate_options (line 79) | def _validate_options(cls, opts): method _parse_opts (line 86) | def _parse_opts(self): method _repr_args (line 99) | def _repr_args(self): class SubwordStats (line 115) | class SubwordStats(ObservableStats): method __init__ (line 120) | def __init__(self, subwords: int, words: int): method update (line 124) | def update(self, other: "SubwordStats"): method __str__ (line 128) | def __str__(self) -> str: class SentencePieceTransform (line 135) | class SentencePieceTransform(TokenizerTransform): method __init__ (line 138) | def __init__(self, opts): method _set_seed (line 142) | def _set_seed(self, seed): method warm_up (line 147) | def warm_up(self, vocabs=None): method _tokenize (line 174) | def _tokenize(self, tokens, side='src', is_train=False): method apply (line 193) | def apply(self, example, is_train=False, stats=None, **kwargs): method _repr_args (line 204) | def _repr_args(self): class BPETransform (line 214) | class BPETransform(TokenizerTransform): method __init__ (line 217) | def __init__(self, opts): method _parse_opts (line 221) | def _parse_opts(self): method _set_seed (line 226) | def _set_seed(self, seed): method warm_up (line 231) | def warm_up(self, vocabs=None): method _tokenize (line 259) | def _tokenize(self, tokens, side='src', is_train=False): method apply (line 266) | def apply(self, example, is_train=False, stats=None, **kwargs): class RoBERTaTransform (line 279) | class RoBERTaTransform(TokenizerTransform): method __init__ (line 282) | def __init__(self, opts): method _parse_opts (line 288) | def _parse_opts(self): method add_options (line 300) | def add_options(cls, parser): method _set_seed (line 308) | def _set_seed(self, seed): method _load_tokenizer (line 313) | def _load_tokenizer(self, vocab_path): method warm_up (line 316) | def warm_up(self, vocabs=None): method _tokenize (line 337) | def _tokenize(self, tokens_str, side='src', is_train=False): method _encode (line 359) | def _encode(self, tokens_str, side='src', is_train=False): method apply (line 382) | def apply(self, example, is_train=False, stats=None, **kwargs): class ONMTTokenizerTransform (line 399) | class ONMTTokenizerTransform(TokenizerTransform): method __init__ (line 402) | def __init__(self, opts): method _set_seed (line 406) | def _set_seed(self, seed): method add_options (line 412) | def add_options(cls, parser): method _validate_options (line 435) | def _validate_options(cls, opts): method _parse_opts (line 447) | def _parse_opts(self): method get_specials (line 459) | def get_specials(cls, opts): method _get_subword_kwargs (line 473) | def _get_subword_kwargs(self, side='src'): method warm_up (line 502) | def warm_up(self, vocabs=None): method _tokenize (line 531) | def _tokenize(self, tokens, side='src', is_train=False): method apply (line 538) | def apply(self, example, is_train=False, stats=None, **kwargs): method _repr_args (line 549) | def _repr_args(self): FILE: onmt/transforms/transform.py class Transform (line 8) | class Transform(object): method __init__ (line 11) | def __init__(self, opts): method _set_seed (line 16) | def _set_seed(self, seed): method require_vocab (line 21) | def require_vocab(cls): method warm_up (line 25) | def warm_up(self, vocabs=None): method add_options (line 40) | def add_options(cls, parser): method _validate_options (line 45) | def _validate_options(cls, opts): method get_specials (line 50) | def get_specials(cls, opts): method apply (line 53) | def apply(self, example, is_train=False, stats=None, **kwargs): method __getstate__ (line 63) | def __getstate__(self): method _parse_opts (line 70) | def _parse_opts(self): method __setstate__ (line 80) | def __setstate__(self, state): method stats (line 87) | def stats(self): method _repr_args (line 91) | def _repr_args(self): method __repr__ (line 95) | def __repr__(self): class ObservableStats (line 101) | class ObservableStats: method name (line 106) | def name(self) -> str: method update (line 110) | def update(self, other: "ObservableStats"): method __str__ (line 114) | def __str__(self) -> str: class TransformStatistics (line 123) | class TransformStatistics: method __init__ (line 125) | def __init__(self): method update (line 128) | def update(self, observable: ObservableStats): method report (line 136) | def report(self): class TransformPipe (line 149) | class TransformPipe(Transform): method __init__ (line 152) | def __init__(self, opts, transform_list): method build_from (line 159) | def build_from(cls, transform_list): method warm_up (line 167) | def warm_up(self, vocabs): method get_specials (line 173) | def get_specials(cls, opts, transforms): method apply (line 182) | def apply(self, example, is_train=False, **kwargs): method __getstate__ (line 196) | def __getstate__(self): method __setstate__ (line 200) | def __setstate__(self, state): method stats (line 204) | def stats(self): method _repr_args (line 208) | def _repr_args(self): function make_transforms (line 216) | def make_transforms(opts, transforms_cls, fields): function get_specials (line 232) | def get_specials(opts, transforms_cls_dict): function save_transforms (line 243) | def save_transforms(transforms, save_data, overwrite=True): function load_transforms (line 251) | def load_transforms(opts): FILE: onmt/translate/beam.py class Beam (line 8) | class Beam(object): method __init__ (line 31) | def __init__(self, size, pad, bos, eos, method current_predictions (line 78) | def current_predictions(self): method current_origin (line 82) | def current_origin(self): method advance (line 86) | def advance(self, word_probs, attn_out): method done (line 168) | def done(self): method sort_finished (line 171) | def sort_finished(self, minimum=None): method get_hyp (line 186) | def get_hyp(self, timestep, k): class GNMTGlobalScorer (line 196) | class GNMTGlobalScorer(object): method from_opt (line 215) | def from_opt(cls, opt): method __init__ (line 222) | def __init__(self, alpha, beta, length_penalty, coverage_penalty): method _validate (line 237) | def _validate(cls, alpha, beta, length_penalty, coverage_penalty): method score (line 260) | def score(self, beam, logprobs): method update_score (line 271) | def update_score(self, beam, attn): method update_global_state (line 279) | def update_global_state(self, beam): FILE: onmt/translate/beam_search.py class BeamSearchBase (line 9) | class BeamSearchBase(DecodeStrategy): method __init__ (line 57) | def __init__(self, beam_size, batch_size, pad, bos, eos, n_best, method initialize (line 100) | def initialize(self, *args, **kwargs): method initialize_ (line 103) | def initialize_(self, memory_bank, memory_lengths, src_map, device, method current_predictions (line 125) | def current_predictions(self): method current_backptr (line 130) | def current_backptr(self): method batch_offset (line 136) | def batch_offset(self): method _pick (line 139) | def _pick(self, log_probs): method update_finished (line 158) | def update_finished(self, last_step=None): method remove_finished_batches (line 238) | def remove_finished_batches(self, _B_new, _B_old, non_finished, method advance (line 267) | def advance(self, log_probs, attn): class BeamSearch (line 349) | class BeamSearch(BeamSearchBase): method initialize (line 353) | def initialize(self, memory_bank, src_lengths, src_map=None, device=None, class BeamSearchLM (line 384) | class BeamSearchLM(BeamSearchBase): method initialize (line 388) | def initialize(self, src, src_lengths, src_map=None, device=None, method advance (line 412) | def advance(self, log_probs, attn): method remove_finished_batches (line 419) | def remove_finished_batches(self, _B_new, _B_old, non_finished, class GNMTGlobalScorer (line 433) | class GNMTGlobalScorer(object): method from_opt (line 452) | def from_opt(cls, opt): method __init__ (line 459) | def __init__(self, alpha, beta, length_penalty, coverage_penalty): method _validate (line 474) | def _validate(cls, alpha, beta, length_penalty, coverage_penalty): FILE: onmt/translate/decode_strategy.py class DecodeStrategy (line 5) | class DecodeStrategy(object): method __init__ (line 60) | def __init__(self, pad, bos, eos, batch_size, parallel_paths, method initialize (line 90) | def initialize(self, memory_bank, src_lengths, src_map=None, device=None, method __len__ (line 118) | def __len__(self): method ensure_min_length (line 121) | def ensure_min_length(self, log_probs): method ensure_max_length (line 125) | def ensure_max_length(self): method block_ngram_repeats (line 131) | def block_ngram_repeats(self, log_probs): method maybe_update_forbidden_tokens (line 169) | def maybe_update_forbidden_tokens(self): method target_prefixing (line 202) | def target_prefixing(self, log_probs): method maybe_update_target_prefix (line 240) | def maybe_update_target_prefix(self, select_index): method advance (line 250) | def advance(self, log_probs, attn): method update_finished (line 259) | def update_finished(self): FILE: onmt/translate/greedy_search.py function sample_with_temperature (line 6) | def sample_with_temperature(logits, sampling_temp, keep_topk): class GreedySearch (line 59) | class GreedySearch(DecodeStrategy): method __init__ (line 83) | def __init__(self, pad, bos, eos, batch_size, min_length, method initialize (line 94) | def initialize(self, memory_bank, src_lengths, src_map=None, device=None, method current_predictions (line 116) | def current_predictions(self): method batch_offset (line 120) | def batch_offset(self): method _pick (line 123) | def _pick(self, log_probs): method advance (line 135) | def advance(self, log_probs, attn): method update_finished (line 163) | def update_finished(self, last_step=None): class GreedySearchLM (line 186) | class GreedySearchLM(GreedySearch): method update_finished (line 187) | def update_finished(self): method update_memory_lengths (line 191) | def update_memory_lengths(self): method advance (line 195) | def advance(self, log_probs, attn): FILE: onmt/translate/penalties.py class PenaltyBuilder (line 5) | class PenaltyBuilder(object): method __init__ (line 25) | def __init__(self, cov_pen, length_pen): method _pen_is_none (line 32) | def _pen_is_none(pen): method _coverage_penalty (line 35) | def _coverage_penalty(self, cov_pen): method _length_penalty (line 46) | def _length_penalty(self, length_pen): method coverage_wu (line 61) | def coverage_wu(self, cov, beta=0.): method coverage_summary (line 74) | def coverage_summary(self, cov, beta=0.): method coverage_none (line 80) | def coverage_none(self, cov, beta=0.): method length_wu (line 88) | def length_wu(self, cur_len, alpha=0.): method length_average (line 96) | def length_average(self, cur_len, alpha=0.): method length_none (line 100) | def length_none(self, cur_len, alpha=0.): FILE: onmt/translate/process_zh.py function wrap_str_func (line 6) | def wrap_str_func(func): function zh_segmentator (line 18) | def zh_segmentator(line, server_model): function zh_traditional_standard (line 24) | def zh_traditional_standard(line, server_model): function zh_traditional_hk (line 30) | def zh_traditional_hk(line, server_model): function zh_traditional_tw (line 36) | def zh_traditional_tw(line, server_model): function zh_simplify (line 42) | def zh_simplify(line, server_model): function zh_simplify_v2 (line 48) | def zh_simplify_v2(line, server_model): FILE: onmt/translate/random_sampling.py function sample_with_temperature (line 6) | def sample_with_temperature(logits, sampling_temp, keep_topk): class RandomSampling (line 59) | class RandomSampling(DecodeStrategy): method __init__ (line 86) | def __init__(self, pad, bos, eos, batch_size, device, method advance (line 104) | def advance(self, log_probs, attn): method update_finished (line 132) | def update_finished(self, last_step=None): FILE: onmt/translate/translation.py class TranslationBuilder (line 12) | class TranslationBuilder(object): method __init__ (line 28) | def __init__(self, data, fields, n_best=1, replace_unk=False, method _build_target_tokens (line 48) | def _build_target_tokens(self, src, src_vocab, src_raw, pred, attn): method from_batch (line 82) | def from_batch(self, translation_batch): class Translation (line 178) | class Translation(object): method __init__ (line 204) | def __init__(self, src, src_raw, pred_sents, method log (line 241) | def log(self, sent_number): method __dict__ (line 270) | def __dict__(self): method log_kp (line 315) | def log_kp(self, sent_number): method add_copied_flags (line 356) | def add_copied_flags(self, vocab_size): FILE: onmt/translate/translation_server.py function critical (line 28) | def critical(func): class Timer (line 48) | class Timer: method __init__ (line 49) | def __init__(self, start=False): method start (line 56) | def start(self): method tick (line 61) | def tick(self, name=None, tot=False): class ServerModelError (line 74) | class ServerModelError(Exception): class CTranslate2Translator (line 78) | class CTranslate2Translator(object): method __init__ (line 84) | def __init__(self, model_path, device, device_index, batch_size, method translate (line 106) | def translate(self, texts_to_translate, batch_size=8, tgt=None): method to_cpu (line 122) | def to_cpu(self): method to_gpu (line 125) | def to_gpu(self): class TranslationServer (line 129) | class TranslationServer(object): method __init__ (line 130) | def __init__(self): method start (line 134) | def start(self, config_file): method clone_model (line 166) | def clone_model(self, model_id, opt, timeout=-1): method load_model (line 180) | def load_model(self, opt, model_id=None, **model_kwargs): method preload_model (line 188) | def preload_model(self, opt, model_id=None, **model_kwargs): method run (line 207) | def run(self, inputs): method unload_model (line 223) | def unload_model(self, model_id): method list_models (line 234) | def list_models(self): class ServerModel (line 243) | class ServerModel(object): method __init__ (line 263) | def __init__(self, opt, model_id, preprocess_opt=None, tokenizer_opt=N... method parse_opt (line 337) | def parse_opt(self, opt): method loaded (line 376) | def loaded(self): method load (line 379) | def load(self, preload=False): method run (line 410) | def run(self, inputs): method rebuild_seg_packages (line 544) | def rebuild_seg_packages(self, all_preprocessed, results, method do_timeout (line 570) | def do_timeout(self): method unload (line 586) | def unload(self): method stop_unload_timer (line 594) | def stop_unload_timer(self): method reset_unload_timer (line 598) | def reset_unload_timer(self): method to_dict (line 606) | def to_dict(self): method to_cpu (line 620) | def to_cpu(self): method to_gpu (line 629) | def to_gpu(self): method maybe_preprocess (line 637) | def maybe_preprocess(self, sequence): method preprocess (line 651) | def preprocess(self, sequence): method build_tokenizer (line 666) | def build_tokenizer(self, tokenizer_opt): method maybe_tokenize (line 702) | def maybe_tokenize(self, sequence, side='src'): method tokenize (line 712) | def tokenize(self, sequence, side='src'): method tokenizer_marker (line 733) | def tokenizer_marker(self, side='src'): method maybe_detokenize_with_align (line 749) | def maybe_detokenize_with_align(self, sequence, src, side='tgt'): method maybe_detokenize (line 770) | def maybe_detokenize(self, sequence, side='tgt'): method detokenize (line 780) | def detokenize(self, sequence, side='tgt'): method maybe_convert_align (line 796) | def maybe_convert_align(self, src, tgt, align): method maybe_postprocess (line 817) | def maybe_postprocess(self, sequence): method postprocess (line 826) | def postprocess(self, sequence): function get_function_by_path (line 842) | def get_function_by_path(path, args=[], kwargs={}): FILE: onmt/translate/translator.py function build_translator (line 35) | def build_translator(opt, report_score=True, logger=None, out_file=None): function max_tok_len (line 79) | def max_tok_len(new, count, sofar): class Inference (line 98) | class Inference(object): method __init__ (line 138) | def __init__( method from_opt (line 263) | def from_opt( method _log (line 339) | def _log(self, msg): method _gold_score (line 345) | def _gold_score( method translate (line 370) | def translate( method _align_pad_prediction (line 634) | def _align_pad_prediction(self, predictions, bos, pad): method _report_score (line 666) | def _report_score(self, name, score_total, words_total): method _decode_and_generate (line 680) | def _decode_and_generate( method translate_batch (line 741) | def translate_batch(self, batch, src_vocabs, attn_debug): method _score_target (line 745) | def _score_target( method report_results (line 750) | def report_results( class Translator (line 781) | class Translator(Inference): method validate_task (line 783) | def validate_task(cls, task): method _align_forward (line 790) | def _align_forward(self, batch, predictions): method translate_batch (line 838) | def translate_batch(self, batch, src_vocabs, attn_debug): method _run_encoder (line 878) | def _run_encoder(self, batch): method _decode_and_generate (line 908) | def _decode_and_generate( method _translate_batch_with_strategy (line 1006) | def _translate_batch_with_strategy( method _score_target (line 1155) | def _score_target( method _report_kpeval (line 1185) | def _report_kpeval(self, src_path, tgt_path, pred_path): method add_trans_stats (line 1195) | def add_trans_stats(self, trans, kp_concat_type): method segment_one2seq_trans (line 1208) | def segment_one2seq_trans(self, trans): class GeneratorLM (line 1285) | class GeneratorLM(Inference): method validate_task (line 1287) | def validate_task(cls, task): method _align_forward (line 1294) | def _align_forward(self, batch, predictions): method translate (line 1301) | def translate( method translate_batch (line 1332) | def translate_batch(self, batch, src_vocabs, attn_debug): method split_src_to_prevent_padding (line 1372) | def split_src_to_prevent_padding(self, src, src_lengths): method _translate_batch_with_strategy (line 1386) | def _translate_batch_with_strategy( method _score_target (line 1498) | def _score_target( FILE: onmt/utils/alignment.py function make_batch_align_matrix (line 8) | def make_batch_align_matrix(index_tensor, size=None, normalize=False): function extract_alignment (line 30) | def extract_alignment(align_matrix, tgt_mask, src_lens, n_best): function build_align_pharaoh (line 72) | def build_align_pharaoh(valid_alignment): function to_word_align (line 87) | def to_word_align(src, tgt, subword_align, m_src='joiner', m_tgt='joiner'): function subword_map_by_joiner (line 123) | def subword_map_by_joiner(subwords, marker=SubwordMarker.JOINER): function subword_map_by_spacer (line 139) | def subword_map_by_spacer(subwords, marker=SubwordMarker.SPACER): FILE: onmt/utils/cnn_factory.py function shape_transform (line 13) | def shape_transform(x): class GatedConv (line 18) | class GatedConv(nn.Module): method __init__ (line 21) | def __init__(self, input_size, width=3, dropout=0.2, nopad=False): method forward (line 29) | def forward(self, x_var): class StackedCNN (line 37) | class StackedCNN(nn.Module): method __init__ (line 40) | def __init__(self, num_layers, input_size, cnn_kernel_width=3, method forward (line 50) | def forward(self, x): FILE: onmt/utils/distributed.py function is_master (line 20) | def is_master(opt, device_id): function multi_init (line 24) | def multi_init(opt, device_id): function all_reduce_and_rescale_tensors (line 39) | def all_reduce_and_rescale_tensors(tensors, rescale_denom, function all_gather_list (line 93) | def all_gather_list(data, max_size=4096): class ErrorHandler (line 129) | class ErrorHandler(object): method __init__ (line 133) | def __init__(self, error_queue): method add_child (line 144) | def add_child(self, pid): method error_listener (line 148) | def error_listener(self): method signal_handler (line 154) | def signal_handler(self, signalnum, stackframe): function batch_producer (line 165) | def batch_producer(generator_to_serve, queue, semaphore, opt): function consumer (line 201) | def consumer(process_fn, opt, device_id, error_queue, batch_queue, semap... FILE: onmt/utils/earlystopping.py class PatienceEnum (line 6) | class PatienceEnum(Enum): class Scorer (line 12) | class Scorer(object): method __init__ (line 13) | def __init__(self, best_score, name): method is_improving (line 17) | def is_improving(self, stats): method is_decreasing (line 20) | def is_decreasing(self, stats): method update (line 23) | def update(self, stats): method __call__ (line 26) | def __call__(self, stats, **kwargs): method _caller (line 29) | def _caller(self, stats): class PPLScorer (line 33) | class PPLScorer(Scorer): method __init__ (line 35) | def __init__(self): method is_improving (line 38) | def is_improving(self, stats): method is_decreasing (line 41) | def is_decreasing(self, stats): method _caller (line 44) | def _caller(self, stats): class AccuracyScorer (line 48) | class AccuracyScorer(Scorer): method __init__ (line 50) | def __init__(self): method is_improving (line 53) | def is_improving(self, stats): method is_decreasing (line 56) | def is_decreasing(self, stats): method _caller (line 59) | def _caller(self, stats): function scorers_from_opts (line 72) | def scorers_from_opts(opt): class EarlyStopping (line 84) | class EarlyStopping(object): method __init__ (line 86) | def __init__(self, tolerance, scorers=DEFAULT_SCORERS): method __call__ (line 102) | def __call__(self, valid_stats, step): method _update_stalled (line 133) | def _update_stalled(self): method _update_increasing (line 148) | def _update_increasing(self, valid_stats, step): method _update_decreasing (line 165) | def _update_decreasing(self): method _log_best_step (line 181) | def _log_best_step(self): method _decreasing_or_stopped_status_update (line 185) | def _decreasing_or_stopped_status_update(self, tolerance): method is_improving (line 190) | def is_improving(self): method has_stopped (line 193) | def has_stopped(self): FILE: onmt/utils/logging.py function init_logger (line 11) | def init_logger(log_file=None, log_file_level=logging.NOTSET, rotate=Fal... FILE: onmt/utils/loss.py function build_loss_compute (line 20) | def build_loss_compute(model, tgt_field, opt, train=True): class LossComputeBase (line 118) | class LossComputeBase(nn.Module): method __init__ (line 137) | def __init__(self, criterion, generator): method padding_idx (line 143) | def padding_idx(self): method _make_shard_state (line 146) | def _make_shard_state(self, batch, output, range_, attns=None): method _compute_loss (line 160) | def _compute_loss(self, batch, output, target, **kwargs): method __call__ (line 173) | def __call__(self, method _stats (line 226) | def _stats(self, loss, scores, target, batch_size): method _bottle (line 242) | def _bottle(self, _v): method _unbottle (line 245) | def _unbottle(self, _v, batch_size): class LabelSmoothingLoss (line 249) | class LabelSmoothingLoss(nn.Module): method __init__ (line 255) | def __init__(self, label_smoothing, tgt_vocab_size, ignore_index=-100): method forward (line 267) | def forward(self, output, target): class ReplayMemory (line 279) | class ReplayMemory(object): method __init__ (line 281) | def __init__(self, capacity=300): method push (line 286) | def push(self, stuff): method sample (line 292) | def sample(self, batch_size): method __len__ (line 295) | def __len__(self): class CommonLossCompute (line 299) | class CommonLossCompute(LossComputeBase): method __init__ (line 305) | def __init__(self, criterion, generator, lambda_coverage, lambda_align... method _make_shard_state (line 317) | def _make_shard_state(self, batch, output, range_, attns=None): method _add_coverage_shard_state (line 360) | def _add_coverage_shard_state(self, shard_state, attns): method _compute_loss (line 376) | def _compute_loss(self, batch, output, target, method _compute_coverage_loss (line 436) | def _compute_coverage_loss(self, std_attn, coverage_attn): method _add_align_shard_state (line 441) | def _add_align_shard_state(self, shard_state, batch, range_start, method _compute_alignement_loss (line 471) | def _compute_alignement_loss(self, align_head, ref_align): method orthogonal_penalty (line 481) | def orthogonal_penalty(self, _m, l_n_norm=2): method _compute_orthogonal_regularization_loss (line 495) | def _compute_orthogonal_regularization_loss(self, tgt_indices, decoder... method random_insert (line 531) | def random_insert(self, _list, elem): method _compute_semantic_coverage_loss (line 535) | def _compute_semantic_coverage_loss(self, model, src_states, tgtenc_st... method _make_shard_state (line 611) | def _make_shard_state(self, batch, output, range_, attns=None): class NMTLossCompute (line 630) | class NMTLossCompute(CommonLossCompute): method __init__ (line 634) | def __init__(self, criterion, generator, normalization="sents", class LMLossCompute (line 643) | class LMLossCompute(CommonLossCompute): method __init__ (line 647) | def __init__(self, criterion, generator, normalization="sents", function filter_shard_state (line 656) | def filter_shard_state(state, shard_size=None): function shards (line 671) | def shards(state, shard_size, eval_only=False): FILE: onmt/utils/misc.py function str2bool (line 11) | def str2bool(v): function check_path (line 21) | def check_path(path, exist_ok=False, log=print): function split_corpus (line 32) | def split_corpus(path, shard_size, default=None): function _split_corpus (line 42) | def _split_corpus(path, shard_size): function aeq (line 56) | def aeq(*args): function sequence_mask (line 66) | def sequence_mask(lengths, max_len=None): function tile (line 78) | def tile(x, count, dim=0): function use_gpu (line 100) | def use_gpu(opt): function set_random_seed (line 108) | def set_random_seed(seed, is_cuda): function generate_relative_positions_matrix (line 126) | def generate_relative_positions_matrix(length, max_relative_positions, function relative_matmul (line 144) | def relative_matmul(x, z, transpose): function fn_args (line 161) | def fn_args(fun): function report_matrix (line 166) | def report_matrix(row_label, column_label, matrix): function check_model_config (line 181) | def check_model_config(model_config, root): FILE: onmt/utils/optimizers.py function build_torch_optimizer (line 14) | def build_torch_optimizer(model, opt): function make_learning_rate_decay_fn (line 101) | def make_learning_rate_decay_fn(opt): function noam_decay (line 135) | def noam_decay(step, warmup_steps, model_size): function noam_simple_decay (line 144) | def noam_simple_decay(step, warmup_steps): function noamwd_decay (line 152) | def noamwd_decay(step, warmup_steps, function exponential_decay (line 162) | def exponential_decay(step, rate, decay_steps, start_step=0): function rsqrt_decay (line 169) | def rsqrt_decay(step, warmup_steps): function linear_decay (line 173) | def linear_decay(step, warmup_steps, train_steps): class MultipleOptimizer (line 182) | class MultipleOptimizer(object): method __init__ (line 185) | def __init__(self, op): method param_groups (line 190) | def param_groups(self): method zero_grad (line 196) | def zero_grad(self): method step (line 201) | def step(self): method state (line 207) | def state(self): method state_dict (line 211) | def state_dict(self): method load_state_dict (line 215) | def load_state_dict(self, state_dicts): class Optimizer (line 222) | class Optimizer(object): method __init__ (line 231) | def __init__(self, method from_opt (line 255) | def from_opt(cls, model, opt, checkpoint=None): method training_step (line 314) | def training_step(self): method amp (line 319) | def amp(self): method learning_rate (line 323) | def learning_rate(self): method state_dict (line 330) | def state_dict(self): method load_state_dict (line 337) | def load_state_dict(self, state_dict): method zero_grad (line 345) | def zero_grad(self): method backward (line 349) | def backward(self, loss): method step (line 362) | def step(self): class AdaFactor (line 399) | class AdaFactor(torch.optim.Optimizer): method __init__ (line 401) | def __init__(self, params, lr=None, beta1=0.9, beta2=0.999, eps1=1e-30, method __setstate__ (line 419) | def __setstate__(self, state): method _experimental_reshape (line 422) | def _experimental_reshape(self, shape): method _check_shape (line 434) | def _check_shape(self, shape): method _rms (line 448) | def _rms(self, x): method step (line 451) | def step(self, closure=None): class FusedAdam (line 569) | class FusedAdam(torch.optim.Optimizer): method __init__ (line 598) | def __init__(self, params, method step (line 613) | def step(self, closure=None, grads=None, output_params=None, FILE: onmt/utils/parse.py class DataOptsCheckerMixin (line 11) | class DataOptsCheckerMixin(object): method _validate_file (line 15) | def _validate_file(file_path, info): method _validate_data (line 21) | def _validate_data(cls, opt): method _validate_transforms_opts (line 82) | def _validate_transforms_opts(cls, opt): method _get_all_transform (line 89) | def _get_all_transform(cls, opt): method _validate_fields_opts (line 110) | def _validate_fields_opts(cls, opt, build_vocab_only=False): method _validate_language_model_compatibilities_opts (line 141) | def _validate_language_model_compatibilities_opts(cls, opt): method validate_prepare_opts (line 156) | def validate_prepare_opts(cls, opt, build_vocab_only=False): method validate_model_opts (line 167) | def validate_model_opts(cls, opt): class ArgumentParser (line 171) | class ArgumentParser(cfargparse.ArgumentParser, DataOptsCheckerMixin): method __init__ (line 174) | def __init__( method defaults (line 185) | def defaults(cls, *args): method update_model_opts (line 194) | def update_model_opts(cls, model_opt): method validate_model_opts (line 224) | def validate_model_opts(cls, model_opt): method ckpt_model_opts (line 252) | def ckpt_model_opts(cls, ckpt_opt): method validate_train_opts (line 261) | def validate_train_opts(cls, opt): method validate_translate_opts (line 293) | def validate_translate_opts(cls, opt): FILE: onmt/utils/report_manager.py function build_report_manager (line 12) | def build_report_manager(opt, gpu_rank): class ReportMgrBase (line 34) | class ReportMgrBase(object): method __init__ (line 42) | def __init__(self, report_every, start_time=-1.): method start (line 52) | def start(self): method log (line 55) | def log(self, *args, **kwargs): method report_training (line 58) | def report_training(self, step, num_steps, learning_rate, patience, method _report_training (line 86) | def _report_training(self, *args, **kwargs): method report_step (line 90) | def report_step(self, lr, patience, step, train_stats=None, method _report_step (line 107) | def _report_step(self, *args, **kwargs): class ReportMgr (line 111) | class ReportMgr(ReportMgrBase): method __init__ (line 112) | def __init__(self, report_every, start_time=-1., method maybe_log_tensorboard (line 127) | def maybe_log_tensorboard(self, stats, prefix, learning_rate, method _report_training (line 133) | def _report_training(self, step, num_steps, learning_rate, patience, method _report_step (line 154) | def _report_step(self, lr, patience, step, FILE: onmt/utils/rnn_factory.py function rnn_factory (line 8) | def rnn_factory(rnn_type, **kwargs): FILE: onmt/utils/statistics.py class Statistics (line 11) | class Statistics(object): method __init__ (line 20) | def __init__(self, loss=0, n_words=0, n_correct=0, n_examples=0): method all_gather_stats (line 29) | def all_gather_stats(stat, max_size=4096): method all_gather_stats_list (line 45) | def all_gather_stats_list(stat_list, max_size=4096): method update (line 72) | def update(self, stat, update_n_src_words=False): method accuracy (line 90) | def accuracy(self): method xent (line 94) | def xent(self): method ppl (line 98) | def ppl(self): method elapsed_time (line 102) | def elapsed_time(self): method to_dict (line 106) | def to_dict(self, learning_rate, prefix=''): method output (line 120) | def output(self, step, num_steps, learning_rate, start): method log_tensorboard (line 146) | def log_tensorboard(self, prefix, writer, learning_rate, patience, step): FILE: tools/apply_bpe.py class BPE (line 31) | class BPE(object): method __init__ (line 33) | def __init__(self, codes, separator='@@', vocab=None, glossaries=None): method segment (line 61) | def segment(self, sentence): method _isolate_glossaries (line 81) | def _isolate_glossaries(self, word): function create_parser (line 89) | def create_parser(): function get_pairs (line 126) | def get_pairs(word): function encode (line 139) | def encode(orig, bpe_codes, bpe_codes_reverse, vocab, separator, version... function recursive_split (line 204) | def recursive_split(segment, bpe_codes, vocab, separator, final=False): function check_vocab_and_split (line 232) | def check_vocab_and_split(orig, bpe_codes, vocab, separator): function read_vocabulary (line 257) | def read_vocabulary(vocab_file, threshold): function isolate_glossary (line 272) | def isolate_glossary(word, glossary): FILE: tools/embeddings_to_torch.py function get_vocabs (line 10) | def get_vocabs(dict_path): function read_embeddings (line 29) | def read_embeddings(file_enc, skip_lines=0, filter_set=None): function convert_to_torch_tensor (line 52) | def convert_to_torch_tensor(word_to_float_list_dict, vocab): function calc_vocab_load_stats (line 60) | def calc_vocab_load_stats(vocab, loaded_embed_dict): function main (line 68) | def main(): FILE: tools/extract_embeddings.py function write_embeddings (line 24) | def write_embeddings(filename, dict, embeddings): function main (line 33) | def main(): FILE: tools/extract_vocabulary.py function read_files_batch (line 7) | def read_files_batch(file_list): function main (line 37) | def main(): FILE: tools/learn_bpe.py function create_parser (line 30) | def create_parser(): function get_vocabulary (line 59) | def get_vocabulary(fobj, is_dict=False): function update_pair_statistics (line 73) | def update_pair_statistics(pair, changed, stats, indices): function get_pair_statistics (line 132) | def get_pair_statistics(vocab): function replace_pair (line 151) | def replace_pair(pair, vocab, indices): function prune_stats (line 177) | def prune_stats(stats, big_stats, threshold): function main (line 193) | def main(infile, outfile, num_symbols, min_frequency=2, verbose=False, i... FILE: tools/spm_to_vocab.py function convert (line 11) | def convert(lines): FILE: tools/stanfordcorenlp/corenlp.py class StanfordCoreNLP (line 24) | class StanfordCoreNLP: method __init__ (line 25) | def __init__(self, path_or_host, port=None, memory='4g', lang='en', ti... method __enter__ (line 120) | def __enter__(self): method __exit__ (line 123) | def __exit__(self, exc_type, exc_val, exc_tb): method close (line 126) | def close(self): method annotate (line 149) | def annotate(self, text, properties=None): method tregex (line 157) | def tregex(self, sentence, pattern): method tokensregex (line 162) | def tokensregex(self, sentence, pattern): method semgrex (line 167) | def semgrex(self, sentence, pattern): method sentence_segment (line 172) | def sentence_segment(self, text): method word_tokenize (line 176) | def word_tokenize(self, sentence, span=False): method pos_tag (line 188) | def pos_tag(self, sentence): method ner (line 198) | def ner(self, sentence): method parse (line 208) | def parse(self, sentence): method dependency_parse (line 212) | def dependency_parse(self, sentence): method coref (line 217) | def coref(self, text): method switch_language (line 228) | def switch_language(self, language="en"): method _request (line 232) | def _request(self, annotators=None, data=None, *args, **kwargs): method _check_args (line 247) | def _check_args(self): method _check_language (line 252) | def _check_language(self, lang): FILE: tools/test_rouge.py function test_rouge (line 13) | def test_rouge(cand, ref): function rouge_results_to_str (line 51) | def rouge_results_to_str(results_dict):