![](\"images/logo.png\"){kind=logo}
\n\n \n
\n \n ![\"Documentation](\"https://img.shields.io/badge/docs-passing-brightgreen.svg\"){kind=icon}
\n \n \n 
\n \n \n ![\"Code](\"https://img.shields.io/badge/code%20style-black-000000.svg\"){kind=icon}
\n \n ![](\"https://codecov.io/gh/naver/claf/branch/master/graph/badge.svg\"){kind=icon}
\n \n
\n\nimport argparse\nfrom argparse import RawTextHelpFormatter\nimport json\nimport os\nimport sys\n\nimport torch\n\nfrom claf import nsml\nfrom claf.config import utils\nfrom claf.config.namespace import NestedNamespace\nfrom claf.learn.mode import Mode\n\n\n[docs]def config(argv=None, mode=None):\n if argv is None:\n argv = sys.argv[1:] # 0 is excute file_name\n\n parser = argparse.ArgumentParser(formatter_class=RawTextHelpFormatter)\n\n general(parser)\n\n if mode == Mode.EVAL:\n evaluate(parser)\n return parser.parse_args(argv, namespace=NestedNamespace())\n\n if mode == Mode.PREDICT:\n predict(parser)\n return parser.parse_args(argv, namespace=NestedNamespace())\n\n if mode == Mode.MACHINE:\n machine(parser)\n config = parser.parse_args(argv, namespace=NestedNamespace())\n\n if config.machine_config is None:\n raise ValueError("--machine_config is required.")\n machine_config_path = os.path.join("machine_config", config.machine_config + ".json")\n with open(machine_config_path, "r") as f:\n defined_config = json.load(f)\n config.overwrite(defined_config)\n return config\n\n return train_config(parser, input_argv=argv)\n\n\n[docs]def train_config(parser, input_argv=None):\n """ Add argument only for hyperparameter tuning. """\n\n data(parser)\n token(parser)\n model(parser)\n if nsml.IS_ON_NSML:\n nsml_for_internal(parser)\n trainer(parser)\n\n # Use from config file\n base_config(parser)\n\n config = parser.parse_args(input_argv, namespace=NestedNamespace())\n\n use_base_config = config.base_config\n # use pre-defined base_config\n if use_base_config:\n base_config_path = os.path.join("base_config", config.base_config + ".json")\n with open(base_config_path, "r") as f:\n defined_config = json.load(f)\n # config.overwrite(defined_config)\n\n config = NestedNamespace()\n config.load_from_json(defined_config)\n\n # overwrite input argument when base_config and arguments are provided.\n # (eg. --base_config bidaf --learning_rate 2) -> set bidaf.json then overwrite learning_rate 2)\n input_args = get_input_arguments(parser, input_argv)\n for k, v in input_args.items():\n setattr(config, k, v)\n\n if not use_base_config:\n config = optimize_config(config)\n\n set_gpu_env(config)\n set_batch_size(config)\n return config\n\n\n[docs]def get_input_arguments(parser, input_arguments):\n flat_config = parser.parse_args(input_arguments)\n config_dict = utils.convert_config2dict(flat_config)\n config_default_none = {k: None for k in config_dict.keys()}\n\n input_parser = argparse.ArgumentParser(parents=[parser], conflict_handler="resolve")\n input_parser.set_defaults(**config_default_none)\n\n input_config = input_parser.parse_args(input_arguments)\n input_config = utils.convert_config2dict(input_config)\n\n if "base_config" in input_config:\n del input_config["base_config"]\n return {k: v for k, v in input_config.items() if v is not None}\n\n\n[docs]def optimize_config(config, is_test=False):\n if not is_test:\n # Remove unselected argument\n token_excepts = config.token.names + ["names", "types", "tokenizer"]\n config.delete_unselected(config.token, excepts=token_excepts)\n config.delete_unselected(config.model, excepts=["name", config.model.name])\n config.delete_unselected(\n config.optimizer,\n excepts=[\n "op_type",\n config.optimizer.op_type,\n "learning_rate",\n "lr_scheduler_type",\n config.optimizer.lr_scheduler_type,\n "exponential_moving_average",\n ],\n )\n\n return config\n\n\n[docs]def set_gpu_env(config):\n # GPU & NSML\n config.use_gpu = torch.cuda.is_available() or nsml.IS_ON_NSML\n\n if nsml.IS_ON_NSML:\n if getattr(config, "nsml", None) is None:\n config.nsml = NestedNamespace()\n config.nsml.dataset_path = nsml.DATASET_PATH\n config.gpu_num = int(nsml.GPU_NUM)\n else:\n config.gpu_num = len(getattr(config, "cuda_devices", []))\n\n if not config.use_gpu:\n config.gpu_num = 0\n config.cuda_devices = None\n\n\n[docs]def set_batch_size(config):\n # dynamic batch_size (multi-gpu and gradient_accumulation_steps)\n batch_size = config.iterator.batch_size\n if config.gpu_num > 1:\n batch_size *= config.gpu_num\n if getattr(config.optimizer, "gradient_accumulation_steps", None):\n batch_size = batch_size // config.optimizer.gradient_accumulation_steps\n config.iterator.batch_size = int(batch_size)\n\n\n[docs]def arg_str2bool(v):\n if v.lower() in ("yes", "true", "True", "t", "y", "1"):\n return True\n elif v.lower() in ("no", "false", "False", "f", "n", "0"):\n return False\n else:\n raise argparse.ArgumentTypeError("Boolean value expected.")\n\n\n# fmt: off\n[docs]def general(parser):\n\n group = parser.add_argument_group("General")\n group.add_argument(\n "--seed_num",\n type=int, default=21, dest="seed_num",\n help=""" Manually set seed_num (Python, Numpy, Pytorch) default is 21 """,\n )\n group.add_argument(\n "--cuda_devices", nargs="+",\n type=int, default=[], dest="cuda_devices",\n help=""" Set cuda_devices ids (use GPU). if you use NSML, use GPU_NUM""",\n )\n group.add_argument(\n "--slack_url",\n type=str, default=None, dest="slack_url",\n help=""" Slack notification (Incoming Webhook) """,\n )\n\n\n[docs]def data(parser):\n\n group = parser.add_argument_group("Data Reader")\n group.add_argument(\n "--dataset",\n type=str, default="squad", dest="data_reader.dataset",\n help=""" Dataset Name [squad|squad2] """,\n )\n group.add_argument(\n "--train_file_path",\n type=str, default="train-v1.1.json", dest="data_reader.train_file_path",\n help=""" train file path. """,\n )\n group.add_argument(\n "--valid_file_path",\n type=str, default="dev-v1.1.json", dest="data_reader.valid_file_path",\n help=""" validation file path. """,\n )\n group.add_argument(\n "--test_file_path",\n type=str, default=None, dest="data_reader.test_file_path",\n help=""" test file path. """,\n )\n\n group = parser.add_argument_group(" # SQuAD DataSet")\n group.add_argument(\n "--squad.context_max_length",\n type=int, default=None, dest="data_reader.squad.context_max_length",\n help=""" The number of SQuAD Context maximum length. """,\n )\n\n group = parser.add_argument_group(" # HistoryQA DataSet")\n group.add_argument(\n "--history.context_max_length",\n type=int, default=None, dest="data_reader.history.context_max_length",\n help=""" The number of HistoryQA Context maximum length. """,\n )\n\n group = parser.add_argument_group(" # SeqCls DataSet")\n group.add_argument(\n "--seq_cls.class_key",\n type=int, default=None, dest="data_reader.seq_cls.class_key",\n help=""" Name of the label to use for classification. """,\n )\n group.add_argument(\n "--seq_cls.sequence_max_length",\n type=int, default=None, dest="data_reader.seq_cls.sequence_max_length",\n help=""" The number of maximum sequence length. """,\n )\n\n group = parser.add_argument_group(" # SeqClsBert DataSet")\n group.add_argument(\n "--seq_cls_bert.class_key",\n type=int, default=None, dest="data_reader.seq_cls_bert.class_key",\n help=""" Name of the label to use for classification. """,\n )\n group.add_argument(\n "--seq_cls_bert.sequence_max_length",\n type=int, default=None, dest="data_reader.seq_cls_bert.sequence_max_length",\n help=""" The number of maximum sequence length. """,\n )\n\n group = parser.add_argument_group(" # TokClsBert DataSet")\n group.add_argument(\n "--tok_cls_bert.tag_key",\n type=int, default=None, dest="data_reader.tok_cls_bert.tag_key",\n help=""" Name of the label to use for classification. """,\n )\n group.add_argument(\n "--tok_cls_bert.ignore_tag_idx",\n type=int, default=None, dest="data_reader.tok_cls_bert.ignore_tag_idx",\n help=""" Index of the tag to ignore when calculating loss. (tag pad value) """,\n )\n group.add_argument(\n "--tok_cls_bert.sequence_max_length",\n type=int, default=None, dest="data_reader.tok_cls_bert.sequence_max_length",\n help=""" The number of maximum sequence length. """,\n )\n\n group = parser.add_argument_group("Iterator")\n group.add_argument(\n "--batch_size", type=int, default=32, dest="iterator.batch_size",\n help=""" Maximum batch size for trainer""",\n )\n\n\n[docs]def token(parser):\n\n group = parser.add_argument_group("Token")\n group.add_argument(\n "--token_names", nargs="+",\n type=str, default=["char", "word"], dest="token.names",\n help=""" Define tokens name""",\n )\n group.add_argument(\n "--token_types", nargs="+",\n type=str, default=["char", "word"], dest="token.types",\n help="""\\\n Use pre-defined token\n (tokenizer -> indexer -> embedder)\n\n [char|cove|elmo|exact_match|frequent_word|word]""",\n )\n\n group = parser.add_argument_group(" # Vocabulary")\n\n group.add_argument(\n "--char.pad_token",\n type=str, default=None, dest="token.char.vocab.pad_token",\n help=""" Padding Token value""",\n )\n group.add_argument(\n "--char.oov_token",\n type=str, default=None, dest="token.char.vocab.oov_token",\n help=""" Out-of-Vocabulary Token value""",\n )\n group.add_argument(\n "--char.start_token",\n type=str, default=None, dest="token.char.vocab.start_token",\n help=""" Start Token value""",\n )\n group.add_argument(\n "--char.end_token",\n type=str, default=None, dest="token.char.vocab.end_token",\n help=""" End Token value""",\n )\n group.add_argument(\n "--char.min_count",\n type=int, default=None, dest="token.char.vocab.min_count",\n help=""" The number of token's min count""",\n )\n group.add_argument(\n "--char.max_vocab_size",\n type=int, default=260, dest="token.char.vocab.max_vocab_size",\n help=""" The number of vocab's max size""",\n )\n\n group.add_argument(\n "--feature.pretrained_path",\n type=str, default=None, dest="token.feature.vocab.pretrained_path",\n help=""" Add pretrained vocab_path""",\n )\n group.add_argument(\n "--feature.pad_token",\n type=str, default=None, dest="token.feature.vocab.pad_token",\n help=""" Set pad_token""",\n )\n group.add_argument(\n "--feature.oov_token",\n type=str, default=None, dest="token.feature.vocab.oov_token",\n help=""" Set oov_token""",\n )\n group.add_argument(\n "--feature.cls_token",\n type=str, default=None, dest="token.feature.vocab.cls_token",\n help=""" Set cls_token""",\n )\n group.add_argument(\n "--feature.sep_token",\n type=str, default=None, dest="token.feature.vocab.sep_token",\n help=""" Set sep_token""",\n )\n\n group.add_argument(\n "--word.pad_token",\n type=str, default=None, dest="token.word.vocab.pad_token",\n help=""" Padding Token value""",\n )\n group.add_argument(\n "--word.oov_token",\n type=str, default=None, dest="token.word.vocab.oov_token",\n help=""" Out-of-Vocabulary Token value""",\n )\n group.add_argument(\n "--word.min_count",\n type=int, default=None, dest="token.word.vocab.min_count",\n help=""" The number of token's min count""",\n )\n group.add_argument(\n "--word.max_vocab_size",\n type=int, default=None, dest="token.word.vocab.max_vocab_size",\n help=""" The number of vocab's max size""",\n )\n\n group.add_argument(\n "--frequent_word.frequent_count",\n type=int, default=1000, dest="token.frequent_word.vocab.frequent_count",\n help="""\\\n The number of threshold frequent count\n (>= threshold -> fine-tune, < threshold -> fixed)""",\n )\n\n group = parser.add_argument_group(" # Tokenizer")\n\n group.add_argument(\n "--tokenizer.bpe.name",\n type=str, default="roberta", dest="token.tokenizer.bpe.name",\n help="""\\\n BPE Tokenizer package name [roberta]\n Default is 'roberta' """,\n )\n group.add_argument(\n "--tokenizer.bpe.roberta.vocab_path",\n type=str, default=None, dest="token.tokenizer.bpe.roberta.vocab_path",\n help="""\\\n RoBERTa BPE Tokenizer vocab_path\n Default is 'None' """,\n )\n group.add_argument(\n "--tokenizer.bpe.roberta.merges_path",\n type=str, default=None, dest="token.tokenizer.bpe.roberta.merges_path",\n help="""\\\n RoBERTa BPE Tokenizer merges_path\n Default is 'None' """,\n )\n\n group.add_argument(\n "--tokenizer.char.name",\n type=str, default="character", dest="token.tokenizer.char.name",\n help="""\\\n CharTokenizer package name [character|jamo_ko]\n Default is 'character' """,\n )\n\n group.add_argument(\n "--tokenizer.subword.name",\n type=str, default="wordpiece", dest="token.tokenizer.subword.name",\n help="""\\\n SubWordTokenizer package name [wordpiece]\n Default is 'wordpiece' """,\n )\n group.add_argument(\n "--tokenizer.subword.wordpiece.vocab_path",\n type=str, default=None, dest="token.tokenizer.subword.wordpiece.vocab_path",\n help="""\\\n Wordpiece Tokenizer vocab_path\n Default is 'None' """,\n )\n\n group.add_argument(\n "--tokenizer.word.name",\n type=str, default="treebank_en", dest="token.tokenizer.word.name",\n help="""\\\n WordTokenizer package name [treebank_en|spacy_en|mecab_ko]\n Default is 'treebank_en' """,\n )\n group.add_argument(\n "--tokenizer.word.split_with_regex",\n type=arg_str2bool, default=False, dest="token.tokenizer.word.split_with_regex",\n help=""" preprocess for SQuAD Context data (simple regex) """,\n )\n group.add_argument(\n "--tokenizer.word.bert_basic.do_lower_case",\n type=arg_str2bool, default=True, dest="token.tokenizer.word.bert_basic.do_lower_case",\n help="""\\\n Wordpiece Tokenizer do_lower_case or not\n Default is 'True' """,\n )\n\n group.add_argument(\n "--tokenizer.sent.name",\n type=str, default="punkt", dest="token.tokenizer.sent.name",\n help="""\\\n SentTokenizer package name [punkt]\n Default is 'punkt' """,\n )\n\n group = parser.add_argument_group(" # Indexer")\n group.add_argument(\n "--char.insert_char_start",\n type=arg_str2bool, default=False, dest="token.char.indexer.insert_char_start",\n help=""" insert first start_token to tokens""",\n )\n group.add_argument(\n "--char.insert_char_end",\n type=arg_str2bool, default=False, dest="token.char.indexer.insert_char_end",\n help=""" append end_token to tokens""",\n )\n\n group.add_argument(\n "--exact_match.lower",\n type=arg_str2bool, default=True, dest="token.exact_match.indexer.lower",\n help=""" add lower case feature """,\n )\n group.add_argument(\n "--exact_match.lemma",\n type=arg_str2bool, default=True, dest="token.exact_match.indexer.lemma",\n help=""" add lemma case feature """,\n )\n\n group.add_argument(\n "--linguistic.pos_tag",\n type=arg_str2bool, default=True, dest="token.linguistic.indexer.pos_tag",\n help=""" add POS Tagging feature """,\n )\n group.add_argument(\n "--linguistic.ner",\n type=arg_str2bool, default=True, dest="token.linguistic.indexer.ner",\n help=""" add Named Entity Recognition feature """,\n )\n group.add_argument(\n "--linguistic.dep",\n type=arg_str2bool, default=False, dest="token.linguistic.indexer.dep",\n help=""" add Dependency Parser feature """,\n )\n\n group.add_argument(\n "--word.lowercase",\n type=arg_str2bool, default=False, dest="token.word.indexer.lowercase",\n help=""" Apply word token to lowercase""",\n )\n group.add_argument(\n "--word.insert_start",\n type=arg_str2bool, default=False, dest="token.word.indexer.insert_start",\n help=""" insert first start_token to tokens""",\n )\n group.add_argument(\n "--word.insert_end",\n type=arg_str2bool, default=False, dest="token.word.indexer.insert_end",\n help=""" append end_token to tokens""",\n )\n\n group = parser.add_argument_group(" # Embedding")\n\n group.add_argument(\n "--char.embed_dim",\n type=int, default=16, dest="token.char.embedding.embed_dim",\n help=""" The number of Embedding dimension""",\n )\n group.add_argument(\n "--char.kernel_sizes", nargs="+",\n type=int, default=[5], dest="token.char.embedding.kernel_sizes",\n help=""" CharCNN kernel_sizes (n-gram)""",\n )\n group.add_argument(\n "--char.num_filter",\n type=int, default=100, dest="token.char.embedding.num_filter",\n help=""" The number of CNN filter""",\n )\n group.add_argument(\n "--char.activation",\n type=str, default="relu", dest="token.char.embedding.activation",\n help=""" CharCNN activation Function (default: ReLU)""",\n )\n group.add_argument(\n "--char.dropout",\n type=float, default=0.2, dest="token.char.embedding.dropout",\n help=""" Embedding dropout prob (default: 0.2)""",\n )\n\n group.add_argument(\n "--cove.glove_pretrained_path",\n type=str, default=None, dest="token.cove.embedding.glove_pretrained_path",\n help=""" CoVe's word embedding pretrained_path (GloVE 840B.300d)""",\n )\n group.add_argument(\n "--cove.model_pretrained_path",\n type=str, default=None, dest="token.cove.embedding.model_pretrained_path",\n help=""" CoVe Model pretrained_path """,\n )\n group.add_argument(\n "--cove.trainable",\n type=arg_str2bool, default=True, dest="token.cove.embedding.trainable",\n help=""" CoVe Embedding Trainable""",\n )\n group.add_argument(\n "--cove.dropout",\n type=float, default=0.2, dest="token.cove.embedding.dropout",\n help=""" Embedding dropout prob (default: 0.2)""",\n )\n group.add_argument(\n "--cove.project_dim",\n type=int, default=None, dest="token.cove.embedding.project_dim",\n help=""" The number of projection dimension""",\n )\n\n group.add_argument(\n "--elmo.options_file",\n type=str, default="elmo_2x4096_512_2048cnn_2xhighway_options.json", dest="token.elmo.embedding.options_file",\n help=""" The option file path of ELMo""",\n )\n group.add_argument(\n "--elmo.weight_file",\n type=str, default="elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5", dest="token.elmo.embedding.weight_file",\n help=""" The weight file path of ELMo""",\n )\n group.add_argument(\n "--elmo.trainable",\n type=arg_str2bool, default=False, dest="token.elmo.embedding.trainable",\n help=""" elmo Embedding Trainable""",\n )\n group.add_argument(\n "--elmo.dropout",\n type=float, default=0.5, dest="token.elmo.embedding.dropout",\n help=""" Embedding dropout prob (default: 0.5)""",\n )\n group.add_argument(\n "--elmo.project_dim",\n type=int, default=None, dest="token.elmo.embedding.project_dim",\n help=""" The number of projection dimension (default is None)""",\n )\n\n group.add_argument(\n "--word_permeability.memory_clip",\n type=int, default=3, dest="token.word_permeability.embedding.memory_clip",\n help=""" The number of memory cell clip value """,\n )\n group.add_argument(\n "--word_permeability.proj_clip",\n type=int, default=3, dest="token.word_permeability.embedding.proj_clip",\n help=""" The number of p clip value after projection """,\n )\n group.add_argument(\n "--word_permeability.embed_dim",\n type=int, default=1024, dest="token.word_permeability.embedding.embed_dim",\n help=""" The number of Embedding dimension""",\n )\n group.add_argument(\n "--word_permeability.linear_dim",\n type=int, default=None, dest="token.word_permeability.embedding.linear_dim",\n help=""" The number of linear projection dimension""",\n )\n group.add_argument(\n "--word_permeability.trainable",\n type=arg_str2bool, default=False, dest="token.word_permeability.embedding.trainable",\n help=""" word_permeability Embedding Trainable """,\n )\n group.add_argument(\n "--word_permeability.dropout",\n type=float, default=0.5, dest="token.word_permeability.embedding.dropout",\n help=""" Embedding dropout prob (default: 0.5)""",\n )\n group.add_argument(\n "--word_permeability.activation",\n type=str, default="tanh", dest="token.word_permeability.embedding.activation",\n help=""" Activation Function (default is 'tanh') """,\n )\n group.add_argument(\n "--word_permeability.bidirectional",\n type=arg_str2bool, default=False, dest="token.word_permeability.embedding.bidirectional",\n help=""" bidirectional use or not ([forward;backward]) (default is False) """,\n )\n\n group.add_argument(\n "--frequent_word.embed_dim",\n type=int, default=100, dest="token.frequent_word.embedding.embed_dim",\n help=""" The number of Embedding dimension""",\n )\n group.add_argument(\n "--frequent_word.pretrained_path",\n type=str, default=None, dest="token.frequent_word.embedding.pretrained_path",\n help=""" Add pretrained Word vector model's path. (support file format like Glove)""",\n )\n group.add_argument(\n "--frequent_word.dropout",\n type=float, default=0.2, dest="token.frequent_word.embedding.dropout",\n help=""" Embedding dropout prob (default: 0.2)""",\n )\n\n group.add_argument(\n "--word.embed_dim",\n type=int, default=100, dest="token.word.embedding.embed_dim",\n help=""" The number of Embedding dimension""",\n )\n group.add_argument(\n "--word.pretrained_path",\n type=str, default=None, dest="token.word.embedding.pretrained_path",\n help=""" Add pretrained word vector model's path. (support file format like Glove)""",\n )\n group.add_argument(\n "--word.trainable",\n type=arg_str2bool, default=True, dest="token.word.embedding.trainable",\n help=""" Word Embedding Trainable""",\n )\n group.add_argument(\n "--word.dropout",\n type=float, default=0.2, dest="token.word.embedding.dropout",\n help=""" Embedding dropout prob (default: 0.2)""",\n )\n\n\n[docs]def model(parser):\n\n group = parser.add_argument_group("Model")\n group.add_argument(\n "--model_name",\n type=str, default="bidaf", dest="model.name",\n help="""\\\n\n Pre-defined model\n\n * Reading Comprehension\n [bert_for_qa|bidaf|bidaf_no_answer|docqa|docqa_no_answer|dclaf|qanet|simple]\n\n * Regression\n [bert_for_reg|roberta_for_reg]\n\n * Semantic Parsing\n [sqlnet]\n\n * Sequence Classification\n [bert_for_seq_cls|roberta_for_seq_cls|structured_self_attention]\n\n * Token Classification\n [bert_for_tok_cls]\n """,\n )\n\n reading_comprehension_title = "ㅁReading Comprehension"\n group = parser.add_argument_group(f"{reading_comprehension_title}\\n # BERT for QuestionAnswering")\n group.add_argument(\n "--bert_for_qa.pretrained_model_name",\n type=str, default=None, dest="model.bert_for_qa.pretrained_model_name",\n help=""" A str with the name of a pre-trained model to load selected in the list of (default: None):\n . `bert-base-uncased`\n . `bert-large-uncased`\n . `bert-base-cased`\n . `bert-base-multilingual`\n . `bert-base-chinese` """,\n )\n group.add_argument(\n "--bert_for_qa.answer_maxlen",\n type=int, default=None, dest="model.bert_for_qa.answer_maxlen",\n help=""" The number of maximum answer's length (default: None)""",\n )\n\n group = parser.add_argument_group(f" # RoBERTa")\n group.add_argument(\n "--roberta_for_qa.pretrained_model_name",\n type=str, default=None, dest="model.roberta_for_qa.pretrained_model_name",\n help=""" A str with the name of a pre-trained model to load selected in the list of (default: None):\n . `roberta-base`\n . `roberta-large` """,\n )\n group.add_argument(\n "--roberta_for_qa.answer_maxlen",\n type=int, default=None, dest="model.roberta_for_qa.answer_maxlen",\n help=""" The number of maximum answer's length (default: None)""",\n )\n\n group = parser.add_argument_group(f" # BiDAF")\n group.add_argument(\n "--bidaf.aligned_query_embedding",\n type=int, default=False, dest="model.bidaf.aligned_query_embedding",\n help=""" Aligned Question Embedding (default: False)""",\n )\n group.add_argument(\n "--bidaf.answer_maxlen",\n type=int, default=None, dest="model.bidaf.answer_maxlen",\n help=""" The number of maximum answer's length (default: None)""",\n )\n group.add_argument(\n "--bidaf.model_dim",\n type=int, default=100, dest="model.bidaf.model_dim",\n help=""" The number of BiDAF model dimension""",\n )\n group.add_argument(\n "--bidaf.contextual_rnn_num_layer",\n type=int, default=1, dest="model.bidaf.contextual_rnn_num_layer",\n help=""" The number of BiDAF model contextual_rnn's recurrent layers""",\n )\n group.add_argument(\n "--bidaf.modeling_rnn_num_layer",\n type=int, default=2, dest="model.bidaf.modeling_rnn_num_layer",\n help=""" The number of BiDAF model modeling_rnn's recurrent layers""",\n )\n group.add_argument(\n "--bidaf.predict_rnn_num_layer",\n type=int, default=1, dest="model.bidaf.predict_rnn_num_layer",\n help=""" The number of BiDAF model predict_rnn's recurrent layers""",\n )\n group.add_argument(\n "--bidaf.dropout",\n type=float, default=0.2, dest="model.bidaf.dropout",\n help=""" The prob of BiDAF dropout""",\n )\n\n group = parser.add_argument_group(" # BiDAF + Simple bias")\n group.add_argument(\n "--bidaf_no_answer.aligned_query_embedding",\n type=int, default=False, dest="model.bidaf_no_answer.aligned_query_embedding",\n help=""" Aligned Question Embedding (default: False)""",\n )\n group.add_argument(\n "--bidaf_no_answer.answer_maxlen",\n type=int, default=None, dest="model.bidaf_no_answer.answer_maxlen",\n help=""" The number of maximum answer's length (default: None)""",\n )\n group.add_argument(\n "--bidaf_no_answer.model_dim",\n type=int, default=100, dest="model.bidaf_no_answer.model_dim",\n help=""" The number of BiDAF model dimension""",\n )\n group.add_argument(\n "--bidaf_no_answer.contextual_rnn_num_layer",\n type=int, default=1, dest="model.bidaf_no_answer.contextual_rnn_num_layer",\n help=""" The number of BiDAF model contextual_rnn's recurrent layers""",\n )\n group.add_argument(\n "--bidaf_no_answer.modeling_rnn_num_layer",\n type=int, default=2, dest="model.bidaf_no_answer.modeling_rnn_num_layer",\n help=""" The number of BiDAF model modeling_rnn's recurrent layers""",\n )\n group.add_argument(\n "--bidaf_no_answer.predict_rnn_num_layer",\n type=int, default=1, dest="model.bidaf_no_answer.predict_rnn_num_layer",\n help=""" The number of BiDAF model predict_rnn's recurrent layers""",\n )\n group.add_argument(\n "--bidaf_no_answer.dropout",\n type=float, default=0.2, dest="model.bidaf_no_answer.dropout",\n help=""" The prob of BiDAF dropout""",\n )\n\n group = parser.add_argument_group(" # Simple")\n group.add_argument(\n "--simple.answer_maxlen",\n type=int, default=None, dest="model.simple.answer_maxlen",\n help=""" The number of maximum answer's length (default: None)""",\n )\n group.add_argument(\n "--simple.model_dim",\n type=int, default=100, dest="model.simple.model_dim",\n help=""" The number of Simple model dimension""",\n )\n group.add_argument(\n "--simple.dropout",\n type=float, default=0.2, dest="model.simple.dropout",\n help=""" The prob of Simple dropout""",\n )\n\n group = parser.add_argument_group(" # QANet")\n group.add_argument(\n "--qanet.aligned_query_embedding",\n type=int, default=False, dest="model.qanet.aligned_query_embedding",\n help=""" Aligned Question Embedding (default: False)""",\n )\n group.add_argument(\n "--qanet.answer_maxlen",\n type=int, default=30, dest="model.qanet.answer_maxlen",\n help=""" The number of maximum answer's length (default: 30)""",\n )\n group.add_argument(\n "--qanet.model_dim",\n type=int, default=128, dest="model.qanet.model_dim",\n help=""" The number of QANet model dimension""",\n )\n group.add_argument(\n "--qanet.kernel_size_in_embedding",\n type=int, default=7, dest="model.qanet.kernel_size_in_embedding",\n help=""" The number of QANet model Embed Encoder kernel_size""",\n )\n group.add_argument(\n "--qanet.num_head_in_embedding",\n type=int, default=8, dest="model.qanet.num_head_in_embedding",\n help=""" The number of QANet model Multi-Head Attention's head in Embedding Block""",\n )\n group.add_argument(\n "--qanet.num_conv_block_in_embedding",\n type=int, default=4, dest="model.qanet.num_conv_block_in_embedding",\n help=""" The number of QANet model Conv Blocks in Embedding Block""",\n )\n group.add_argument(\n "--qanet.num_embedding_encoder_block",\n type=int, default=1, dest="model.qanet.num_embedding_encoder_block",\n help=""" The number of QANet model Embedding Encoder Blocks""",\n )\n group.add_argument(\n "--qanet.kernel_size_in_modeling",\n type=int, default=5, dest="model.qanet.kernel_size_in_modeling",\n help=""" The number of QANet model Model Encoder kernel_size""",\n )\n group.add_argument(\n "--qanet.num_head_in_modeling",\n type=int, default=8, dest="model.qanet.num_head_in_modeling",\n help=""" The number of QANet model Multi-Head Attention's head in Modeling Block""",\n )\n group.add_argument(\n "--qanet.num_conv_block_in_modeling",\n type=int, default=2, dest="model.qanet.num_conv_block_in_modeling",\n help=""" The number of QANet model Conv Blocks in Modeling Block""",\n )\n group.add_argument(\n "--qanet.num_modeling_encoder_block",\n type=int, default=7, dest="model.qanet.num_modeling_encoder_block",\n help=""" The number of QANet model Modeling Encoder Blocks""",\n )\n group.add_argument(\n "--qanet.layer_dropout",\n type=float, default=0.9, dest="model.qanet.layer_dropout",\n help=""" The prob of QANet model layer dropout""",\n )\n group.add_argument(\n "--qanet.dropout",\n type=float, default=0.1, dest="model.qanet.dropout",\n help=""" The prob of QANet dropout""",\n )\n\n group = parser.add_argument_group(" # DocQA")\n group.add_argument(\n "--docqa.aligned_query_embedding",\n type=arg_str2bool, default=False, dest="model.docqa.aligned_query_embedding",\n help=""" Aligned Question Embedding (default: False)""",\n )\n group.add_argument(\n "--docqa.answer_maxlen",\n type=int, default=17, dest="model.docqa.answer_maxlen",\n help=""" The number of maximum answer's length (default: 17)""",\n )\n group.add_argument(\n "--docqa.rnn_dim",\n type=int, default=100, dest="model.docqa.rnn_dim",\n help=""" The number of DocQA model rnn dimension""",\n )\n group.add_argument(\n "--docqa.linear_dim",\n type=int, default=200, dest="model.docqa.linear_dim",\n help=""" The number of DocQA model linear dimension""",\n )\n group.add_argument(\n "--docqa.preprocess_rnn_num_layer",\n type=int, default=1, dest="model.docqa.preprocess_rnn_num_layer",\n help=""" The number of DocQA model preprocess_rnn's recurrent layers""",\n )\n group.add_argument(\n "--docqa.modeling_rnn_num_layer",\n type=int, default=1, dest="model.docqa.modeling_rnn_num_layer",\n help=""" The number of DocQA model modeling_rnn's recurrent layers""",\n )\n group.add_argument(\n "--docqa.predict_rnn_num_layer",\n type=int, default=1, dest="model.docqa.predict_rnn_num_layer",\n help=""" The number of DocQA model predict_rnn's recurrent layers""",\n )\n group.add_argument(\n "--docqa.dropout",\n type=float, default=0.2, dest="model.docqa.dropout",\n help=""" The prob of DocQA dropout""",\n )\n group.add_argument(\n "--docqa.weight_init",\n type=arg_str2bool, default=True, dest="model.docqa.weight_init",\n help=""" Weight Init""",\n )\n\n group = parser.add_argument_group(" # DocQA + No_Answer Option")\n group.add_argument(\n "--docqa_no_answer.aligned_query_embedding",\n type=arg_str2bool, default=False, dest="model.docqa_no_answer.aligned_query_embedding",\n help=""" Aligned Question Embedding (default: False)""",\n )\n group.add_argument(\n "--docqa_no_answer.answer_maxlen",\n type=int, default=17, dest="model.docqa_no_answer.answer_maxlen",\n help=""" The number of maximum answer's length (default: None)""",\n )\n group.add_argument(\n "--docqa_no_answer.rnn_dim",\n type=int, default=100, dest="model.docqa_no_answer.rnn_dim",\n help=""" The number of docqa_no_answer model rnn dimension""",\n )\n group.add_argument(\n "--docqa_no_answer.linear_dim",\n type=int, default=200, dest="model.docqa_no_answer.linear_dim",\n help=""" The number of docqa_no_answer model linear dimension""",\n )\n group.add_argument(\n "--docqa_no_answer.dropout",\n type=float, default=0.2, dest="model.docqa_no_answer.dropout",\n help=""" The prob of QANet dropout""",\n )\n group.add_argument(\n "--docqa_no_answer.weight_init",\n type=arg_str2bool, default=True, dest="model.docqa_no_answer.weight_init",\n help=""" Weight Init""",\n )\n\n group = parser.add_argument_group(" # DrQA")\n group.add_argument(\n "--drqa.aligned_query_embedding",\n type=int, default=True, dest="model.drqa.aligned_query_embedding",\n help=""" Aligned Question Embedding (default: True)""",\n )\n group.add_argument(\n "--drqa.answer_maxlen",\n type=int, default=15, dest="model.drqa.answer_maxlen",\n help=""" The number of maximum answer's length (default: None)""",\n )\n group.add_argument(\n "--drqa.model_dim",\n type=int, default=128, dest="model.drqa.model_dim",\n help=""" The number of document reader model dimension""",\n )\n group.add_argument(\n "--drqa.dropout",\n type=int, default=0.3, dest="model.drqa.dropout",\n help=""" The number of document reader model dropout""",\n )\n\n\n regression_title = "ㅁRegression"\n group = parser.add_argument_group(f"{regression_title}\\n # BERT for Regression")\n group.add_argument(\n "--bert_for_reg.pretrained_model_name",\n type=str, default=None, dest="model.bert_for_reg.pretrained_model_name",\n help=""" A str with the name of a pre-trained model to load selected in the list of (default: None):\n . `bert-base-uncased`\n . `bert-large-uncased`\n . `bert-base-cased`\n . `bert-base-multilingual`\n . `bert-base-chinese` """,\n )\n group.add_argument(\n "--bert_for_reg.dropout",\n type=float, default=0.2, dest="model.bert_for_reg.dropout",\n help=""" The prob of fc layer dropout """\n )\n\n group = parser.add_argument_group(f" # RoBERTa")\n group.add_argument(\n "--roberta_for_reg.pretrained_model_name",\n type=str, default=None, dest="model.roberta_for_reg.pretrained_model_name",\n help=""" A str with the name of a pre-trained model to load selected in the list of (default: None):\n . `roberta-base`\n . `roberta-large` """,\n )\n group.add_argument(\n "--roberta_for_reg.dropout",\n type=float, default=0.2, dest="model.roberta_for_reg.dropout",\n help=""" The prob of fc layer dropout """\n )\n\n\n semantic_parsing_title = "ㅁSemantic Parsing"\n group = parser.add_argument_group(f"{semantic_parsing_title}\\n # SQLNet")\n group.add_argument(\n "--sqlnet.column_attention",\n type=int, default=True, dest="model.sqlnet.column_attention",\n help=""" Compute attention map on a question conditioned on the column names (default: True)""",\n )\n group.add_argument(\n "--sqlnet.model_dim",\n type=int, default=100, dest="model.sqlnet.model_dim",\n help=""" The number of document reader model dimension""",\n )\n group.add_argument(\n "--sqlnet.rnn_num_layer",\n type=int, default=2, dest="model.sqlnet.rnn_num_layer",\n help=""" The number of SQLNet model rnn's recurrent layers""",\n )\n group.add_argument(\n "--sqlnet.dropout",\n type=int, default=0.3, dest="model.sqlnet.dropout",\n help=""" The prob of model dropout """,\n )\n group.add_argument(\n "--sqlnet.column_maxlen",\n type=int, default=4, dest="model.sqlnet.column_maxlen",\n help=""" The number of maximum column's length (default: 4)""",\n )\n group.add_argument(\n "--sqlnet.token_maxlen",\n type=int, default=200, dest="model.sqlnet.token_maxlen",\n help=""" An upper-bound N on the number of decoder tokeni """,\n )\n group.add_argument(\n "--sqlnet.conds_column_loss_alpha",\n type=int, default=0.3, dest="model.sqlnet.conds_column_loss_alpha",\n help=""" balance the positive data versus negative data """,\n )\n\n sequence_classification_title = "ㅁSequence Classification"\n group = parser.add_argument_group(f"{sequence_classification_title}\\n # BERT for Sequence Classification")\n group.add_argument(\n "--bert_for_seq_cls.pretrained_model_name",\n type=str, default=None, dest="model.bert_for_seq_cls.pretrained_model_name",\n help=""" A str with the name of a pre-trained model to load selected in the list of (default: None):\n . `bert-base-uncased`\n . `bert-large-uncased`\n . `bert-base-cased`\n . `bert-base-multilingual`\n . `bert-base-chinese` """,\n )\n group.add_argument(\n "--bert_for_seq_cls.dropout",\n type=float, default=0.2, dest="model.bert_for_seq_cls.dropout",\n help=""" The prob of fc layer dropout """\n )\n\n group = parser.add_argument_group(f" # RoBERTa")\n group.add_argument(\n "--roberta_for_seq_cls.pretrained_model_name",\n type=str, default=None, dest="model.roberta_for_seq_cls.pretrained_model_name",\n help=""" A str with the name of a pre-trained model to load selected in the list of (default: None):\n . `roberta-base`\n . `roberta-large` """,\n )\n group.add_argument(\n "--roberta_for_seq_cls.dropout",\n type=float, default=0.2, dest="model.roberta_for_seq_cls.dropout",\n help=""" The prob of fc layer dropout """\n )\n\n group = parser.add_argument_group(f"{sequence_classification_title}\\n # Structured Self Attention")\n group.add_argument(\n "--structured_self_attention.token_encoder",\n type=str, default="bilstm", dest="model.structured_self_attention.token_encoder",\n help=""" Token encoder type [none|bilstm] """\n )\n group.add_argument(\n "--structured_self_attention.encoding_rnn_hidden_dim",\n type=int, default=600, dest="model.structured_self_attention.encoding_rnn_hidden_dim",\n help=""" The number of hidden dimension for each token """\n )\n group.add_argument(\n "--structured_self_attention.encoding_rnn_num_layer",\n type=int, default=2, dest="model.structured_self_attention.encoding_rnn_num_layer",\n help=""" The number of layers of token encoding rnn """\n )\n group.add_argument(\n "--structured_self_attention.encoding_rnn_dropout",\n type=float, default=0., dest="model.structured_self_attention.encoding_rnn_dropout",\n help=""" The prob of token encoding rnn dropout (between layers) """\n )\n group.add_argument(\n "--structured_self_attention.attention_dim",\n type=int, default=350, dest="model.structured_self_attention.attention_dim",\n help=""" The number of embedding dimension for attention """\n )\n group.add_argument(\n "--structured_self_attention.num_attention_heads",\n type=int, default=30, dest="model.structured_self_attention.num_attention_heads",\n help=""" The number of rows for attention (attention heads) """\n )\n group.add_argument(\n "--structured_self_attention.project_dim",\n type=int, default=2000, dest="model.structured_self_attention.project_dim",\n help=""" The number of bottleneck layer embedding dimension """\n )\n group.add_argument(\n "--structured_self_attention.dropout",\n type=float, default=0.5, dest="model.structured_self_attention.dropout",\n help=""" The prob of bottleneck-making fnn dropout """\n )\n group.add_argument(\n "--structured_self_attention.penalization_coefficient",\n type=float, default=1., dest="model.structured_self_attention.penalization_coefficient",\n help=""" The coefficient of penalization term """\n )\n\n token_classification_title = "ㅁToken Classification"\n group = parser.add_argument_group(f"{token_classification_title}\\n # BERT for Token Classification")\n group.add_argument(\n "--bert_for_tok_cls.pretrained_model_name",\n type=str, default=None, dest="model.bert_for_tok_cls.pretrained_model_name",\n help=""" A str with the name of a pre-trained model to load selected in the list of (default: None):\n . `bert-base-uncased`\n . `bert-large-uncased`\n . `bert-base-cased`\n . `bert-base-multilingual`\n . `bert-base-chinese` """,\n )\n group.add_argument(\n "--bert_for_tok_cls.dropout",\n type=float, default=0.2, dest="model.bert_for_tok_cls.dropout",\n help=""" The prob of fc layer dropout """\n )\n\n group = parser.add_argument_group(f" # RoBERTa")\n group.add_argument(\n "--roberta_for_tok_cls.pretrained_model_name",\n type=str, default=None, dest="model.roberta_for_tok_cls.pretrained_model_name",\n help=""" A str with the name of a pre-trained model to load selected in the list of (default: None):\n . `roberta-base`\n . `roberta-large` """,\n )\n group.add_argument(\n "--roberta_for_tok_cls.dropout",\n type=float, default=0.2, dest="model.roberta_for_tok_cls.dropout",\n help=""" The prob of fc layer dropout """\n )\n\n\n[docs]def nsml_for_internal(parser):\n\n group = parser.add_argument_group("NSML")\n group.add_argument(\n "--pause",\n type=int, default=0, dest="nsml.pause",\n help=""" NSML default setting"""\n )\n group.add_argument(\n "--iteration",\n type=int, default=0, dest="nsml.iteration",\n help=""" Start from NSML epoch count""",\n )\n\n\n[docs]def trainer(parser):\n\n group = parser.add_argument_group("Trainer")\n group.add_argument(\n "--num_epochs",\n type=int, default=20, dest="trainer.num_epochs",\n help=""" The number of training epochs""",\n )\n group.add_argument(\n "--patience",\n type=int, default=10, dest="trainer.early_stopping_threshold",\n help=""" The number of early stopping threshold""",\n )\n group.add_argument(\n "--metric_key",\n type=str, default="em", dest="trainer.metric_key",\n help=""" The key of metric for model's score""",\n )\n group.add_argument(\n "--verbose_step_count",\n type=int, default=100, dest="trainer.verbose_step_count",\n help=""" The number of training verbose""",\n )\n group.add_argument(\n "--eval_and_save_step_count",\n type=int, default=1, dest="trainer.eval_and_save_step_count",\n help=""" The number of save and evaluate step_count (e.g. 'epoch' or 1000)""",\n )\n group.add_argument(\n "--save_checkpoint",\n type=arg_str2bool, default=True, dest="trainer.save_checkpoint",\n help=""" The boolean value of save checkpoint""",\n )\n group.add_argument(\n "--log_dir",\n type=str, default="logs/experiment_1", dest="trainer.log_dir",\n help=""" TensorBoard and Checkpoint log directory""",\n )\n\n group = parser.add_argument_group("Gradient")\n group.add_argument(\n "--grad_max_norm",\n type=float, default=None, dest="trainer.grad_max_norm",\n help=""" Clips gradient norm of an iterable of parameters. (Default: None)""")\n\n group = parser.add_argument_group("Optimizer")\n group.add_argument(\n "--optimizer_type",\n type=str, default="adam", dest="optimizer.op_type",\n help=""" Optimizer\n (https://pytorch.org/docs/stable/optim.html#algorithms)\n\n - adadelta: ADADELTA: An Adaptive Learning Rate Method\n (https://arxiv.org/abs/1212.5701)\n - adagrad: Adaptive Subgradient Methods for Online Learning and Stochastic Optimization\n (http://jmlr.org/papers/v12/duchi11a.html)\n - adam: Adam: A Method for Stochastic Optimization\n (https://arxiv.org/abs/1412.6980)\n - adamw: Adam: Adam algorithm with weight decay fix. (BertAdam)\n - sparse_adam: Implements lazy version of Adam algorithm suitable for sparse tensors.\n In this variant, only moments that show up in the gradient get updated,\n and only those portions of the gradient get applied to the parameters.\n - adamax: Implements Adamax algorithm (a variant of Adam based on infinity norm).\n - averaged_sgd: Acceleration of stochastic approximation by averaging\n (http://dl.acm.org/citation.cfm?id=131098)\n - rmsprop: Implements RMSprop algorithm.\n (https://arxiv.org/pdf/1308.0850v5.pdf)\n - rprop: Implements the resilient backpropagation algorithm.\n - sgd: Implements stochastic gradient descent (optionally with momentum).\n Nesterov momentum: (http://www.cs.toronto.edu/~hinton/absps/momentum.pdf)\n\n [adadelta|adagrad|adam|adamw|sparse_adam|adamax|averaged_sgd|rmsprop|rprop|sgd]""",\n )\n group.add_argument(\n "--learning_rate",\n type=float, default=0.5, dest="optimizer.learning_rate",\n help="""\\\n Starting learning rate.\n Recommended settings: sgd = 1, adagrad = 0.1, adadelta = 1, adam = 0.001 """,\n )\n\n group = parser.add_argument_group(" # Adadelta")\n group.add_argument(\n "--adadelta.rho",\n type=float, default=0.9, dest="optimizer.adadelta.rho",\n help="""\\\n coefficient used for computing a running average of squared gradients\n Default: 0.9 """,\n )\n group.add_argument(\n "--adadelta.eps",\n type=float, default=1e-6, dest="optimizer.adadelta.eps",\n help="""\\\n term added to the denominator to improve numerical stability\n Default: 1e-6 """,\n )\n group.add_argument(\n "--adadelta.weight_decay",\n type=float,\n default=0,\n dest="optimizer.adadelta.weight_decay",\n help="""\\\n weight decay (L2 penalty)\n Default: 0 """,\n )\n\n group = parser.add_argument_group(" # Adagrad")\n group.add_argument(\n "--adagrad.lr_decay",\n type=float, default=0, dest="optimizer.adagrad.lr_decay",\n help="""\\\n learning rate decay\n Default: 0 """,\n )\n group.add_argument(\n "--adagrad.weight_decay",\n type=float,\n default=0,\n dest="optimizer.adagrad.weight_decay",\n help="""\\\n weight decay (L2 penalty)\n Default: 0 """,\n )\n\n group = parser.add_argument_group(" # Adam")\n group.add_argument(\n "--adam.betas", nargs="+",\n type=float, default=[0.9, 0.999], dest="optimizer.adam.betas",\n help="""\\\n coefficients used for computing running averages of gradient and its square\n Default: (0.9, 0.999) """,\n )\n group.add_argument(\n "--adam.eps",\n type=float, default=1e-8, dest="optimizer.adam.eps",\n help="""\\\n term added to the denominator to improve numerical stability\n Default: 1e-8 """,\n )\n group.add_argument(\n "--adam.weight_decay",\n type=float,\n default=0,\n dest="optimizer.adam.weight_decay",\n help="""\\\n weight decay (L2 penalty)\n Default: 0 """,\n )\n\n group = parser.add_argument_group(" # AdamW")\n group.add_argument(\n "--adamw.betas", nargs="+",\n type=float, default=[0.9, 0.999], dest="optimizer.adamw.betas",\n help="""\\\n coefficients used for computing running averages of gradient and its square\n Default: (0.9, 0.999) """,\n )\n group.add_argument(\n "--adamw.eps",\n type=float, default=1e-6, dest="optimizer.adamw.eps",\n help="""\\\n term added to the denominator to improve numerical stability\n Default: 1e-8 """,\n )\n group.add_argument(\n "--adamw.weight_decay",\n type=float,\n default=0.0,\n dest="optimizer.adamw.weight_decay",\n help="""\\\n weight decay (L2 penalty)\n Default: 0 """,\n )\n group.add_argument(\n "--adamw.correct_bias",\n type=arg_str2bool,\n default=True,\n dest="optimizer.adamw.correct_bias",\n help="""\\\n can be set to False to avoid correcting bias in Adam (e.g. like in Bert TF repository).\n Default: True """,\n )\n\n group = parser.add_argument_group(" # SparseAdam")\n group.add_argument(\n "--sparse_adam.betas", nargs="+",\n type=float, default=[0.9, 0.999], dest="optimizer.sparse_adam.betas",\n help="""\\\n coefficients used for computing running averages of gradient and its square\n Default: (0.9, 0.999) """,\n )\n group.add_argument(\n "--sparse_adam.eps",\n type=float, default=1e-8, dest="optimizer.sparse_adam.eps",\n help="""\\\n term added to the denominator to improve numerical stability\n Default: 1e-8 """,\n )\n\n group = parser.add_argument_group(" # Adamax")\n group.add_argument(\n "--adamax.betas", nargs="+",\n type=float, default=[0.9, 0.999], dest="optimizer.adamax.betas",\n help="""\\\n coefficients used for computing running averages of gradient and its square.\n Default: (0.9, 0.999) """,\n )\n group.add_argument(\n "--adamax.eps",\n type=float, default=1e-8, dest="optimizer.adamax.eps",\n help="""\\\n term added to the denominator to improve numerical stability.\n Default: 1e-8 """,\n )\n group.add_argument(\n "--adamax.weight_decay",\n type=float, default=0, dest="optimizer.adamax.weight_decay",\n help="""\\\n weight decay (L2 penalty)\n Default: 0 """,\n )\n\n group = parser.add_argument_group(" # ASGD (Averaged Stochastic Gradient Descent)")\n group.add_argument(\n "--averaged_sgd.lambd",\n type=float, default=1e-4, dest="optimizer.averaged_sgd.lambd",\n help="""\\\n decay term\n Default: 1e-4 """,\n )\n group.add_argument(\n "--averaged_sgd.alpha",\n type=float, default=0.75, dest="optimizer.averaged_sgd.alpha",\n help="""\\\n power for eta update\n Default: 0.75 """,\n )\n group.add_argument(\n "--averaged_sgd.t0",\n type=float, default=1e6, dest="optimizer.averaged_sgd.t0",\n help="""\\\n point at which to start averaging\n Default: 1e6 """,\n )\n group.add_argument(\n "--averaged_sgd.weight_decay",\n type=float, default=0, dest="optimizer.averaged_sgd.weight_decay",\n help="""\\\n weight decay (L2 penalty)\n Default: 0 """,\n )\n\n group = parser.add_argument_group(" # RMSprop")\n group.add_argument(\n "--rmsprop.momentum",\n type=float, default=0, dest="optimizer.rmsprop.momentum",\n help="""\\\n momentum factor\n Default: 0 """,\n )\n group.add_argument(\n "--rmsprop.alpha",\n type=float, default=0.99, dest="optimizer.rmsprop.alpha",\n help="""\\\n smoothing constant\n Default: 0.99 """,\n )\n group.add_argument(\n "--rmsprop.eps",\n type=float, default=1e-8, dest="optimizer.rmsprop.eps",\n help="""\\\n term added to the denominator to improve numerical stability.\n Default: 1e-8 """,\n )\n group.add_argument(\n "--rmsprop.centered",\n type=arg_str2bool, default=False, dest="optimizer.rmsprop.centered",\n help="""\\\n if True, compute the centered RMSProp,\n the gradient is normalized by an estimation of its variance\n Default: False """,\n )\n group.add_argument(\n "--rmsprop.weight_decay",\n type=float, default=0, dest="optimizer.rmsprop.weight_decay",\n help="""\\\n weight decay (L2 penalty)\n Default: 0 """,\n )\n\n group = parser.add_argument_group(" # SGD (Stochastic Gradient Descent)")\n group.add_argument(\n "--sgd.momentum",\n type=float, default=0, dest="optimizer.sgd.momentum",\n help="""\\\n momentum factor\n Default: 0 """,\n )\n group.add_argument(\n "--sgd.dampening",\n type=float, default=0, dest="optimizer.sgd.dampening",\n help="""\\\n dampening for momentum\n Default: 0 """,\n )\n group.add_argument(\n "--sgd.nesterov",\n type=arg_str2bool, default=False, dest="optimizer.sgd.nesterov",\n help="""\\\n enables Nesterov momentum\n Default: False """,\n )\n group.add_argument(\n "--sgd.weight_decay",\n type=float, default=0, dest="optimizer.sgd.weight_decay",\n help="""\\\n weight decay (L2 penalty)\n Default: 0 """,\n )\n\n group = parser.add_argument_group("Learning Rate Scheduler")\n group.add_argument(\n "--lr_scheduler_type",\n type=str, default=None, dest="optimizer.lr_scheduler_type",\n help="""Learning Rate Schedule\n (https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate) \\n\n\n - lambda: Sets the learning rate of each parameter group to the\n initial lr times a given function.\n - step: Sets the learning rate of each parameter group to the\n initial lr decayed by gamma every step_size epochs.\n - multi_step: Set the learning rate of each parameter group to\n the initial lr decayed by gamma once the number of epoch\n reaches one of the milestones.\n - exponential: Set the learning rate of each parameter group to\n the initial lr decayed by gamma every epoch.\n - cosine: Set the learning rate of each parameter group using\n a cosine annealing schedule, where ηmax is set to the initial\n lr and Tcur is the number of epochs since the last restart in SGDR:\n SGDR: Stochastic Gradient Descent with Warm Restarts\n (https://arxiv.org/abs/1608.03983)\n When last_epoch=-1, sets initial lr as lr.\n\n - reduce_on_plateau: Reduce learning rate when a metric has\n stopped improving. Models often benefit from reducing the\n learning rate by a factor of 2-10 once learning stagnates.\n This scheduler reads a metrics quantity and if no improvement\n is seen for a ‘patience’ number of epochs, the learning rate is reduced.\n - warmup_constant: Linear warmup and then constant.\n Linearly increases learning rate schedule from 0 to 1 over `warmup_steps` training steps.\n Keeps learning rate schedule equal to 1. after warmup_steps.\n - warmup_linear: Linear warmup and then linear decay.\n Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.\n Linearly decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps.\n - warmup_consine: Linear warmup and then cosine decay.\n Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.\n Decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps following a cosine curve.\n If `cycles` (default=0.5) is different from default, learning rate follows cosine function after warmup.\n - warmup_consine_with_hard_restart: Linear warmup and then cosine cycles with hard restarts.\n Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.\n If `cycles` (default=1.) is different from default, learning rate follows `cycles` times a cosine decaying\n learning rate (with hard restarts).\n\n [step|multi_step|exponential|reduce_on_plateau|cosine|\n warmup_constant|warmup_linear|warmup_consine|warmup_consine_with_hard_restart]\n """,\n )\n\n group = parser.add_argument_group(" # StepLR")\n group.add_argument(\n "--step.step_size",\n type=int, default=1, dest="optimizer.step.step_size",\n help="""\\\n Period of learning rate decay.\n Default: 1""",\n )\n group.add_argument(\n "--step.gamma",\n type=float, default=0.1, dest="optimizer.step.gamma",\n help="""\\\n Multiplicative factor of learning rate decay.\n Default: 0.1. """,\n )\n group.add_argument(\n "--step.last_epoch",\n type=int, default=-1, dest="optimizer.step.last_epoch",\n help="""\\\n The index of last epoch.\n Default: -1. """\n )\n\n group = parser.add_argument_group(" # MultiStepLR")\n group.add_argument(\n "--multi_step.milestones", nargs="+",\n type=int, dest="optimizer.multi_step.milestones",\n help="""\\\n List of epoch indices. Must be increasing\n list of int""",\n )\n group.add_argument(\n "--multi_step.gamma",\n type=float, default=0.1, dest="optimizer.multi_step.gamma",\n help="""\\\n Multiplicative factor of learning rate decay.\n Default: 0.1. """,\n )\n group.add_argument(\n "--multi_step.last_epoch",\n type=int, default=-1, dest="optimizer.multi_step.last_epoch",\n help="""\\\n The index of last epoch.\n Default: -1. """\n )\n\n group = parser.add_argument_group(" # ExponentialLR")\n group.add_argument(\n "--exponential.gamma",\n type=float, default=0.1, dest="optimizer.exponential.gamma",\n help="""\\\n Multiplicative factor of learning rate decay.\n Default: 0.1. """,\n )\n group.add_argument(\n "--exponential.last_epoch",\n type=int, default=-1, dest="optimizer.exponential.last_epoch",\n help="""\\\n The index of last epoch.\n Default: -1. """\n )\n\n group = parser.add_argument_group(" # CosineAnnealingLR")\n group.add_argument(\n "--cosine.T_max",\n type=int, default=50, dest="optimizer.cosine.T_max",\n help="""\\\n Maximum number of iterations.\n Default: 50""",\n )\n group.add_argument(\n "--cosine.eta_min",\n type=float, default=0, dest="optimizer.cosine.eta_min",\n help="""\\\n Minimum learning rate.\n Default: 0. """,\n )\n group.add_argument(\n "--cosine.last_epoch",\n type=int, default=-1, dest="optimizer.cosine.last_epoch",\n help="""\\\n The index of last epoch.\n Default: -1. """\n )\n\n group = parser.add_argument_group(" # ReduceLROnPlateau")\n group.add_argument(\n "--reduce_on_plateau.factor",\n type=float, default=0.1, dest="optimizer.reduce_on_plateau.factor",\n help=""" Factor by which the learning rate will be reduced. new_lr = lr * factor. Default: 0.1. """,\n )\n group.add_argument(\n "--reduce_on_plateau.mode",\n type=str, default="min", dest="optimizer.reduce_on_plateau.mode",\n help="""\\\n One of `min`, `max`. In `min` mode, lr will\n be reduced when the quantity monitored has stopped\n decreasing; in `max` mode it will be reduced when the\n quantity monitored has stopped increasing.\n Default: 'min'. """,\n )\n group.add_argument(\n "--reduce_on_plateau.patience",\n type=int, default=10, dest="optimizer.reduce_on_plateau.patience",\n help="""\\\n Number of epochs with no improvement after which learning rate will be reduced.\n Default: 10. """,\n )\n group.add_argument(\n "--reduce_on_plateau.threshold",\n type=float, default=1e-4, dest="optimizer.reduce_on_plateau.threshold",\n help="""\\\n Threshold for measuring the new optimum, to only focus on significant changes.\n Default: 1e-4 """,\n )\n group.add_argument(\n "--reduce_on_plateau.threshold_mode",\n type=str, default="rel", dest="optimizer.reduce_on_plateau.threshold_mode",\n help="""\\\n One of rel, abs. In rel mode, dynamic_threshold = best * ( 1 + threshold ) in ‘max’ mode or\n best * ( 1 - threshold ) in min mode. In abs mode, dynamic_threshold = best + threshold\n in max mode or best - threshold in min mode.\n Default: ‘rel’. """\n )\n group.add_argument(\n "--reduce_on_plateau.cooldown",\n type=int, default=0, dest="optimizer.reduce_on_plateau.cooldown",\n help="""\\\n Number of epochs to wait before resuming normal operation after lr has been reduced.\n Default: 0. """,\n )\n group.add_argument(\n "--reduce_on_plateau.min_lr", nargs="+",\n type=float, default=0, dest="optimizer.reduce_on_plateau.min_lr",\n help="""\\\n A scalar or a list of scalars. A lower bound on the learning rate of\n all param groups or each group respectively.\n Default: 0. """,\n )\n group.add_argument(\n "--reduce_on_plateau.eps",\n type=float, default=1e-8, dest="optimizer.reduce_on_plateau.eps",\n help="""\\\n Minimal decay applied to lr. If the difference between new and\n old lr is smaller than eps, the update is ignored.\n Default: 1e-8 """,\n )\n\n group = parser.add_argument_group(" # WarmUp Constant")\n group.add_argument(\n "--warmup_constant.warmup_steps",\n type=int, default=None, dest="optimizer.warmup_constant.warmup_steps",\n help="""\\\n The number of steps to increase the learning rate from 0 to 1.\n Default: None """,\n )\n group.add_argument(\n "--warmup_constant.last_epoch",\n type=int, default=-1, dest="optimizer.warmup_constant.last_epoch",\n help="""\\\n The index of last epoch.\n Default: -1. """\n )\n\n group = parser.add_argument_group(" # WarmUp Linear")\n group.add_argument(\n "--warmup_linear.warmup_steps",\n type=int, default=None, dest="optimizer.warmup_linear.warmup_steps",\n help="""\\\n The number of steps to increase the learning rate from 0 to 1.\n Default: None """,\n )\n group.add_argument(\n "--warmup_linear_warmup_proportion",\n type=float, default=None, dest="optimizer.warmup_linear.warmup_proportion",\n help="""\\\n The number of steps (proportion of total_step) to increase the learning rate from 0 to 1.\n Default: None """,\n )\n group.add_argument(\n "--warmup_linear.last_epoch",\n type=int, default=-1, dest="optimizer.warmup_linear.last_epoch",\n help="""\\\n The index of last epoch.\n Default: -1. """\n )\n\n group = parser.add_argument_group(" # WarmUp Cosine")\n group.add_argument(\n "--warmup_cosine.warmup_steps",\n type=int, default=None, dest="optimizer.warmup_cosine.warmup_steps",\n help="""\\\n The number of steps to increase the learning rate from 0 to 1.\n Default: None """,\n )\n group.add_argument(\n "--warmup_cosine.last_epoch",\n type=int, default=-1, dest="optimizer.warmup_cosine.last_epoch",\n help="""\\\n The index of last epoch.\n Default: -1. """\n )\n group.add_argument(\n "--warmup_cosine.cycles",\n type=float, default=.5, dest="optimizer.warmup_cosine.cycles",\n help="""\\\n If `cycles` is different from default, learning rate follows cosine function after warmup\n Default: .5 """\n )\n\n group = parser.add_argument_group(" # WarmUp Cosine with hard restarts")\n group.add_argument(\n "--warmup_cosine_with_hard_restart.warmup_steps",\n type=int, default=None, dest="optimizer.warmup_cosine_with_hard_restart.warmup_steps",\n help="""\\\n The number of steps to increase the learning rate from 0 to 1.\n Default: None """,\n )\n group.add_argument(\n "--warmup_cosine_with_hard_restart.last_epoch",\n type=int, default=-1, dest="optimizer.warmup_cosine_with_hard_restart.last_epoch",\n help="""\\\n The index of last epoch.\n Default: -1. """\n )\n group.add_argument(\n "--warmup_cosine_with_hard_restart.cycles",\n type=float, default=1., dest="optimizer.warmup_cosine_with_hard_restart.cycles",\n help="""\\\n If `cycles` is different from default, learning rate follows cosine_with_hard_restart function after warmup\n Default: 1. """\n )\n\n group = parser.add_argument_group("Exponential Moving Average")\n group.add_argument(\n "--ema",\n type=float, default=None, dest="optimizer.exponential_moving_average",\n help="""\\\n Exponential Moving Average\n Default: None (don't use)""",\n )\n\n\n[docs]def base_config(parser):\n\n group = parser.add_argument_group("Base Config")\n group.add_argument(\n "--base_config",\n type=str, default=None, dest="base_config",\n help=f"""\\\n Use pre-defined base_config:\n {_get_define_config()}\n\n\n * CoNLL 2003:\n {_get_define_config(category='conll2003')}\n\n * GLUE:\n {_get_define_config(category='glue')}\n\n * KorQuAD:\n {_get_define_config(category='korquad')}\n\n * SQuAD:\n {_get_define_config(category='squad')}\n\n * WikiSQL:\n {_get_define_config(category='wikisql')}\n """,\n )\n\n\ndef _get_define_config(category=None, config_dir="base_config"):\n if category is not None:\n config_dir = os.path.join(config_dir, category)\n\n config_files = [\n config_path.replace(".json", "")\n for config_path in os.listdir(config_dir)\n if config_path.endswith(".json")\n ]\n\n if category is not None:\n config_files = [category + "/" + fname for fname in config_files]\n return config_files\n\n\n[docs]def evaluate(parser):\n\n group = parser.add_argument_group("Run evaluate")\n group.add_argument(\n "data_file_path",\n type=str,\n help=" Path to the file containing the evaluation data"\n )\n group.add_argument("checkpoint_path", type=str, help="Path to an checkpoint trained model")\n group.add_argument(\n "--infer",\n default=None, dest="inference_latency", type=int,\n help=""" Evaluate with inference-latency with maximum value (ms)""",\n )\n group.add_argument(\n "--prev_cuda_device_id",\n type=int, default=0, dest="prev_cuda_device_id",\n help=""" Previous cuda device id (need to mapping)""",\n )\n\n\n[docs]def predict(parser):\n\n group = parser.add_argument_group("Run inference")\n group.add_argument(\n "checkpoint_path",\n type=str,\n help=" Path to an checkpoint trained model")\n group.add_argument(\n "-i", "--interactive",\n default=False, dest="interactive", action="store_true",\n help=""" Interactive Mode """,\n )\n group.add_argument(\n "--prev_cuda_device_id",\n type=int, default=0, dest="prev_cuda_device_id",\n help=""" Previous cuda device id (need to mapping)""",\n )\n\n group.add_argument("--question",\n type=str, dest="question",\n help=""" Input Question (required)""")\n\n group = parser.add_argument_group(" # Reading Comprehension")\n group.add_argument("--context",\n type=str, dest="context",\n help=""" Input Context """)\n\n group = parser.add_argument_group(" # Semantic Parsing")\n group.add_argument("--column", nargs="+",\n type=str, dest="column",\n help=""" Input Database Columns """)\n group.add_argument("--db_path",\n type=str, dest="db_path",\n help=""" Input Database file path """)\n group.add_argument("--table_id",\n type=str, dest="table_id",\n help=""" Input Database Table Id """)\n\n group = parser.add_argument_group(" # Document Retrieval")\n group.add_argument("--doc_path",\n type=str, dest="doc_path",\n help=""" Document file Path """)\n\n group.add_argument(\n "--retrieval",\n type=str, default=None, dest="doc_retrieval",\n help=""" Document Retrieval Model [tfidf] """,\n )\n group.add_argument("--k",\n type=int, default=1, dest="top_k",\n help=""" Return Top K results """)\n\n group = parser.add_argument_group(" # Sequence/Token Classification")\n group.add_argument("--sequence",\n type=str, dest="sequence",\n help=""" Input Sequence """)\n\n\n[docs]def machine(parser):\n\n group = parser.add_argument_group("Machine Config")\n group.add_argument(\n "--machine_config",\n type=str, default=None, dest="machine_config",\n help=f"""\\\n Use pre-defined machine_config (.json)\n\n {_get_define_config(config_dir="./machine_config")}\n """)\n\n# fmt: on\n\n[docs]class Factory:\n """\n Factory Base Class\n\n Factory method pattern\n\n In class-based programming, the factory method pattern is a creational pattern that\n uses factory methods to deal with the problem of creating objects without having to\n specify the exact class of the object that will be created. This is done by creating\n objects by calling a factory method—either specified in an interface and implemented\n by child classes, or implemented in a base class and optionally overridden by derived\n classes—rather than by calling a constructor.\n """\n\n def __init__(self):\n pass\n\n\n\n\nfrom overrides import overrides\nfrom torch.utils.data import DataLoader\n\nfrom .base import Factory\n\n\n[docs]def make_data_loader(dataset, batch_size=32, shuffle=True, cuda_device_id=None):\n is_cpu = cuda_device_id is None\n\n return DataLoader(\n dataset,\n batch_size=batch_size,\n shuffle=shuffle,\n collate_fn=dataset.collate_fn(cuda_device_id=cuda_device_id),\n num_workers=0,\n pin_memory=is_cpu, # only CPU memory can be pinned\n )\n\n\n[docs]class DataLoaderFactory(Factory):\n """\n DataLoader Factory Class\n\n * Args:\n config: data_loader config from argument (config.data_loader)\n """\n\n def __init__(self, config):\n self.batch_size = config.batch_size\n self.cuda_device_id = None\n if config.cuda_devices:\n self.cuda_device_id = config.cuda_devices[0]\n\n[docs] @overrides\n def create(self, datasets):\n """ create train, valid and test iterator """\n dataset_key = next(iter(datasets))\n dataset = datasets[dataset_key]\n\n if getattr(dataset, "name", None) is None:\n raise ValueError("unknown dataset.")\n\n train_loader = None\n if "train" in datasets:\n train_loader = make_data_loader(\n datasets["train"],\n batch_size=self.batch_size,\n shuffle=True,\n cuda_device_id=self.cuda_device_id,\n )\n valid_loader = None\n if "valid" in datasets:\n valid_loader = make_data_loader(\n datasets["valid"],\n batch_size=self.batch_size,\n shuffle=False,\n cuda_device_id=self.cuda_device_id,\n )\n test_loader = None\n if "test" in datasets:\n test_loader = make_data_loader(\n datasets["test"],\n batch_size=self.batch_size,\n shuffle=False,\n cuda_device_id=self.cuda_device_id,\n )\n return train_loader, valid_loader, test_loader\n\n\nfrom overrides import overrides\n\nfrom claf.config.registry import Registry\n\nfrom .base import Factory\n\n\n[docs]class DataReaderFactory(Factory):\n """\n DataReader Factory Class\n\n Create Concrete reader according to config.dataset\n Get reader from reader registries (eg. @register("reader:{reader_name}"))\n\n * Args:\n config: data_reader config from argument (config.data_reader)\n """\n\n def __init__(self, config):\n self.registry = Registry()\n\n self.dataset_name = config.dataset\n file_paths = {}\n if getattr(config, "train_file_path", None) and config.train_file_path != "":\n file_paths["train"] = config.train_file_path\n if getattr(config, "valid_file_path", None) and config.valid_file_path != "":\n file_paths["valid"] = config.valid_file_path\n\n self.reader_config = {"file_paths": file_paths}\n if "params" in config and type(config.params) == dict:\n self.reader_config.update(config.params)\n if "tokenizers" in config:\n self.reader_config["tokenizers"] = config.tokenizers\n\n dataset_config = getattr(config, config.dataset, None)\n if dataset_config is not None:\n dataset_config = vars(dataset_config)\n self.reader_config.update(dataset_config)\n\n[docs] @overrides\n def create(self):\n reader = self.registry.get(f"reader:{self.dataset_name.lower()}")\n return reader(**self.reader_config)\n\n\nfrom overrides import overrides\n\nfrom claf.config.registry import Registry\nfrom claf.model.base import ModelWithTokenEmbedder, ModelWithoutTokenEmbedder\nfrom claf.model.reading_comprehension.mixin import ReadingComprehension\nfrom claf.tokens import token_embedder\n\nfrom .base import Factory\n\n\n[docs]class ModelFactory(Factory):\n """\n Model Factory Class\n\n Create Concrete model according to config.model_name\n Get model from model registries (eg. @register("model:{model_name}"))\n\n * Args:\n config: model config from argument (config.model)\n """\n\n def __init__(self, config):\n self.registry = Registry()\n\n self.name = config.name\n self.model_config = {}\n if getattr(config, config.name, None):\n self.model_config = vars(getattr(config, config.name))\n\n self.is_independent = getattr(config, "independent", False)\n\n[docs] @overrides\n def create(self, token_makers, **params):\n model = self.registry.get(f"model:{self.name}")\n\n if issubclass(model, ModelWithTokenEmbedder):\n token_embedder = self.create_token_embedder(model, token_makers)\n self.model_config["token_embedder"] = token_embedder\n elif issubclass(model, ModelWithoutTokenEmbedder):\n self.model_config["token_makers"] = token_makers\n else:\n raise ValueError(\n "Model must have inheritance. (ModelWithTokenEmbedder or ModelWithoutTokenEmbedder)"\n )\n\n return model(**self.model_config, **params)\n\n[docs] def create_token_embedder(self, model, token_makers):\n # 1. Specific case\n # ...\n\n # 2. Base case\n if issubclass(model, ReadingComprehension):\n return token_embedder.RCTokenEmbedder(token_makers)\n else:\n return token_embedder.BasicTokenEmbedder(token_makers)\n\n\nfrom overrides import overrides\nimport torch\n\nfrom claf.config.namespace import NestedNamespace\nfrom claf.learn.optimization.learning_rate_scheduler import get_lr_schedulers\nfrom claf.learn.optimization.learning_rate_scheduler import (\n LearningRateWithoutMetricsWrapper,\n LearningRateWithMetricsWrapper,\n)\nfrom claf.learn.optimization.optimizer import get_optimizer_by_name\nfrom claf.model.sequence_classification import BertForSeqCls, RobertaForSeqCls\n\nfrom .base import Factory\n\n\n[docs]class OptimizerFactory(Factory):\n """\n Optimizer Factory Class\n\n include optimizer, learning_rate_scheduler and exponential_moving_average\n\n * Args:\n config: optimizer config from argument (config.optimizer)\n """\n\n def __init__(self, config):\n # Optimizer\n self.op_type = config.op_type\n self.optimizer_params = {"lr": config.learning_rate}\n\n op_config = getattr(config, self.op_type, None)\n if op_config is not None:\n op_config = vars(op_config)\n self.optimizer_params.update(op_config)\n\n # LearningRate Scheduler\n self.lr_scheduler_type = getattr(config, "lr_scheduler_type", None)\n if self.lr_scheduler_type is not None:\n self.lr_scheduler_config = getattr(config, self.lr_scheduler_type, {})\n if type(self.lr_scheduler_config) == NestedNamespace:\n self.lr_scheduler_config = vars(self.lr_scheduler_config)\n\n if "warmup" in self.lr_scheduler_type:\n self.lr_scheduler_config["t_total"] = config.num_train_steps\n self.set_warmup_steps(config)\n\n # EMA\n self.ema = getattr(config, "exponential_moving_average", 0)\n\n[docs] def set_warmup_steps(self, config):\n warmup_proportion = self.lr_scheduler_config.get("warmup_proportion", None)\n warmup_steps = self.lr_scheduler_config.get("warmup_steps", None)\n\n if warmup_steps and warmup_proportion:\n raise ValueError("Check 'warmup_steps' and 'warmup_proportion'.")\n elif not warmup_steps and warmup_proportion:\n self.lr_scheduler_config["warmup_steps"] = int(config.num_train_steps * warmup_proportion) + 1\n del self.lr_scheduler_config["warmup_proportion"]\n elif warmup_steps and not warmup_proportion:\n pass\n else:\n raise ValueError("Check 'warmup_steps' and 'warmup_proportion'.")\n\n[docs] @overrides\n def create(self, model):\n if not issubclass(type(model), torch.nn.Module):\n raise ValueError("optimizer model is must be subclass of torch.nn.Module.")\n\n if getattr(model, "use_pytorch_transformers", False):\n weight_decay = self.optimizer_params.get("weight_decay", 0)\n model_parameters = self._group_parameters_for_transformers(model, weight_decay=weight_decay)\n else:\n model_parameters = [param for param in model.parameters() if param.requires_grad]\n\n optimizer = get_optimizer_by_name(self.op_type)(model_parameters, **self.optimizer_params)\n op_dict = {"optimizer": optimizer}\n\n # learning_rate_scheduler\n if self.lr_scheduler_type:\n self.lr_scheduler_config["optimizer"] = op_dict["optimizer"]\n lr_scheduler = get_lr_schedulers()[self.lr_scheduler_type](**self.lr_scheduler_config)\n\n if self.lr_scheduler_type == "reduce_on_plateau":\n lr_scheduler = LearningRateWithMetricsWrapper(lr_scheduler)\n else:\n lr_scheduler = LearningRateWithoutMetricsWrapper(lr_scheduler)\n\n op_dict["learning_rate_scheduler"] = lr_scheduler\n\n # exponential_moving_average\n if self.ema and self.ema > 0:\n op_dict["exponential_moving_average"] = self.ema\n\n return op_dict\n\n def _group_parameters_for_transformers(self, model, weight_decay=0):\n # Prepare optimizer\n param_optimizer = list(model.named_parameters())\n\n # hack to remove pooler, which is not used\n # thus it produce None grad that break apex\n if not isinstance(model, BertForSeqCls) or not isinstance(model, RobertaForSeqCls):\n param_optimizer = [n for n in param_optimizer if "pooler" not in n[0]]\n\n no_decay = ["bias", "LayerNorm.weight"]\n optimizer_grouped_parameters = [\n {\n "params": [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n "weight_decay": weight_decay,\n },\n {\n "params": [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n "weight_decay": 0.0,\n },\n ]\n return optimizer_grouped_parameters\n\n\nfrom overrides import overrides\n\nfrom claf.config.registry import Registry\nfrom claf.config.utils import convert_config2dict\nfrom claf.tokens import tokenizer\n\nfrom .base import Factory\n\n\n[docs]def make_tokenizer(tokenizer_cls, tokenizer_config, parent_tokenizers={}):\n if tokenizer_config is None or "name" not in tokenizer_config:\n return None\n\n package_name = tokenizer_config["name"]\n package_config = tokenizer_config.get(package_name, {})\n tokenizer_config["config"] = package_config\n if package_name in tokenizer_config:\n del tokenizer_config[package_name]\n\n tokenizer_config.update(parent_tokenizers)\n\n return tokenizer_cls(**tokenizer_config)\n\n\n[docs]def make_all_tokenizers(all_tokenizer_config):\n """ Tokenizer is resource used all token together """\n\n sent_tokenizer = make_tokenizer(\n tokenizer.SentTokenizer, all_tokenizer_config.get("sent", {"name": "punkt"})\n )\n word_tokenizer = make_tokenizer(\n tokenizer.WordTokenizer,\n all_tokenizer_config.get("word", None),\n parent_tokenizers={"sent_tokenizer": sent_tokenizer},\n )\n subword_tokenizer = make_tokenizer(\n tokenizer.SubwordTokenizer,\n all_tokenizer_config.get("subword", None),\n parent_tokenizers={"word_tokenizer": word_tokenizer},\n )\n char_tokenizer = make_tokenizer(\n tokenizer.CharTokenizer,\n all_tokenizer_config.get("char", None),\n parent_tokenizers={"word_tokenizer": word_tokenizer},\n )\n bpe_tokenizer = make_tokenizer(\n tokenizer.BPETokenizer,\n all_tokenizer_config.get("bpe", None),\n )\n\n return {\n "bpe": bpe_tokenizer,\n "char": char_tokenizer,\n "subword": subword_tokenizer,\n "word": word_tokenizer,\n "sent": sent_tokenizer,\n }\n\n\n[docs]class TokenMakersFactory(Factory):\n """\n TokenMakers Factory Class\n\n * Args:\n config: token config from argument (config.token)\n """\n\n LANGS = ["eng", "kor"]\n\n def __init__(self, config):\n self.config = config\n self.registry = Registry()\n\n[docs] @overrides\n def create(self):\n if getattr(self.config, "tokenizer", None):\n tokenizers = make_all_tokenizers(convert_config2dict(self.config.tokenizer))\n else:\n tokenizers = {}\n\n token_names, token_types = self.config.names, self.config.types\n\n if len(token_names) != len(token_types):\n raise ValueError("token_names and token_types must be same length.")\n\n token_makers = {"tokenizers": tokenizers}\n for token_name, token_type in sorted(zip(token_names, token_types)):\n token_config = getattr(self.config, token_name, {})\n if token_config != {}:\n token_config = convert_config2dict(token_config)\n\n # Token (tokenizer, indexer, embedding, vocab)\n token_config = {\n "tokenizers": tokenizers,\n "indexer_config": token_config.get("indexer", {}),\n "embedding_config": token_config.get("embedding", {}),\n "vocab_config": token_config.get("vocab", {}),\n }\n token_makers[token_name] = self.registry.get(f"token:{token_type}")(**token_config)\n return token_makers\n\n\nimport argparse\n\n\n[docs]class NestedNamespace(argparse.Namespace):\n """\n Nested Namespace\n (Simple class used by default by parse_args() to create\n an object holding attributes and return it.)\n """\n\n def __setattr__(self, name, value):\n if "." in name:\n group, name = name.split(".", 1)\n namespace = getattr(self, group, NestedNamespace())\n setattr(namespace, name, value)\n self.__dict__[group] = namespace\n else:\n self.__dict__[name] = value\n\n[docs] def delete_unselected(self, namespace, excepts=[]):\n delete_keys = []\n for key in namespace.__dict__:\n if key not in excepts:\n delete_keys.append(key)\n\n for key in delete_keys:\n delattr(namespace, key)\n\n[docs] def overwrite(self, config):\n def _overwrite(namespace, d):\n for k, v in d.items():\n if type(v) == dict:\n nested_namespace = getattr(namespace, k, None)\n if nested_namespace is None:\n nested_namespace = NestedNamespace()\n nested_namespace.load_from_json(v)\n\n setattr(namespace, k, nested_namespace)\n else:\n _overwrite(nested_namespace, v)\n else:\n setattr(namespace, k, v)\n return namespace\n\n return _overwrite(self, config)\n\n[docs] def load_from_json(self, dict_data):\n\n name_value_pairs = []\n\n def make_key_value_pairs(d, prefix=""):\n for k, v in d.items():\n if type(v) == dict:\n next_prefix = k\n if prefix != "":\n next_prefix = f"{prefix}.{k}"\n make_key_value_pairs(v, prefix=next_prefix)\n else:\n key_with_prefix = k\n if prefix != "":\n key_with_prefix = f"{prefix}.{k}"\n name_value_pairs.append((key_with_prefix, v))\n\n make_key_value_pairs(dict_data)\n for (name, value) in name_value_pairs:\n self.__setattr__(name, value)\n\n[docs]class Singleton(type):\n """\n Design Pattern Base\n\n Singleton Meta Class\n the singleton pattern is a software design pattern that restricts the\n instantiation of a class to one object.\n """\n\n _instances = {}\n\n def __call__(cls, *args, **kwargs):\n if cls not in cls._instances:\n cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)\n return cls._instances[cls]\n\n\nimport logging\n\nfrom claf.config.pattern import Singleton\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]class Registry(metaclass=Singleton):\n """\n Registry class (Singleton)\n """\n\n def __init__(self):\n self._name_to_subclass = {\n "component": {},\n "reader": {},\n "machine": {},\n "model": {},\n "token": {},\n }\n\n[docs] def add(self, name, obj):\n component_type, component_name = self._split_component_type_and_name(name)\n\n if component_name in self._name_to_subclass[component_type]:\n logger.info(\n f"{component_name} is already included in Registry. It override with {obj}."\n )\n self._name_to_subclass[component_type][component_name] = obj\n\n[docs] def get(self, name):\n component_type, component_name = self._split_component_type_and_name(name)\n\n if component_type not in self._name_to_subclass:\n raise ValueError(f"There is no {component_type} in _name_to_subclass.")\n if component_name not in self._name_to_subclass[component_type]:\n raise ValueError(f"There is no {component_name} object in {component_type}.")\n return self._name_to_subclass[component_type][component_name]\n\n def _split_component_type_and_name(self, name):\n if ":" in name:\n names = name.split(":")\n return names[0], names[1]\n else:\n raise ValueError("do not recognize component_type.")\n\n\nfrom argparse import Namespace\nimport copy\nimport json\n\nimport jsbeautifier\nimport numpy as np\nimport random\nimport torch\n\n\n[docs]def pretty_json_dumps(inputs):\n js_opts = jsbeautifier.default_options()\n js_opts.indent_size = 2\n\n inputs = remove_none(inputs)\n return jsbeautifier.beautify(json.dumps(inputs))\n\n\n[docs]def remove_none(obj):\n if isinstance(obj, (list, tuple, set)):\n return type(obj)(remove_none(x) for x in obj if x is not None)\n elif isinstance(obj, dict):\n return type(obj)(\n (remove_none(k), remove_none(v))\n for k, v in obj.items()\n if k is not None and v is not None\n )\n else:\n return obj\n\n\n[docs]def convert_config2dict(config):\n config_dict = copy.deepcopy(config)\n if isinstance(config_dict, Namespace):\n config_dict = vars(config_dict)\n\n for k, v in config_dict.items():\n if isinstance(v, Namespace):\n config_dict[k] = convert_config2dict(v)\n return config_dict\n\n\n[docs]def set_global_seed(seed=21):\n # Tensorflow\n try:\n import tensorflow as tf\n except ImportError:\n pass\n else:\n tf.set_random_seed(seed)\n\n # PyTorch\n torch.manual_seed(seed)\n if torch.cuda.is_available():\n torch.cuda.manual_seed_all(seed)\n\n # NumPy\n np.random.seed(seed)\n\n # Python\n random.seed(seed)\n\n\nimport logging\n\nlogger = logging.getLogger(__name__)\n\n\n\n\n\n[docs]class Batch:\n """\n Batch Data Transfer Object (DTO) Class\n\n dictionary consisting of\n - features: (dict) input\n - labels: (dict) output\n """\n\n def __init__(self, **kwargs):\n if set(kwargs.keys()) != set(["features", "labels"]):\n raise ValueError("You can use only 'features' and 'labels' as dictionary key.")\n self.__dict__ = kwargs\n\n def __repr__(self):\n return str(self.__dict__)\n\n def __len__(self):\n return len(self.__dict__)\n\n[docs] def sort_by_key(self, sort_key):\n logger.info(f"Start sort by key: {sort_key}'s length")\n\n zipped = zip(self.__dict__["features"], self.__dict__["labels"])\n\n features = self.__dict__["features"]\n if type(features) == list:\n feature_keys = list(features[0].keys())\n else:\n feature_keys = features.keys()\n\n key_index = 0 if sort_key in feature_keys else 1 # sort_key in features or labels\n\n sorted_features, sorted_labels = [], []\n for data in sorted(zipped, key=lambda x: len(x[key_index][sort_key])):\n feature, label = data\n sorted_features.append(feature)\n sorted_labels.append(label)\n\n self.__dict__["features"] = sorted_features\n self.__dict__["labels"] = sorted_labels\n zipped = None\n logger.info("Complete sorting...")\n\n[docs] def to_dict(self, flatten=False, recursive=True):\n def _flatten(d):\n if d == {}:\n return d\n\n k, v = d.popitem()\n if isinstance(v, dict):\n flat_v = _flatten(v)\n for f_k in list(flat_v.keys()):\n flat_v[k + "#" + f_k] = flat_v[f_k]\n del flat_v[f_k]\n return {**flat_v, **_flatten(d)}\n else:\n return {k: v, **_flatten(d)}\n\n def _recursive(d):\n if not isinstance(d, dict):\n return d\n\n for k, v in d.items():\n if isinstance(v, dict):\n dict_v = dict(v)\n d[k] = _recursive(dict_v)\n return d\n\n if flatten:\n d = {}\n d.update(_flatten(self.__dict__["features"]))\n d.update(_flatten(self.__dict__["labels"]))\n return d\n\n if recursive:\n return _recursive(self.__dict__)\n\n return dict(self.__dict__)\n\n\nfrom overrides import overrides\n\nimport torch\nfrom torch.autograd import Variable\n\nfrom claf.data import utils\n\n\n[docs]class PadCollator:\n """\n Collator apply pad and make tensor\n Minimizes amount of padding needed while producing mini-batch.\n\n * Kwargs:\n cuda_device_id: tensor assign to cuda device id\n Default is None (CPU)\n skip_keys: skip to make tensor\n """\n\n def __init__(self, cuda_device_id=None, pad_value=0, skip_keys=["text"]):\n self.cuda_device_id = cuda_device_id\n self.pad_value = pad_value\n self.skip_keys = skip_keys\n\n def __call__(self, features, labels):\n self.collate(features, pad_value=self.pad_value)\n self.collate(labels, apply_pad=False, pad_value=self.pad_value)\n\n return utils.make_batch(features, labels)\n\n[docs] def collate(self, datas, apply_pad=True, pad_value=0):\n for data_name, data in datas.items():\n if isinstance(data, dict):\n for key, value in data.items():\n data[key] = self._collate(\n value, apply_pad=apply_pad, token_name=key, pad_value=pad_value)\n else:\n datas[data_name] = self._collate(data, apply_pad=apply_pad)\n\n def _collate(self, value, apply_pad=True, token_name=None, pad_value=0):\n if apply_pad:\n value = self._apply_pad(value, token_name=token_name, pad_value=pad_value)\n return self._make_tensor(value)\n\n def _apply_pad(self, value, token_name=None, pad_value=0):\n return utils.padding_tokens(value, token_name=token_name, pad_value=pad_value)\n\n def _make_tensor(self, value):\n if not isinstance(value, torch.Tensor):\n value_type = utils.get_token_type(value)\n if value_type == int:\n value = torch.LongTensor(value)\n else:\n value = torch.FloatTensor(value)\n\n value = Variable(value, requires_grad=False)\n if self.cuda_device_id is not None:\n value = value.cuda(self.cuda_device_id)\n return value\n\n\n[docs]class FeatLabelPadCollator(PadCollator):\n """\n Collator apply pad and make tensor\n Minimizes amount of padding needed while producing mini-batch.\n\n FeatLabelPadCollator allows applying pad to not only features, but also labels.\n\n * Kwargs:\n cuda_device_id: tensor assign to cuda device id\n Default is None (CPU)\n skip_keys: skip to make tensor\n """\n\n @overrides\n def __call__(self, features, labels, apply_pad_labels=(), apply_pad_values=()):\n self.collate(features)\n self.collate(labels, apply_pad=False,\n apply_pad_labels=apply_pad_labels, apply_pad_values=apply_pad_values)\n\n return utils.make_batch(features, labels)\n\n[docs] @overrides\n def collate(self, datas, apply_pad=True, apply_pad_labels=(), apply_pad_values=()):\n for data_name, data in datas.items():\n if not apply_pad and data_name in apply_pad_labels:\n _apply_pad = True # ignore apply_pad\n pad_value = apply_pad_values[apply_pad_labels.index(data_name)]\n else:\n _apply_pad = apply_pad\n pad_value = 0\n\n if isinstance(data, dict):\n for key, value in data.items():\n data[key] = self._collate(\n value, apply_pad=_apply_pad, token_name=key, pad_value=pad_value)\n else:\n datas[data_name] = self._collate(data, apply_pad=_apply_pad, pad_value=pad_value)\n\n\nimport logging\nimport pickle\nimport os\nfrom pathlib import Path, PosixPath\nimport shutil\nimport tempfile\n\nimport msgpack\nimport requests\nfrom tqdm import tqdm\n\nfrom claf import nsml\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]class CachePath:\n if nsml.IS_ON_NSML:\n ROOT = Path("./claf_cache")\n else:\n ROOT = Path.home() / ".claf_cache"\n DATASET = ROOT / "dataset"\n MACHINE = ROOT / "machine"\n PRETRAINED_VECTOR = ROOT / "pretrained_vector"\n TOKEN_COUNTER = ROOT / "token_counter"\n VOCAB = ROOT / "vocab"\n\n\n[docs]class DataHandler:\n """\n DataHandler with CachePath\n\n - read (from_path, from_http)\n - dump (.msgpack or .pkl (pickle))\n - load\n """\n\n def __init__(self, cache_path=CachePath.ROOT):\n if type(cache_path) != PosixPath:\n raise ValueError(f"cache_path type is PosixPath (use pathlib.Path). not f{type(cache_path)}")\n\n self.cache_path = cache_path\n cache_path.mkdir(parents=True, exist_ok=True)\n\n[docs] def convert_cache_path(self, path):\n cache_data_path = self.cache_path / Path(path)\n return cache_data_path\n\n\n\n[docs] def read(self, file_path, encoding="utf-8", return_path=False):\n if file_path.startswith("http"):\n file_path = self._read_from_http(file_path, encoding)\n\n path = Path(file_path)\n if path.exists():\n if return_path:\n return path\n return path.read_bytes().decode(encoding)\n\n if nsml.IS_ON_NSML:\n dataset_path = Path(nsml.DATASET_PATH)\n\n path = dataset_path / file_path\n if not path.exists():\n path = dataset_path / "train" / file_path\n if not path.exists():\n raise FileNotFoundError(path)\n\n if path.exists():\n if return_path:\n return path\n return path.read_bytes().decode(encoding)\n else:\n raise FileNotFoundError(f"{file_path} is not found.")\n\n def _read_from_http(self, file_path, encoding, return_path=False):\n cache_data_path = self.cache_path / Path(file_path).name\n if cache_data_path.exists():\n logger.info(f"'{file_path}' is already downloaded.")\n pass\n else:\n with tempfile.TemporaryFile() as temp_file:\n self._download_from_http(temp_file, file_path)\n temp_file.flush()\n temp_file.seek(0)\n\n with open(cache_data_path, 'wb') as cache_file:\n shutil.copyfileobj(temp_file, cache_file)\n\n return cache_data_path\n\n def _download_from_http(self, temp_file, url):\n req = requests.get(url, stream=True)\n content_length = req.headers.get('Content-Length')\n total = int(content_length) if content_length is not None else None\n with tqdm(total=total, unit="B", unit_scale=True, desc="download...") as pbar:\n for chunk in req.iter_content(chunk_size=1024):\n if chunk: # filter out keep-alive new chunks\n temp_file.write(chunk)\n pbar.update(len(chunk))\n\n[docs] def cache_token_counter(self, data_reader_config, tokenizer_name, obj=None):\n data_paths = os.path.basename(data_reader_config.train_file_path)\n if getattr(data_reader_config, "valid_file_path", None):\n data_paths += "#" + os.path.basename(data_reader_config.valid_file_path)\n\n path = self.cache_path / data_reader_config.dataset / data_paths\n path.mkdir(parents=True, exist_ok=True)\n path = path / tokenizer_name\n\n if obj:\n self.dump(path, obj)\n else:\n return self.load(path)\n\n[docs] def load(self, file_path, encoding="utf-8"):\n path = self.cache_path / file_path\n logger.info(f"load path: {path}")\n\n msgpack_path = path.with_suffix(".msgpack")\n if msgpack_path.exists():\n return self._load_msgpack(msgpack_path, encoding)\n\n pickle_path = path.with_suffix(".pkl")\n if pickle_path.exists():\n return self._load_pickle(pickle_path, encoding)\n\n return None\n\n def _load_msgpack(self, path, encoding):\n with open(path, "rb") as in_file:\n return msgpack.unpack(in_file, encoding=encoding)\n\n def _load_pickle(self, path, encoding):\n with open(path, "rb") as in_file:\n return pickle.load(in_file, encoding=encoding)\n\n[docs] def dump(self, file_path, obj, encoding="utf-8"):\n path = self.cache_path / file_path\n path.parent.mkdir(parents=True, exist_ok=True)\n\n try:\n with open(path.with_suffix(".msgpack"), "wb") as out_file:\n msgpack.pack(obj, out_file, encoding=encoding)\n except TypeError:\n os.remove(path.with_suffix(".msgpack"))\n with open(path.with_suffix(".pkl"), "wb") as out_file:\n pickle.dump(obj, out_file, protocol=pickle.HIGHEST_PROTOCOL)\n\n\nfrom torch.utils.data.dataset import Dataset\n\nfrom claf.data import utils\n\n\n[docs]class DatasetBase(Dataset):\n """\n Dataset Base Model\n An abstract class representing a Dataset.\n """\n\n def __init__(self):\n # Features - Lazy Evaluation\n self.f_count = 0\n self.features = []\n\n def __getitem__(self, index):\n raise NotImplementedError\n\n def _get_feature_maxlen(self, features):\n max_len = -1\n for feature in features:\n for token_name, sentence in feature.items():\n if token_name == "text":\n continue\n if callable(sentence):\n continue\n\n max_len = max(max_len, len(sentence))\n return max_len\n\n\n\n[docs] def get_ground_truths(self, data_idxs):\n data_idxs_dim = utils.get_token_dim(data_idxs)\n if data_idxs_dim > 2:\n raise ValueError(f"data_idxs dimension can't be larger than 2.({data_idxs_dim})")\n\n if data_idxs_dim == 2:\n return [self.get_ground_truth(data_id) for data_id in data_idxs]\n elif data_idxs_dim == 1:\n return self.get_ground_truth(data_idxs)\n else:\n raise ValueError(f"data_idxs dimension must be 1 or 2. not {data_idxs_dim}")\n\n\n\n\n\n[docs] def lazy_evaluation(self, index):\n if self.f_count < self.__len__():\n self.f_count += 1\n\n for feature in self.features:\n for k, v in feature[index].items():\n if utils.is_lazy(v):\n feature[index][k] = v()\n\n\nimport json\nfrom overrides import overrides\nimport torch\nimport random\n\nfrom claf.config.factory.data_loader import make_data_loader\nfrom claf.data.dataset.base import DatasetBase\n\n\n[docs]class MultiTaskBertDataset(DatasetBase):\n """\n Dataset for Multi-Task GLUE using BERT\n\n * Args:\n batch: Batch DTO (claf.data.batch)\n\n * Kwargs:\n helper: helper from data_reader\n """\n\n def __init__(self, batches, vocab, helper=None):\n super(MultiTaskBertDataset, self).__init__()\n\n self.name = "multitask_bert"\n self.vocab = vocab\n\n task_helpers = helper["task_helpers"]\n\n self.multi_dataset_size = 0\n self.batch_sizes = []\n self.task_datasets = []\n\n for b, h in zip(batches, task_helpers):\n batch_size = h["batch_size"]\n self.batch_sizes.append(batch_size)\n\n dataset_cls = h["dataset"]\n dataset = dataset_cls(b, vocab, helper=h)\n self.task_datasets.append(dataset)\n\n task_dataset_size, remain = divmod(len(dataset), batch_size)\n if remain > 0:\n task_dataset_size += 1\n self.multi_dataset_size += task_dataset_size\n\n self.init_iterators()\n\n[docs] def init_iterators(self):\n cuda_device_id = None\n if torch.cuda.is_available():\n cuda_device_id = 0 # TODO: Hard-code\n\n self.iterators = []\n for batch_size, dataset in zip(self.batch_sizes, self.task_datasets):\n data_loader = make_data_loader(dataset, batch_size=batch_size, cuda_device_id=cuda_device_id) # TODO: cuda_device_id\n self.iterators.append(iter(data_loader))\n\n self.available_iterators = list(range(len(self.iterators)))\n\n[docs] @overrides\n def collate_fn(self, cuda_device_id=None):\n\n def pass_tensor(data):\n task_idx, tensor_datas = zip(*data)\n tensor_batch = tensor_datas[0]\n\n task_id_tensor = torch.LongTensor(list(task_idx))\n if torch.cuda.is_available():\n task_id_tensor.cuda(cuda_device_id)\n tensor_batch.features["task_index"] = task_id_tensor\n return tensor_batch\n return pass_tensor\n\n @overrides\n def __getitem__(self, index):\n # self.lazy_evaluation(index)\n if len(self.available_iterators) == 0:\n self.init_iterators()\n\n random_index = random.choice(self.available_iterators)\n task_iterator = self.iterators[random_index]\n try:\n return random_index, next(task_iterator)\n except StopIteration:\n self.available_iterators.remove(random_index)\n return self.__getitem__(index)\n\n def __len__(self):\n return self.multi_dataset_size\n\n def __repr__(self):\n dataset_properties = {\n "name": self.name,\n "total_count": self.__len__(),\n "dataset_count": len(self.iterators),\n "task_dataset_sizes": [len(dataset) for dataset in self.task_datasets],\n }\n return json.dumps(dataset_properties, indent=4)\n\n\nimport json\nfrom overrides import overrides\n\nfrom claf.data import utils\nfrom claf.data.collate import PadCollator\nfrom claf.data.dataset.base import DatasetBase\n\n\n[docs]class RegressionBertDataset(DatasetBase):\n """\n Dataset for Regression using BERT\n\n * Args:\n batch: Batch DTO (claf.data.batch)\n\n * Kwargs:\n helper: helper from data_reader\n """\n\n def __init__(self, batch, vocab, helper=None):\n super(RegressionBertDataset, self).__init__()\n\n self.name = "reg_bert"\n self.vocab = vocab\n self.helper = helper\n\n # Features\n self.bert_input_idx = [feature["bert_input"] for feature in batch.features]\n SEP_token = self.helper.get("sep_token", "[SEP]")\n self.token_type_idx = utils.make_bert_token_types(self.bert_input_idx, SEP_token=SEP_token)\n\n self.features = [self.bert_input_idx, self.token_type_idx] # for lazy evaluation\n\n # Labels\n self.data_ids = {data_index: label["id"] for (data_index, label) in enumerate(batch.labels)}\n self.data_indices = list(self.data_ids.keys())\n\n self.labels = {\n label["id"]: {\n "score": label["score"],\n }\n for label in batch.labels\n }\n\n self.label_scores = [label["score"] for label in batch.labels]\n\n[docs] @overrides\n def collate_fn(self, cuda_device_id=None):\n """ collate: indexed features and labels -> tensor """\n collator = PadCollator(cuda_device_id=cuda_device_id, pad_value=self.vocab.pad_index)\n\n def make_tensor_fn(data):\n data_idxs, bert_input_idxs, token_type_idxs, label_scores = zip(*data)\n\n features = {\n "bert_input": utils.transpose(bert_input_idxs, skip_keys=["text"]),\n "token_type": utils.transpose(token_type_idxs, skip_keys=["text"]),\n }\n labels = {\n "data_idx": data_idxs,\n "score": label_scores,\n }\n return collator(features, labels)\n\n return make_tensor_fn\n\n @overrides\n def __getitem__(self, index):\n self.lazy_evaluation(index)\n\n return (\n self.data_indices[index],\n self.bert_input_idx[index],\n self.token_type_idx[index],\n self.label_scores[index],\n )\n\n def __len__(self):\n return len(self.data_ids)\n\n def __repr__(self):\n dataset_properties = {\n "name": self.name,\n "total_count": self.__len__(),\n "sequence_maxlen": self.sequence_maxlen,\n }\n return json.dumps(dataset_properties, indent=4)\n\n @property\n def sequence_maxlen(self):\n return self._get_feature_maxlen(self.bert_input_idx)\n\n\n\n\n\n\nimport json\nfrom overrides import overrides\n\nfrom claf.data import utils\nfrom claf.data.collate import PadCollator\nfrom claf.data.dataset.base import DatasetBase\n\n\n[docs]class SeqClsBertDataset(DatasetBase):\n """\n Dataset for Sequence Classification using BERT\n\n * Args:\n batch: Batch DTO (claf.data.batch)\n\n * Kwargs:\n helper: helper from data_reader\n """\n\n def __init__(self, batch, vocab, helper=None):\n super(SeqClsBertDataset, self).__init__()\n\n self.name = "seq_cls_bert"\n self.vocab = vocab\n self.helper = helper\n\n self.class_idx2text = helper["class_idx2text"]\n\n # Features\n self.bert_input_idx = [feature["bert_input"] for feature in batch.features]\n SEP_token = self.helper.get("sep_token", "[SEP]")\n self.token_type_idx = utils.make_bert_token_types(self.bert_input_idx, SEP_token=SEP_token)\n\n self.features = [self.bert_input_idx, self.token_type_idx] # for lazy evaluation\n\n # Labels\n self.data_ids = {data_index: label["id"] for (data_index, label) in enumerate(batch.labels)}\n self.data_indices = list(self.data_ids.keys())\n\n self.classes = {\n label["id"]: {\n "class_idx": label["class_idx"],\n "class_text": label["class_text"],\n }\n for label in batch.labels\n }\n\n self.class_text = [label["class_text"] for label in batch.labels]\n self.class_idx = [label["class_idx"] for label in batch.labels]\n\n[docs] @overrides\n def collate_fn(self, cuda_device_id=None):\n """ collate: indexed features and labels -> tensor """\n collator = PadCollator(cuda_device_id=cuda_device_id, pad_value=self.vocab.pad_index)\n\n def make_tensor_fn(data):\n data_idxs, bert_input_idxs, token_type_idxs, class_idxs = zip(*data)\n\n features = {\n "bert_input": utils.transpose(bert_input_idxs, skip_keys=["text"]),\n "token_type": utils.transpose(token_type_idxs, skip_keys=["text"]),\n }\n labels = {\n "class_idx": class_idxs,\n "data_idx": data_idxs,\n }\n return collator(features, labels)\n\n return make_tensor_fn\n\n @overrides\n def __getitem__(self, index):\n self.lazy_evaluation(index)\n\n return (\n self.data_indices[index],\n self.bert_input_idx[index],\n self.token_type_idx[index],\n self.class_idx[index],\n )\n\n def __len__(self):\n return len(self.data_ids)\n\n def __repr__(self):\n dataset_properties = {\n "name": self.name,\n "total_count": self.__len__(),\n "num_classes": self.num_classes,\n "sequence_maxlen": self.sequence_maxlen,\n "classes": self.class_idx2text,\n }\n return json.dumps(dataset_properties, indent=4)\n\n @property\n def num_classes(self):\n return len(self.class_idx2text)\n\n @property\n def sequence_maxlen(self):\n return self._get_feature_maxlen(self.bert_input_idx)\n\n\n\n\n\n[docs] def get_class_text_with_idx(self, class_index):\n if class_index is None:\n raise ValueError("class_index is required.")\n\n return self.class_idx2text[class_index]\n\n\nimport json\nfrom overrides import overrides\n\nfrom claf.data import utils\nfrom claf.data.collate import PadCollator\nfrom claf.data.dataset.base import DatasetBase\n\n\n[docs]class SQuADBertDataset(DatasetBase):\n """\n SQuAD Dataset for BERT\n compatible with v1.1 and v2.0\n\n * Args:\n batch: Batch DTO (claf.data.batch)\n\n * Kwargs:\n helper: helper from data_reader\n """\n\n def __init__(self, batch, vocab, helper=None):\n super(SQuADBertDataset, self).__init__()\n\n self.name = "squad_bert"\n self.vocab = vocab\n self.helper = helper\n self.raw_dataset = helper["raw_dataset"]\n\n # Features\n self.bert_input_idx = [feature["bert_input"] for feature in batch.features]\n SEP_token = self.helper.get("sep_token", "[SEP]")\n self.token_type_idx = utils.make_bert_token_types(self.bert_input_idx, SEP_token=SEP_token)\n\n self.features = [self.bert_input_idx, self.token_type_idx] # for lazy_evaluation\n\n # Labels\n self.qids = {data_index: label["id"] for (data_index, label) in enumerate(batch.labels)}\n self.data_indices = list(self.qids.keys())\n\n self.answers = {\n label["id"]: (\n label["answerable"],\n (label["answer_start"], label["answer_end"]),\n )\n for label in batch.labels\n }\n self.answer_starts = [label["answer_start"] for label in batch.labels]\n self.answer_ends = [label["answer_end"] for label in batch.labels]\n self.answerables = [label["answerable"] for label in batch.labels]\n\n[docs] @overrides\n def collate_fn(self, cuda_device_id=None):\n """ collate: indexed features and labels -> tensor """\n collator = PadCollator(cuda_device_id=cuda_device_id, pad_value=self.vocab.pad_index)\n\n def make_tensor_fn(data):\n bert_input_idxs, token_type_idxs, data_idxs, answer_starts, answer_ends, answerables = zip(\n *data\n )\n\n features = {\n "bert_input": utils.transpose(bert_input_idxs, skip_keys=["text"]),\n "token_type": utils.transpose(token_type_idxs, skip_keys=["text"]),\n }\n labels = {\n "data_idx": data_idxs,\n "answer_start_idx": answer_starts,\n "answer_end_idx": answer_ends,\n "answerable": answerables,\n }\n return collator(features, labels)\n\n return make_tensor_fn\n\n @overrides\n def __getitem__(self, index):\n self.lazy_evaluation(index)\n\n return (\n self.bert_input_idx[index],\n self.token_type_idx[index],\n self.data_indices[index],\n self.answer_starts[index],\n self.answer_ends[index],\n self.answerables[index],\n )\n\n def __len__(self):\n return len(self.qids)\n\n def __repr__(self):\n dataset_properties = {\n "name": self.name,\n "total_count": self.__len__(),\n "HasAns_count": len([True for k, v in self.answers.items() if v[1] == 1]),\n "NoAns_count": len([False for k, v in self.answers.items() if v[1] == 0]),\n "bert_input_maxlen": self.bert_input_maxlen,\n }\n return json.dumps(dataset_properties, indent=4)\n\n @property\n def bert_input_maxlen(self):\n return self._get_feature_maxlen(self.bert_input_idx)\n\n[docs] def get_qid(self, data_index):\n qid = self.qids[data_index]\n if "#" in qid:\n qid = qid.split("#")[0]\n return qid\n\n\n\n[docs] def get_qid_index(self, data_index):\n qid = self.qids[data_index]\n if "#" in qid:\n return qid.split("#")[1]\n return None\n\n[docs] def get_context(self, data_index):\n qid = self.get_qid(data_index)\n return self.helper["examples"][qid]["context"]\n\n[docs] @overrides\n def get_ground_truths(self, data_index):\n qid = self.get_qid(data_index)\n answer_texts = self.helper["examples"][qid]["answers"]\n answerable, answer_span = self.answers[qid]\n return answer_texts, answerable, answer_span\n\n[docs] @overrides\n def get_predict(self, data_index, start, end):\n return self.get_text_with_index(data_index, start, end)\n\n[docs] def get_text_with_index(self, data_index, start, end):\n if data_index is None:\n raise ValueError("data_id or text is required.")\n\n context_text = self.get_context(data_index)\n bert_token = self.get_bert_tokens(data_index)\n\n if (\n start <= 0\n or end >= len(bert_token)\n or bert_token[start].text_span is None\n or bert_token[end].text_span is None\n ):\n # No_Answer Case\n return "<noanswer>"\n\n char_start = bert_token[start].text_span[0]\n char_end = bert_token[end].text_span[1]\n if char_start > char_end or len(context_text) <= char_end:\n return ""\n return context_text[char_start:char_end]\n\n[docs] def get_bert_tokens(self, data_index):\n qid = self.get_qid(data_index)\n index = self.get_qid_index(data_index)\n\n if index is None:\n raise ValueError("bert_qid must have 'bert_index' (bert_id: qid#bert_index)")\n\n bert_index = f"bert_tokens_{index}"\n return self.helper["examples"][qid][bert_index]\n\n\nimport json\nfrom overrides import overrides\n\nfrom claf.data import utils\nfrom claf.data.collate import FeatLabelPadCollator\nfrom claf.data.dataset.base import DatasetBase\n\n\n[docs]class TokClsBertDataset(DatasetBase):\n """\n Dataset for Token Classification\n\n * Args:\n batch: Batch DTO (claf.data.batch)\n\n * Kwargs:\n helper: helper from data_reader\n """\n\n def __init__(self, batch, vocab, helper=None):\n super(TokClsBertDataset, self).__init__()\n\n self.name = "tok_cls_bert"\n self.vocab = vocab\n self.helper = helper\n\n self.tag_idx2text = helper["tag_idx2text"]\n\n # Features\n self.bert_input_idx = [feature["bert_input"] for feature in batch.features]\n SEP_token = self.helper.get("sep_token", "[SEP]")\n self.token_type_idx = utils.make_bert_token_types(self.bert_input_idx, SEP_token=SEP_token)\n\n self.tagged_sub_token_idxs = [{"feature": feature["tagged_sub_token_idxs"]} for feature in batch.features]\n self.num_tokens = [{"feature": feature["num_tokens"]} for feature in batch.features]\n\n self.features = [self.bert_input_idx, self.token_type_idx] # for lazy evaluation\n\n # Labels\n self.data_ids = {data_index: label["id"] for (data_index, label) in enumerate(batch.labels)}\n self.data_indices = list(self.data_ids.keys())\n\n self.tags = {\n label["id"]: {\n "tag_idxs": label["tag_idxs"],\n "tag_texts": label["tag_texts"],\n }\n for label in batch.labels\n }\n self.tag_texts = [label["tag_texts"] for label in batch.labels]\n self.tag_idxs = [label["tag_idxs"] for label in batch.labels]\n\n self.ignore_tag_idx = helper["ignore_tag_idx"]\n\n[docs] @overrides\n def collate_fn(self, cuda_device_id=None):\n """ collate: indexed features and labels -> tensor """\n collator = FeatLabelPadCollator(cuda_device_id=cuda_device_id, pad_value=self.vocab.pad_index)\n\n def make_tensor_fn(data):\n data_idxs, bert_input_idxs, token_type_idxs, tagged_token_idxs, num_tokens, tag_idxs_list = zip(*data)\n\n features = {\n "bert_input": utils.transpose(bert_input_idxs, skip_keys=["text"]),\n "token_type": utils.transpose(token_type_idxs, skip_keys=["text"]),\n "tagged_sub_token_idxs": utils.transpose(tagged_token_idxs, skip_keys=["text"]),\n "num_tokens": utils.transpose(num_tokens, skip_keys=["text"]),\n }\n labels = {\n "tag_idxs": tag_idxs_list,\n "data_idx": data_idxs,\n }\n return collator(\n features,\n labels,\n apply_pad_labels=["tag_idxs"],\n apply_pad_values=[self.ignore_tag_idx]\n )\n\n return make_tensor_fn\n\n @overrides\n def __getitem__(self, index):\n self.lazy_evaluation(index)\n\n return (\n self.data_indices[index],\n self.bert_input_idx[index],\n self.token_type_idx[index],\n self.tagged_sub_token_idxs[index],\n self.num_tokens[index],\n self.tag_idxs[index],\n )\n\n def __len__(self):\n return len(self.data_ids)\n\n def __repr__(self):\n dataset_properties = {\n "name": self.name,\n "total_count": self.__len__(),\n "num_tags": self.num_tags,\n "sequence_maxlen": self.sequence_maxlen,\n "tags": self.tag_idx2text,\n }\n return json.dumps(dataset_properties, indent=4)\n\n @property\n def num_tags(self):\n return len(self.tag_idx2text)\n\n @property\n def sequence_maxlen(self):\n return self._get_feature_maxlen(self.bert_input_idx)\n\n\n\n\n\n[docs] def get_tag_texts_with_idxs(self, tag_idxs):\n return [self.get_tag_text_with_idx(tag_idx)for tag_idx in tag_idxs]\n\n[docs] def get_tag_text_with_idx(self, tag_index):\n if tag_index is None:\n raise ValueError("tag_index is required.")\n\n return self.tag_idx2text[tag_index]\n\n\nimport json\nfrom overrides import overrides\nimport torch\n\nfrom claf.data import utils\nfrom claf.data.collate import PadCollator\nfrom claf.data.dataset.base import DatasetBase\n\n\n[docs]class SeqClsDataset(DatasetBase):\n """\n Dataset for Sequence Classification\n\n * Args:\n batch: Batch DTO (claf.data.batch)\n\n * Kwargs:\n helper: helper from data_reader\n """\n\n def __init__(self, batch, vocab, helper=None):\n super(SeqClsDataset, self).__init__()\n\n self.name = "seq_cls"\n self.vocab = vocab\n self.helper = helper\n\n self.class_idx2text = helper["class_idx2text"]\n\n self.sequences = {feature["id"]: feature["sequence"]["text"] for feature in batch.features}\n\n # Features\n self.sequence_idxs = [feature["sequence"] for feature in batch.features]\n\n self.features = [self.sequence_idxs] # for lazy evaluation\n\n # Labels\n self.data_ids = {data_index: label["id"] for (data_index, label) in enumerate(batch.labels)}\n self.data_indices = list(self.data_ids.keys())\n\n self.classes = {\n label["id"]: {\n "class_idx": label["class_idx"],\n "class_text": label["class_text"],\n }\n for label in batch.labels\n }\n\n self.class_text = [label["class_text"] for label in batch.labels]\n self.class_idx = [label["class_idx"] for label in batch.labels]\n\n[docs] @overrides\n def collate_fn(self, cuda_device_id=None):\n """ collate: indexed features and labels -> tensor """\n collator = PadCollator(cuda_device_id=cuda_device_id, pad_value=self.vocab.pad_index)\n\n def make_tensor_fn(data):\n data_idxs, sequence_idxs, class_idxs = zip(*data)\n\n features = {\n "sequence": utils.transpose(sequence_idxs, skip_keys=["text"]),\n }\n labels = {\n "class_idx": class_idxs,\n "data_idx": data_idxs,\n }\n return collator(features, labels)\n\n return make_tensor_fn\n\n @overrides\n def __getitem__(self, index):\n self.lazy_evaluation(index)\n\n return (\n self.data_indices[index],\n self.sequence_idxs[index],\n self.class_idx[index],\n )\n\n def __len__(self):\n return len(self.data_ids)\n\n def __repr__(self):\n dataset_properties = {\n "name": self.name,\n "total_count": self.__len__(),\n "num_classes": self.num_classes,\n "sequence_maxlen": self.sequence_maxlen,\n "classes": self.class_idx2text,\n }\n return json.dumps(dataset_properties, indent=4)\n\n @property\n def num_classes(self):\n return len(self.class_idx2text)\n\n @property\n def sequence_maxlen(self):\n return self._get_feature_maxlen(self.sequence_idxs)\n\n\n\n\n\n[docs] def get_class_text_with_idx(self, class_index):\n if class_index is None:\n raise ValueError("class_index is required.")\n\n return self.class_idx2text[class_index]\n\n\nimport json\nfrom overrides import overrides\nimport torch\n\nfrom claf.data import utils\nfrom claf.data.collate import PadCollator\nfrom claf.data.dataset.base import DatasetBase\n\n\n[docs]class SeqClsBertDataset(DatasetBase):\n """\n Dataset for Sequence Classification using BERT\n\n * Args:\n batch: Batch DTO (claf.data.batch)\n\n * Kwargs:\n helper: helper from data_reader\n """\n\n def __init__(self, batch, helper=None):\n super(SeqClsBertDataset, self).__init__()\n\n self.name = "seq_cls_bert"\n self.helper = helper\n self.raw_dataset = helper["raw_dataset"]\n\n self.class_idx2text = helper["class_idx2text"]\n\n # Features\n self.bert_input_idx = [feature["bert_input"] for feature in batch.features]\n SEP_token = self.helper.get("sep_token", "[SEP]")\n self.token_type_idx = utils.make_bert_token_types(self.bert_input_idx, SEP_token=SEP_token)\n\n self.features = [self.bert_input_idx, self.token_type_idx] # for lazy evaluation\n\n # Labels\n self.data_ids = {data_index: label["id"] for (data_index, label) in enumerate(batch.labels)}\n self.data_indices = list(self.data_ids.keys())\n\n self.classes = {\n label["id"]: {\n "class_idx": label["class_idx"],\n "class_text": label["class_text"],\n }\n for label in batch.labels\n }\n\n self.class_text = [label["class_text"] for label in batch.labels]\n self.class_idx = [label["class_idx"] for label in batch.labels]\n\n[docs] @overrides\n def collate_fn(self, cuda_device_id=None):\n """ collate: indexed features and labels -> tensor """\n collator = PadCollator(cuda_device_id=cuda_device_id)\n\n def make_tensor_fn(data):\n data_idxs, bert_input_idxs, token_type_idxs, class_idxs = zip(*data)\n\n features = {\n "bert_input": utils.transpose(bert_input_idxs, skip_keys=["text"]),\n "token_type": utils.transpose(token_type_idxs, skip_keys=["text"]),\n }\n labels = {\n "class_idx": class_idxs,\n "data_idx": data_idxs,\n }\n return collator(features, labels)\n\n return make_tensor_fn\n\n @overrides\n def __getitem__(self, index):\n self.lazy_evaluation(index)\n\n return (\n self.data_indices[index],\n self.bert_input_idx[index],\n self.token_type_idx[index],\n self.class_idx[index],\n )\n\n def __len__(self):\n return len(self.data_ids)\n\n def __repr__(self):\n dataset_properties = {\n "name": self.name,\n "total_count": self.__len__(),\n "num_classes": self.num_classes,\n "sequence_maxlen": self.sequence_maxlen,\n "classes": self.class_idx2text,\n }\n return json.dumps(dataset_properties, indent=4)\n\n @property\n def num_classes(self):\n return len(self.class_idx2text)\n\n @property\n def sequence_maxlen(self):\n return self._get_feature_maxlen(self.bert_input_idx)\n\n\n\n\n\n[docs] def get_class_text_with_idx(self, class_index):\n if class_index is None:\n raise ValueError("class_index is required.")\n\n return self.class_idx2text[class_index]\n\n\nimport json\nfrom overrides import overrides\n\nfrom claf.data import utils\nfrom claf.data.collate import PadCollator\nfrom claf.data.dataset.base import DatasetBase\n\n\n[docs]class SQuADDataset(DatasetBase):\n """\n SQuAD Dataset\n compatible with v1.1 and v2.0\n\n * Args:\n batch: Batch DTO (claf.data.batch)\n\n * Kwargs:\n helper: helper from data_reader\n """\n\n def __init__(self, batch, vocab, helper=None):\n super(SQuADDataset, self).__init__()\n\n self.name = "squad"\n self.vocab = vocab\n self.helper = helper\n self.raw_dataset = helper["raw_dataset"] # for SQuAD official metric\n\n # Features\n self.context_idx = [feature["context"] for feature in batch.features]\n self.question_idx = [feature["question"] for feature in batch.features]\n\n self.features = [self.context_idx, self.question_idx] # for lazy_evaluation\n\n # Labels\n self.qids = {data_index: label["id"] for (data_index, label) in enumerate(batch.labels)}\n self.data_indices = list(self.qids.keys())\n\n self.answers = {\n label["id"]: (\n label["answerable"],\n (label["answer_start"], label["answer_end"]),\n )\n for label in batch.labels\n }\n self.answer_starts = [label["answer_start"] for label in batch.labels]\n self.answer_ends = [label["answer_end"] for label in batch.labels]\n self.answerables = [label["answerable"] for label in batch.labels]\n\n[docs] @overrides\n def collate_fn(self, cuda_device_id=None):\n """ collate: indexed features and labels -> tensor """\n collator = PadCollator(cuda_device_id=cuda_device_id, pad_value=self.vocab.pad_index)\n\n def make_tensor_fn(data):\n context_idxs, question_idxs, data_idxs, \\\n answer_starts, answer_ends, answerables = zip(*data)\n\n features = {\n "context": utils.transpose(context_idxs, skip_keys=["text"]),\n "question": utils.transpose(question_idxs, skip_keys=["text"]),\n }\n labels = {\n "data_idx": data_idxs,\n "answer_start_idx": answer_starts,\n "answer_end_idx": answer_ends,\n "answerable": answerables,\n }\n return collator(features, labels)\n\n return make_tensor_fn\n\n @overrides\n def __getitem__(self, index):\n self.lazy_evaluation(index)\n\n return (\n self.context_idx[index],\n self.question_idx[index],\n self.data_indices[index],\n self.answer_starts[index],\n self.answer_ends[index],\n self.answerables[index],\n )\n\n def __len__(self):\n return len(self.qids)\n\n def __repr__(self):\n dataset_properties = {\n "name": self.name,\n "total_count": self.__len__(),\n "HasAns_count": len([True for item in self.answerables if item == 1]),\n "NoAns_count": len([False for item in self.answerables if item == 0]),\n "context_maxlen": self.context_maxlen,\n "question_maxlen": self.question_maxlen,\n }\n return json.dumps(dataset_properties, indent=4)\n\n @property\n def context_maxlen(self):\n return self._get_feature_maxlen(self.context_idx)\n\n @property\n def question_maxlen(self):\n return self._get_feature_maxlen(self.question_idx)\n\n\n\n[docs] def get_context(self, data_index):\n qid = self.get_qid(data_index)\n return self.helper["examples"][qid]["context"]\n\n[docs] def get_text_span(self, data_index):\n qid = self.get_qid(data_index)\n return self.helper["examples"][qid]["text_span"]\n\n[docs] @overrides\n def get_ground_truths(self, data_index):\n qid = self.get_qid(data_index)\n answer_texts = self.helper["examples"][qid]["answers"]\n answerable, answer_span = self.answers[qid]\n return answer_texts, answerable, answer_span\n\n[docs] @overrides\n def get_predict(self, data_index, start, end):\n return self.get_text_with_index(data_index, start, end)\n\n[docs] def get_text_with_index(self, data_index, start, end):\n if data_index is None:\n raise ValueError("qid or text is required.")\n\n context_text = self.get_context(data_index)\n text_span = self.get_text_span(data_index)\n\n if start >= len(text_span) or end >= len(text_span):\n # No_Answer Case\n return "<noanswer>"\n\n char_start = text_span[start][0]\n char_end = text_span[end][1]\n if char_start > char_end or len(context_text) <= char_end:\n return ""\n return context_text[char_start:char_end]\n\n\nimport json\nfrom overrides import overrides\n\nfrom claf.data import utils\nfrom claf.data.collate import PadCollator\nfrom claf.data.dataset.base import DatasetBase\n\n\n[docs]class SQuADBertDataset(DatasetBase):\n """\n SQuAD Dataset for BERT\n compatible with v1.1 and v2.0\n\n * Args:\n batch: Batch DTO (claf.data.batch)\n\n * Kwargs:\n helper: helper from data_reader\n """\n\n def __init__(self, batch, helper=None):\n super(SQuADBertDataset, self).__init__()\n\n self.name = "squad_bert"\n self.helper = helper\n self.raw_dataset = helper["raw_dataset"]\n\n # Features\n self.bert_input_idx = [feature["bert_input"] for feature in batch.features]\n SEP_token = self.helper.get("sep_token", "[SEP]")\n self.token_type_idx = utils.make_bert_token_types(self.bert_input_idx, SEP_token=SEP_token)\n\n self.features = [self.bert_input_idx, self.token_type_idx] # for lazy_evaluation\n\n # Labels\n self.qids = {data_index: label["id"] for (data_index, label) in enumerate(batch.labels)}\n self.data_indices = list(self.qids.keys())\n\n self.answers = {\n label["id"]: (\n label["answerable"],\n (label["answer_start"], label["answer_end"]),\n )\n for label in batch.labels\n }\n self.answer_starts = [label["answer_start"] for label in batch.labels]\n self.answer_ends = [label["answer_end"] for label in batch.labels]\n self.answerables = [label["answerable"] for label in batch.labels]\n\n[docs] @overrides\n def collate_fn(self, cuda_device_id=None):\n """ collate: indexed features and labels -> tensor """\n collator = PadCollator(cuda_device_id=cuda_device_id)\n\n def make_tensor_fn(data):\n bert_input_idxs, token_type_idxs, data_idxs, answer_starts, answer_ends, answerables = zip(\n *data\n )\n\n features = {\n "bert_input": utils.transpose(bert_input_idxs, skip_keys=["text"]),\n "token_type": utils.transpose(token_type_idxs, skip_keys=["text"]),\n }\n labels = {\n "answer_idx": data_idxs,\n "answer_start_idx": answer_starts,\n "answer_end_idx": answer_ends,\n "answerable": answerables,\n }\n return collator(features, labels)\n\n return make_tensor_fn\n\n @overrides\n def __getitem__(self, index):\n self.lazy_evaluation(index)\n\n return (\n self.bert_input_idx[index],\n self.token_type_idx[index],\n self.data_indices[index],\n self.answer_starts[index],\n self.answer_ends[index],\n self.answerables[index],\n )\n\n def __len__(self):\n return len(self.qids)\n\n def __repr__(self):\n dataset_properties = {\n "name": self.name,\n "total_count": self.__len__(),\n "HasAns_count": len([True for k, v in self.answers.items() if v[1] == 1]),\n "NoAns_count": len([False for k, v in self.answers.items() if v[1] == 0]),\n "bert_input_maxlen": self.bert_input_maxlen,\n }\n return json.dumps(dataset_properties, indent=4)\n\n @property\n def bert_input_maxlen(self):\n return self._get_feature_maxlen(self.bert_input_idx)\n\n[docs] def get_qid(self, data_index):\n qid = self.qids[data_index]\n if "#" in qid:\n qid = qid.split("#")[0]\n return qid\n\n[docs] def get_qid_index(self, data_index):\n qid = self.qids[data_index]\n if "#" in qid:\n return qid.split("#")[1]\n return None\n\n[docs] def get_context(self, data_index):\n qid = self.get_qid(data_index)\n return self.helper["examples"][qid]["context"]\n\n[docs] @overrides\n def get_ground_truths(self, data_index):\n qid = self.get_qid(data_index)\n answer_texts = self.helper["examples"][qid]["answers"]\n answerable, answer_span = self.answers[qid]\n return answer_texts, answerable, answer_span\n\n[docs] @overrides\n def get_predict(self, data_index, start, end):\n return self.get_text_with_index(data_index, start, end)\n\n[docs] def get_text_with_index(self, data_index, start, end):\n if data_index is None:\n raise ValueError("data_id or text is required.")\n\n context_text = self.get_context(data_index)\n bert_token = self.get_bert_tokens(data_index)\n\n if (\n start <= 0\n or end >= len(bert_token)\n or bert_token[start].text_span is None\n or bert_token[end].text_span is None\n ):\n # No_Answer Case\n return "<noanswer>"\n\n char_start = bert_token[start].text_span[0]\n char_end = bert_token[end].text_span[1]\n if char_start > char_end or len(context_text) <= char_end:\n return ""\n return context_text[char_start:char_end]\n\n[docs] def get_bert_tokens(self, data_index):\n qid = self.get_qid(data_index)\n index = self.get_qid_index(data_index)\n\n if index is None:\n raise ValueError("bert_qid must have 'bert_index' (bert_id: qid#bert_index)")\n\n bert_index = f"bert_tokens_{index}"\n return self.helper["examples"][qid][bert_index]\n\n\nimport json\nfrom collections import defaultdict\nfrom overrides import overrides\nimport torch\nfrom seqeval.metrics.sequence_labeling import get_entities\n\nfrom claf.data import utils\nfrom claf.data.collate import FeatLabelPadCollator\nfrom claf.data.dataset.base import DatasetBase\n\n\n[docs]class TokClsBertDataset(DatasetBase):\n """\n Dataset for Token Classification\n\n * Args:\n batch: Batch DTO (claf.data.batch)\n\n * Kwargs:\n helper: helper from data_reader\n """\n\n def __init__(self, batch, helper=None):\n super(TokClsBertDataset, self).__init__()\n\n self.name = "tok_cls_bert"\n self.helper = helper\n self.raw_dataset = helper["raw_dataset"]\n\n self.tag_idx2text = helper["tag_idx2text"]\n\n # Features\n self.bert_input_idx = [feature["bert_input"] for feature in batch.features]\n SEP_token = self.helper.get("sep_token", "[SEP]")\n self.token_type_idx = utils.make_bert_token_types(self.bert_input_idx, SEP_token=SEP_token)\n\n self.tagged_sub_token_idxs = [{"feature": feature["tagged_sub_token_idxs"]} for feature in batch.features]\n self.num_tokens = [{"feature": feature["num_tokens"]} for feature in batch.features]\n\n self.features = [self.bert_input_idx, self.token_type_idx] # for lazy evaluation\n\n # Labels\n self.data_ids = {data_index: label["id"] for (data_index, label) in enumerate(batch.labels)}\n self.data_indices = list(self.data_ids.keys())\n\n self.tags = {\n label["id"]: {\n "tag_idxs": label["tag_idxs"],\n "tag_texts": label["tag_texts"],\n }\n for label in batch.labels\n }\n self.tag_texts = [label["tag_texts"] for label in batch.labels]\n self.tag_idxs = [label["tag_idxs"] for label in batch.labels]\n\n self.ignore_tag_idx = helper["ignore_tag_idx"]\n\n[docs] @overrides\n def collate_fn(self, cuda_device_id=None):\n """ collate: indexed features and labels -> tensor """\n collator = FeatLabelPadCollator(cuda_device_id=cuda_device_id)\n\n def make_tensor_fn(data):\n data_idxs, bert_input_idxs, token_type_idxs, tagged_token_idxs, num_tokens, tag_idxs_list = zip(*data)\n\n features = {\n "bert_input": utils.transpose(bert_input_idxs, skip_keys=["text"]),\n "token_type": utils.transpose(token_type_idxs, skip_keys=["text"]),\n "tagged_sub_token_idxs": utils.transpose(tagged_token_idxs, skip_keys=["text"]),\n "num_tokens": utils.transpose(num_tokens, skip_keys=["text"]),\n }\n labels = {\n "tag_idxs": tag_idxs_list,\n "data_idx": data_idxs,\n }\n return collator(\n features,\n labels,\n apply_pad_labels=["tag_idxs"],\n apply_pad_values=[self.ignore_tag_idx]\n )\n\n return make_tensor_fn\n\n @overrides\n def __getitem__(self, index):\n self.lazy_evaluation(index)\n\n return (\n self.data_indices[index],\n self.bert_input_idx[index],\n self.token_type_idx[index],\n self.tagged_sub_token_idxs[index],\n self.num_tokens[index],\n self.tag_idxs[index],\n )\n\n def __len__(self):\n return len(self.data_ids)\n\n def __repr__(self):\n dataset_properties = {\n "name": self.name,\n "total_count": self.__len__(),\n "num_tags": self.num_tags,\n "sequence_maxlen": self.sequence_maxlen,\n "tags": self.tag_idx2text,\n }\n return json.dumps(dataset_properties, indent=4)\n\n @property\n def num_tags(self):\n return len(self.tag_idx2text)\n\n @property\n def sequence_maxlen(self):\n return self._get_feature_maxlen(self.bert_input_idx)\n\n\n\n\n\n[docs] def get_tag_texts_with_idxs(self, tag_idxs):\n return [self.get_tag_text_with_idx(tag_idx)for tag_idx in tag_idxs]\n\n[docs] def get_tag_text_with_idx(self, tag_index):\n if tag_index is None:\n raise ValueError("tag_index is required.")\n\n return self.tag_idx2text[tag_index]\n\n\nimport json\nfrom overrides import overrides\n\nimport torch\n\nfrom claf.data import utils\nfrom claf.data.collate import PadCollator\nfrom claf.data.dataset.base import DatasetBase\n\n\n\n[docs]class WikiSQLDataset(DatasetBase):\n """\n WikiSQL Dataset\n\n * Args:\n batch: Batch DTO (claf.data.batch)\n\n * Kwargs:\n helper: helper from data_reader\n """\n\n def __init__(self, batch, vocab, helper=None):\n super(WikiSQLDataset, self).__init__()\n\n self.name = "wikisql"\n self.vocab = vocab\n self.helper = helper\n\n # Features\n self.column_idx = [feature["column"] for feature in batch.features]\n self.question_idx = [feature["question"] for feature in batch.features]\n\n self.features = [self.column_idx, self.question_idx]\n\n # Labels\n self.data_idx = {data_index: label["id"] for (data_index, label) in enumerate(batch.labels)}\n self.data_indices = list(self.data_idx.keys())\n\n self.table_idx = {data_index: label["table_id"] for (data_index, label) in enumerate(batch.labels)}\n\n self.tokenized_question = {label["id"]: label["tokenized_question"] for label in batch.labels}\n\n self.labels = {\n label["id"]: {\n "agg_idx": label["aggregator_idx"],\n "sel_idx": label["select_column_idx"],\n "conds_num": label["conditions_num"],\n "conds_col": label["conditions_column_idx"],\n "conds_op": label["conditions_operator_idx"],\n "conds_val_str": label["conditions_value_string"],\n "conds_val_pos": label["conditions_value_position"],\n "sql_query": label["sql_query"],\n "execution_result": label["execution_result"],\n }\n for label in batch.labels\n }\n\n[docs] @overrides\n def collate_fn(self, cuda_device_id=None):\n """ collate: indexed features and labels -> tensor """\n collator = PadCollator(cuda_device_id=cuda_device_id, pad_value=self.vocab.pad_index)\n\n def make_tensor_fn(data):\n column_idxs, question_idxs, data_idxs = zip(*data)\n\n features = {\n "column": utils.transpose(column_idxs, skip_keys=["text"]),\n "question": utils.transpose(question_idxs, skip_keys=["text"]),\n }\n labels = {\n "data_idx": data_idxs,\n }\n return collator(features, labels)\n\n return make_tensor_fn\n\n @overrides\n def __getitem__(self, index):\n self.lazy_evaluation(index)\n\n return (\n self.column_idx[index],\n self.question_idx[index],\n self.data_indices[index],\n )\n\n def __len__(self):\n return len(self.data_idx)\n\n def __repr__(self):\n dataset_properties = {\n "name": self.name,\n "total_count": self.__len__(),\n "question_maxlen": self.question_maxlen,\n }\n return json.dumps(dataset_properties, indent=4)\n\n @property\n def question_maxlen(self):\n return self._get_feature_maxlen(self.question_idx)\n\n[docs] def get_id(self, data_index):\n if type(data_index) == torch.Tensor:\n data_index = data_index.item()\n return self.data_idx[data_index]\n\n[docs] def get_table_id(self, data_index):\n if type(data_index) == torch.Tensor:\n data_index = data_index.item()\n return self.table_idx[data_index]\n\n[docs] def get_tokenized_question(self, data_index):\n data_id = self.get_id(data_index)\n return self.tokenized_question[data_id]\n\n[docs] @overrides\n def get_ground_truth(self, data_index):\n if type(data_index) == torch.Tensor:\n data_id = self.get_id(data_index)\n else:\n data_id = data_index\n return self.labels[data_id]\n\n\nimport logging\n\nfrom claf.data.data_handler import CachePath, DataHandler\nfrom claf import utils as common_utils\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]class DataReader:\n """\n DataReader Base Class\n\n * Args:\n file_paths: dictionary of consisting ('train' and 'vaild') file_path\n dataset_obj: Dataset Object (claf.data.dataset.base)\n """\n\n def __init__(self, file_paths, dataset_obj):\n self.file_paths = file_paths\n self.dataset_obj = dataset_obj\n\n self.data_handler = DataHandler(cache_path=CachePath.DATASET) # for Concrete DataReader\n self.text_columns = None\n\n[docs] def filter_texts(self, dataset):\n texts = []\n\n def append_texts(datas):\n for data in datas:\n for key, value in data.items():\n if key in self.text_columns:\n texts.append(value)\n\n for data_type, dataset in dataset.items():\n append_texts(dataset.features)\n # append_texts(dataset.labels)\n\n texts = list(common_utils.flatten(texts))\n texts = list(set(texts)) # remove duplicate\n return texts\n\n[docs] def read(self):\n """ read with Concrete DataReader each type """\n\n if type(self.file_paths) != dict:\n raise ValueError(f"file_paths type is must be dict. not {type(self.file_paths)}")\n\n logger.info("Start read dataset")\n datasets, helpers = {}, {}\n for data_type, file_path in self.file_paths.items():\n if data_type is None:\n continue\n\n batch, helper = self._read(file_path, data_type=data_type)\n\n datasets[data_type] = batch\n helpers[data_type] = helper\n logger.info("Complete read dataset...\\n")\n return datasets, helpers\n\n def _read(self, file_path, desc=None):\n raise NotImplementedError\n\n\n\n[docs] def convert_to_dataset(self, datas, vocab, helpers=None):\n """ Batch to Dataset """\n datasets = {}\n for k, batch in datas.items():\n if batch is None:\n continue\n datasets[k] = self.dataset_obj(batch, vocab, helper=helpers[k])\n logger.info(f"{k} dataset. {datasets[k]}")\n return datasets\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:cola_bert")\nclass CoLABertReader(SeqClsBertReader):\n """\n CoLA DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = [0, 1]\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(CoLABertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n line_tokens = line.split("\\t")\n if len(line_tokens) <= 3:\n continue\n data.append({\n "uid": f"{data_type}-{i}",\n "sequence_a": line_tokens[3],\n self.class_key: str(line_tokens[1])\n })\n\n return data\n\n\nimport logging\nfrom itertools import chain\n\nfrom overrides import overrides\n\nfrom claf.data.reader import TokClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:conll2003_bert")\nclass CoNLL2003BertReader(TokClsBertReader):\n """\n CoNLL2003 for BERT\n\n * Args:\n file_paths: file paths (train and dev)\n\n * Kwargs:\n ignore_tag_idx: prediction results that have this number as ground-truth idx are ignored\n """\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n ignore_tag_idx=-1,\n ):\n\n super(CoNLL2003BertReader, self).__init__(\n file_paths,\n tokenizers,\n lang_code=None,\n sequence_max_length=sequence_max_length,\n cls_token=cls_token,\n sep_token=sep_token,\n ignore_tag_idx=ignore_tag_idx,\n )\n\n @overrides\n def _get_data(self, file_path):\n _file = self.data_handler.read(file_path)\n texts = _file.split("\\n\\n")\n texts.pop(0)\n\n data = []\n for text in texts:\n tokens = text.split("\\n")\n if len(tokens) > 1:\n example = list(zip(*[token.split() for token in tokens]))\n data.append({\n "sequence": " ".join(example[0]),\n self.tag_key: list(example[-1]),\n })\n\n return data, data\n\n @overrides\n def _get_tag_dicts(self, **kwargs):\n data = kwargs["data"]\n tags = sorted(list(set(chain.from_iterable(d[self.tag_key] for d in data))))\n\n tag_idx2text = {tag_idx: tag_text for tag_idx, tag_text in enumerate(tags)}\n tag_text2idx = {tag_text: tag_idx for tag_idx, tag_text in tag_idx2text.items()}\n\n return tag_idx2text, tag_text2idx\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:cola_bert")\nclass CoLABertReader(SeqClsBertReader):\n """\n CoLA DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = [0, 1]\n METRIC_KEY = "matthews_corr"\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(CoLABertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n line_tokens = line.split("\\t")\n if len(line_tokens) <= 3:\n continue\n data.append({\n "uid": f"cola-{file_path}-{data_type}-{i}",\n "sequence_a": line_tokens[3],\n self.class_key: str(line_tokens[1])\n })\n\n return data\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:mnli_bert")\nclass MNLIBertReader(SeqClsBertReader):\n """\n MNLI DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = ["contradiction", "entailment", "neutral"]\n METRIC_KEY = "accuracy"\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(MNLIBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n if len(line_tokens) <= 1:\n continue\n data.append({\n "uid": f"mnli-{file_path}-{data_type}-{i}",\n "sequence_a": line_tokens[8],\n "sequence_b": line_tokens[9],\n self.class_key: str(line_tokens[-1]),\n })\n\n return data\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:mrpc_bert")\nclass MRPCBertReader(SeqClsBertReader):\n """\n MRPC DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = [0, 1]\n METRIC_KEY = "f1"\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(MRPCBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n if len(line_tokens) != 5:\n continue\n data.append({\n "uid": f"mrpc-{file_path}-{data_type}-{i}",\n "sequence_a": line_tokens[3],\n "sequence_b": line_tokens[4],\n self.class_key: str(line_tokens[0]),\n })\n\n return data\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:qnli_bert")\nclass QNLIBertReader(SeqClsBertReader):\n """\n QNLI DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = ["entailment", "not_entailment"]\n METRIC_KEY = "accuracy"\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(QNLIBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n if len(line_tokens) <= 1:\n continue\n data.append({\n "uid": f"qnli-{file_path}-{data_type}-{i}",\n "sequence_a": line_tokens[1],\n "sequence_b": line_tokens[2],\n self.class_key: str(line_tokens[-1]),\n })\n\n return data\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:qqp_bert")\nclass QQPBertReader(SeqClsBertReader):\n """\n Quora Question Pairs DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = [0, 1]\n METRIC_KEY = "f1"\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(QQPBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n try:\n data.append({\n "uid": f"qqp-{file_path}-{data_type}-{i}",\n "sequence_a": line_tokens[3],\n "sequence_b": line_tokens[4],\n self.class_key: str(line_tokens[5])\n })\n except IndexError:\n continue\n\n return data\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:rte_bert")\nclass RTEBertReader(SeqClsBertReader):\n """\n RTE (Recognizing Textual Entailment) DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = ["entailment", "not_entailment"]\n METRIC_KEY = "accuracy"\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(RTEBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n if len(line_tokens) <= 1:\n continue\n data.append({\n "uid": f"rte-{file_path}-{data_type}-{i}",\n "sequence_a": line_tokens[1],\n "sequence_b": line_tokens[2],\n self.class_key: str(line_tokens[-1]),\n })\n\n return data\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:sst_bert")\nclass SSTBertReader(SeqClsBertReader):\n """\n SST DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = [0, 1]\n METRIC_KEY = "accuracy"\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(SSTBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n if data_type == "train":\n lines.pop(0)\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n if len(line_tokens) <= 1:\n continue\n data.append({\n "uid": f"sst-{file_path}-{data_type}-{i}",\n "sequence_a": line_tokens[0],\n self.class_key: str(line_tokens[1]),\n })\n\n return data\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import RegressionBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:stsb_bert")\nclass STSBBertReader(RegressionBertReader):\n """\n STS-B (Semantic Textual Similarity Benchmark) DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n METRIC_KEY = "pearson_spearman_corr"\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(STSBBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n label_key="score",\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n if len(line_tokens) <= 1:\n continue\n data.append({\n "uid": f"stsb-{file_path}-{data_type}-{i}",\n "sequence_a": line_tokens[7],\n "sequence_b": line_tokens[8],\n "score": float(line_tokens[-1]),\n })\n\n return data\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:wnli_bert")\nclass WNLIBertReader(SeqClsBertReader):\n """\n WNLI (Winograd NLI) DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = [0, 1]\n METRIC_KEY = "accuracy"\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(WNLIBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n if len(line_tokens) <= 1:\n continue\n data.append({\n "uid": f"wnli-{file_path}-{data_type}-{i}",\n "sequence_a": line_tokens[1],\n "sequence_b": line_tokens[2],\n self.class_key: str(line_tokens[-1]),\n })\n\n return data\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:mnli_bert")\nclass MNLIBertReader(SeqClsBertReader):\n """\n MNLI DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = ["contradiction", "entailment", "neutral"]\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(MNLIBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n if len(line_tokens) <= 1:\n continue\n data.append({\n "uid": f"{data_type}-{i}",\n "sequence_a": line_tokens[8],\n "sequence_b": line_tokens[9],\n self.class_key: str(line_tokens[-1]),\n })\n\n return data\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:mrpc_bert")\nclass MRPCBertReader(SeqClsBertReader):\n """\n MRPC DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = [0, 1]\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(MRPCBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n if len(line_tokens) != 5:\n continue\n data.append({\n "uid": f"{data_type}-{i}",\n "sequence_a": line_tokens[3],\n "sequence_b": line_tokens[4],\n self.class_key: str(line_tokens[0]),\n })\n\n return data\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.config.factory import DataReaderFactory\nfrom claf.config.namespace import NestedNamespace\nfrom claf.config.registry import Registry\nfrom claf.data.dataset import MultiTaskBertDataset\nfrom claf.data.dto import Helper\nfrom claf.data.reader.base import DataReader\nfrom claf.decorator import register\nfrom claf.model.multi_task.category import TaskCategory\n\nfrom .seq_cls import SeqClsBertReader\nfrom .squad import SQuADBertReader\nfrom .regression import RegressionBertReader\nfrom .tok_cls import TokClsBertReader\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:multitask_bert")\nclass MultiTaskBertReader(DataReader):\n """\n DataReader for Multi-Task using BERT\n\n * Args:\n file_paths: .json file paths (train and dev)\n tokenizers: define tokenizers config (subword)\n\n * Kwargs:\n class_key: name of the label in .json file to use for classification\n """\n\n CLASS_DATA = None\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n batch_sizes=[],\n readers=[],\n ):\n\n super(MultiTaskBertReader, self).__init__(file_paths, MultiTaskBertDataset)\n assert len(batch_sizes) == len(readers)\n\n self.registry = Registry()\n\n self.text_columns = ["bert_input"]\n\n self.tokenizers = tokenizers\n self.batch_sizes = batch_sizes\n\n self.dataset_batches = []\n self.dataset_helpers = []\n self.tasks = []\n\n for reader in readers:\n data_reader = self.make_data_reader(reader)\n batches, helpers = data_reader.read()\n\n self.dataset_batches.append(batches)\n self.dataset_helpers.append(helpers)\n\n dataset_name = reader["dataset"]\n helper = helpers["train"]\n task = self.make_task_by_reader(dataset_name, data_reader, helper)\n self.tasks.append(task)\n\n[docs] def make_data_reader(self, config_dict):\n config = NestedNamespace()\n config.load_from_json(config_dict)\n config.tokenizers = self.tokenizers\n\n data_reader_factory = DataReaderFactory(config)\n return data_reader_factory.create()\n\n[docs] def make_task_by_reader(self, name, data_reader, helper):\n task = {}\n task["name"] = name\n task["metric_key"] = data_reader.METRIC_KEY\n\n if isinstance(data_reader, SeqClsBertReader):\n task["category"] = TaskCategory.SEQUENCE_CLASSIFICATION\n task["num_label"] = helper["model"]["num_classes"]\n elif isinstance(data_reader, SQuADBertReader):\n task["category"] = TaskCategory.READING_COMPREHENSION\n task["num_label"] = None\n elif isinstance(data_reader, RegressionBertReader):\n task["category"] = TaskCategory.REGRESSION\n task["num_label"] = 1\n elif isinstance(data_reader, TokClsBertReader):\n task["category"] = TaskCategory.TOKEN_CLASSIFICATION\n task["num_label"] = helper["model"]["num_tags"]\n else:\n raise ValueError("Check data_reader.")\n\n task["model_params"] = helper.get("model", {})\n return task\n\n @overrides\n def _read(self, file_path, data_type=None):\n """ TODO: Doc-String """\n\n batches = []\n helper = Helper()\n helper.task_helpers = []\n\n for b in self.dataset_batches:\n batches.append(b[data_type])\n for i, h in enumerate(self.dataset_helpers):\n task_helper = h[data_type]\n task_helper["batch_size"] = self.batch_sizes[i]\n\n helper.task_helpers.append(task_helper)\n\n helper.set_model_parameter({\n "tasks": self.tasks,\n })\n return batches, helper.to_dict()\n\n\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:qnli_bert")\nclass QNLIBertReader(SeqClsBertReader):\n """\n QNLI DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = ["entailment", "not_entailment"]\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(QNLIBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n if len(line_tokens) <= 1:\n continue\n data.append({\n "uid": f"{data_type}-{i}",\n "sequence_a": line_tokens[1],\n "sequence_b": line_tokens[2],\n self.class_key: str(line_tokens[-1]),\n })\n\n return data\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:qqp_bert")\nclass QQPBertReader(SeqClsBertReader):\n """\n Quora Question Pairs DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = [0, 1]\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(QQPBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n try:\n data.append({\n "uid": f"{data_type}-{i}",\n "sequence_a": line_tokens[3],\n "sequence_b": line_tokens[4],\n self.class_key: str(line_tokens[5])\n })\n except IndexError:\n continue\n\n return data\n\n\nimport logging\nimport json\nimport uuid\n\nfrom overrides import overrides\nfrom tqdm import tqdm\n\nfrom claf.data.dataset import RegressionBertDataset\nfrom claf.data.dto import BertFeature, Helper\nfrom claf.data.reader.base import DataReader\nfrom claf.data import utils\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:regression_bert")\nclass RegressionBertReader(DataReader):\n """\n Regression DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n METRIC_KEY = None\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n label_key="score",\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(RegressionBertReader, self).__init__(file_paths, RegressionBertDataset)\n\n self.sequence_max_length = sequence_max_length\n self.text_columns = ["bert_input", "sequence"]\n\n # Tokenizers\n # - BERT: Word + Subword or Word + Char\n # - RoBERTa: BPE\n\n if input_type == "bert":\n self.tokenizer = tokenizers.get("subword", None)\n if self.tokenizer is None:\n self.tokenizer["char"]\n elif input_type == "roberta":\n self.tokenizer = tokenizers["bpe"]\n else:\n raise ValueError("'bert' and 'roberta' are available input_type.")\n\n self.label_key = label_key\n self.cls_token = cls_token\n self.sep_token = sep_token\n self.input_type = input_type\n self.is_test = is_test\n\n def _get_data(self, file_path, **kwargs):\n data = self.data_handler.read(file_path)\n seq_cls_data = json.loads(data)\n\n return seq_cls_data["data"]\n\n @overrides\n def _read(self, file_path, data_type=None):\n """\n .json file structure should be something like this:\n\n {\n "data": [\n {\n "sequence_a": "what a wonderful day!",\n "sequence_b": "what a great day!",\n "score": 0.9\n },\n ...\n ]\n }\n """\n\n data = self._get_data(file_path, data_type=data_type)\n\n helper = Helper(**{\n "file_path": file_path,\n "cls_token": self.cls_token,\n "sep_token": self.sep_token,\n "dataset": RegressionBertDataset,\n "metric_key": self.METRIC_KEY,\n })\n\n features, labels = [], []\n\n for example in tqdm(data, desc=data_type):\n sequence_a = utils.get_sequence_a(example)\n sequence_b = example.get("sequence_b", None)\n\n sequence_a_tokens = self.tokenizer.tokenize(sequence_a)\n sequence_b_tokens = None\n if sequence_b:\n sequence_b_tokens = self.tokenizer.tokenize(sequence_b)\n\n bert_input = utils.make_bert_input(\n sequence_a,\n sequence_b,\n self.tokenizer,\n max_seq_length=self.sequence_max_length,\n data_type=data_type,\n cls_token=self.cls_token,\n sep_token=self.sep_token,\n input_type=self.input_type,\n )\n\n if bert_input is None:\n continue\n\n if "uid" in example:\n data_uid = example["uid"]\n else:\n data_uid = str(uuid.uuid1())\n\n feature_row = {\n "id": data_uid,\n "bert_input": bert_input,\n }\n features.append(feature_row)\n\n score = example[self.label_key]\n label_row = {\n "id": data_uid,\n "score": score,\n }\n labels.append(label_row)\n\n helper.set_example(data_uid, {\n "sequence_a": sequence_a,\n "sequence_a_tokens": sequence_a_tokens,\n "sequence_b": sequence_b,\n "sequence_b_tokens": sequence_b_tokens,\n "score": score,\n })\n\n if self.is_test and len(features) >= 10:\n break\n\n return utils.make_batch(features, labels), helper.to_dict()\n\n[docs] def read_one_example(self, inputs):\n """ inputs keys: sequence_a and sequence_b """\n sequence_a = utils.get_sequence_a(inputs)\n sequence_b = inputs.get("sequence_b", None)\n\n bert_feature = BertFeature()\n bert_feature.set_input_with_speical_token(\n sequence_a,\n sequence_b,\n self.tokenizer,\n max_seq_length=self.sequence_max_length,\n data_type="predict",\n cls_token=self.cls_token,\n sep_token=self.sep_token,\n input_type=self.input_type,\n )\n\n features = [bert_feature.to_dict()]\n helper = {}\n return features, helper\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:rte_bert")\nclass RTEBertReader(SeqClsBertReader):\n """\n RTE (Recognizing Textual Entailment) DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = ["entailment", "not_entailment"]\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(RTEBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n if len(line_tokens) <= 1:\n continue\n data.append({\n "uid": f"{data_type}-{i}",\n "sequence_a": line_tokens[1],\n "sequence_b": line_tokens[2],\n self.class_key: str(line_tokens[-1]),\n })\n\n return data\n\n\nimport json\nimport logging\nimport uuid\n\nfrom overrides import overrides\nfrom tqdm import tqdm\n\nfrom claf.data.dataset import SeqClsBertDataset\nfrom claf.data.dto import BertFeature, Helper\nfrom claf.data.reader.base import DataReader\nfrom claf.data import utils\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:seq_cls_bert")\nclass SeqClsBertReader(DataReader):\n """\n DataReader for Sequence (Single and Pair) Classification using BERT\n\n * Args:\n file_paths: .json file paths (train and dev)\n tokenizers: define tokenizers config (subword)\n\n * Kwargs:\n class_key: name of the label in .json file to use for classification\n """\n\n CLASS_DATA = None\n METRIC_KEY = None\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n class_key="class",\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(SeqClsBertReader, self).__init__(file_paths, SeqClsBertDataset)\n\n self.sequence_max_length = sequence_max_length\n self.text_columns = ["bert_input", "sequence"]\n\n # Tokenizers\n # - BERT: Word + Subword or Word + Char\n # - RoBERTa: BPE\n\n if input_type == "bert":\n self.tokenizer = tokenizers.get("subword", None)\n if self.tokenizer is None:\n self.tokenizer["char"]\n elif input_type == "roberta":\n self.tokenizer = tokenizers["bpe"]\n else:\n raise ValueError("'bert' and 'roberta' are available input_type.")\n\n self.class_key = class_key\n self.cls_token = cls_token\n self.sep_token = sep_token\n self.input_type = input_type\n self.is_test = is_test\n\n def _get_data(self, file_path, **kwargs):\n data = self.data_handler.read(file_path)\n seq_cls_data = json.loads(data)\n\n return seq_cls_data["data"]\n\n def _get_class_dicts(self, **kwargs):\n seq_cls_data = kwargs["data"]\n if self.class_key is None:\n class_data = self.CLASS_DATA\n else:\n class_data = [item[self.class_key] for item in seq_cls_data]\n class_data = list(set(class_data)) # remove duplicate\n\n class_idx2text = {\n class_idx: str(class_text)\n for class_idx, class_text in enumerate(class_data)\n }\n class_text2idx = {class_text: class_idx for class_idx, class_text in class_idx2text.items()}\n\n return class_idx2text, class_text2idx\n\n @overrides\n def _read(self, file_path, data_type=None):\n """\n .json file structure should be something like this:\n\n {\n "data": [\n {\n "sequence": "what a wonderful day!",\n "emotion": "happy"\n },\n ...\n ],\n "emotion": [ // class_key\n "angry",\n "happy",\n "sad",\n ...\n ]\n }\n """\n\n data = self._get_data(file_path, data_type=data_type)\n class_idx2text, class_text2idx = self._get_class_dicts(data=data)\n\n helper = Helper(**{\n "file_path": file_path,\n "class_idx2text": class_idx2text,\n "class_text2idx": class_text2idx,\n "cls_token": self.cls_token,\n "sep_token": self.sep_token,\n "dataset": SeqClsBertDataset,\n "metric_key": self.METRIC_KEY,\n })\n helper.set_model_parameter({\n "num_classes": len(class_idx2text),\n })\n helper.set_predict_helper({\n "class_idx2text": class_idx2text,\n })\n\n features, labels = [], []\n\n for example in tqdm(data, desc=data_type):\n sequence_a = utils.get_sequence_a(example)\n sequence_b = example.get("sequence_b", None)\n\n sequence_a_tokens = self.tokenizer.tokenize(sequence_a)\n sequence_b_tokens = None\n if sequence_b:\n sequence_b_tokens = self.tokenizer.tokenize(sequence_b)\n\n bert_input = utils.make_bert_input(\n sequence_a,\n sequence_b,\n self.tokenizer,\n max_seq_length=self.sequence_max_length,\n data_type=data_type,\n cls_token=self.cls_token,\n sep_token=self.sep_token,\n input_type=self.input_type,\n )\n\n if bert_input is None:\n continue\n\n if "uid" in example:\n data_uid = example["uid"]\n else:\n data_uid = str(uuid.uuid1())\n\n # token_type(segment_ids) will be added in dataset\n feature_row = {\n "id": data_uid,\n "bert_input": bert_input,\n }\n features.append(feature_row)\n\n class_text = example[self.class_key]\n label_row = {\n "id": data_uid,\n "class_idx": class_text2idx[class_text],\n "class_text": class_text,\n }\n labels.append(label_row)\n\n helper.set_example(data_uid, {\n "sequence_a": sequence_a,\n "sequence_a_tokens": sequence_a_tokens,\n "sequence_b": sequence_b,\n "sequence_b_tokens": sequence_b_tokens,\n "class_idx": class_text2idx[class_text],\n "class_text": class_text,\n })\n\n if self.is_test and len(features) >= 10:\n break\n\n return utils.make_batch(features, labels), helper.to_dict()\n\n[docs] def read_one_example(self, inputs):\n """ inputs keys: sequence_a and sequence_b """\n sequence_a = utils.get_sequence_a(inputs)\n sequence_b = inputs.get("sequence_b", None)\n\n bert_feature = BertFeature()\n bert_feature.set_input_with_speical_token(\n sequence_a,\n sequence_b,\n self.tokenizer,\n max_seq_length=self.sequence_max_length,\n data_type="predict",\n cls_token=self.cls_token,\n sep_token=self.sep_token,\n input_type=self.input_type,\n )\n\n features = [bert_feature.to_dict()]\n helper = {}\n return features, helper\n\n\nfrom collections import Counter\nimport json\nimport logging\nimport re\n\nfrom overrides import overrides\nfrom tqdm import tqdm\n\nfrom claf.data.dataset import SQuADBertDataset\nfrom claf.data.dto import BertFeature, Helper\nfrom claf.data.reader.base import DataReader\nfrom claf.data import utils\nfrom claf.decorator import register\nfrom claf.metric.squad_v1_official import normalize_answer\nfrom claf.tokens.tokenizer import SentTokenizer, WordTokenizer\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]class Token:\n def __init__(self, text, text_span=None):\n self.text = text\n self.text_span = text_span\n\n\n[docs]@register("reader:squad_bert")\nclass SQuADBertReader(DataReader):\n """\n SQuAD DataReader for BERT\n\n * Args:\n file_paths: .json file paths (train and dev)\n tokenizers: defined tokenizers config (char/word)\n """\n\n METRIC_KEY = "f1"\n\n def __init__(\n self,\n file_paths,\n lang_code,\n tokenizers,\n max_seq_length=384,\n context_stride=128,\n max_question_length=64,\n cls_token="[CLS]",\n sep_token="[SEP]",\n ):\n\n super(SQuADBertReader, self).__init__(file_paths, SQuADBertDataset)\n self.lang_code = lang_code\n self.max_seq_length = max_seq_length\n self.context_stride = context_stride\n self.max_question_length = max_question_length\n self.cls_token = cls_token\n self.sep_token = sep_token\n\n self.text_columns = ["bert_input", "context", "question"]\n\n sent_tokenizer = SentTokenizer("punkt", {})\n if lang_code == "ko":\n self.word_tokenizer = WordTokenizer("mecab_ko", sent_tokenizer, split_with_regex=True)\n else:\n self.word_tokenizer = WordTokenizer(\n "treebank_en", sent_tokenizer, split_with_regex=True\n )\n\n if tokenizers["bpe"] is not None:\n self.sub_level_tokenizer = tokenizers["bpe"] # RoBERTa\n elif tokenizers["subword"] is not None:\n self.sub_level_tokenizer = tokenizers["subword"] # BERT\n else:\n raise ValueError("'bpe' or 'subword' tokenizer is required.")\n\n @overrides\n def _read(self, file_path, data_type=None):\n word_tokenized_error_count, sub_level_tokenized_error_count = 0, 0\n\n data = self.data_handler.read(file_path)\n squad = json.loads(data)\n if "data" in squad:\n squad = squad["data"]\n\n helper = Helper(**{\n "file_path": file_path,\n "raw_dataset": squad,\n "cls_token": self.cls_token,\n "sep_token": self.sep_token,\n "dataset": SQuADBertDataset,\n })\n helper.set_model_parameter({\n "lang_code": self.lang_code,\n })\n\n features, labels = [], []\n is_training = data_type == "train"\n\n for article in tqdm(squad, desc=data_type):\n for paragraph in article["paragraphs"]:\n context_text = paragraph["context"].replace("``", '" ').replace("''", '" ')\n context_tokens = self.word_tokenizer.tokenize(context_text)\n\n context_spans, char_to_word_offset = self._convert_to_spans(\n context_text, context_tokens\n )\n context_tokens = [\n Token(text, span) for (text, span) in zip(context_tokens, context_spans)\n ]\n\n context_sub_tokens = []\n for token in context_tokens:\n for sub_token in self.sub_level_tokenizer.tokenize(token.text):\n context_sub_tokens.append(Token(sub_token, token.text_span))\n\n for qa in paragraph["qas"]:\n question_text = qa["question"]\n question_text = " ".join(self.word_tokenizer.tokenize(question_text))\n question_sub_tokens = [\n Token(sub_token) for sub_token in self.sub_level_tokenizer.tokenize(question_text)\n ]\n\n id_ = qa["id"]\n answers = qa["answers"]\n\n answer_texts, answer_indices = [], []\n\n if qa.get("is_impossible", None):\n answers = qa["plausible_answers"]\n answerable = 0\n else:\n answers = qa["answers"]\n answerable = 1\n\n for answer in answers:\n answer_start = answer["answer_start"]\n answer_end = answer_start + len(answer["text"]) - 1\n\n answer_texts.append(answer["text"])\n answer_indices.append((answer_start, answer_end))\n\n if len(answer_indices) > 0:\n answer_char_start, answer_char_end = self._find_one_most_common(\n answer_indices\n )\n answer_word_start = char_to_word_offset[answer_char_start]\n answer_word_end = char_to_word_offset[answer_char_end]\n\n char_answer_text = context_text[answer_char_start : answer_char_end + 1]\n word_answer_text = context_text[\n context_spans[answer_word_start][0] : context_spans[answer_word_end][1]\n ]\n\n if not self._is_rebuild(char_answer_text, word_answer_text):\n logger.warning(f"word_tokenized_error: {char_answer_text} ### {word_answer_text}")\n word_tokenized_error_count += 1\n else:\n # Unanswerable\n answers = ["<noanswer>"]\n answer_char_start, answer_char_end = -1, -1\n answer_word_start, answer_word_end = -1, -1\n\n bert_features, bert_labels = self._make_features_and_labels(\n context_sub_tokens,\n question_sub_tokens,\n answer_char_start,\n answer_char_end + 1,\n )\n\n for (index, (feature, label)) in enumerate(zip(bert_features, bert_labels)):\n bert_tokens = feature\n answer_start, answer_end = label\n\n if is_training and (\n answer_start < 0\n or answer_start >= len(bert_tokens)\n or answer_end >= len(bert_tokens)\n or bert_tokens[answer_start].text_span is None\n or bert_tokens[answer_end].text_span is None\n ):\n continue\n\n if is_training:\n char_start = bert_tokens[answer_start].text_span[0]\n char_end = bert_tokens[answer_end].text_span[1]\n bert_answer = context_text[char_start:char_end]\n\n if char_answer_text != bert_answer:\n logger.warning(f"sub_level_tokenized_error: {char_answer_text} ### {word_answer_text})")\n sub_level_tokenized_error_count += 1\n\n feature_row = {\n "bert_input": [token.text for token in bert_tokens],\n "bert_token": bert_tokens,\n }\n features.append(feature_row)\n\n bert_id = id_ + f"#{index}"\n label_row = {\n "id": bert_id, # question_id + bert_index\n "answer_texts": "\\t".join(answer_texts),\n "answer_start": answer_start,\n "answer_end": answer_end,\n "answerable": answerable,\n }\n labels.append(label_row)\n\n if id_ not in helper.examples:\n helper.set_example(id_, {\n "context": context_text,\n "question": question_text,\n "answers": answer_texts,\n })\n helper.set_example(id_, {\n f"bert_tokens_{index}": bert_tokens,\n }, update=True)\n\n logger.info(\n f"tokenized_error_count - word: {word_tokenized_error_count} | sub_level: {sub_level_tokenized_error_count}"\n )\n return utils.make_batch(features, labels), helper.to_dict()\n\n[docs] @overrides\n def read_one_example(self, inputs):\n """ inputs keys: question, context """\n context_text = inputs["context"].replace("``", '" ').replace("''", '" ')\n tokenized_context = self.word_tokenizer.tokenize(context_text)\n context_spans, char_to_word_offset = self._convert_to_spans(context_text, tokenized_context)\n context_tokens = [\n Token(text, span) for (text, span) in zip(tokenized_context, context_spans)\n ]\n\n context_sub_tokens = []\n for token in context_tokens:\n for sub_token in self.sub_level_tokenizer.tokenize(token.text):\n context_sub_tokens.append(Token(sub_token, token.text_span))\n\n question_text = inputs["question"]\n question_text = " ".join(self.word_tokenizer.tokenize(question_text))\n question_sub_tokens = [\n Token(sub_token) for sub_token in self.sub_level_tokenizer.tokenize(question_text)\n ]\n\n bert_tokens, _ = self._make_features_and_labels(\n context_sub_tokens, question_sub_tokens, -1, -1\n )\n\n features = []\n helper = Helper(**{\n "bert_token": [],\n "tokenized_context": tokenized_context,\n "token_key": "tokenized_context" # for 1-example inference latency key\n })\n\n for bert_token in bert_tokens:\n bert_input = [token.text for token in bert_token]\n\n bert_feature = BertFeature()\n bert_feature.set_input(bert_input)\n\n features.append(bert_feature.to_dict())\n helper.bert_token.append(bert_token)\n return features, helper.to_dict()\n\n def _find_one_most_common(self, answers):\n answer_counter = Counter(answers)\n value = answer_counter.most_common(1)[0][0]\n return value[0], value[1]\n\n def _convert_to_spans(self, raw_text, tokenized_text):\n """ Convert a tokenized version of `raw_text` into a series character spans referencing the `raw_text` """\n double_quote_re = re.compile("\\"|``|''")\n\n curr_idx = 0\n spans = []\n char_to_words = [-1 for _ in range(len(raw_text))]\n\n for token in tokenized_text:\n # Tokenizer might transform double quotes, for this case search over several\n # possible encodings\n if double_quote_re.match(token):\n span = double_quote_re.search(raw_text[curr_idx:])\n temp = curr_idx + span.start()\n token_length = span.end() - span.start()\n else:\n temp = raw_text.find(token, curr_idx)\n token_length = len(token)\n if temp < curr_idx:\n joined_tokenized_text = " ".join(tokenized_text)\n raise ValueError(\n f"\\n{raw_text} \\n\\n{joined_tokenized_text} \\nToken: {token}, Index: {temp}, Current Index: {curr_idx}"\n )\n curr_idx = temp\n spans.append((curr_idx, curr_idx + token_length))\n curr_idx += token_length\n\n start, end = spans[-1]\n for i in range(start, end):\n char_to_words[i] = len(spans) - 1\n\n for i in range(len(raw_text)):\n if char_to_words[i] != -1:\n continue\n\n for j, span in enumerate(spans):\n start, end = span\n if start < i <= end:\n char_to_words[i] = j\n\n return spans, char_to_words\n\n def _is_rebuild(self, char_answer_text, word_answer_text):\n norm_char_answer_text = normalize_answer(char_answer_text)\n norm_word_answer_text = normalize_answer(word_answer_text)\n\n if norm_char_answer_text != norm_word_answer_text:\n return False\n else:\n return True\n\n def _make_features_and_labels(\n self, context_sub_tokens, question_sub_tokens, answer_char_start, answer_char_end\n ):\n # sub_token, context_stride logic with context_max_length\n context_max_length = (\n self.max_seq_length - len(question_sub_tokens) - 3\n ) # [CLS], [SEP], [SEP]\n start_offset = 0\n\n context_stride_spans = []\n while start_offset < len(context_sub_tokens):\n strided_context_length = len(context_sub_tokens) - start_offset\n if strided_context_length > context_max_length:\n strided_context_length = context_max_length\n\n context_stride_spans.append((start_offset, strided_context_length))\n if start_offset + strided_context_length == len(context_sub_tokens):\n break\n start_offset += min(strided_context_length, self.context_stride)\n\n features, labels = [], []\n for (start_offset, length) in context_stride_spans:\n bert_tokens = [Token(self.cls_token)]\n bert_tokens += question_sub_tokens[: self.max_question_length]\n bert_tokens += [Token(self.sep_token)]\n bert_tokens += context_sub_tokens[start_offset : start_offset + length]\n bert_tokens += [Token(self.sep_token)]\n features.append(bert_tokens)\n\n if answer_char_start == -1 and answer_char_end == -1:\n answer_start, answer_end = 0, 0\n else:\n answer_start, answer_end = self._get_closest_answer_spans(\n bert_tokens, answer_char_start, answer_char_end\n )\n\n labels.append((answer_start, answer_end))\n return features, labels\n\n def _get_closest_answer_spans(self, tokens, char_start, char_end):\n NONE_VALUE, DISTANCE_THRESHOLD = -100, 2\n\n text_spans = [\n (NONE_VALUE, NONE_VALUE) if token.text_span is None else token.text_span\n for token in tokens\n ]\n\n start_distances = [abs(span[0] - char_start) for span in text_spans]\n end_distances = [abs(span[1] - char_end) for span in text_spans]\n\n min_start_distance, min_end_distance = min(start_distances), min(end_distances)\n if min_start_distance < DISTANCE_THRESHOLD:\n answer_start = start_distances.index(min_start_distance)\n else:\n answer_start = 0\n\n if min_end_distance < DISTANCE_THRESHOLD:\n answer_end = end_distances.index(min_end_distance)\n start_from = answer_end + 1\n try:\n # e.g.) end_distances: [3, 1, 1, 4], min_end_distance = 1 => use 2 index instead of 1\n answer_end = end_distances.index(min_end_distance, start_from)\n except ValueError:\n pass\n else:\n answer_end = 0\n return answer_start, answer_end\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:sst_bert")\nclass SSTBertReader(SeqClsBertReader):\n """\n SST DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = [0, 1]\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(SSTBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n if data_type == "train":\n lines.pop(0)\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n if len(line_tokens) <= 1:\n continue\n data.append({\n "uid": f"{data_type}-{i}",\n "sequence_a": line_tokens[0],\n self.class_key: str(line_tokens[1]),\n })\n\n return data\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import RegressionBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:stsb_bert")\nclass STSBBertReader(RegressionBertReader):\n """\n STS-B (Semantic Textual Similarity Benchmark) DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(STSBBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n label_key="score",\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n if len(line_tokens) <= 1:\n continue\n data.append({\n "uid": f"{data_type}-{i}",\n "sequence_a": line_tokens[7],\n "sequence_b": line_tokens[8],\n "score": float(line_tokens[-1]),\n })\n\n return data\n\n\nfrom itertools import chain\nimport json\nimport logging\nimport uuid\n\nfrom overrides import overrides\nfrom tqdm import tqdm\n\nfrom claf.data.dataset import TokClsBertDataset\nfrom claf.data.dto import BertFeature, Helper\nfrom claf.data.reader.base import DataReader\nfrom claf.decorator import register\nfrom claf.tokens.tokenizer import WordTokenizer\nimport claf.data.utils as utils\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:tok_cls_bert")\nclass TokClsBertReader(DataReader):\n """\n DataReader for Token Classification using BERT\n\n * Args:\n file_paths: .json file paths (train and dev)\n tokenizers: define tokenizers config (subword)\n\n * Kwargs:\n lang_code: language code: set as 'ko' if using BERT model trained with mecab-tokenized data\n tag_key: name of the label in .json file to use for classification\n ignore_tag_idx: prediction results that have this number as ground-truth idx are ignored\n """\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n lang_code=None,\n sequence_max_length=None,\n tag_key="tags",\n cls_token="[CLS]",\n sep_token="[SEP]",\n ignore_tag_idx=-1,\n ):\n\n super(TokClsBertReader, self).__init__(file_paths, TokClsBertDataset)\n\n self.sequence_max_length = sequence_max_length\n self.text_columns = ["bert_input", "sequence"]\n\n if "subword" not in tokenizers:\n raise ValueError("WordTokenizer and SubwordTokenizer is required.")\n\n self.subword_tokenizer = tokenizers["subword"]\n\n self.sent_tokenizer = tokenizers["sent"]\n self.word_tokenizer = tokenizers["word"]\n if lang_code == "ko":\n self.mecab_tokenizer = WordTokenizer("mecab_ko", self.sent_tokenizer, split_with_regex=False)\n\n self.lang_code = lang_code\n self.tag_key = tag_key\n self.cls_token = cls_token\n self.sep_token = sep_token\n self.ignore_tag_idx = ignore_tag_idx\n\n def _get_data(self, file_path):\n data = self.data_handler.read(file_path)\n tok_cls_data = json.loads(data)\n\n return tok_cls_data["data"]\n\n def _get_tag_dicts(self, **kwargs):\n data = kwargs["data"]\n\n if type(data) == dict:\n tag_idx2text = {tag_idx: tag_text for tag_idx, tag_text in enumerate(data[self.tag_key])}\n elif type(data) == list:\n tags = sorted(list(set(chain.from_iterable(d[self.tag_key] for d in data))))\n tag_idx2text = {tag_idx: tag_text for tag_idx, tag_text in enumerate(tags)}\n else:\n raise ValueError("check _get_data return type.")\n\n tag_text2idx = {tag_text: tag_idx for tag_idx, tag_text in tag_idx2text.items()}\n\n return tag_idx2text, tag_text2idx\n\n @overrides\n def _read(self, file_path, data_type=None):\n """\n .json file structure should be something like this:\n\n {\n "data": [\n {\n "sequence": "i'm looking for a flight from New York to London.",\n "slots": ["O", "O", "O", "O", "O", "O", "B-city.dept", "I-city.dept" "O", "B-city.dest"]\n // the number of tokens in sequence.split() and tags must match\n },\n ...\n ],\n "slots": [ // tag_key\n "O", // tags should be in IOB format\n "B-city.dept",\n "I-city.dept",\n "B-city.dest",\n "I-city.dest",\n ...\n ]\n }\n """\n\n data = self._get_data(file_path)\n tag_idx2text, tag_text2idx = self._get_tag_dicts(data=data)\n\n helper = Helper(**{\n "file_path": file_path,\n "tag_idx2text": tag_idx2text,\n "ignore_tag_idx": self.ignore_tag_idx,\n "cls_token": self.cls_token,\n "sep_token": self.sep_token,\n })\n helper.set_model_parameter({\n "num_tags": len(tag_idx2text),\n "ignore_tag_idx": self.ignore_tag_idx,\n })\n helper.set_predict_helper({\n "tag_idx2text": tag_idx2text,\n })\n\n features, labels = [], []\n\n for example in tqdm(data, desc=data_type):\n sequence_text = example["sequence"].strip().replace("\\n", "")\n\n sequence_tokens = self.word_tokenizer.tokenize(sequence_text)\n naive_tokens = sequence_text.split()\n is_head_word = utils.get_is_head_of_word(naive_tokens, sequence_tokens)\n\n sequence_sub_tokens = []\n tagged_sub_token_idxs = []\n curr_sub_token_idx = 1 # skip CLS_TOKEN\n for token_idx, token in enumerate(sequence_tokens):\n for sub_token_pos, sub_token in enumerate(\n self.subword_tokenizer.tokenize(token, unit="word")\n ):\n sequence_sub_tokens.append(sub_token)\n if is_head_word[token_idx] and sub_token_pos == 0:\n tagged_sub_token_idxs.append(curr_sub_token_idx)\n curr_sub_token_idx += 1\n\n bert_input = [self.cls_token] + sequence_sub_tokens + [self.sep_token]\n\n if (\n self.sequence_max_length is not None\n and data_type == "train"\n and len(bert_input) > self.sequence_max_length\n ):\n continue\n\n if "uid" in example:\n data_uid = example["uid"]\n else:\n data_uid = str(uuid.uuid1())\n\n tag_texts = example[self.tag_key]\n tag_idxs = [tag_text2idx[tag_text] for tag_text in tag_texts]\n\n utils.sanity_check_iob(naive_tokens, tag_texts)\n assert len(naive_tokens) == len(tagged_sub_token_idxs), \\\n f"""Wrong tagged_sub_token_idxs: followings mismatch.\n naive_tokens: {naive_tokens}\n sequence_sub_tokens: {sequence_sub_tokens}\n tagged_sub_token_idxs: {tagged_sub_token_idxs}"""\n\n feature_row = {\n "id": data_uid,\n "bert_input": bert_input,\n "tagged_sub_token_idxs": tagged_sub_token_idxs,\n "num_tokens": len(naive_tokens),\n }\n features.append(feature_row)\n\n label_row = {\n "id": data_uid,\n "tag_idxs": tag_idxs,\n "tag_texts": tag_texts,\n }\n labels.append(label_row)\n\n helper.set_example(data_uid, {\n "sequence": sequence_text,\n "sequence_sub_tokens": sequence_sub_tokens,\n "tag_idxs": tag_idxs,\n "tag_texts": tag_texts,\n })\n\n return utils.make_batch(features, labels), helper.to_dict()\n\n[docs] def read_one_example(self, inputs):\n """ inputs keys: sequence """\n sequence_text = inputs["sequence"].strip().replace("\\n", "")\n sequence_tokens = self.word_tokenizer.tokenize(sequence_text)\n naive_tokens = sequence_text.split()\n is_head_word = utils.get_is_head_of_word(naive_tokens, sequence_tokens)\n\n sequence_sub_tokens = []\n tagged_sub_token_idxs = []\n curr_sub_token_idx = 1 # skip CLS_TOKEN\n for token_idx, token in enumerate(sequence_tokens):\n for sub_token_pos, sub_token in enumerate(\n self.subword_tokenizer.tokenize(token, unit="word")\n ):\n sequence_sub_tokens.append(sub_token)\n if is_head_word[token_idx] and sub_token_pos == 0:\n tagged_sub_token_idxs.append(curr_sub_token_idx)\n curr_sub_token_idx += 1\n\n if len(sequence_sub_tokens) > self.sequence_max_length:\n sequence_sub_tokens = sequence_sub_tokens[:self.sequence_max_length]\n\n bert_input = [self.cls_token] + sequence_sub_tokens + [self.sep_token]\n assert len(naive_tokens) == len(tagged_sub_token_idxs), \\\n f"""Wrong tagged_sub_token_idxs: followings mismatch.\n naive_tokens: {naive_tokens}\n sequence_sub_tokens: {sequence_sub_tokens}\n tagged_sub_token_idxs: {tagged_sub_token_idxs}"""\n\n bert_feature = BertFeature()\n bert_feature.set_input(bert_input)\n bert_feature.set_feature("tagged_sub_token_idxs", tagged_sub_token_idxs)\n bert_feature.set_feature("num_tokens", len(naive_tokens))\n\n features = [bert_feature.to_dict()]\n helper = {}\n return features, helper\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:wnli_bert")\nclass WNLIBertReader(SeqClsBertReader):\n """\n WNLI (Winograd NLI) DataReader for BERT\n\n * Args:\n file_paths: .tsv file paths (train and dev)\n tokenizers: defined tokenizers config\n """\n\n CLASS_DATA = [0, 1]\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n is_test=False,\n ):\n\n super(WNLIBertReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length,\n class_key=None,\n cls_token=cls_token,\n sep_token=sep_token,\n input_type=input_type,\n is_test=is_test,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n data = []\n for i, line in enumerate(lines):\n if i == 0:\n continue\n line_tokens = line.split("\\t")\n if len(line_tokens) <= 1:\n continue\n data.append({\n "uid": f"{data_type}-{i}",\n "sequence_a": line_tokens[1],\n "sequence_b": line_tokens[2],\n self.class_key: str(line_tokens[-1]),\n })\n\n return data\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsReader\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:cola")\nclass CoLAReader(SeqClsReader):\n """\n CoLA DataReader\n\n * Args:\n file_paths: .json file paths (train and dev)\n tokenizers: define tokenizers config (word)\n """\n\n CLASS_DATA = [0, 1]\n\n def __init__(\n self,\n file_paths,\n tokenizers,\n sequence_max_length=None,\n ):\n\n super(CoLAReader, self).__init__(\n file_paths,\n tokenizers,\n sequence_max_length=sequence_max_length,\n )\n\n @overrides\n def _get_data(self, file_path, **kwargs):\n data_type = kwargs["data_type"]\n\n _file = self.data_handler.read(file_path)\n lines = _file.split("\\n")\n\n if data_type == "train":\n lines.pop(0)\n\n data = []\n for i, line in enumerate(lines):\n line_tokens = line.split("\\t")\n if len(line_tokens) > 1:\n data.append({\n "uid": f"{data_type}-{i}",\n "sequence": line_tokens[1] if data_type == "test" else line_tokens[3],\n self.class_key: str(0) if data_type == "test" else str(line_tokens[1])\n })\n\n return data\n\n\nimport json\nimport logging\nimport uuid\n\nfrom overrides import overrides\nfrom tqdm import tqdm\n\nfrom claf.data.dataset.seq_cls import SeqClsDataset\nfrom claf.data.dto import Helper\nfrom claf.data.reader.base import DataReader\nfrom claf.data import utils\nfrom claf.decorator import register\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:seq_cls")\nclass SeqClsReader(DataReader):\n """\n DataReader for Sequence Classification\n\n * Args:\n file_paths: .json file paths (train and dev)\n tokenizers: define tokenizers config (word)\n\n * Kwargs:\n class_key: name of the label in .json file to use for classification\n """\n\n CLASS_DATA = None\n\n def __init__(self, file_paths, tokenizers, sequence_max_length=None, class_key="class"):\n super(SeqClsReader, self).__init__(file_paths, SeqClsDataset)\n\n self.sequence_max_length = sequence_max_length\n self.text_columns = ["sequence"]\n\n if "word" not in tokenizers:\n raise ValueError("WordTokenizer is required. define WordTokenizer")\n\n self.word_tokenizer = tokenizers["word"]\n self.class_key = class_key\n\n def _get_data(self, file_path, **kwargs):\n data = self.data_handler.read(file_path)\n seq_cls_data = json.loads(data)\n\n return seq_cls_data["data"]\n\n def _get_class_dicts(self, **kwargs):\n seq_cls_data = kwargs["data"]\n if self.class_key is None:\n class_data = self.CLASS_DATA\n else:\n class_data = [item[self.class_key] for item in seq_cls_data]\n class_data = list(set(class_data)) # remove duplicate\n\n class_idx2text = {\n class_idx: str(class_text)\n for class_idx, class_text in enumerate(class_data)\n }\n class_text2idx = {class_text: class_idx for class_idx, class_text in class_idx2text.items()}\n\n return class_idx2text, class_text2idx\n\n @overrides\n def _read(self, file_path, data_type=None):\n """\n .json file structure should be something like this:\n\n {\n "data": [\n {\n "sequence": "what a wonderful day!",\n "emotion": "happy"\n },\n ...\n ],\n "emotion": [ // class_key\n "angry",\n "happy",\n "sad",\n ...\n ]\n }\n """\n\n data = self._get_data(file_path, data_type=data_type)\n class_idx2text, class_text2idx = self._get_class_dicts(data=data)\n\n helper = Helper(**{\n "file_path": file_path,\n "class_idx2text": class_idx2text,\n "class_text2idx": class_text2idx,\n })\n helper.set_model_parameter({\n "num_classes": len(class_idx2text),\n })\n helper.set_predict_helper({\n "class_idx2text": class_idx2text,\n })\n\n features, labels = [], []\n\n for example in tqdm(data, desc=data_type):\n sequence = example["sequence"].strip().replace("\\n", "")\n sequence_words = self.word_tokenizer.tokenize(sequence)\n\n if (\n self.sequence_max_length is not None\n and data_type == "train"\n and len(sequence_words) > self.sequence_max_length\n ):\n continue\n\n if "uid" in example:\n data_uid = example["uid"]\n else:\n data_uid = str(uuid.uuid1())\n\n feature_row = {\n "id": data_uid,\n "sequence": sequence,\n }\n features.append(feature_row)\n\n class_text = example[self.class_key]\n label_row = {\n "id": data_uid,\n "class_idx": class_text2idx[class_text],\n "class_text": class_text,\n }\n labels.append(label_row)\n\n helper.set_example(data_uid, {\n "sequence": sequence,\n "class_idx": class_text2idx[class_text],\n "class_text": class_text,\n })\n\n return utils.make_batch(features, labels), helper.to_dict()\n\n[docs] def read_one_example(self, inputs):\n """ inputs keys: sequence """\n sequence = inputs["sequence"].strip().replace("\\n", "")\n\n inputs["sequence"] = sequence\n\n return inputs, {}\n\n\nfrom collections import Counter\nimport json\nimport logging\nimport re\n\nfrom overrides import overrides\nfrom tqdm import tqdm\n\nfrom claf.data.dataset import SQuADDataset\nfrom claf.data.dto import Helper\nfrom claf.data.reader.base import DataReader\nfrom claf.data import utils\nfrom claf.decorator import register\nfrom claf.metric.squad_v1_official import normalize_answer\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:squad")\nclass SQuADReader(DataReader):\n """\n SQuAD DataReader\n\n * Args:\n file_paths: .json file paths (train and dev)\n tokenizers: defined tokenizers config (char/word)\n """\n\n def __init__(self, file_paths, lang_code, tokenizers, context_max_length=None):\n super(SQuADReader, self).__init__(file_paths, SQuADDataset)\n self.lang_code = lang_code\n self.context_max_length = context_max_length\n\n self.text_columns = ["context", "question"]\n\n if "word" not in tokenizers:\n raise ValueError("WordTokenizer is required. define English WordTokenizer")\n self.word_tokenizer = tokenizers["word"]\n\n @overrides\n def _read(self, file_path, data_type=None):\n tokenized_error_count = 0\n\n data = self.data_handler.read(file_path)\n squad = json.loads(data)\n if "data" in squad:\n squad = squad["data"]\n\n helper = Helper(**{\n "file_path": file_path,\n "raw_dataset": squad,\n })\n helper.set_model_parameter({\n "lang_code": self.lang_code,\n })\n\n features, labels = [], []\n\n for article in tqdm(squad, desc=data_type):\n for paragraph in article["paragraphs"]:\n context = paragraph["context"].replace("``", '" ').replace("''", '" ')\n context_words = self.word_tokenizer.tokenize(context)\n\n if (\n self.context_max_length is not None\n and data_type == "train"\n and len(context_words) > self.context_max_length\n ):\n continue\n\n for qa in paragraph["qas"]:\n question = qa["question"].strip().replace("\\n", "")\n id_ = qa["id"]\n\n answer_texts, answer_indices = [], []\n\n if qa.get("is_impossible", None):\n answers = qa["plausible_answers"]\n answerable = 0\n else:\n answers = qa["answers"]\n answerable = 1\n\n for answer in answers:\n answer_start = answer["answer_start"]\n answer_end = answer_start + len(answer["text"])\n\n answer_texts.append(answer["text"])\n answer_indices.append((answer_start, answer_end))\n\n feature_row = {\n "context": self._clean_text(context),\n "question": question,\n }\n features.append(feature_row)\n\n if len(answer_indices) > 0:\n answer_start, answer_end = self._find_one_most_common(answer_indices)\n text_spans = self._convert_to_spans(context, context_words)\n word_idxs = self._get_word_span_idxs(text_spans, answer_start, answer_end)\n\n word_answer_start = word_idxs[0]\n word_answer_end = word_idxs[-1]\n\n # To check rebuild answer: char_answer_text - word_answer_text\n char_answer_text = context[answer_start:answer_end]\n word_answer_text = context[\n text_spans[word_answer_start][0] : text_spans[word_answer_end][1]\n ]\n\n if not self._is_rebuild(char_answer_text, word_answer_text):\n logger.warning(f"word_tokenized_error: {char_answer_text} ### {word_answer_text}")\n tokenized_error_count += 1\n\n else:\n # Unanswerable\n answers = ["<noanswer>"]\n text_spans = []\n answer_start, answer_end = 0, 0\n word_answer_start, word_answer_end = 0, 0\n\n label_row = {\n "id": id_,\n "answer_start": word_answer_start,\n "answer_end": word_answer_end,\n "answerable": answerable,\n }\n labels.append(label_row)\n\n helper.set_example(id_, {\n "context": context,\n "text_span": text_spans,\n "question": question,\n "answers": answer_texts,\n })\n\n logger.info(f"tokenized_error_count: {tokenized_error_count} ")\n return utils.make_batch(features, labels), helper.to_dict()\n\n[docs] @overrides\n def read_one_example(self, inputs):\n """ inputs keys: question, context """\n context_text = inputs["context"]\n tokenized_context = self.word_tokenizer.tokenize(context_text)\n question_text = inputs["question"].strip().replace("\\n", "")\n\n features = {}\n features["context"] = self._clean_text(context_text)\n features["question"] = self._clean_text(question_text)\n\n helper = {\n "text_span": self._convert_to_spans(context_text, tokenized_context),\n "tokenized_context": tokenized_context,\n "token_key": "tokenized_context" # for 1-example inference latency key\n }\n return features, helper\n\n def _clean_text(self, text):\n # https://github.com/allenai/document-qa/blob/2f9fa6878b60ed8a8a31bcf03f802cde292fe48b/docqa/data_processing/text_utils.py#L124\n # be consistent with quotes, and replace \\u2014 and \\u2212 which I have seen being mapped to UNK\n # by glove word vecs\n return (\n text.replace("''", '"')\n .replace("``", '"')\n .replace("\\u2212", "-")\n .replace("\\u2014", "\\u2013")\n )\n\n def _find_one_most_common(self, answers):\n answer_counter = Counter(answers)\n value = answer_counter.most_common(1)[0][0]\n return value[0], value[1]\n\n def _convert_to_spans(self, raw_text, tokenized_text):\n """ Convert a tokenized version of `raw_text` into a series character spans referencing the `raw_text` """\n double_quote_re = re.compile("\\"|``|''")\n\n curr_idx = 0\n spans = []\n for token in tokenized_text:\n # Tokenizer might transform double quotes, for this case search over several\n # possible encodings\n if double_quote_re.match(token):\n span = double_quote_re.search(raw_text[curr_idx:])\n temp = curr_idx + span.start()\n token_length = span.end() - span.start()\n else:\n temp = raw_text.find(token, curr_idx)\n token_length = len(token)\n if temp < curr_idx:\n raise ValueError(f"{raw_text} \\n{tokenized_text} \\n{token}")\n curr_idx = temp\n spans.append((curr_idx, curr_idx + token_length))\n curr_idx += token_length\n return spans\n\n def _get_word_span_idxs(self, spans, start, end):\n idxs = []\n for word_ix, (s, e) in enumerate(spans):\n if e > start:\n if s < end:\n idxs.append(word_ix)\n else:\n break\n return idxs\n\n def _is_rebuild(self, char_answer_text, word_answer_text):\n norm_char_answer_text = normalize_answer(char_answer_text)\n norm_word_answer_text = normalize_answer(word_answer_text)\n\n if norm_char_answer_text != norm_word_answer_text:\n return False\n else:\n return True\n\n\nimport json\nimport logging\nfrom pathlib import Path\nimport uuid\n\nfrom overrides import overrides\nfrom tqdm import tqdm\n\nfrom claf.data.dataset import WikiSQLDataset\nfrom claf.data.dto import Helper\nfrom claf.data.reader.base import DataReader\nfrom claf.data import utils\nfrom claf.decorator import register\nfrom claf.metric.wikisql_lib.dbengine import DBEngine\nfrom claf.metric.wikisql_lib.query import Query\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("reader:wikisql")\nclass WikiSQLReader(DataReader):\n """\n WikiSQL DataReader\n (http://arxiv.org/abs/1709.00103)\n\n * Args:\n file_paths: .json file paths (train and dev)\n tokenizers: defined tokenizers config (char/word)\n """\n\n def __init__(self, file_paths, tokenizers, context_max_length=None, is_test=None):\n super(WikiSQLReader, self).__init__(file_paths, WikiSQLDataset)\n self.is_test = is_test\n self.text_columns = ["column", "question"]\n\n if "word" not in tokenizers:\n raise ValueError("WordTokenizer is required. define English WordTokenizer")\n self.word_tokenizer = tokenizers["word"]\n self.dbengine = None\n\n @overrides\n def _read(self, file_path, data_type=None):\n file_path = self.data_handler.read(file_path, return_path=True)\n file_path = Path(file_path)\n\n data_dir = file_path.parent\n file_name = file_path.stem\n\n db_path = data_dir / f"{file_name}.db"\n table_path = data_dir / f"{file_name}.tables.jsonl"\n\n self.dbengine = DBEngine(db_path)\n\n helper = Helper(**{\n "file_path": file_path,\n "db_path": db_path,\n })\n\n features, labels = [], []\n\n sql_datas, table_data = self.load_data(file_path, table_path, data_type=data_type)\n for sql_data in tqdm(sql_datas, desc=data_type):\n question = sql_data["question"]\n table_id = sql_data["table_id"]\n column_headers = table_data[table_id]["header"]\n\n feature_row = {"column": column_headers, "question": question}\n\n data_uid = str(uuid.uuid1())\n conditions_value_position = self.get_coditions_value_position(\n sql_data["question"], [x[2] for x in sql_data["sql"]["conds"]]\n )\n\n sql_query = Query.from_dict(sql_data["sql"], ordered=True)\n execution_result = self.dbengine.execute_query(table_id, sql_query, lower=True)\n\n label_row = {\n "id": data_uid,\n "table_id": table_id,\n "tokenized_question": self.word_tokenizer.tokenize(question),\n "aggregator_idx": sql_data["sql"]["agg"],\n "select_column_idx": sql_data["sql"]["sel"],\n "conditions_num": len(sql_data["sql"]["conds"]),\n "conditions_column_idx": [x[0] for x in sql_data["sql"]["conds"]],\n "conditions_operator_idx": [x[1] for x in sql_data["sql"]["conds"]],\n "conditions_value_string": [str(x[2]) for x in sql_data["sql"]["conds"]],\n "conditions_value_position": conditions_value_position,\n "sql_query": sql_query,\n "execution_result": execution_result,\n }\n\n features.append(feature_row)\n labels.append(label_row)\n\n helper.set_example(data_uid, {\n "question": question,\n "sql_query": sql_query,\n "execution_result": execution_result,\n })\n\n if self.is_test and len(labels) == 10:\n break\n\n return utils.make_batch(features, labels), helper.to_dict()\n\n[docs] @overrides\n def read_one_example(self, inputs):\n """ inputs keys: question, column, db_path, table_id """\n question_text = inputs["question"]\n helper = {"tokenized_question": self.word_tokenizer.tokenize(question_text)}\n return inputs, helper\n\n[docs] def load_data(self, sql_path, table_path, data_type=None):\n sql_data = []\n table_data = {}\n\n logger.info(f"Loading data from {sql_path}")\n with open(sql_path) as inf:\n for line in tqdm(inf, desc=f"sql_{data_type}"):\n sql = json.loads(line.strip())\n sql_data.append(sql)\n\n logger.info(f"Loading data from {table_path}")\n with open(table_path) as inf:\n for line in tqdm(inf, desc=f"table_{data_type}"):\n tab = json.loads(line.strip())\n table_data[tab["id"]] = tab\n\n for sql in sql_data:\n assert sql["table_id"] in table_data\n return sql_data, table_data\n\n[docs] def get_coditions_value_position(self, question, values):\n tokenized_question = self.word_tokenizer.tokenize(question.lower())\n tokenized_values = [self.word_tokenizer.tokenize(str(value).lower()) for value in values]\n\n START_TOKEN, END_TOKEN = "<BEG>", "<END>"\n\n token_to_index = {START_TOKEN: 0}\n for token in tokenized_question:\n token_to_index[token] = len(token_to_index)\n token_to_index[END_TOKEN] = len(token_to_index)\n\n position_tokens = []\n for value in tokenized_values:\n position_token = [token_to_index[START_TOKEN]]\n for token in value:\n if token in token_to_index:\n position_token.append(token_to_index[token])\n else:\n for i in range(len(tokenized_question)):\n q_token = tokenized_question[i]\n if token in q_token:\n position_token.append(token_to_index[q_token])\n position_token.append(token_to_index[END_TOKEN])\n\n assert len(position_token) != 2\n position_tokens.append(position_token)\n\n return position_tokens\n\n\nfrom collections import defaultdict\n\nimport numpy as np\nimport torch\n\nfrom claf.data.dto.batch import Batch\n\n\n\n\n\n[docs]def make_bert_input(\n sequence_a,\n sequence_b,\n bert_tokenizer,\n max_seq_length=128,\n data_type="train",\n cls_token="[CLS]",\n sep_token="[SEP]",\n input_type="bert",\n):\n sequence_a_tokens = bert_tokenizer.tokenize(sequence_a)\n bert_input = [cls_token] + sequence_a_tokens + [sep_token]\n\n if sequence_b:\n if input_type == "roberta":\n bert_input += [sep_token]\n\n sequence_b_tokens = bert_tokenizer.tokenize(sequence_b)\n bert_input += sequence_b_tokens + [sep_token]\n\n if len(bert_input) > max_seq_length:\n if data_type == "train":\n return None # for skip\n else:\n return bert_input[:max_seq_length-1] + [sep_token]\n return bert_input\n\n\n[docs]def make_bert_token_types(bert_inputs, SEP_token="[SEP]"):\n """\n Bert Inputs segment_ids\n\n ex) [CLS] hi [SEP] he ##llo [SEP] => 0 0 0 1 1 1\n\n * Args:\n bert_inputs: feature dictionary consisting of\n - text: text from data_reader\n - token_name: text converted to corresponding token_type\n\n * Kwargs:\n SEP_token: SEP special token for BERT\n """\n\n feature_keys = list(bert_inputs[0].keys()) # TODO: hard-code\n if "text" in feature_keys:\n feature_keys.remove("text")\n\n feature_key = feature_keys[0]\n\n token_types = []\n for bert_input in bert_inputs:\n token_type = make_bert_token_type(bert_input["text"], SEP_token=SEP_token)\n token_types.append({feature_key: token_type})\n return token_types\n\n\n[docs]def make_bert_token_type(bert_input_text, SEP_token="[SEP]"):\n SEP_index = bert_input_text.index(SEP_token) + 1\n\n token_type = [0] * SEP_index\n token_type += [1] * (len(bert_input_text) - SEP_index)\n\n assert len(token_type) == len(bert_input_text)\n return token_type\n\n\n[docs]def padding_tokens(tokens, max_len=None, token_name=None, pad_value=0):\n """ Padding tokens according to token's dimension """\n\n def _pad_tokens(seqs, maxlen, pad_id=0):\n lens = [len(seq) for seq in seqs]\n\n if pad_id == 0:\n padded_seqs = torch.zeros(len(seqs), maxlen).long()\n else:\n padded_seqs = torch.ones(len(seqs), maxlen).long() * pad_id\n\n for i, seq in enumerate(seqs):\n if type(seq[0]) == dict:\n pass\n else:\n seq = [int(s) for s in seq]\n end = lens[i]\n padded_seqs[i, :end] = torch.LongTensor(seq)\n return padded_seqs\n\n def _pad_char_tokens(seqs, seq_maxlen, char_minlen=10, char_maxlen=None, pad_value=0):\n if char_maxlen is None:\n char_maxlen = max([len(chars) for seq in seqs for chars in seq])\n if char_maxlen < char_minlen:\n char_maxlen = char_minlen\n\n padded_chars = torch.zeros(len(seqs), seq_maxlen, char_maxlen).long()\n for i in range(len(seqs)):\n char_tokens = _pad_with_value(seqs[i], seq_maxlen, pad_value=[[pad_value]])\n padded_chars[i] = _pad_tokens(char_tokens, char_maxlen, pad_id=pad_value)\n return padded_chars\n\n def _pad_with_value(data, size, pad_value=[0]):\n if type(pad_value) != list:\n raise ValueError("pad_value data type is list.")\n\n return data + pad_value * (size - len(data))\n\n token_dim = get_token_dim(tokens)\n if token_dim > 1 and max_len is None:\n max_len = max(len(token) for token in tokens)\n\n if token_dim == 2: # word\n return _pad_tokens(tokens, max_len, pad_id=pad_value)\n elif token_dim == 3: # char\n if token_name == "elmo":\n return _pad_char_tokens(\n tokens, max_len, char_maxlen=50, pad_value=261,\n ) # 260: padding_character, +1 for mask\n elif token_name == "char":\n return _pad_char_tokens(tokens, max_len, char_minlen=10, pad_value=pad_value)\n else:\n return _pad_char_tokens(tokens, max_len, char_minlen=1, pad_value=pad_value)\n else:\n return tokens\n\n\n[docs]def get_sequence_a(example):\n if "sequence" in example:\n return example["sequence"]\n elif "sequence_a" in example:\n return example["sequence_a"]\n else:\n raise ValueError("'sequence' or 'sequence_a' key is required.")\n\n\n[docs]def get_token_dim(tokens, dim=0):\n if type(tokens) == torch.Tensor:\n dim = tokens.dim()\n if tokens.size(-1) > 1:\n dim += 1\n return dim\n\n if type(tokens) == np.ndarray:\n dim = tokens.ndim\n if tokens.shape[-1] > 1:\n dim += 1\n return dim\n\n if type(tokens) == list or type(tokens) == tuple:\n dim = get_token_dim(tokens[0], dim + 1)\n return dim\n\n\n[docs]def get_token_type(tokens):\n token = tokens[0]\n while isinstance(token, np.ndarray) and isinstance(token, list):\n token = token[0]\n return type(token)\n\n\n[docs]def is_lazy(tokens):\n if type(tokens) == list:\n tokens = tokens[0]\n\n if callable(tokens):\n return True\n else:\n return False\n\n\n[docs]def transpose(list_of_dict, skip_keys=[]):\n if type(skip_keys) != list:\n raise ValueError(f"skip_keys type must be list. not {type(skip_keys)}")\n\n dict_of_list = defaultdict(lambda: [])\n for dic in list_of_dict:\n for key, value in dic.items():\n if key in skip_keys:\n continue\n dict_of_list[key].append(value)\n return dict_of_list\n\n\n[docs]def sanity_check_iob(naive_tokens, tag_texts):\n """\n Check if the IOB tags are valid.\n\n * Args:\n naive_tokens: tokens split by .split()\n tag_texts: list of tags in IOB format\n """\n def prefix(tag):\n if tag == "O":\n return tag\n return tag.split("-")[0]\n\n def body(tag):\n if tag == "O":\n return None\n return tag.split("-")[1]\n\n # same number check\n assert len(naive_tokens) == len(tag_texts), \\\n f"""Number of tokens and tags doest not match.\n original tokens: {naive_tokens}\n tags: {tag_texts}"""\n\n # IOB format check\n prev_tag = None\n for tag_text in tag_texts:\n curr_tag = tag_text\n\n if prev_tag is None: # first tag\n assert prefix(curr_tag) in ["B", "O"], \\\n f"""Wrong tag: first tag starts with I.\n tag: {curr_tag}"""""\n\n else: # following tags\n if prefix(prev_tag) in ["B", "I"]:\n assert (\n (prefix(curr_tag) == "I" and body(curr_tag) == body(prev_tag))\n or (prefix(curr_tag) == "B")\n or (prefix(curr_tag) == "O")\n ), f"""Wrong tag: following tag mismatch.\n previous tag: {prev_tag}\n current tag: {curr_tag}"""\n\n elif prefix(prev_tag) in ["O"]:\n assert prefix(curr_tag) in ["B", "O"], \\\n f"""Wrong tag: following tag mismatch.\n previous tag: {prev_tag}\n current tag: {curr_tag}"""\n else:\n raise RuntimeError(f"Encountered unknown tag: {prev_tag}.")\n\n prev_tag = curr_tag\n\n[docs]def get_is_head_of_word(naive_tokens, sequence_tokens):\n """\n Return a list of flags whether the token is head(prefix) of naively split tokens\n\n ex) naive_tokens: ["hello.", "how", "are", "you?"]\n sequence_tokens: ["hello", ".", "how", "are", "you", "?"]\n\n => [1, 0, 1, 1, 1, 0]\n\n * Args:\n naive_tokens: a list of tokens, naively split by whitespace\n sequence_tokens: a list of tokens, split by 'word_tokenizer'\n\n * Returns:\n is_head_of_word: a list with its length the same as that of 'sequence_tokens'.\n has 1 if the tokenized word at the position is head(prefix) of a `naive_token`\n and 0 if otherwise.\n """\n\n is_head_of_word = []\n for naive_token in naive_tokens:\n consumed_chars = 0\n consumed_words = 0\n for sequence_token in sequence_tokens:\n if naive_token[consumed_chars:].startswith(sequence_token):\n is_head_of_word.append(0 if consumed_chars else 1)\n consumed_chars += len(sequence_token)\n consumed_words += 1\n else:\n break\n sequence_tokens = sequence_tokens[consumed_words:]\n return is_head_of_word\n\n[docs]class arguments_required:\n """\n Decorator Class\n check required arguments for predict function\n (eg. @arguments_required(["db_path", "table_id"]))\n """\n\n def __init__(self, required_fields):\n self.required_fields = required_fields\n\n def __call__(self, fn):\n def wrapper(*args, **kwargs):\n arguments = args[2]\n for item in self.required_fields:\n if arguments.get(item, None) is None:\n raise ValueError(f"--{item} is required argument.")\n return fn(*args, **kwargs)\n\n return wrapper\n\n\nfrom claf.config.registry import Registry\n\n\n[docs]class register:\n """\n Decorator Class\n register subclass with decorator.\n (eg. @register("model:bidaf"), @register("reader:squad") )\n """\n\n def __init__(self, name):\n self.name = name\n\n def __call__(self, obj):\n registry = Registry()\n registry.add(self.name, obj)\n return obj\n\n\nimport atexit\nimport logging\nfrom pathlib import Path\n\nimport torch\n\nfrom claf import nsml\nfrom claf.config.factory import (\n DataReaderFactory,\n DataLoaderFactory,\n TokenMakersFactory,\n ModelFactory,\n OptimizerFactory,\n)\nfrom claf import utils as common_utils\nfrom claf.config.args import NestedNamespace\nfrom claf.config.utils import convert_config2dict, pretty_json_dumps, set_global_seed\nfrom claf.tokens.text_handler import TextHandler\nfrom claf.learn.mode import Mode\nfrom claf.learn.trainer import Trainer\nfrom claf.learn import utils\n\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]class Experiment:\n """\n Experiment settings with config.\n\n * Args:\n mode: Mode (ex. TRAIN, EVAL, INFER_EVAL, PREDICT)\n config: (NestedNamespace) Argument config according to mode\n """\n\n def __init__(self, mode, config):\n common_utils.set_logging_config(mode, config)\n\n self.argument = (\n config\n ) # self.config (experiment overall config) / config (argument according to mode)\n self.config = config\n self.mode = mode\n\n self.common_setting(mode, config)\n if mode != Mode.TRAIN: # evaluate and predict\n self.load_setting()\n\n # Set evaluation config\n if mode.endswith(Mode.EVAL):\n self.config.data_reader.train_file_path = ""\n self.config.data_reader.valid_file_path = self.argument.data_file_path\n self.config.cuda_devices = self.argument.cuda_devices\n self.config.iterator.cuda_devices = self.argument.cuda_devices\n\n if getattr(self.argument, "inference_latency", None):\n self.config.max_latency = self.argument.inference_latency\n\n self.predict_settings = None\n\n[docs] def common_setting(self, mode, config):\n """ Common Setting - experiment config, use_gpu and cuda_device_ids """\n self.config_dict = convert_config2dict(config)\n\n cuda_devices = self._get_cuda_devices()\n self.config.cuda_devices = cuda_devices\n self.config.slack_url = getattr(self.config, "slack_url", False)\n\n def _get_cuda_devices(self):\n if getattr(self.config, "use_gpu", None) is None:\n self.config.use_gpu = torch.cuda.is_available() or nsml.IS_ON_NSML\n\n if self.config.use_gpu:\n if nsml.IS_ON_NSML:\n return list(range(self.config.gpu_num))\n else:\n return self.config.cuda_devices\n else:\n return None\n\n[docs] def load_setting(self):\n """ Load Setting - need to load checkpoint case (ex. evaluate and predict) """\n cuda_devices = self.argument.cuda_devices\n checkpoint_path = self.argument.checkpoint_path\n prev_cuda_device_id = getattr(self.argument, "prev_cuda_device_id", None)\n\n self.model_checkpoint = self._read_checkpoint(\n cuda_devices, checkpoint_path, prev_cuda_device_id=prev_cuda_device_id\n )\n self._set_saved_config(cuda_devices)\n\n def _read_checkpoint(self, cuda_devices, checkpoint_path, prev_cuda_device_id=None):\n if cuda_devices == "cpu":\n return torch.load(checkpoint_path, map_location="cpu") # use CPU\n\n if torch.cuda.is_available():\n checkpoint = torch.load(\n checkpoint_path,\n map_location={\n f"cuda:{prev_cuda_device_id}": f"cuda:{cuda_devices[0]}"\n }, # different cuda_device id case (save/load)\n )\n else:\n checkpoint = torch.load(checkpoint_path, map_location="cpu") # use CPU\n return checkpoint\n\n def _set_saved_config(self, cuda_devices):\n saved_config_dict = self.model_checkpoint["config"]\n self.config_dict = saved_config_dict\n\n logger.info("Load saved_config ...")\n logger.info(pretty_json_dumps(saved_config_dict))\n\n saved_config = NestedNamespace()\n saved_config.load_from_json(saved_config_dict)\n\n is_use_gpu = self.config.use_gpu\n\n self.config = saved_config\n self.config.use_gpu = is_use_gpu\n self.config.cuda_devices = cuda_devices\n\n def __call__(self):\n """ Run Trainer """\n\n set_global_seed(self.config.seed_num) # For Reproducible\n\n if self.mode == Mode.TRAIN:\n # exit trigger slack notification\n if self.config.slack_url:\n atexit.register(utils.send_message_to_slack)\n\n train_loader, valid_loader, optimizer = self.set_train_mode()\n\n assert train_loader is not None\n assert optimizer is not None\n\n if valid_loader is None:\n self.trainer.train(train_loader, optimizer)\n else:\n self.trainer.train_and_evaluate(train_loader, valid_loader, optimizer)\n self._summary_experiments()\n\n elif self.mode == Mode.EVAL:\n valid_loader = self.set_eval_mode()\n\n assert valid_loader is not None\n return self.trainer.evaluate(valid_loader)\n\n elif self.mode == Mode.INFER_EVAL:\n raw_examples, raw_to_tensor_fn = self.set_eval_inference_latency_mode()\n\n assert raw_examples is not None\n assert raw_to_tensor_fn is not None\n return self.trainer.evaluate_inference_latency(raw_examples, raw_to_tensor_fn, max_latency=self.config.max_latency)\n\n elif self.mode.endswith(Mode.PREDICT):\n raw_features, raw_to_tensor_fn, arguments = self.set_predict_mode()\n\n assert raw_features is not None\n assert raw_to_tensor_fn is not None\n return self.trainer.predict(\n raw_features,\n raw_to_tensor_fn,\n arguments,\n interactive=arguments.get("interactive", False),\n )\n else:\n raise ValueError(f"unknown mode: {self.mode}")\n\n[docs] def set_train_mode(self):\n """\n Training Mode\n\n - Pipeline\n 1. read raw_data (DataReader)\n 2. build vocabs (DataReader, Token)\n 3. indexing tokens (DataReader, Token)\n 4. convert to DataSet (DataReader)\n 5. create DataLoader (DataLoader)\n 6. define model and optimizer\n 7. run!\n """\n logger.info("Config. \\n" + pretty_json_dumps(self.config_dict) + "\\n")\n\n data_reader, token_makers = self._create_data_and_token_makers()\n datas, helpers = data_reader.read()\n\n # Token & Vocab\n text_handler = TextHandler(token_makers, lazy_indexing=True)\n if text_handler.is_all_vocab_use_pretrained():\n token_counters = token_makers\n else:\n texts = data_reader.filter_texts(datas)\n token_counters = text_handler.make_token_counters(texts, config=self.config)\n\n vocabs = text_handler.build_vocabs(token_counters)\n text_handler.index(datas, data_reader.text_columns)\n\n # iterator\n vocab = vocabs[next(iter(vocabs))]\n datasets = data_reader.convert_to_dataset(datas, vocab, helpers=helpers) # with name\n\n self.config.iterator.cuda_devices = self.config.cuda_devices\n train_loader, valid_loader, test_loader = self._create_by_factory(\n DataLoaderFactory, self.config.iterator, param={"datasets": datasets}\n )\n\n # calculate 'num_train_steps'\n num_train_steps = self._get_num_train_steps(train_loader)\n self.config.optimizer.num_train_steps = num_train_steps\n\n checkpoint_dir = Path(self.config.trainer.log_dir) / "checkpoint"\n checkpoints = None\n if checkpoint_dir.exists():\n checkpoints = self._load_exist_checkpoints(checkpoint_dir) # contain model and optimizer\n\n if checkpoints is None:\n model = self._create_model(token_makers, helpers=helpers)\n op_dict = self._create_by_factory(\n OptimizerFactory, self.config.optimizer, param={"model": model}\n )\n else:\n model = self._create_model(token_makers, checkpoint=checkpoints)\n op_dict = self._create_by_factory(\n OptimizerFactory, self.config.optimizer, param={"model": model}\n )\n utils.load_optimizer_checkpoint(op_dict["optimizer"], checkpoints)\n\n self.set_trainer(model, op_dict=op_dict)\n return train_loader, valid_loader, op_dict["optimizer"]\n\n def _create_data_and_token_makers(self):\n token_makers = self._create_by_factory(TokenMakersFactory, self.config.token)\n tokenizers = token_makers["tokenizers"]\n del token_makers["tokenizers"]\n\n self.config.data_reader.tokenizers = tokenizers\n data_reader = self._create_by_factory(DataReaderFactory, self.config.data_reader)\n return data_reader, token_makers\n\n def _create_by_factory(self, factory, item_config, param={}):\n return factory(item_config).create(**param)\n\n def _get_num_train_steps(self, train_loader):\n train_set_size = len(train_loader.dataset)\n batch_size = self.config.iterator.batch_size\n gradient_accumulation_steps = getattr(self.config.optimizer, "gradient_accumulation_steps", 1)\n num_epochs = self.config.trainer.num_epochs\n\n one_epoch_steps = int(train_set_size / batch_size / gradient_accumulation_steps)\n if one_epoch_steps == 0:\n one_epoch_steps = 1\n num_train_steps = one_epoch_steps * num_epochs\n return num_train_steps\n\n def _load_exist_checkpoints(self, checkpoint_dir): # pragma: no cover\n checkpoints = utils.get_sorted_path(checkpoint_dir, both_exist=True)\n\n train_counts = list(checkpoints.keys())\n if not train_counts:\n return None\n\n seperator = "-" * 50\n message = f"{seperator}\\n !! Find exist checkpoints {train_counts}.\\n If you want to recover, input train_count in list.\\n If you don't want to recover, input 0.\\n{seperator}"\n selected_train_count = common_utils.get_user_input(message)\n\n if selected_train_count == 0:\n return None\n\n model_path = checkpoints[selected_train_count]["model"]\n model_checkpoint = self._read_checkpoint(self.config.cuda_devices, model_path)\n\n optimizer_path = checkpoints[selected_train_count]["optimizer"]\n optimizer_checkpoint = self._read_checkpoint("cpu", optimizer_path)\n\n checkpoints = {}\n checkpoints.update(model_checkpoint)\n checkpoints.update(optimizer_checkpoint)\n return checkpoints\n\n def _create_model(self, token_makers, checkpoint=None, helpers=None):\n if checkpoint is None:\n assert helpers is not None\n first_key = next(iter(helpers))\n helper = helpers[first_key] # get first helper\n model_init_params = helper.get("model", {})\n predict_helper = helper.get("predict_helper", {})\n else:\n model_init_params = checkpoint.get("init_params", {})\n predict_helper = checkpoint.get("predict_helper", {})\n\n model_params = {"token_makers": token_makers}\n model_params.update(model_init_params)\n\n model = self._create_by_factory(\n ModelFactory, self.config.model, param=model_params\n )\n # Save params\n model.init_params = model_init_params\n model.predict_helper = predict_helper\n\n if checkpoint is not None:\n model = utils.load_model_checkpoint(model, checkpoint)\n model = self._set_gpu_env(model)\n return model\n\n def _set_gpu_env(self, model):\n if self.config.use_gpu:\n cuda_devices = self._get_cuda_devices()\n num_gpu = len(cuda_devices)\n\n use_multi_gpu = num_gpu > 1\n if use_multi_gpu:\n model = torch.nn.DataParallel(model, device_ids=cuda_devices)\n model.cuda()\n else:\n num_gpu = 0\n\n num_gpu_state = str(num_gpu)\n if num_gpu > 1:\n num_gpu_state += " (Multi-GPU)"\n\n # TODO: distributed training and 16-bits training (FP16)\n logger.info(f"use_gpu: {self.config.use_gpu} num_gpu: {num_gpu_state}, distributed training: False, 16-bits training: False")\n return model\n\n[docs] def set_trainer(self, model, op_dict={}, save_params={}):\n trainer_config = vars(self.config.trainer)\n trainer_config["config"] = self.config_dict\n trainer_config["model"] = model\n trainer_config["learning_rate_scheduler"] = op_dict.get("learning_rate_scheduler", None)\n trainer_config["exponential_moving_average"] = op_dict.get(\n "exponential_moving_average", None\n )\n self.trainer = Trainer(**trainer_config)\n\n # Set NSML\n if nsml.IS_ON_NSML:\n utils.bind_nsml(model, optimizer=op_dict.get("optimizer", None))\n if getattr(self.config.nsml, "pause", None):\n nsml.paused(scope=locals())\n\n def _summary_experiments(self):\n hr_text = "-" * 50\n summary_logs = f"\\n\\n\\nExperiment Summary. {nsml.SESSION_NAME}\\n{hr_text}\\n"\n summary_logs += f"Config.\\n{pretty_json_dumps(self.config_dict)}\\n{hr_text}\\n"\n summary_logs += (\n f"Training Logs.\\n{pretty_json_dumps(self.trainer.training_logs)}\\n{hr_text}\\n"\n )\n summary_logs += f"Metric Logs.\\n{pretty_json_dumps(self.trainer.metric_logs)}"\n\n logger.info(summary_logs)\n\n if self.config.slack_url: # pragma: no cover\n simple_summary_title = f"Session Name: {nsml.SESSION_NAME} "\n if getattr(self.config, "base_config", None):\n simple_summary_title += f"({self.config.base_config})"\n\n simple_summary_logs = f" - Dataset: {self.config.data_reader.dataset} \\n"\n simple_summary_logs += f" - Model: {self.config.model.name}"\n\n best_metrics = {"epoch": self.trainer.metric_logs["best_epoch"]}\n best_metrics.update(self.trainer.metric_logs["best"])\n\n simple_summary_logs += f" - Best metrics.\\n {pretty_json_dumps(best_metrics)} "\n\n utils.send_message_to_slack(self.config.slack_url, title=simple_summary_title, message=simple_summary_logs)\n\n[docs] def set_eval_mode(self):\n """\n Evaluate Mode\n\n - Pipeline\n 1. read raw_data (DataReader)\n 2. load vocabs from checkpoint (DataReader, Token)\n 3. indexing tokens (DataReader, Token)\n 4. convert to DataSet (DataReader)\n 5. create DataLoader (DataLoader)\n 6. define and load model\n 7. run!\n """\n\n data_reader, token_makers = self._create_data_and_token_makers()\n\n # DataReader\n datas, helpers = data_reader.read()\n\n # Token & Vocab\n vocabs = utils.load_vocabs(self.model_checkpoint)\n for token_name, token_maker in token_makers.items():\n token_maker.set_vocab(vocabs[token_name])\n\n text_handler = TextHandler(token_makers, lazy_indexing=False)\n text_handler.index(datas, data_reader.text_columns)\n\n # iterator\n vocab = vocabs[next(iter(vocabs))]\n datasets = data_reader.convert_to_dataset(datas, vocab, helpers=helpers) # with name\n\n self.config.iterator.cuda_devices = self.config.cuda_devices\n _, valid_loader, _ = self._create_by_factory(\n DataLoaderFactory, self.config.iterator, param={"datasets": datasets}\n )\n\n # Model\n model = self._create_model(token_makers, checkpoint=self.model_checkpoint)\n self.set_trainer(model)\n\n return valid_loader\n\n[docs] def set_eval_inference_latency_mode(self):\n """\n Evaluate Inference Latency Mode\n\n - Pipeline\n 1. read raw_data (DataReader)\n 2. load vocabs from checkpoint (DataReader, Token)\n 3. define raw_to_tensor_fn (DataReader, Token)\n 4. define and load model\n 5. run!\n """\n data_reader, token_makers = self._create_data_and_token_makers()\n\n # Token & Vocab\n vocabs = utils.load_vocabs(self.model_checkpoint)\n for token_name, token_maker in token_makers.items():\n token_maker.set_vocab(vocabs[token_name])\n\n text_handler = TextHandler(token_makers, lazy_indexing=False)\n\n _, helpers = data_reader.read()\n raw_examples = helpers["valid"]["examples"]\n\n cuda_device = self.config.cuda_devices[0] if self.config.use_gpu else None\n raw_to_tensor_fn = text_handler.raw_to_tensor_fn(data_reader, cuda_device=cuda_device)\n\n # Model\n model = self._create_model(token_makers, checkpoint=self.model_checkpoint)\n self.set_trainer(model)\n\n return raw_examples, raw_to_tensor_fn\n\n[docs] def predict(self, raw_features):\n if self.predict_settings is None:\n raise ValueError(\n "To use 'predict()', you must call 'set_predict_mode()' first, with preload=True parameter"\n )\n\n raw_to_tensor_fn = self.predict_settings["raw_to_tensor_fn"]\n arguments = self.predict_settings["arguments"]\n arguments.update(raw_features)\n\n assert raw_features is not None\n assert raw_to_tensor_fn is not None\n return self.trainer.predict(\n raw_features,\n raw_to_tensor_fn,\n arguments,\n interactive=arguments.get("interactive", False),\n )\n\n[docs] def set_predict_mode(self, preload=False):\n """\n Predict Mode\n\n - Pipeline\n 1. read raw_data (Argument)\n 2. load vocabs from checkpoint (DataReader, Token)\n 3. define raw_to_tensor_fn (DataReader, Token)\n 4. define and load model\n 5. run!\n """\n\n data_reader, token_makers = self._create_data_and_token_makers()\n\n # Token & Vocab\n vocabs = utils.load_vocabs(self.model_checkpoint)\n for token_name, token_maker in token_makers.items():\n token_maker.set_vocab(vocabs[token_name])\n\n text_handler = TextHandler(token_makers, lazy_indexing=False)\n\n # Set predict config\n if self.argument.interactive:\n raw_features = {feature_name: "" for feature_name in data_reader.text_columns}\n else:\n raw_features = {}\n for feature_name in data_reader.text_columns:\n feature = getattr(self.argument, feature_name, None)\n # if feature is None:\n # raise ValueError(f"--{feature_name} argument is required!")\n raw_features[feature_name] = feature\n\n cuda_device = self.config.cuda_devices[0] if self.config.use_gpu else None\n raw_to_tensor_fn = text_handler.raw_to_tensor_fn(\n data_reader,\n cuda_device=cuda_device,\n helper=self.model_checkpoint.get("predict_helper", {})\n )\n\n # Model\n model = self._create_model(token_makers, checkpoint=self.model_checkpoint)\n self.set_trainer(model)\n\n arguments = vars(self.argument)\n\n if preload:\n self.predict_settings = {"raw_to_tensor_fn": raw_to_tensor_fn, "arguments": arguments}\n else:\n return raw_features, raw_to_tensor_fn, arguments\n\n[docs]class Mode:\n """ Experiment Flag class """\n\n TRAIN = "train"\n EVAL = "eval"\n INFER_EVAL = "infer_eval"\n PREDICT = "predict"\n MACHINE = "machine"\n\n\nimport os\n\nfrom tensorboardX import SummaryWriter\n\nfrom claf import nsml\n\n\n[docs]class TensorBoard:\n """ TensorBoard Wrapper for Pytorch """\n\n def __init__(self, log_dir):\n if not os.path.exists(log_dir):\n os.makedirs(log_dir)\n self.writer = SummaryWriter(log_dir=log_dir)\n\n[docs] def scalar_summaries(self, step, summary):\n if nsml.IS_ON_NSML:\n if type(summary) != dict:\n raise ValueError(f"summary type is dict. not {type(summary)}")\n kwargs = {"summary": True, "scope": locals(), "step": step}\n kwargs.update(summary)\n\n nsml.report(**kwargs)\n else:\n for tag, value in summary.items():\n self.scalar_summary(step, tag, value)\n\n[docs] def scalar_summary(self, step, tag, value):\n """Log a scalar variable."""\n if nsml.IS_ON_NSML:\n nsml.report(**{"summary": True, "scope": locals(), "step": step, tag: value})\n else:\n self.writer.add_scalar(tag, value, step)\n\n[docs] def image_summary(self, tag, images, step):\n """Log a list of images."""\n raise NotImplementedError()\n\n[docs] def embedding_summary(self, features, metadata=None, label_img=None):\n raise NotImplementedError()\n\n[docs] def histogram_summary(self, tag, values, step, bins=1000):\n """Log a histogram of the tensor of values."""\n raise NotImplementedError()\n\n\n\n# -*- coding: utf-8 -*-\n\nimport json\nimport logging\nimport os\nimport time\nimport random\n\nimport torch\nfrom torch.nn.utils import clip_grad_norm_\nfrom tqdm import tqdm\n\nfrom claf import nsml\nfrom claf.config.utils import pretty_json_dumps\nfrom claf.learn.optimization.exponential_moving_avarage import EMA\nfrom claf.learn.tensorboard import TensorBoard\nfrom claf.learn import utils\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]class Trainer:\n """\n Trainer\n Run experiment\n\n - train\n - train_and_evaluate\n - evaluate\n - evaluate_inference_latency\n - predict\n\n * Args:\n config: experiment overall config\n model: Model based on torch.nn.Module\n\n * Kwargs:\n log_dir: path to directory for save model and other options\n grad_max_norm: Clips gradient norm of an iterable of parameters.\n learning_rate_scheduler: PyTorch's Learning Rate Scheduler.\n (https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html)\n exponential_moving_average: the moving averages of all weights of the model are maintained\n with the exponential decay rate of {ema}.\n num_epochs: the number of maximun epochs (Default is 20)\n early_stopping_threshold: the number of early stopping threshold (Default is 10)\n max_eval_examples: print evaluation examples\n metric_key: metric score's control point\n verbose_step_count: print verbose step count (Default is 100)\n eval_and_save_step_count: evaluate valid_dataset then save every n step_count (Default is 'epoch')\n """\n\n def __init__(\n self,\n model,\n config={},\n log_dir="logs/experiment",\n grad_max_norm=None,\n gradient_accumulation_steps=1,\n learning_rate_scheduler=None,\n exponential_moving_average=None,\n num_epochs=20,\n early_stopping_threshold=10,\n max_eval_examples=5,\n metric_key=None,\n verbose_step_count=100,\n eval_and_save_step_count="epoch",\n save_checkpoint=True,\n ):\n assert metric_key is not None\n\n # CUDA\n self.use_multi_gpu = type(model) == torch.nn.DataParallel\n\n if getattr(model, "train_counter", None):\n self.train_counter = model.train_counter\n else:\n self.train_counter = utils.TrainCounter(display_unit=eval_and_save_step_count)\n\n self.model = model\n model_config = config.get("model", {})\n self.model_name = model_config.get("name", "model")\n self.set_model_base_properties(config, log_dir)\n\n # Logs\n os.makedirs(log_dir, exist_ok=True)\n self.tensorboard = TensorBoard(log_dir)\n self.metric_logs = {"best_epoch": 0, "best_global_step": 0, "best": None, "best_score": 0}\n self.training_logs = {"early_stopping_count": 0}\n\n # optimization options\n self.grad_max_norm = grad_max_norm\n\n if gradient_accumulation_steps is None:\n gradient_accumulation_steps = 1\n self.gradient_accumulation_steps = gradient_accumulation_steps\n\n self.learning_rate_scheduler = learning_rate_scheduler\n self.exponential_moving_average = exponential_moving_average\n if exponential_moving_average:\n self.exponential_moving_average = EMA(model, self.exponential_moving_average)\n\n # property\n self.num_epochs = num_epochs\n self.early_stopping = False\n self.early_stopping_threshold = early_stopping_threshold\n self.max_eval_examples = max_eval_examples\n self.metric_key = metric_key\n self.verbose_step_count = verbose_step_count\n self.eval_and_save_step_count = eval_and_save_step_count\n self.save_checkpoint = save_checkpoint\n self.log_dir = log_dir\n\n[docs] def set_model_base_properties(self, config, log_dir):\n model = self.model\n if self.use_multi_gpu:\n model = self.model.module\n\n model.config = config\n model.log_dir = log_dir\n model.train_counter = self.train_counter\n assert model.is_ready() == True\n\n[docs] def train_and_evaluate(self, train_loader, valid_loader, optimizer):\n """ Train and Evaluate """\n start_time = time.time()\n\n for epoch in range(1, self.num_epochs + 1):\n self.train_counter.epoch = epoch\n\n # Training with metrics\n train_metrics = self._run_epoch(\n train_loader,\n valid_loader=valid_loader,\n is_training=True,\n optimizer=optimizer,\n verbose_step_count=self.verbose_step_count,\n eval_and_save_step_count=self.eval_and_save_step_count,\n )\n\n valid_metrics = None\n if self.eval_and_save_step_count == "epoch":\n valid_metrics = self._run_epoch(valid_loader, is_training=False)\n self._check_valid_results(valid_metrics, report=False)\n self.save(optimizer)\n\n self._report_metrics(train_metrics=train_metrics, valid_metrics=valid_metrics)\n self._estimate_remainig_time(start_time)\n\n if self.early_stopping:\n break\n\n self._report_trainings(start_time, train_loader=train_loader, valid_loader=valid_loader)\n\n[docs] def train(self, data_loader, optimizer):\n """ Train """\n start_time = time.time()\n\n for epoch in range(1, self.num_epochs + 1):\n self.train_counter.epoch = epoch\n\n metrics = self._run_epoch(\n data_loader,\n is_training=True,\n optimizer=optimizer,\n verbose_step_count=self.verbose_step_count,\n )\n\n self._report_metrics(train_metrics=metrics)\n self._estimate_remainig_time(start_time)\n self.save(optimizer)\n\n self._report_trainings(start_time, train_loader=data_loader)\n\n[docs] def evaluate(self, data_loader):\n """ Evaluate """\n print("evaluate:", type(data_loader), data_loader)\n eval_metrics = self._run_epoch(data_loader, is_training=False, disable_prograss_bar=False)\n\n self._report_metrics(tensorboard=False, valid_metrics=eval_metrics)\n\n[docs] def evaluate_inference_latency(self, raw_examples, raw_to_tensor_fn, token_key=None, max_latency=1000):\n """\n Evaluate with focusing inferece latency\n (Note: must use sorted synthetic data)\n\n * inference_latency: raw_data -> pre-processing -> model -> predict_value\n (elapsed_time) (elapsed_time)\n """\n\n logger.info("\\n# Evaluate Inference Latency Mode.")\n self.model.eval()\n\n total_raw_to_tensor_time = 0\n tensor_to_predicts = []\n\n raw_example_items = tqdm(raw_examples.items())\n for _, raw_example in raw_example_items:\n # raw_data -> tensor\n raw_to_tensor_start_time = time.time()\n feature, helper = raw_to_tensor_fn(raw_example)\n raw_to_tensor_elapsted_time = time.time() - raw_to_tensor_start_time\n raw_to_tensor_elapsted_time *= 1000 # unit: sec -> ms\n\n total_raw_to_tensor_time += raw_to_tensor_elapsted_time\n\n # tensor to predict\n tensor_to_predict_start_time = time.time()\n output_dict = self.model(feature)\n tensor_to_predict_elapsed_time = time.time() - tensor_to_predict_start_time\n\n if "token_key" not in helper:\n raise ValueError(\n "helper must have 'token_key' data for 1-example inference latency."\n )\n\n tensor_to_predict_elapsed_time *= 1000 # unit: sec -> ms\n tensor_to_predict = {\n "elapsed_time": tensor_to_predict_elapsed_time,\n "token_count": len(helper[helper["token_key"]]),\n }\n tensor_to_predicts.append(tensor_to_predict)\n\n if tensor_to_predict_elapsed_time > max_latency:\n raw_example_items.close()\n break\n\n total_tensor_to_predict = sum(\n [tensor_to_predict["elapsed_time"] for tensor_to_predict in tensor_to_predicts]\n )\n\n max_token_count_per_times = {}\n max_times = list(range(0, max_latency+1, 100))\n for t2p in sorted(tensor_to_predicts, key=lambda x: x["token_count"]):\n elapsed_time = t2p["elapsed_time"]\n token_count = t2p["token_count"]\n\n for max_time in max_times:\n if elapsed_time < max_time:\n max_token_count_per_times[max_time] = token_count\n\n result = {\n "average_raw_to_tensor": total_raw_to_tensor_time / len(raw_examples),\n "average_tensor_to_predict": total_tensor_to_predict / len(raw_examples),\n "average_end_to_end": (total_raw_to_tensor_time + total_tensor_to_predict)\n / len(raw_examples),\n "tensor_to_predicts": tensor_to_predicts,\n "max_token_count_per_time": max_token_count_per_times\n }\n\n env = "gpu" if torch.cuda.is_available() else "cpu"\n file_name = f"{self.model_name}-{env}.json"\n with open(file_name, "w") as f:\n json.dump(result, f, indent=4)\n\n logger.info(f"saved inference_latency results. {file_name}")\n\n def _is_early_stopping(self, metrics):\n score = metrics[self.metric_key]\n\n if score > self.metric_logs["best_score"]:\n self.training_logs["early_stopping_count"] = 0\n else:\n self.training_logs["early_stopping_count"] += 1\n\n if self.training_logs["early_stopping_count"] >= self.early_stopping_threshold:\n self.training_logs["early_stopping"] = True\n return True\n else:\n return False\n\n def _report_metrics(self, tensorboard=True, train_metrics=None, valid_metrics=None):\n\n total_metrics = {}\n\n def update_metrics(metrics, category=""):\n if metrics is not None:\n for k, v in metrics.items():\n total_metrics[f"{category}/{k}"] = v\n\n update_metrics(train_metrics, "train")\n update_metrics(valid_metrics, "valid")\n\n # TensorBoard\n if tensorboard:\n self.tensorboard.scalar_summaries(self.train_counter.get_display(), total_metrics)\n\n # Console\n metric_console = ""\n if train_metrics:\n metric_console += (\n f"\\n# Epoch: [{self.train_counter.epoch}/{self.num_epochs}]: Metrics \\n"\n )\n metric_console += json.dumps(total_metrics, indent=4)\n logger.info(metric_console)\n\n if valid_metrics:\n self._update_metric_logs(total_metrics)\n\n def _update_metric_logs(self, total_metrics):\n for k, v in total_metrics.items():\n if self.metric_logs.get(k, None) is None:\n self.metric_logs[k] = [v]\n else:\n self.metric_logs[k].append(v)\n\n valid_score = total_metrics.get(f"valid/{self.metric_key}", None)\n if valid_score and valid_score > self.metric_logs["best_score"]:\n logger.info(f" * Best validation score so far. ({self.metric_key}) : {valid_score}")\n self.metric_logs["best_score"] = valid_score\n self.metric_logs["best"] = total_metrics\n self.metric_logs["best_epoch"] = self.train_counter.epoch\n self.metric_logs["best_global_step"] = self.train_counter.global_step\n else:\n logger.info(\n f" * Current best validation score. ({self.metric_key}) : {self.metric_logs['best_score']}"\n )\n\n def _estimate_remainig_time(self, start_time):\n elapsed_time = time.time() - start_time\n estimated_time_remaining = elapsed_time * (\n (self.num_epochs - self.train_counter.epoch) / float(self.train_counter.epoch) - 1\n )\n formatted_time = time.strftime("%H:%M:%S", time.gmtime(estimated_time_remaining))\n logger.info(f"Estimated training time remaining: {formatted_time} ")\n\n def _report_trainings(self, start_time, train_loader=None, valid_loader=None):\n elapsed_time = time.time() - start_time\n self.training_logs["elapsed_time"] = (time.strftime("%H:%M:%S", time.gmtime(elapsed_time)),)\n\n if train_loader is not None:\n self.training_logs["train_dataset"] = json.loads(str(train_loader.dataset))\n if valid_loader is not None:\n self.training_logs["valid_dataset"] = json.loads(str(valid_loader.dataset))\n\n def _run_epoch(\n self,\n data_loader,\n valid_loader=None,\n is_training=True,\n optimizer=None,\n disable_prograss_bar=True,\n verbose_step_count=100,\n eval_and_save_step_count=None,\n ):\n """\n Run Epoch\n\n 1. forward inputs to model\n 2. (training) backpropagation\n 3. update predictions\n 4. make metrics\n """\n\n if is_training:\n logger.info("# Train Mode.")\n self.model.train()\n else:\n logger.info("# Evaluate Mode.")\n self.model.eval()\n\n # set dataset (train/valid)\n self._set_dataset_to_model(data_loader.dataset)\n\n metrics = {}\n predictions = {}\n\n epoch_loss = 0\n epoch_start_time = time.time()\n step_start_time = time.time()\n\n eval_example_count = 0\n\n for step, batch in enumerate(tqdm(data_loader, disable=disable_prograss_bar)):\n inputs = batch.to_dict() # for DataParallel\n output_dict = self.model(**inputs)\n\n loss = output_dict["loss"]\n if self.use_multi_gpu:\n loss = loss.mean()\n if self.gradient_accumulation_steps > 1:\n loss = loss / self.gradient_accumulation_steps\n\n epoch_loss += loss.item()\n\n if is_training:\n # Training Verbose\n if self.train_counter.global_step == 0:\n logger.info(f" Start - Batch Loss: {loss.item():.5f}")\n\n if (\n self.train_counter.global_step != 0\n and self.train_counter.global_step % verbose_step_count == 0\n ):\n step_elapsed_time = time.time() - step_start_time\n\n logger.info(\n f" Step: {self.train_counter.global_step} Batch Loss: {loss.item():.5f} {step_elapsed_time:.5f} sec"\n )\n self.tensorboard.scalar_summary(\n self.train_counter.global_step, "train/batch_loss", loss.item()\n )\n\n step_start_time = time.time()\n\n loss.backward()\n\n if self.grad_max_norm:\n clip_grad_norm_(self._get_model_parameters(), self.grad_max_norm)\n\n if (step + 1) % self.gradient_accumulation_steps == 0:\n # Backpropagation\n if self.learning_rate_scheduler:\n self.learning_rate_scheduler.step_batch(self.train_counter.global_step)\n\n optimizer.step()\n optimizer.zero_grad()\n self.train_counter.global_step += 1\n\n if self.exponential_moving_average:\n for name, param in self.model.named_parameters():\n if param.requires_grad:\n param.data = self.exponential_moving_average(name, param.data)\n\n # Evaluate then Save checkpoint\n if (\n valid_loader\n and type(eval_and_save_step_count) == int\n and self.train_counter.global_step % eval_and_save_step_count == 0\n ):\n with torch.no_grad():\n valid_metrics = self._run_epoch(valid_loader, is_training=False)\n self._check_valid_results(valid_metrics, report=True)\n self.save(optimizer)\n\n if is_training: # roll-back to train mode\n self.model.train()\n self._set_dataset_to_model(data_loader.dataset)\n else:\n if eval_example_count < self.max_eval_examples:\n total_step_count = int(len(data_loader) / data_loader.batch_size)\n random_num = random.randint(0, total_step_count)\n\n if random_num <= self.max_eval_examples:\n eval_example_predictions = {}\n self._update_predictions(eval_example_predictions, output_dict)\n\n random_index = random.randint(0, data_loader.batch_size)\n self._print_examples(random_index, inputs, eval_example_predictions)\n eval_example_count += 1\n\n self._update_predictions(predictions, output_dict)\n\n epoch_loss /= len(data_loader)\n epoch_elapsed_time = time.time() - epoch_start_time\n\n logger.info("Epoch duration: " + time.strftime("%H:%M:%S", time.gmtime(epoch_elapsed_time)))\n\n # Updat metrics\n metrics["loss"] = epoch_loss\n metrics["epoch_time"] = epoch_elapsed_time\n metrics.update(self._make_metrics(predictions)) # model metric\n\n return metrics\n\n def _set_dataset_to_model(self, dataset):\n if self.use_multi_gpu:\n self.model.module.dataset = dataset\n else:\n self.model.dataset = dataset\n\n def _get_model_parameters(self):\n if self.use_multi_gpu:\n return self.model.module.parameters()\n else:\n return self.model.parameters()\n\n def _check_valid_results(self, metrics, report=False):\n if self.learning_rate_scheduler:\n # The LRScheduler API is agnostic to whether your schedule requires a validation metric -\n # if it doesn't, the validation metric passed here is ignored.\n this_epoch_val_metric = metrics[self.metric_key]\n self.learning_rate_scheduler.step(this_epoch_val_metric, self.train_counter.global_step)\n\n if self._is_early_stopping(metrics):\n self.early_stopping = True\n logger.info(" --- Early Stopping. --- ")\n\n if report:\n self._report_metrics(valid_metrics=metrics)\n\n def _make_metrics(self, predictions):\n model = self.model\n if self.use_multi_gpu:\n model = model.module\n\n model.train_counter = self.train_counter\n return model.make_metrics(predictions)\n\n def _update_predictions(self, predictions, output_dict):\n if self.use_multi_gpu:\n predictions.update(self.model.module.make_predictions(output_dict))\n else:\n predictions.update(self.model.make_predictions(output_dict))\n\n def _print_examples(self, index, inputs, predictions):\n try:\n if self.use_multi_gpu:\n self.model.module.print_examples(index, inputs, predictions)\n else:\n self.model.print_examples(index, inputs, predictions)\n except IndexError:\n pass\n\n[docs] def predict(self, raw_feature, raw_to_tensor_fn, arguments, interactive=False):\n """ Inference / Predict """\n\n self.model.eval()\n with torch.no_grad():\n if interactive: # pragma: no cover\n while True:\n for k in raw_feature:\n raw_feature[k] = utils.get_user_input(k)\n\n tensor_feature, helper = raw_to_tensor_fn(raw_feature)\n output_dict = self.model(tensor_feature)\n\n arguments.update(raw_feature)\n predict = self.model.predict(output_dict, arguments, helper)\n print(f"Predict: {pretty_json_dumps(predict)} \\n")\n else:\n tensor_feature, helper = raw_to_tensor_fn(raw_feature)\n output_dict = self.model(tensor_feature)\n\n return self.model.predict(output_dict, arguments, helper)\n\n[docs] def save(self, optimizer):\n if not self.save_checkpoint:\n return\n\n # set all config to model\n model = self.model\n if self.use_multi_gpu:\n model = self.model.module\n\n model.train_counter = self.train_counter\n model.metrics = self.metric_logs\n\n if nsml.IS_ON_NSML:\n nsml.save(self.train_counter.get_display())\n else:\n utils.save_checkpoint(self.log_dir, model, optimizer)\n\n\nfrom collections import OrderedDict\nimport json\nimport logging\nfrom pathlib import Path\nimport os\nimport re\n\nimport torch\nfrom torch.nn import DataParallel\nimport requests\n\nfrom claf import nsml\nfrom claf.tokens.vocabulary import Vocab\n\n\nlogger = logging.getLogger(__name__)\n\n\n""" Train Counter """\n\n\n[docs]class TrainCounter:\n\n global_step = 0\n epoch = 0\n\n def __init__(self, display_unit="epoch"):\n if type(display_unit) == int:\n display_unit = f"every_{display_unit}_global_step"\n self.display_unit = display_unit\n\n[docs] def get_display(self):\n if self.display_unit == "epoch":\n return self.epoch\n else:\n return self.global_step\n\n\n""" Save and Load checkpoint """\n\n\n[docs]def load_model_checkpoint(model, checkpoint):\n model.load_state_dict(checkpoint["weights"])\n model.config = checkpoint["config"]\n model.metrics = checkpoint["metrics"]\n model.init_params = checkpoint["init_params"]\n model.predict_helper = checkpoint["predict_helper"]\n model.train_counter = checkpoint["train_counter"]\n model.vocabs = load_vocabs(checkpoint)\n\n logger.info(f"Load model checkpoints...!")\n return model\n\n\n[docs]def load_optimizer_checkpoint(optimizer, checkpoint):\n optimizer.load_state_dict(checkpoint["optimizer"])\n\n logger.info(f"Load optimizer checkpoints...!")\n return optimizer\n\n\n[docs]def load_vocabs(model_checkpoint):\n vocabs = {}\n token_config = model_checkpoint["config"]["token"]\n for token_name in token_config["names"]:\n token = token_config[token_name]\n vocab_config = token.get("vocab", {})\n\n texts = model_checkpoint["vocab_texts"][token_name]\n vocabs[token_name] = Vocab(token_name, **vocab_config).from_texts(texts)\n return vocabs\n\n\n[docs]def save_checkpoint(path, model, optimizer, max_to_keep=10):\n path = Path(path)\n\n checkpoint_dir = path / "checkpoint"\n checkpoint_dir.mkdir(exist_ok=True)\n\n # Remove old checkpoints\n sorted_path = get_sorted_path(checkpoint_dir)\n if len(sorted_path) > max_to_keep:\n remove_train_counts = list(sorted_path.keys())[: -(max_to_keep - 1)]\n for train_count in remove_train_counts:\n optimizer_path = sorted_path[train_count].get("optimizer", None)\n if optimizer_path:\n os.remove(optimizer_path)\n\n model_path = sorted_path[train_count].get("model", None)\n if model_path:\n os.remove(model_path)\n\n train_counter = model.train_counter\n\n optimizer_path = checkpoint_dir / f"optimizer_{train_counter.get_display()}.pkl"\n torch.save({"optimizer": optimizer.state_dict()}, optimizer_path)\n\n model_path = checkpoint_dir / f"model_{train_counter.get_display()}.pkl"\n torch.save(\n {\n "config": model.config,\n "init_params": model.init_params,\n "predict_helper": model.predict_helper,\n "metrics": model.metrics,\n "train_counter": model.train_counter,\n "vocab_texts": {k: v.to_text() for k, v in model.vocabs.items()},\n "weights": model.state_dict(),\n },\n model_path,\n )\n\n # Write Vocab as text file (Only once)\n vocab_dir = path / "vocab"\n vocab_dir.mkdir(exist_ok=True)\n\n for token_name, vocab in model.vocabs.items():\n vocab_path = vocab_dir / f"{token_name}.txt"\n if not vocab_path.exists():\n vocab.dump(vocab_path)\n\n logger.info(f"Save {train_counter.global_step} global_step checkpoints...!")\n\n\n[docs]def get_sorted_path(checkpoint_dir, both_exist=False):\n paths = []\n for root, dirs, files in os.walk(checkpoint_dir):\n for f_name in files:\n if "model" in f_name or "optimizer" in f_name:\n paths.append(Path(root) / f_name)\n\n path_with_train_count = {}\n for path in paths:\n train_count = re.findall("\\d+", path.name)[0]\n train_count = int(train_count)\n if train_count not in path_with_train_count:\n path_with_train_count[train_count] = {}\n\n if "model" in path.name:\n path_with_train_count[train_count]["model"] = path\n if "optimizer" in path.name:\n path_with_train_count[train_count]["optimizer"] = path\n\n if both_exist:\n remove_keys = []\n for key, checkpoint in path_with_train_count.items():\n if not ("model" in checkpoint and "optimizer" in checkpoint):\n remove_keys.append(key)\n\n for key in remove_keys:\n del path_with_train_count[key]\n\n return OrderedDict(sorted(path_with_train_count.items()))\n\n\n""" NSML """\n\n\n[docs]def bind_nsml(model, **kwargs): # pragma: no cover\n if type(model) == DataParallel:\n model = model.module\n\n CHECKPOINT_FNAME = "checkpoint.bin"\n\n def infer(raw_data, **kwargs):\n print("raw_data:", raw_data)\n\n def load(dir_path, *args):\n checkpoint_path = os.path.join(dir_path, CHECKPOINT_FNAME)\n checkpoint = torch.load(checkpoint_path)\n\n model.load_state_dict(checkpoint["weights"])\n model.config = checkpoint["config"]\n model.metrics = checkpoint["metrics"]\n model.init_params = checkpoint["init_params"],\n model.predict_helper = checkpoint["predict_helper"],\n model.train_counter = checkpoint["train_counter"]\n model.vocabs = load_vocabs(checkpoint)\n\n if "optimizer" in kwargs:\n kwargs["optimizer"].load_state_dict(checkpoint["optimizer"])\n logger.info(f"Load checkpoints...! {checkpoint_path}")\n\n def save(dir_path, *args):\n # save the model with 'checkpoint' dictionary.\n checkpoint_path = os.path.join(dir_path, CHECKPOINT_FNAME)\n checkpoint = {\n "config": model.config,\n "init_params": model.init_params,\n "predict_helper": model.predict_helper,\n "metrics": model.metrics,\n "train_counter": model.train_counter,\n "vocab_texts": {k: v.to_text() for k, v in model.vocabs.items()},\n "weights": model.state_dict(),\n }\n\n if "optimizer" in kwargs:\n checkpoint["optimizer"] = kwargs["optimizer"].state_dict()\n\n torch.save(checkpoint, checkpoint_path)\n\n train_counter = model.train_counter\n logger.info(f"Save {train_counter.global_step} global_step checkpoints...! {checkpoint_path}")\n\n # function in function is just used to divide the namespace.\n nsml.bind(save, load, infer)\n\n\n""" Notification """\n\n\n[docs]def get_session_name():\n session_name = "local"\n if nsml.IS_ON_NSML:\n session_name = nsml.SESSION_NAME\n return session_name\n\n\n[docs]def send_message_to_slack(webhook_url, title=None, message=None): # pragma: no cover\n if message is None:\n data = {"text": f"{get_session_name()} session is exited."}\n else:\n data = {"attachments": [{"title": title, "text": message, "color": "#438C56"}]}\n\n try:\n if webhook_url == "":\n print(data["text"])\n else:\n requests.post(webhook_url, data=json.dumps(data))\n except Exception as e:\n print(str(e))\n\n\nfrom argparse import Namespace\nimport json\n\nfrom claf.config.namespace import NestedNamespace\nfrom claf.config.registry import Registry\nfrom claf.learn.experiment import Experiment\nfrom claf.learn.mode import Mode\nfrom claf.machine.module import Module\n\n\n[docs]class Machine:\n """\n Machine: Combine modules then make a NLP Machine\n\n * Args:\n config: machine_config\n """\n\n def __init__(self, config):\n self.config = config\n self.registry = Registry()\n\n\n\n[docs] @classmethod\n def load_from_config(cls, config_path):\n with open(config_path, "r", encoding="utf-8") as in_file:\n machine_config = NestedNamespace()\n machine_config.load_from_json(json.load(in_file))\n\n machine_name = machine_config.name\n config = getattr(machine_config, machine_name, {})\n return cls(config)\n\n def __call__(self, text):\n raise NotImplementedError("")\n\n[docs] def make_module(self, config):\n """\n Make component or experiment for claf Machine's module\n\n * Args:\n - config: module's config (claf.config.namespace.NestedNamespace)\n """\n\n module_type = config.type\n if module_type == Module.COMPONENT:\n name = config.name\n module_config = getattr(config, name, {})\n if isinstance(module_config, Namespace):\n module_config = vars(module_config)\n\n if getattr(config, "params", None):\n module_config.update(config.params)\n return self.registry.get(f"component:{name}")(**module_config)\n elif module_type == Module.EXPERIMENT:\n experiment_config = Namespace()\n experiment_config.checkpoint_path = config.checkpoint_path\n experiment_config.cuda_devices = getattr(config, "cuda_devices", None)\n experiment_config.interactive = False\n\n experiment = Experiment(Mode.PREDICT, experiment_config)\n experiment.set_predict_mode(preload=True)\n return experiment\n else:\n raise ValueError(\n f"module_type is available only [component|experiment]. not '{module_type}'"\n )\n\n\nfrom pathlib import Path\n\nfrom gensim.corpora import Dictionary\nfrom gensim.models import TfidfModel\nfrom gensim.similarities import MatrixSimilarity, SparseMatrixSimilarity\n\nfrom tqdm import tqdm\n\nfrom claf.decorator import register\n\n\n[docs]@register("component:tfidf")\nclass TFIDF:\n """\n TF-IDF document retrieval model\n\n - Term Frequency\n - Inverse Document Frequency\n - log(tf + 1) * log((N - Nt + 0.5) / (Nt + 0.5))\n\n * Kwargs:\n k: the number of top k results\n """\n\n VOCAB_FNAME = "vocab.txt"\n TFIDF_FNAME = "tfidf.model"\n INDEX_FNAME = "similarities.index"\n\n def __init__(self, texts, word_tokenizer, k=1):\n super(TFIDF, self).__init__()\n self.k = k\n\n self.texts = texts\n self.word_tokenizer = word_tokenizer\n\n[docs] def init(self):\n corpus = [\n self.word_tokenizer.tokenize(text)\n for text in tqdm(self.texts, desc="make corpus (Tokenize)")\n ]\n self.vocab = Dictionary(corpus)\n self.init_model()\n\n[docs] def init_model(self):\n corpus = []\n for text in tqdm(self.texts, desc="make corpus (BoW)"):\n corpus.append(self.parse(text))\n\n self.model = TfidfModel(corpus)\n self.index = SparseMatrixSimilarity(self.model[corpus], num_features=len(self.vocab))\n\n[docs] def get_closest(self, query):\n query_tfidf = self.text_to_tfidf(query)\n\n self.index.num_best = self.k\n results = self.index[query_tfidf]\n\n return [\n (text_index, self.texts[text_index], score) # return (index, text, score)\n for (text_index, score) in results\n ]\n\n[docs] def parse(self, query, ngram=1):\n query_tokens = self.word_tokenizer.tokenize(query)\n return self.vocab.doc2bow(query_tokens)\n\n[docs] def text_to_tfidf(self, query):\n """\n Create a tfidf-weighted word vector from query.\n\n tfidf = log(tf + 1) * log((N - Nt + 0.5) / (Nt + 0.5))\n """\n\n query_bow = self.parse(query)\n return self.model[query_bow]\n\n[docs] def save(self, dir_path):\n dir_path = Path(dir_path)\n dir_path.mkdir(parents=True, exist_ok=True)\n\n vocab_path = str(dir_path / self.VOCAB_FNAME)\n model_path = str(dir_path / self.TFIDF_FNAME)\n index_path = str(dir_path / self.INDEX_FNAME)\n\n self.vocab.save(vocab_path)\n self.model.save(model_path)\n self.index.save(index_path)\n\n[docs] def load(self, dir_path):\n dir_path = Path(dir_path)\n\n vocab_path = str(dir_path / self.VOCAB_FNAME)\n model_path = str(dir_path / self.TFIDF_FNAME)\n index_path = str(dir_path / self.INDEX_FNAME)\n\n self.vocab = Dictionary.load(vocab_path)\n self.model = TfidfModel.load(model_path)\n self.index = SparseMatrixSimilarity.load(index_path)\n\n[docs]class Module:\n """ Machine Flag class """\n\n KNOWLEDGE_BASE = "knowledge_base"\n COMPONENT = "component"\n EXPERIMENT = "experiment"\n\n\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.data_handler import CachePath, DataHandler\nfrom claf.decorator import register\n\nfrom claf.machine.base import Machine\n\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("machine:nlu")\nclass NLU(Machine):\n """\n Natural Language Understanding Machine\n\n * Args:\n config: machine_config\n """\n\n def __init__(self, config):\n super(NLU, self).__init__(config)\n self.data_handler = DataHandler(CachePath.MACHINE / "nlu")\n\n self.load()\n\n[docs] @overrides\n def load(self):\n # NLU\n # - Intent Classification Experiment\n # - Slot Filling Experiment\n\n nlu_config = self.config.nlu\n\n self.ic_experiment = self.make_module(nlu_config.intent)\n self.sf_experiment = self.make_module(nlu_config.slots)\n print("Ready ..! \\n")\n\n @overrides\n def __call__(self, utterance):\n\n nlu_result = dict()\n\n intent_info = self.intent_classification(utterance)\n nlu_result.update({"intent": intent_info["class_text"]})\n\n slots_info = self.slot_filling(utterance)\n nlu_result.update({"slots": slots_info["tag_dict"]})\n\n return nlu_result\n\n[docs] def intent_classification(self, utterance):\n raw_feature = {"sequence": utterance}\n return self.ic_experiment.predict(raw_feature)\n\n[docs] def slot_filling(self, utterance):\n raw_feature = {"sequence": utterance}\n return self.sf_experiment.predict(raw_feature)\n\n\nimport logging\nimport os\n\nfrom overrides import overrides\n\nfrom claf.config.factory.tokens import make_all_tokenizers\nfrom claf.config.utils import convert_config2dict\nfrom claf.data.data_handler import CachePath, DataHandler\nfrom claf.decorator import register\n\nfrom claf.machine.base import Machine\nfrom claf.machine.knowlege_base.docs import read_wiki_articles\n\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]@register("machine:open_qa")\nclass OpenQA(Machine):\n """\n Open-Domain Question Answer Machine (DrQA)\n\n DrQA is a system for reading comprehension applied to open-domain question answering.\n\n * Args:\n config: machine_config\n """\n\n def __init__(self, config):\n super(OpenQA, self).__init__(config)\n self.data_handler = DataHandler(CachePath.MACHINE / "open_qa")\n\n self.load()\n\n[docs] @overrides\n def load(self):\n # Tokenizers\n tokenizers_config = convert_config2dict(self.config.tokenizers)\n tokenizers = make_all_tokenizers(tokenizers_config)\n\n # Knowledge Base\n # - Wiki\n knowledge_base_config = self.config.knowledge_base\n self.docs, doc_name = self._load_knowledge_base(knowledge_base_config)\n\n # Reasoning\n # - Document Retrieval\n # - Reading Comprehension Experiment\n reasoning_config = self.config.reasoning\n\n self.document_retrieval = self._load_document_retrieval(\n reasoning_config.document_retrieval, tokenizers["word"], basename=doc_name\n )\n self.rc_experiment = self.make_module(reasoning_config.reading_comprehension)\n print("Ready ..! \\n")\n\n def _load_knowledge_base(self, config):\n docs = read_wiki_articles(config.wiki) # TODO: fix read whole wiki\n doc_name = f"{os.path.basename(config.wiki)}-{len(docs)}-articles"\n return docs, doc_name\n\n def _load_document_retrieval(self, config, word_tokenizer, basename="docs"):\n dir_path = f"doc-{config.type}-{config.name}-{word_tokenizer.cache_name}"\n doc_retrieval_path = os.path.join(dir_path, basename)\n\n config.params = {\n "texts": [doc.title for doc in self.docs],\n "word_tokenizer": word_tokenizer,\n }\n document_retrieval = self.make_module(config)\n\n doc_retrieval_path = self.data_handler.convert_cache_path(doc_retrieval_path)\n if doc_retrieval_path.exists():\n document_retrieval.load(doc_retrieval_path)\n else:\n print("Start Document Retrieval Indexing ...")\n document_retrieval.init()\n document_retrieval.save(doc_retrieval_path) # Save Cache\n print("Completed!")\n return document_retrieval\n\n @overrides\n def __call__(self, question):\n result_docs = self.search_documents(question)\n print("-" * 50)\n print("Doc Scores:")\n for doc in result_docs:\n print(f" - {doc[1]} : {doc[2]}")\n print("-" * 50)\n\n passages = []\n for result_doc in result_docs:\n doc_index = result_doc[0]\n doc = self.docs[doc_index]\n passages.append(doc.text)\n\n answers = []\n for passage in passages:\n answer_text = self.machine_reading(passage, question)\n answers.append(answer_text)\n\n ranked_answers = sorted(answers, key=lambda x: x["score"], reverse=True)\n return ranked_answers\n\n\n\n[docs] def machine_reading(self, context, question):\n raw_feature = {"context": context, "question": question}\n return self.rc_experiment.predict(raw_feature)\n\n\n[docs]def recall(pycm_obj):\n return {key: pycm_obj.TPR[key] if pycm_obj.TPR[key] != "None" else 0. for key in pycm_obj.TPR}\n\n\n[docs]def precision(pycm_obj):\n return {key: pycm_obj.PPV[key] if pycm_obj.PPV[key] != "None" else 0. for key in pycm_obj.PPV}\n\n\n[docs]def f1(pycm_obj):\n return {key: pycm_obj.F1[key] if pycm_obj.F1[key] != "None" else 0. for key in pycm_obj.F1}\n\n\n\n\n\n[docs]def macro_precision(pycm_obj):\n return sum(precision(pycm_obj).values()) / len(pycm_obj.classes)\n\n\n\n\n""" Official evaluation script for v1.1 of the SQuAD dataset. """\nfrom __future__ import print_function\nfrom collections import Counter\nimport string\nimport re\nimport argparse\nimport json\nimport sys\n\n\n[docs]def normalize_answer(s): # pragma: no cover\n """Lower text and remove punctuation, articles and extra whitespace."""\n\n def remove_articles(text):\n return re.sub(r"\\b(a|an|the)\\b", " ", text)\n\n def white_space_fix(text):\n return " ".join(text.split())\n\n def remove_punc(text):\n exclude = set(string.punctuation)\n return "".join(ch for ch in text if ch not in exclude)\n\n def lower(text):\n return text.lower()\n\n return white_space_fix(remove_articles(remove_punc(lower(s))))\n\n\n[docs]def f1_score(prediction, ground_truth): # pragma: no cover\n prediction_tokens = normalize_answer(prediction).split()\n ground_truth_tokens = normalize_answer(ground_truth).split()\n common = Counter(prediction_tokens) & Counter(ground_truth_tokens)\n num_same = sum(common.values())\n if num_same == 0:\n return 0\n precision = 1.0 * num_same / len(prediction_tokens)\n recall = 1.0 * num_same / len(ground_truth_tokens)\n f1 = (2 * precision * recall) / (precision + recall)\n return f1\n\n\n[docs]def exact_match_score(prediction, ground_truth): # pragma: no cover\n return normalize_answer(prediction) == normalize_answer(ground_truth)\n\n\n[docs]def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): # pragma: no cover\n scores_for_ground_truths = []\n for ground_truth in ground_truths:\n score = metric_fn(prediction, ground_truth)\n scores_for_ground_truths.append(score)\n return max(scores_for_ground_truths)\n\n\n[docs]def evaluate(dataset, predictions):\n f1 = exact_match = total = 0\n for article in dataset:\n for paragraph in article["paragraphs"]:\n for qa in paragraph["qas"]:\n total += 1\n if qa["id"] not in predictions:\n message = "Unanswered question " + qa["id"] + " will receive score 0."\n print(message, file=sys.stderr)\n continue\n ground_truths = list(map(lambda x: x["text"], qa["answers"]))\n prediction = predictions[qa["id"]]\n exact_match += metric_max_over_ground_truths(\n exact_match_score, prediction, ground_truths\n )\n f1 += metric_max_over_ground_truths(f1_score, prediction, ground_truths)\n\n exact_match = 100.0 * exact_match / total\n f1 = 100.0 * f1 / total\n\n return {"em": exact_match, "f1": f1}\n\n\nif __name__ == "__main__": # pragma: no cover\n expected_version = "1.1"\n parser = argparse.ArgumentParser(description="Evaluation for SQuAD " + expected_version)\n parser.add_argument("dataset_file", help="Dataset file")\n parser.add_argument("prediction_file", help="Prediction File")\n args = parser.parse_args()\n with open(args.dataset_file) as dataset_file:\n dataset_json = json.load(dataset_file)\n if dataset_json["version"] != expected_version:\n print(\n "Evaluation expects v-"\n + expected_version\n + ", but got dataset with v-"\n + dataset_json["version"],\n file=sys.stderr,\n )\n dataset = dataset_json["data"]\n with open(args.prediction_file) as prediction_file:\n predictions = json.load(prediction_file)\n print(json.dumps(evaluate(dataset, predictions)))\n\n"""Official evaluation script for SQuAD version 2.0.\n\nIn addition to basic functionality, we also compute additional statistics and\nplot precision-recall curves if an additional na_prob.json file is provided.\nThis file is expected to map question ID's to the model's predicted probability\nthat a question is unanswerable.\n"""\nimport argparse\nimport collections\nimport json\nimport numpy as np\nimport os\nimport re\nimport string\nimport sys\n\nOPTS = None\n\n\n[docs]def parse_args(): # pragma: no cover\n parser = argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0.")\n parser.add_argument("data_file", metavar="data.json", help="Input data JSON file.")\n parser.add_argument("pred_file", metavar="pred.json", help="Model predictions.")\n parser.add_argument(\n "--out-file",\n "-o",\n metavar="eval.json",\n help="Write accuracy metrics to file (default is stdout).",\n )\n parser.add_argument(\n "--na-prob-file",\n "-n",\n metavar="na_prob.json",\n help="Model estimates of probability of no answer.",\n )\n parser.add_argument(\n "--na-prob-thresh",\n "-t",\n type=float,\n default=1.0,\n help='Predict "" if no-answer probability exceeds this (default = 1.0).',\n )\n parser.add_argument(\n "--out-image-dir",\n "-p",\n metavar="out_images",\n default=None,\n help="Save precision-recall curves to directory.",\n )\n parser.add_argument("--verbose", "-v", action="store_true")\n if len(sys.argv) == 1:\n parser.print_help()\n sys.exit(1)\n return parser.parse_args()\n\n\n[docs]def make_qid_to_has_ans(dataset): # pragma: no cover\n qid_to_has_ans = {}\n for article in dataset:\n for p in article["paragraphs"]:\n for qa in p["qas"]:\n qid_to_has_ans[qa["id"]] = bool(qa["answers"])\n return qid_to_has_ans\n\n\n[docs]def normalize_answer(s): # pragma: no cover\n """Lower text and remove punctuation, articles and extra whitespace."""\n\n def remove_articles(text):\n regex = re.compile(r"\\b(a|an|the)\\b", re.UNICODE)\n return re.sub(regex, " ", text)\n\n def white_space_fix(text):\n return " ".join(text.split())\n\n def remove_punc(text):\n exclude = set(string.punctuation)\n return "".join(ch for ch in text if ch not in exclude)\n\n def lower(text):\n return text.lower()\n\n return white_space_fix(remove_articles(remove_punc(lower(s))))\n\n\n[docs]def get_tokens(s): # pragma: no cover\n if not s:\n return []\n return normalize_answer(s).split()\n\n\n[docs]def compute_exact(a_gold, a_pred): # pragma: no cover\n return int(normalize_answer(a_gold) == normalize_answer(a_pred))\n\n\n[docs]def compute_f1(a_gold, a_pred): # pragma: no cover\n gold_toks = get_tokens(a_gold)\n pred_toks = get_tokens(a_pred)\n common = collections.Counter(gold_toks) & collections.Counter(pred_toks)\n num_same = sum(common.values())\n if len(gold_toks) == 0 or len(pred_toks) == 0:\n # If either is no-answer, then F1 is 1 if they agree, 0 otherwise\n return int(gold_toks == pred_toks)\n if num_same == 0:\n return 0\n precision = 1.0 * num_same / len(pred_toks)\n recall = 1.0 * num_same / len(gold_toks)\n f1 = (2 * precision * recall) / (precision + recall)\n return f1\n\n\n[docs]def get_raw_scores(dataset, preds): # pragma: no cover\n exact_scores = {}\n f1_scores = {}\n for article in dataset:\n for p in article["paragraphs"]:\n for qa in p["qas"]:\n qid = qa["id"]\n gold_answers = [a["text"] for a in qa["answers"] if normalize_answer(a["text"])]\n if not gold_answers:\n # For unanswerable questions, only correct answer is empty\n # string\n gold_answers = [""]\n if qid not in preds:\n # print('Missing prediction for %s' % qid)\n continue\n a_pred = preds[qid]\n # Take max over all gold answers\n exact_scores[qid] = max(compute_exact(a, a_pred) for a in gold_answers)\n f1_scores[qid] = max(compute_f1(a, a_pred) for a in gold_answers)\n return exact_scores, f1_scores\n\n\n[docs]def apply_no_ans_threshold(scores, na_probs, qid_to_has_ans, na_prob_thresh): # pragma: no cover\n new_scores = {}\n for qid, s in scores.items():\n pred_na = na_probs[qid] > na_prob_thresh\n if pred_na:\n new_scores[qid] = float(not qid_to_has_ans[qid])\n else:\n new_scores[qid] = s\n return new_scores\n\n\n[docs]def make_eval_dict(exact_scores, f1_scores, qid_list=None): # pragma: no cover\n if not qid_list:\n total = len(exact_scores)\n return collections.OrderedDict(\n [\n ("exact", 100.0 * sum(exact_scores.values()) / total),\n ("f1", 100.0 * sum(f1_scores.values()) / total),\n ("total", total),\n ]\n )\n else:\n total = len(qid_list)\n return collections.OrderedDict(\n [\n ("exact", 100.0 * sum(exact_scores[k] for k in qid_list) / total),\n ("f1", 100.0 * sum(f1_scores[k] for k in qid_list) / total),\n ("total", total),\n ]\n )\n\n\n[docs]def merge_eval(main_eval, new_eval, prefix): # pragma: no cover\n for k in new_eval:\n main_eval["%s_%s" % (prefix, k)] = new_eval[k]\n\n\n[docs]def plot_pr_curve(precisions, recalls, out_image, title): # pragma: no cover\n plt.step(recalls, precisions, color="b", alpha=0.2, where="post")\n plt.fill_between(recalls, precisions, step="post", alpha=0.2, color="b")\n plt.xlabel("Recall")\n plt.ylabel("Precision")\n plt.xlim([0.0, 1.05])\n plt.ylim([0.0, 1.05])\n plt.title(title)\n plt.savefig(out_image)\n plt.clf()\n\n\n[docs]def make_precision_recall_eval(\n scores, na_probs, num_true_pos, qid_to_has_ans, out_image=None, title=None\n): # pragma: no cover\n qid_list = sorted(na_probs, key=lambda k: na_probs[k])\n true_pos = 0.0\n cur_p = 1.0\n cur_r = 0.0\n precisions = [1.0]\n recalls = [0.0]\n avg_prec = 0.0\n for i, qid in enumerate(qid_list):\n if qid_to_has_ans[qid]:\n true_pos += scores[qid]\n cur_p = true_pos / float(i + 1)\n cur_r = true_pos / float(num_true_pos)\n if i == len(qid_list) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:\n # i.e., if we can put a threshold after this point\n avg_prec += cur_p * (cur_r - recalls[-1])\n precisions.append(cur_p)\n recalls.append(cur_r)\n if out_image:\n plot_pr_curve(precisions, recalls, out_image, title)\n return {"ap": 100.0 * avg_prec}\n\n\n[docs]def run_precision_recall_analysis(\n main_eval, exact_raw, f1_raw, na_probs, qid_to_has_ans, out_image_dir\n): # pragma: no cover\n if out_image_dir and not os.path.exists(out_image_dir):\n os.makedirs(out_image_dir)\n num_true_pos = sum(1 for v in qid_to_has_ans.values() if v)\n if num_true_pos == 0:\n return\n pr_exact = make_precision_recall_eval(\n exact_raw,\n na_probs,\n num_true_pos,\n qid_to_has_ans,\n out_image=os.path.join(out_image_dir, "pr_exact.png"),\n title="Precision-Recall curve for Exact Match score",\n )\n pr_f1 = make_precision_recall_eval(\n f1_raw,\n na_probs,\n num_true_pos,\n qid_to_has_ans,\n out_image=os.path.join(out_image_dir, "pr_f1.png"),\n title="Precision-Recall curve for F1 score",\n )\n oracle_scores = {k: float(v) for k, v in qid_to_has_ans.items()}\n pr_oracle = make_precision_recall_eval(\n oracle_scores,\n na_probs,\n num_true_pos,\n qid_to_has_ans,\n out_image=os.path.join(out_image_dir, "pr_oracle.png"),\n title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)",\n )\n merge_eval(main_eval, pr_exact, "pr_exact")\n merge_eval(main_eval, pr_f1, "pr_f1")\n merge_eval(main_eval, pr_oracle, "pr_oracle")\n\n\n[docs]def histogram_na_prob(na_probs, qid_list, image_dir, name): # pragma: no cover\n if not qid_list:\n return\n x = [na_probs[k] for k in qid_list]\n weights = np.ones_like(x) / float(len(x))\n plt.hist(x, weights=weights, bins=20, range=(0.0, 1.0))\n plt.xlabel("Model probability of no-answer")\n plt.ylabel("Proportion of dataset")\n plt.title("Histogram of no-answer probability: %s" % name)\n plt.savefig(os.path.join(image_dir, "na_prob_hist_%s.png" % name))\n plt.clf()\n\n\n[docs]def find_best_thresh(preds, scores, na_probs, qid_to_has_ans): # pragma: no cover\n num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k])\n cur_score = num_no_ans\n best_score = cur_score\n best_thresh = 0.0\n qid_list = sorted(na_probs, key=lambda k: na_probs[k])\n for i, qid in enumerate(qid_list):\n if qid not in scores:\n continue\n if qid_to_has_ans[qid]:\n diff = scores[qid]\n else:\n if preds[qid]:\n diff = -1\n else:\n diff = 0\n cur_score += diff\n if cur_score > best_score:\n best_score = cur_score\n best_thresh = na_probs[qid]\n return 100.0 * best_score / len(scores), best_thresh\n\n\n[docs]def find_all_best_thresh(\n main_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans\n): # pragma: no cover\n best_exact, exact_thresh = find_best_thresh(preds, exact_raw, na_probs, qid_to_has_ans)\n best_f1, f1_thresh = find_best_thresh(preds, f1_raw, na_probs, qid_to_has_ans)\n main_eval["best_exact"] = best_exact\n main_eval["best_exact_thresh"] = exact_thresh\n main_eval["best_f1"] = best_f1\n main_eval["best_f1_thresh"] = f1_thresh\n\n\n[docs]def evaluate(dataset, na_probs, preds, na_prob_thresh=1.0):\n qid_to_has_ans = make_qid_to_has_ans(dataset) # maps qid to True/False\n has_ans_qids = [k for k, v in qid_to_has_ans.items() if v]\n no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v]\n\n exact_raw, f1_raw = get_raw_scores(dataset, preds)\n\n exact_thresh = apply_no_ans_threshold(exact_raw, na_probs, qid_to_has_ans, na_prob_thresh)\n f1_thresh = apply_no_ans_threshold(f1_raw, na_probs, qid_to_has_ans, na_prob_thresh)\n\n out_eval = make_eval_dict(exact_thresh, f1_thresh)\n if has_ans_qids:\n has_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=has_ans_qids)\n merge_eval(out_eval, has_ans_eval, "HasAns")\n if no_ans_qids:\n no_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids)\n merge_eval(out_eval, no_ans_eval, "NoAns")\n\n find_all_best_thresh(out_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans)\n\n return out_eval\n\n\n[docs]def main(): # pragma: no cover\n with open(OPTS.data_file) as f:\n dataset_json = json.load(f)\n dataset = dataset_json["data"]\n with open(OPTS.pred_file) as f:\n preds = json.load(f)\n if OPTS.na_prob_file:\n with open(OPTS.na_prob_file) as f:\n na_probs = json.load(f)\n else:\n na_probs = {k: 0.0 for k in preds}\n qid_to_has_ans = make_qid_to_has_ans(dataset) # maps qid to True/False\n has_ans_qids = [k for k, v in qid_to_has_ans.items() if v]\n no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v]\n exact_raw, f1_raw = get_raw_scores(dataset, preds)\n exact_thresh = apply_no_ans_threshold(exact_raw, na_probs, qid_to_has_ans, OPTS.na_prob_thresh)\n f1_thresh = apply_no_ans_threshold(f1_raw, na_probs, qid_to_has_ans, OPTS.na_prob_thresh)\n out_eval = make_eval_dict(exact_thresh, f1_thresh)\n if has_ans_qids:\n has_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=has_ans_qids)\n merge_eval(out_eval, has_ans_eval, "HasAns")\n if no_ans_qids:\n no_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids)\n merge_eval(out_eval, no_ans_eval, "NoAns")\n if OPTS.na_prob_file:\n find_all_best_thresh(out_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans)\n if OPTS.na_prob_file and OPTS.out_image_dir:\n run_precision_recall_analysis(\n out_eval, exact_raw, f1_raw, na_probs, qid_to_has_ans, OPTS.out_image_dir\n )\n histogram_na_prob(na_probs, has_ans_qids, OPTS.out_image_dir, "hasAns")\n histogram_na_prob(na_probs, no_ans_qids, OPTS.out_image_dir, "noAns")\n if OPTS.out_file:\n with open(OPTS.out_file, "w") as f:\n json.dump(out_eval, f)\n else:\n print(json.dumps(out_eval, indent=2))\n\n\nif __name__ == "__main__": # pragma: no cover\n OPTS = parse_args()\n if OPTS.out_image_dir:\n import matplotlib\n\n matplotlib.use("Agg")\n import matplotlib.pyplot as plt\n main()\n\n""" Official evaluation script for WikiSQL dataset. """\n\nimport json\nfrom argparse import ArgumentParser\nfrom tqdm import tqdm\nfrom claf.metric.wikisql_lib.dbengine import DBEngine\nfrom claf.metric.wikisql_lib.query import Query\n\n\n[docs]def count_lines(fname): # pragma: no cover\n with open(fname) as f:\n return sum(1 for line in f)\n\n\n[docs]def evaluate(labels, predictions, db_path, ordered=True): # pragma: no cover\n """ labels and predictions: dictionary {data_uid: sql_data, ...} """\n engine = DBEngine(db_path)\n\n exact_match, grades = [], []\n for idx, data_uid in enumerate(predictions):\n eg = labels[data_uid]\n ep = predictions[data_uid]\n\n qg = eg["sql_query"]\n gold = eg["execution_result"]\n\n pred = ep.get("error", None)\n qp = None\n if not ep.get("error", None):\n try:\n qp = Query.from_dict(ep["query"], ordered=ordered)\n pred = engine.execute_query(ep["table_id"], qp, lower=True)\n except Exception as e:\n pred = repr(e)\n\n correct = pred == gold\n match = qp == qg\n grades.append(correct)\n exact_match.append(match)\n\n return {\n "ex_accuracy": sum(grades) / len(grades) * 100.0,\n "lf_accuracy": sum(exact_match) / len(exact_match) * 100.0,\n }\n\n\nif __name__ == "__main__": # pragma: no cover\n parser = ArgumentParser()\n parser.add_argument("source_file", help="source file for the prediction")\n parser.add_argument("db_file", help="source database for the prediction")\n parser.add_argument("pred_file", help="predictions by the model")\n parser.add_argument(\n "--ordered",\n action="store_true",\n help="whether the exact match should consider the order of conditions",\n )\n args = parser.parse_args()\n\n engine = DBEngine(args.db_file)\n exact_match = []\n with open(args.source_file) as fs, open(args.pred_file) as fp:\n grades = []\n for ls, lp in tqdm(zip(fs, fp), total=count_lines(args.source_file)):\n eg = json.loads(ls)\n ep = json.loads(lp)\n qg = Query.from_dict(eg["sql"], ordered=args.ordered)\n gold = engine.execute_query(eg["table_id"], qg, lower=True)\n pred = ep.get("error", None)\n qp = None\n if not ep.get("error", None):\n try:\n qp = Query.from_dict(ep["query"], ordered=args.ordered)\n pred = engine.execute_query(eg["table_id"], qp, lower=True)\n except Exception as e:\n pred = repr(e)\n correct = pred == gold\n match = qp == qg\n grades.append(correct)\n exact_match.append(match)\n print(\n json.dumps(\n {\n "ex_accuracy": sum(grades) / len(grades),\n "lf_accuracy": sum(exact_match) / len(exact_match),\n },\n indent=2,\n )\n )\n\n\nimport json\nfrom pathlib import Path\n\nimport torch.nn as nn\n\n\n[docs]class ModelBase(nn.Module):\n """\n Model Base Class\n\n Args:\n token_embedder: (claf.tokens.token_embedder.base) TokenEmbedder\n """\n\n def __init__(self):\n super(ModelBase, self).__init__()\n\n\n\n\n\n[docs] def make_predictions(self, features):\n """\n for Metrics\n """\n\n raise NotImplementedError\n\n\n\n[docs] def print_examples(self, params):\n """\n Print evaluation examples\n """\n\n raise NotImplementedError\n\n[docs] def write_predictions(self, predictions, file_path=None, is_dict=True):\n data_type = "train" if self.training else "valid"\n\n pred_dir = Path(self._log_dir) / "predictions"\n pred_dir.mkdir(exist_ok=True)\n\n if file_path is None:\n file_path = f"predictions-{data_type}-{self._train_counter.get_display()}.json"\n\n pred_path = pred_dir / file_path\n with open(pred_path, "w") as out_file:\n if is_dict:\n out_file.write(json.dumps(predictions, indent=4))\n else:\n out_file.write(predictions)\n\n[docs] def is_ready(self):\n properties = [\n self._config,\n self._log_dir,\n # self._dataset, It's set at _run_epoch()\n # self._metrics, It's set at save()\n self._train_counter,\n self._vocabs\n ]\n\n return all([p is not None for p in properties])\n\n @property\n def config(self):\n return self._config\n\n @config.setter\n def config(self, config):\n self._config = config\n\n @property\n def log_dir(self):\n return self._log_dir\n\n @log_dir.setter\n def log_dir(self, log_dir):\n self._log_dir = log_dir\n\n @property\n def dataset(self):\n return self._dataset\n\n @dataset.setter\n def dataset(self, dataset):\n self._dataset = dataset\n\n @property\n def metrics(self):\n return self._metrics\n\n @metrics.setter\n def metrics(self, metrics):\n self._metrics = metrics\n\n @property\n def train_counter(self):\n return self._train_counter\n\n @train_counter.setter\n def train_counter(self, train_counter):\n self._train_counter = train_counter\n\n @property\n def vocabs(self):\n return self._vocabs\n\n @vocabs.setter\n def vocabs(self, vocabs):\n self._vocabs = vocabs\n\n\n[docs]class ModelWithTokenEmbedder(ModelBase):\n def __init__(self, token_embedder):\n super(ModelWithTokenEmbedder, self).__init__()\n\n self.token_embedder = token_embedder\n if token_embedder is not None:\n self._vocabs = token_embedder.vocabs\n\n\n[docs]class ModelWithoutTokenEmbedder(ModelBase):\n def __init__(self, token_makers):\n super(ModelWithoutTokenEmbedder, self).__init__()\n\n self.token_makers = token_makers\n self._vocabs = {\n token_name: token_maker.vocab for token_name, token_maker in token_makers.items()\n }\n\nimport csv\nfrom collections import defaultdict\n\nfrom seqeval.metrics.sequence_labeling import get_entities\n\n\n# pycm\n[docs]def write_confusion_matrix_to_csv(file_path, pycm_obj):\n with open(file_path + ".csv", "w") as f:\n indicator = "target/predict"\n\n fieldnames = [indicator] + pycm_obj.classes + ["FN"]\n writer = csv.DictWriter(f, fieldnames=fieldnames)\n writer.writeheader()\n\n data = dict(pycm_obj.matrix)\n FN = dict(pycm_obj.FN)\n\n for row_idx in fieldnames[1:-1]: # remove indicator and FN\n row = {indicator: row_idx}\n row.update(\n {\n col_idx: data[row_idx][col_idx]\n for col_idx in data[row_idx]\n if col_idx in fieldnames\n }\n )\n row.update({"FN": FN[row_idx]})\n writer.writerow(row)\n\n row = {indicator: "FP"}\n row.update(dict(pycm_obj.FP))\n writer.writerow(row)\n\n\n# seqeval\n[docs]def get_tag_dict(sequence, tag_texts):\n words = sequence.split()\n entities = get_entities(tag_texts)\n\n slots = defaultdict(list)\n for slot, start_idx, end_idx in entities:\n slots[slot].append(" ".join(words[start_idx : end_idx + 1]))\n return dict(slots)\n\n\n\nfrom overrides import overrides\nfrom pytorch_transformers import BertForQuestionAnswering\nimport torch.nn as nn\n\nfrom claf.data.data_handler import CachePath\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithoutTokenEmbedder\nfrom claf.model.reading_comprehension.mixin import SQuADv1ForBert\n\n\n[docs]@register("model:bert_for_qa")\nclass BertForQA(SQuADv1ForBert, ModelWithoutTokenEmbedder):\n """\n Document Reader Model. `Span Detector`\n\n Implementation of model presented in\n BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding\n (https://arxiv.org/abs/1810.04805)\n\n * Args:\n token_embedder: 'QATokenEmbedder', Used to embed the 'context' and 'question'.\n\n * Kwargs:\n lang_code: Dataset language code [en|ko]\n pretrained_model_name: the name of a pre-trained model\n answer_maxlen: the most probable answer span of length less than or equal to {answer_maxlen}\n """\n\n def __init__(self, token_makers, lang_code="en", pretrained_model_name=None, answer_maxlen=30):\n super(BertForQA, self).__init__(token_makers)\n\n self.lang_code = lang_code\n self.use_pytorch_transformers = True # for optimizer's model parameters\n self.answer_maxlen = answer_maxlen\n\n self.model = BertForQuestionAnswering.from_pretrained(\n pretrained_model_name, cache_dir=str(CachePath.ROOT)\n )\n self.criterion = nn.CrossEntropyLoss()\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {"feature_name1": {\n "token_name1": tensor,\n "toekn_name2": tensor},\n "feature_name2": ...}\n\n * Kwargs:\n label: label dictionary like below.\n {"label_name1": tensor,\n "label_name2": tensor}\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - start_logits: representing unnormalized log probabilities of the span start position.\n - end_logits: representing unnormalized log probabilities of the span end position.\n - best_span: the string from the original passage that the model thinks is the best answer to the question.\n - data_idx: the question id, mapping with answer\n - loss: A scalar loss to be optimised.\n """\n\n bert_inputs = features["bert_input"]["feature"]\n token_type_ids = features["token_type"]["feature"]\n attention_mask = (bert_inputs > 0).long()\n\n span_start_logits, span_end_logits = self.model(\n bert_inputs, token_type_ids=token_type_ids, attention_mask=attention_mask\n )\n\n output_dict = {\n "start_logits": span_start_logits,\n "end_logits": span_end_logits,\n "best_span": self.get_best_span(\n span_start_logits, span_end_logits, answer_maxlen=self.answer_maxlen\n ),\n }\n\n if labels:\n data_idx = labels["data_idx"]\n answer_start_idx = labels["answer_start_idx"]\n answer_end_idx = labels["answer_end_idx"]\n\n output_dict["data_idx"] = data_idx\n\n # If we are on multi-GPU, split add a dimension\n if len(answer_start_idx.size()) > 1:\n answer_start_idx = answer_start_idx.squeeze(-1)\n if len(answer_end_idx.size()) > 1:\n answer_end_idx = answer_end_idx.squeeze(-1)\n # sometimes the start/end positions are outside our model inputs, we ignore these terms\n ignored_index = span_start_logits.size(1)\n\n answer_start_idx.clamp_(0, ignored_index)\n answer_end_idx.clamp_(0, ignored_index)\n\n # Loss\n criterion = nn.CrossEntropyLoss(ignore_index=ignored_index)\n loss = criterion(span_start_logits, answer_start_idx)\n loss += criterion(span_end_logits, answer_end_idx)\n loss /= 2 # (start + end)\n output_dict["loss"] = loss\n\n return output_dict\n\n\n\nfrom overrides import overrides\nfrom pytorch_transformers import BertForQuestionAnswering\nimport torch.nn as nn\n\nfrom claf.data.data_handler import CachePath\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithoutTokenEmbedder\nfrom claf.model.reading_comprehension.mixin import SQuADv1\n\n\n[docs]@register("model:bert_for_qa")\nclass BertForQA(SQuADv1, ModelWithoutTokenEmbedder):\n """\n Document Reader Model. `Span Detector`\n\n Implementation of model presented in\n BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding\n (https://arxiv.org/abs/1810.04805)\n\n * Args:\n token_embedder: 'QATokenEmbedder', Used to embed the 'context' and 'question'.\n\n * Kwargs:\n lang_code: Dataset language code [en|ko]\n pretrained_model_name: the name of a pre-trained model\n answer_maxlen: the most probable answer span of length less than or equal to {answer_maxlen}\n """\n\n def __init__(self, token_makers, lang_code="en", pretrained_model_name=None, answer_maxlen=30):\n super(BertForQA, self).__init__(token_makers)\n\n self.lang_code = lang_code\n self.bert = True # for optimizer's model parameters\n self.answer_maxlen = answer_maxlen\n\n self.model = BertForQuestionAnswering.from_pretrained(\n pretrained_model_name, cache_dir=str(CachePath.ROOT)\n )\n self.criterion = nn.CrossEntropyLoss()\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {"feature_name1": {\n "token_name1": tensor,\n "toekn_name2": tensor},\n "feature_name2": ...}\n\n * Kwargs:\n label: label dictionary like below.\n {"label_name1": tensor,\n "label_name2": tensor}\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - start_logits: representing unnormalized log probabilities of the span start position.\n - end_logits: representing unnormalized log probabilities of the span end position.\n - best_span: the string from the original passage that the model thinks is the best answer to the question.\n - answer_idx: the question id, mapping with answer\n - loss: A scalar loss to be optimised.\n """\n\n bert_inputs = features["bert_input"]["feature"]\n token_type_ids = features["token_type"]["feature"]\n attention_mask = (bert_inputs > 0).long()\n\n span_start_logits, span_end_logits = self.model(\n bert_inputs, token_type_ids=token_type_ids, attention_mask=attention_mask\n )\n\n output_dict = {\n "start_logits": span_start_logits,\n "end_logits": span_end_logits,\n "best_span": self.get_best_span(\n span_start_logits, span_end_logits, answer_maxlen=self.answer_maxlen\n ),\n }\n\n if labels:\n answer_idx = labels["answer_idx"]\n answer_start_idx = labels["answer_start_idx"]\n answer_end_idx = labels["answer_end_idx"]\n\n output_dict["answer_idx"] = answer_idx\n\n # If we are on multi-GPU, split add a dimension\n if len(answer_start_idx.size()) > 1:\n answer_start_idx = answer_start_idx.squeeze(-1)\n if len(answer_end_idx.size()) > 1:\n answer_end_idx = answer_end_idx.squeeze(-1)\n # sometimes the start/end positions are outside our model inputs, we ignore these terms\n ignored_index = span_start_logits.size(1)\n\n answer_start_idx.clamp_(0, ignored_index)\n answer_end_idx.clamp_(0, ignored_index)\n\n # Loss\n criterion = nn.CrossEntropyLoss(ignore_index=ignored_index)\n loss = criterion(span_start_logits, answer_start_idx)\n loss += criterion(span_end_logits, answer_end_idx)\n loss /= 2 # (start + end)\n output_dict["loss"] = loss\n\n return output_dict\n\n[docs] @overrides\n def make_metrics(self, predictions):\n """ BERT predictions need to get nbest result """\n\n best_predictions = {}\n for index, prediction in predictions.items():\n qid = self._dataset.get_qid(index)\n\n predict_text = prediction["predict_text"]\n\n start_logit = prediction["start_logits"][prediction["pred_span_start"]]\n end_logit = prediction["end_logits"][prediction["pred_span_end"]]\n predict_score = start_logit.item() + end_logit.item()\n\n if qid not in best_predictions:\n best_predictions[qid] = []\n best_predictions[qid].append((predict_text, predict_score))\n\n for qid, predictions in best_predictions.items():\n sorted_predictions = sorted(predictions, key=lambda x: x[1], reverse=True)\n best_predictions[qid] = sorted_predictions[0][0]\n\n self.write_predictions(best_predictions)\n return self._make_metrics_with_official(best_predictions)\n\n[docs] @overrides\n def predict(self, output_dict, arguments, helper):\n """\n Inference by raw_feature\n\n * Args:\n output_dict: model's output dictionary consisting of\n - answer_idx: question id\n - best_span: calculate the span_start_logits and span_end_logits to what is the best span\n arguments: arguments dictionary consisting of user_input\n helper: dictionary for helping get answer\n\n * Returns:\n span: predict best_span\n """\n\n context_text = arguments["context"]\n bert_tokens = helper["bert_token"]\n predictions = [\n (best_span, start_logits, end_logits)\n for best_span, start_logits, end_logits in zip(\n list(output_dict["best_span"].data),\n list(output_dict["start_logits"].data),\n list(output_dict["end_logits"].data),\n )\n ]\n\n best_predictions = []\n for index, prediction in enumerate(predictions):\n bert_token = bert_tokens[index]\n best_span, start_logits, end_logits = prediction\n pred_start, pred_end = best_span\n\n predict_text = ""\n if (\n pred_start < len(bert_token)\n and pred_end < len(bert_token)\n and bert_token[pred_start].text_span is not None\n and bert_token[pred_end].text_span is not None\n ):\n char_start = bert_token[pred_start].text_span[0]\n char_end = bert_token[pred_end].text_span[1]\n predict_text = context_text[char_start:char_end]\n\n start_logit = start_logits[pred_start]\n end_logit = end_logits[pred_end]\n predict_score = start_logit.item() + end_logit.item()\n\n best_predictions.append((predict_text, predict_score))\n\n sorted_predictions = sorted(best_predictions, key=lambda x: x[1], reverse=True)\n return {"text": sorted_predictions[0][0], "score": sorted_predictions[0][1]}\n\n\nfrom overrides import overrides\nimport torch\nimport torch.nn as nn\n\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithTokenEmbedder\nfrom claf.model.reading_comprehension.mixin import SQuADv1\nimport claf.modules.functional as f\nimport claf.modules.attention as attention\nimport claf.modules.layer as layer\n\n\n[docs]@register("model:bidaf")\nclass BiDAF(SQuADv1, ModelWithTokenEmbedder):\n """\n Document Reader Model. `Span Detector`\n\n Implementation of model presented in\n BiDAF: Bidirectional Attention Flow for Machine Comprehension\n (https://arxiv.org/abs/1611.01603)\n\n - Embedding (Word + Char -> Contextual)\n - Attention Flow\n - Modeling (RNN)\n - Output\n\n * Args:\n token_embedder: 'QATokenEmbedder', Used to embed the 'context' and 'question'.\n\n * Kwargs:\n lang_code: Dataset language code [en|ko]\n aligned_query_embedding: f_align(p_i) = sum(a_ij, E(qj), where the attention score a_ij\n captures the similarity between pi and each question words q_j.\n these features add soft alignments between similar but non-identical words (e.g., car and vehicle)\n it only apply to 'context_embed'.\n answer_maxlen: the most probable answer span of length less than or equal to {answer_maxlen}\n model_dim: the number of model dimension\n contextual_rnn_num_layer: the number of recurrent layers (contextual)\n modeling_rnn_num_layer: the number of recurrent layers (modeling)\n predict_rnn_num_layer: the number of recurrent layers (predict)\n dropout: the dropout probability\n """\n\n def __init__(\n self,\n token_embedder,\n lang_code="en",\n aligned_query_embedding=False,\n answer_maxlen=None,\n model_dim=100,\n contextual_rnn_num_layer=1,\n modeling_rnn_num_layer=2,\n predict_rnn_num_layer=1,\n dropout=0.2,\n ):\n super(BiDAF, self).__init__(token_embedder)\n\n self.lang_code = lang_code\n self.aligned_query_embedding = aligned_query_embedding\n self.answer_maxlen = answer_maxlen\n self.token_embedder = token_embedder\n self.dropout = nn.Dropout(p=dropout)\n\n context_embed_dim, query_embed_dim = token_embedder.get_embed_dim()\n if self.aligned_query_embedding:\n context_embed_dim += query_embed_dim\n\n if context_embed_dim != query_embed_dim:\n self.context_highway = layer.Highway(context_embed_dim)\n self.context_contextual_rnn = nn.LSTM(\n input_size=context_embed_dim,\n hidden_size=model_dim,\n bidirectional=True,\n num_layers=contextual_rnn_num_layer,\n batch_first=True,\n )\n\n self.query_highway = layer.Highway(query_embed_dim)\n self.query_contextual_rnn = nn.LSTM(\n input_size=query_embed_dim,\n hidden_size=model_dim,\n bidirectional=True,\n num_layers=contextual_rnn_num_layer,\n batch_first=True,\n )\n else:\n highway = layer.Highway(query_embed_dim)\n\n self.context_highway = highway\n self.query_highway = highway\n\n contextual_rnn = nn.LSTM(\n input_size=context_embed_dim,\n hidden_size=model_dim,\n bidirectional=True,\n num_layers=contextual_rnn_num_layer,\n batch_first=True,\n )\n\n self.context_contextual_rnn = contextual_rnn\n self.query_contextual_rnn = contextual_rnn\n\n self.attention = attention.BiAttention(model_dim)\n self.modeling_rnn = nn.LSTM(\n input_size=8 * model_dim,\n hidden_size=model_dim,\n num_layers=modeling_rnn_num_layer,\n bidirectional=True,\n dropout=dropout,\n batch_first=True,\n )\n self.output_end_rnn = nn.LSTM(\n input_size=14 * model_dim,\n hidden_size=model_dim,\n bidirectional=True,\n num_layers=predict_rnn_num_layer,\n batch_first=True,\n )\n\n self.span_start_linear = nn.Linear(10 * model_dim, 1)\n self.span_end_linear = nn.Linear(10 * model_dim, 1)\n\n self.criterion = nn.CrossEntropyLoss()\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {"feature_name1": {\n "token_name1": tensor,\n "toekn_name2": tensor},\n "feature_name2": ...}\n\n * Kwargs:\n label: label dictionary like below.\n {"label_name1": tensor,\n "label_name2": tensor}\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - start_logits: representing unnormalized log probabilities of the span start position.\n - end_logits: representing unnormalized log probabilities of the span end position.\n - best_span: the string from the original passage that the model thinks is the best answer to the question.\n - data_idx: the question id, mapping with answer\n - loss: A scalar loss to be optimised.\n """\n\n context = features["context"]\n question = features["question"]\n\n # Sorted Sequence config (seq_lengths, perm_idx, unperm_idx) for RNN pack_forward\n context_seq_config = f.get_sorted_seq_config(context)\n query_seq_config = f.get_sorted_seq_config(question)\n\n # Embedding Layer (Char + Word -> Contextual)\n query_params = {"frequent_word": {"frequent_tuning": True}}\n context_embed, query_embed = self.token_embedder(\n context, question, query_params=query_params, query_align=self.aligned_query_embedding\n )\n\n context_mask = f.get_mask_from_tokens(context).float()\n query_mask = f.get_mask_from_tokens(question).float()\n\n B, C_L = context_embed.size(0), context_embed.size(1)\n\n context_embed = self.context_highway(context_embed)\n query_embed = self.query_highway(query_embed)\n\n context_encoded = f.forward_rnn_with_pack(\n self.context_contextual_rnn, context_embed, context_seq_config\n )\n context_encoded = self.dropout(context_encoded)\n\n query_encoded = f.forward_rnn_with_pack(\n self.query_contextual_rnn, query_embed, query_seq_config\n )\n query_encoded = self.dropout(query_encoded)\n\n # Attention Flow Layer\n attention_context_query = self.attention(\n context_encoded, context_mask, query_encoded, query_mask\n )\n\n # Modeling Layer\n modeled_context = f.forward_rnn_with_pack(\n self.modeling_rnn, attention_context_query, context_seq_config\n )\n modeled_context = self.dropout(modeled_context)\n\n M_D = modeled_context.size(-1)\n\n # Output Layer\n span_start_input = self.dropout(\n torch.cat([attention_context_query, modeled_context], dim=-1)\n ) # (B, C_L, 10d)\n span_start_logits = self.span_start_linear(span_start_input).squeeze(-1) # (B, C_L)\n span_start_probs = f.masked_softmax(span_start_logits, context_mask)\n\n span_start_representation = f.weighted_sum(\n attention=span_start_probs, matrix=modeled_context\n )\n tiled_span_start_representation = span_start_representation.unsqueeze(1).expand(B, C_L, M_D)\n\n span_end_representation = torch.cat(\n [\n attention_context_query,\n modeled_context,\n tiled_span_start_representation,\n modeled_context * tiled_span_start_representation,\n ],\n dim=-1,\n )\n encoded_span_end = f.forward_rnn_with_pack(\n self.output_end_rnn, span_end_representation, context_seq_config\n )\n encoded_span_end = self.dropout(encoded_span_end)\n\n span_end_input = self.dropout(\n torch.cat([attention_context_query, encoded_span_end], dim=-1)\n )\n span_end_logits = self.span_end_linear(span_end_input).squeeze(-1)\n\n # Masked Value\n span_start_logits = f.add_masked_value(span_start_logits, context_mask, value=-1e7)\n span_end_logits = f.add_masked_value(span_end_logits, context_mask, value=-1e7)\n\n output_dict = {\n "start_logits": span_start_logits,\n "end_logits": span_end_logits,\n "best_span": self.get_best_span(\n span_start_logits, span_end_logits, answer_maxlen=self.answer_maxlen\n ),\n }\n\n if labels:\n data_idx = labels["data_idx"]\n answer_start_idx = labels["answer_start_idx"]\n answer_end_idx = labels["answer_end_idx"]\n\n output_dict["data_idx"] = data_idx\n\n # Loss\n loss = self.criterion(span_start_logits, answer_start_idx)\n loss += self.criterion(span_end_logits, answer_end_idx)\n output_dict["loss"] = loss.unsqueeze(0) # NOTE: DataParallel concat Error\n\n return output_dict\n\n\nfrom overrides import overrides\nimport torch\nimport torch.nn as nn\n\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithTokenEmbedder\nfrom claf.model.reading_comprehension.mixin import SQuADv2\nimport claf.modules.functional as f\nimport claf.modules.attention as attention\nimport claf.modules.layer as layer\n\n\n[docs]@register("model:bidaf_no_answer")\nclass BiDAF_No_Answer(SQuADv2, ModelWithTokenEmbedder):\n """\n Question Answering Model. `Span Detector`, `No Answer`\n\n Bidirectional Attention Flow for Machine Comprehension + Bias (No_Answer)\n\n - Embedding (Word + Char -> Contextual)\n - Attention Flow\n - Modeling (RNN)\n - Output\n\n * Args:\n token_embedder: 'QATokenEmbedder', Used to embed the 'context' and 'question'.\n\n * Kwargs:\n lang_code: Dataset language code [en|ko]\n aligned_query_embedding: f_align(p_i) = sum(a_ij, E(qj), where the attention score a_ij\n captures the similarity between pi and each question words q_j.\n these features add soft alignments between similar but non-identical words (e.g., car and vehicle)\n it only apply to 'context_embed'.\n answer_maxlen: the most probable answer span of length less than or equal to {answer_maxlen}\n model_dim: the number of model dimension\n dropout: the dropout probability\n """\n\n def __init__(\n self,\n token_embedder,\n lang_code="en",\n aligned_query_embedding=False,\n answer_maxlen=None,\n model_dim=100,\n contextual_rnn_num_layer=1,\n modeling_rnn_num_layer=2,\n predict_rnn_num_layer=1,\n dropout=0.2,\n ):\n super(BiDAF_No_Answer, self).__init__(token_embedder)\n\n self.lang_code = lang_code\n self.aligned_query_embedding = aligned_query_embedding\n self.answer_maxlen = answer_maxlen\n self.token_embedder = token_embedder\n self.dropout = nn.Dropout(p=dropout)\n\n context_embed_dim, query_embed_dim = token_embedder.get_embed_dim()\n if self.aligned_query_embedding:\n context_embed_dim += query_embed_dim\n\n if context_embed_dim != query_embed_dim:\n self.context_highway = layer.Highway(context_embed_dim)\n self.context_contextual_rnn = nn.LSTM(\n input_size=context_embed_dim,\n hidden_size=model_dim,\n bidirectional=True,\n num_layers=contextual_rnn_num_layer,\n batch_first=True,\n )\n\n self.query_highway = layer.Highway(query_embed_dim)\n self.query_contextual_rnn = nn.LSTM(\n input_size=query_embed_dim,\n hidden_size=model_dim,\n bidirectional=True,\n num_layers=contextual_rnn_num_layer,\n batch_first=True,\n )\n else:\n highway = layer.Highway(query_embed_dim)\n\n self.context_highway = highway\n self.query_highway = highway\n\n contextual_rnn = nn.LSTM(\n input_size=context_embed_dim,\n hidden_size=model_dim,\n bidirectional=True,\n num_layers=contextual_rnn_num_layer,\n batch_first=True,\n )\n\n self.context_contextual_rnn = contextual_rnn\n self.query_contextual_rnn = contextual_rnn\n\n self.attention = attention.BiAttention(model_dim)\n self.modeling_rnn = nn.LSTM(\n input_size=8 * model_dim,\n hidden_size=model_dim,\n num_layers=modeling_rnn_num_layer,\n bidirectional=True,\n dropout=dropout,\n batch_first=True,\n )\n self.output_end_rnn = nn.LSTM(\n input_size=14 * model_dim,\n hidden_size=model_dim,\n bidirectional=True,\n num_layers=predict_rnn_num_layer,\n batch_first=True,\n )\n\n self.span_start_linear = nn.Linear(10 * model_dim, 1)\n self.span_end_linear = nn.Linear(10 * model_dim, 1)\n\n self.bias = nn.Parameter(torch.randn(1, 1))\n\n self.criterion = nn.CrossEntropyLoss()\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {"feature_name1": {\n "token_name1": tensor,\n "toekn_name2": tensor},\n "feature_name2": ...}\n\n * Kwargs:\n label: label dictionary like below.\n {"label_name1": tensor,\n "label_name2": tensor}\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - start_logits: representing unnormalized log probabilities of the span start position.\n - end_logits: representing unnormalized log probabilities of the span end position.\n - best_span: the string from the original passage that the model thinks is the best answer to the question.\n - data_idx: the question id, mapping with answer\n - loss: A scalar loss to be optimised.\n """\n\n context = features["context"]\n question = features["question"]\n\n # Sorted Sequence config (seq_lengths, perm_idx, unperm_idx) for RNN pack_forward\n context_seq_config = f.get_sorted_seq_config(context)\n query_seq_config = f.get_sorted_seq_config(question)\n\n # Embedding Layer (Char + Word -> Contextual)\n query_params = {"frequent_word": {"frequent_tuning": True}}\n context_embed, query_embed = self.token_embedder(\n context, question, query_params=query_params, query_align=self.aligned_query_embedding\n )\n\n context_mask = f.get_mask_from_tokens(context).float()\n query_mask = f.get_mask_from_tokens(question).float()\n\n B, C_L = context_embed.size(0), context_embed.size(1)\n\n context_embed = self.context_highway(context_embed)\n query_embed = self.query_highway(query_embed)\n\n context_encoded = f.forward_rnn_with_pack(\n self.context_contextual_rnn, context_embed, context_seq_config\n )\n context_encoded = self.dropout(context_encoded)\n\n query_encoded = f.forward_rnn_with_pack(\n self.query_contextual_rnn, query_embed, query_seq_config\n )\n query_encoded = self.dropout(query_encoded)\n\n # Attention Flow Layer\n attention_context_query = self.attention(\n context_encoded, context_mask, query_encoded, query_mask\n )\n\n # Modeling Layer\n modeled_context = f.forward_rnn_with_pack(\n self.modeling_rnn, attention_context_query, context_seq_config\n )\n modeled_context = self.dropout(modeled_context)\n\n M_D = modeled_context.size(-1)\n\n # Output Layer\n span_start_input = self.dropout(\n torch.cat([attention_context_query, modeled_context], dim=-1)\n ) # (B, C_L, 10d)\n span_start_logits = self.span_start_linear(span_start_input).squeeze(-1) # (B, C_L)\n span_start_probs = f.masked_softmax(span_start_logits, context_mask)\n\n span_start_representation = f.weighted_sum(\n attention=span_start_probs, matrix=modeled_context\n )\n tiled_span_start_representation = span_start_representation.unsqueeze(1).expand(B, C_L, M_D)\n\n span_end_representation = torch.cat(\n [\n attention_context_query,\n modeled_context,\n tiled_span_start_representation,\n modeled_context * tiled_span_start_representation,\n ],\n dim=-1,\n )\n encoded_span_end = f.forward_rnn_with_pack(\n self.output_end_rnn, span_end_representation, context_seq_config\n )\n encoded_span_end = self.dropout(encoded_span_end)\n\n span_end_input = self.dropout(\n torch.cat([attention_context_query, encoded_span_end], dim=-1)\n )\n span_end_logits = self.span_end_linear(span_end_input).squeeze(-1)\n\n # Masked Value\n span_start_logits = f.add_masked_value(span_start_logits, context_mask, value=-1e7)\n span_end_logits = f.add_masked_value(span_end_logits, context_mask, value=-1e7)\n\n # No_Answer Bias\n bias = self.bias.expand(B, 1)\n span_start_logits = torch.cat([span_start_logits, bias], dim=-1)\n span_end_logits = torch.cat([span_end_logits, bias], dim=-1)\n\n output_dict = {\n "start_logits": span_start_logits,\n "end_logits": span_end_logits,\n "best_span": self.get_best_span(\n span_start_logits[:, :-1],\n span_end_logits[:, :-1],\n answer_maxlen=self.answer_maxlen, # except no_answer bias\n ),\n }\n\n if labels:\n data_idx = labels["data_idx"]\n answer_start_idx = labels["answer_start_idx"]\n answer_end_idx = labels["answer_end_idx"]\n answerable = labels["answerable"]\n\n # No_Asnwer Case\n C_L = context_mask.size(1)\n answer_start_idx = answer_start_idx.masked_fill(answerable.eq(0), C_L)\n answer_end_idx = answer_end_idx.masked_fill(answerable.eq(0), C_L)\n\n output_dict["data_idx"] = data_idx\n\n # Loss\n loss = self.criterion(span_start_logits, answer_start_idx)\n loss += self.criterion(span_end_logits, answer_end_idx)\n output_dict["loss"] = loss.unsqueeze(0) # NOTE: DataParallel concat Error\n\n return output_dict\n\nfrom overrides import overrides\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithTokenEmbedder\nfrom claf.model.reading_comprehension.mixin import SQuADv1\nfrom claf.modules import attention, initializer\nfrom claf.modules import functional as f\n\n\n[docs]@register("model:docqa")\nclass DocQA(SQuADv1, ModelWithTokenEmbedder):\n """\n Document Reader Model. `Span Detector`\n\n Implementation of model presented in\n Simple and Effective Multi-Paragraph Reading Comprehension\n (https://arxiv.org/abs/1710.10723)\n\n - Embedding (Word + Char -> Contextual)\n - Attention\n - Residual self-attention\n - Output\n\n * Args:\n token_embedder: 'QATokenEmbedder', Used to embed the 'context' and 'question'.\n\n * Kwargs:\n lang_code: Dataset language code [en|ko]\n aligned_query_embedding: f_align(p_i) = sum(a_ij, E(qj), where the attention score a_ij\n captures the similarity between pi and each question words q_j.\n these features add soft alignments between similar but non-identical words (e.g., car and vehicle)\n it only apply to 'context_embed'.\n answer_maxlen: the most probable answer span of length less than or equal to {answer_maxlen}\n rnn_dim: the number of RNN cell dimension\n linear_dim: the number of attention linear dimension\n preprocess_rnn_num_layer: the number of recurrent layers (preprocess)\n modeling_rnn_num_layer: the number of recurrent layers (modeling)\n predict_rnn_num_layer: the number of recurrent layers (predict)\n dropout: the dropout probability\n """\n\n def __init__(\n self,\n token_embedder,\n lang_code="en",\n aligned_query_embedding=False,\n answer_maxlen=17,\n rnn_dim=100,\n linear_dim=200,\n preprocess_rnn_num_layer=1,\n modeling_rnn_num_layer=2,\n predict_rnn_num_layer=1,\n dropout=0.2,\n weight_init=True,\n ):\n super(DocQA, self).__init__(token_embedder)\n\n self.lang_code = lang_code\n self.aligned_query_embedding = aligned_query_embedding\n self.answer_maxlen = answer_maxlen\n self.token_embedder = token_embedder\n self.dropout = nn.Dropout(p=dropout)\n\n context_embed_dim, query_embed_dim = token_embedder.get_embed_dim()\n if self.aligned_query_embedding:\n context_embed_dim += query_embed_dim\n\n if context_embed_dim != query_embed_dim:\n self.context_preprocess_rnn = nn.GRU(\n input_size=context_embed_dim,\n hidden_size=rnn_dim,\n bidirectional=True,\n num_layers=preprocess_rnn_num_layer,\n batch_first=True,\n )\n self.query_preprocess_rnn = nn.GRU(\n input_size=query_embed_dim,\n hidden_size=rnn_dim,\n bidirectional=True,\n num_layers=preprocess_rnn_num_layer,\n batch_first=True,\n )\n else:\n preprocess_rnn = nn.GRU(\n input_size=context_embed_dim,\n hidden_size=rnn_dim,\n bidirectional=True,\n num_layers=preprocess_rnn_num_layer,\n batch_first=True,\n )\n\n self.context_preprocess_rnn = preprocess_rnn\n self.query_preprocess_rnn = preprocess_rnn\n\n self.bi_attention = attention.DocQAAttention(rnn_dim, linear_dim)\n self.attn_linear = nn.Linear(rnn_dim * 8, linear_dim)\n\n self.modeling_rnn = nn.GRU(\n input_size=linear_dim,\n hidden_size=rnn_dim,\n num_layers=modeling_rnn_num_layer,\n bidirectional=True,\n batch_first=True,\n )\n self.self_attention = SelfAttention(rnn_dim, linear_dim, weight_init=weight_init)\n\n self.span_start_rnn = nn.GRU(\n input_size=linear_dim,\n hidden_size=rnn_dim,\n bidirectional=True,\n num_layers=predict_rnn_num_layer,\n batch_first=True,\n )\n self.span_start_linear = nn.Linear(rnn_dim * 2, 1)\n\n self.span_end_rnn = nn.GRU(\n input_size=linear_dim + rnn_dim * 2,\n hidden_size=rnn_dim,\n bidirectional=True,\n num_layers=predict_rnn_num_layer,\n batch_first=True,\n )\n self.span_end_linear = nn.Linear(rnn_dim * 2, 1)\n\n self.activation_fn = F.relu\n self.criterion = nn.CrossEntropyLoss()\n\n if weight_init:\n modules = [\n self.context_preprocess_rnn,\n self.query_preprocess_rnn,\n self.modeling_rnn,\n self.attn_linear,\n self.span_start_rnn,\n self.span_start_linear,\n self.span_end_rnn,\n self.span_end_linear,\n ]\n initializer.weight(modules)\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {"feature_name1": {\n "token_name1": tensor,\n "toekn_name2": tensor},\n "feature_name2": ...}\n\n * Kwargs:\n label: label dictionary like below.\n {"label_name1": tensor,\n "label_name2": tensor}\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - start_logits: representing unnormalized log probabilities of the span start position.\n - end_logits: representing unnormalized log probabilities of the span end position.\n - best_span: the string from the original passage that the model thinks is the best answer to the question.\n - data_idx: the question id, mapping with answer\n - loss: A scalar loss to be optimised.\n """\n\n context = features["context"]\n question = features["question"]\n\n # Sorted Sequence config (seq_lengths, perm_idx, unperm_idx) for RNN pack_forward\n context_seq_config = f.get_sorted_seq_config(context)\n query_seq_config = f.get_sorted_seq_config(question)\n\n # Embedding\n query_params = {"frequent_word": {"frequent_tuning": True}}\n context_embed, query_embed = self.token_embedder(\n context, question, query_params=query_params, query_align=self.aligned_query_embedding\n )\n\n context_mask = f.get_mask_from_tokens(context).float() # B X 1 X C_L\n query_mask = f.get_mask_from_tokens(question).float() # B X 1 X Q_L\n\n # Pre-process\n context_embed = self.dropout(context_embed)\n context_encoded = f.forward_rnn_with_pack(\n self.context_preprocess_rnn, context_embed, context_seq_config\n )\n context_encoded = self.dropout(context_encoded)\n\n query_embed = self.dropout(query_embed)\n query_encoded = f.forward_rnn_with_pack(\n self.query_preprocess_rnn, query_embed, query_seq_config\n )\n query_encoded = self.dropout(query_encoded)\n\n # Attention -> Projection\n context_attnded = self.bi_attention(\n context_encoded, context_mask, query_encoded, query_mask\n )\n context_attnded = self.activation_fn(self.attn_linear(context_attnded)) # B X C_L X dim*2\n\n # Residual Self-Attention\n context_attnded = self.dropout(context_attnded)\n context_encoded = f.forward_rnn_with_pack(\n self.modeling_rnn, context_attnded, context_seq_config\n )\n context_encoded = self.dropout(context_encoded)\n\n context_self_attnded = self.self_attention(context_encoded, context_mask) # B X C_L X dim*2\n context_final = self.dropout(context_attnded + context_self_attnded) # B X C_L X dim*2\n\n # Prediction\n span_start_input = f.forward_rnn_with_pack(\n self.span_start_rnn, context_final, context_seq_config\n ) # B X C_L X dim*2\n span_start_input = self.dropout(span_start_input)\n span_start_logits = self.span_start_linear(span_start_input).squeeze(-1) # B X C_L\n\n span_end_input = torch.cat([span_start_input, context_final], dim=-1) # B X C_L X dim*4\n span_end_input = f.forward_rnn_with_pack(\n self.span_end_rnn, span_end_input, context_seq_config\n ) # B X C_L X dim*2\n span_end_input = self.dropout(span_end_input)\n span_end_logits = self.span_end_linear(span_end_input).squeeze(-1) # B X C_L\n\n # Masked Value\n span_start_logits = f.add_masked_value(span_start_logits, context_mask, value=-1e7)\n span_end_logits = f.add_masked_value(span_end_logits, context_mask, value=-1e7)\n\n output_dict = {\n "start_logits": span_start_logits,\n "end_logits": span_end_logits,\n "best_span": self.get_best_span(\n span_start_logits, span_end_logits, answer_maxlen=self.answer_maxlen\n ),\n }\n\n if labels:\n data_idx = labels["data_idx"]\n answer_start_idx = labels["answer_start_idx"]\n answer_end_idx = labels["answer_end_idx"]\n\n output_dict["data_idx"] = data_idx\n\n # Loss\n loss = self.criterion(span_start_logits, answer_start_idx)\n loss += self.criterion(span_end_logits, answer_end_idx)\n output_dict["loss"] = loss.unsqueeze(0) # NOTE: DataParallel concat Error\n\n return output_dict\n\n\n[docs]class SelfAttention(nn.Module):\n """\n Same bi-attention mechanism, only now between the passage and itself.\n """\n\n def __init__(self, rnn_dim, linear_dim, dropout=0.2, weight_init=True):\n super(SelfAttention, self).__init__()\n\n self.self_attention = attention.DocQAAttention(\n rnn_dim, linear_dim, self_attn=True, weight_init=weight_init\n )\n self.self_attn_Linear = nn.Linear(rnn_dim * 6, linear_dim)\n self.dropout = nn.Dropout(p=dropout)\n self.activation_fn = F.relu\n\n if weight_init:\n initializer.weight(self.self_attn_Linear)\n\n[docs] def forward(self, context, context_mask):\n context_self_attnded = self.self_attention(context, context_mask, context, context_mask)\n return self.activation_fn(self.self_attn_Linear(context_self_attnded))\n\nfrom overrides import overrides\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithTokenEmbedder\nfrom claf.model.reading_comprehension.mixin import SQuADv2\nfrom claf.modules import attention, initializer\nfrom claf.modules import functional as f\n\n\n[docs]@register("model:docqa_no_answer")\nclass DocQA_No_Answer(SQuADv2, ModelWithTokenEmbedder):\n """\n Question Answering Model. `Span Detector`, `No Answer`\n\n Implementation of model presented in\n Simple and Effective Multi-Paragraph Reading Comprehension + No_Asnwer\n (https://arxiv.org/abs/1710.10723)\n\n - Embedding (Word + Char -> Contextual)\n - Attention\n - Residual self-attention\n - Output\n\n * Args:\n token_embedder: 'QATokenEmbedder', Used to embed the 'context' and 'question'.\n\n * Kwargs:\n lang_code: Dataset language code [en|ko]\n aligned_query_embedding: f_align(p_i) = sum(a_ij, E(qj), where the attention score a_ij\n captures the similarity between pi and each question words q_j.\n these features add soft alignments between similar but non-identical words (e.g., car and vehicle)\n it only apply to 'context_embed'.\n answer_maxlen: the most probable answer span of length less than or equal to {answer_maxlen}\n rnn_dim: the number of RNN cell dimension\n linear_dim: the number of attention linear dimension\n preprocess_rnn_num_layer: the number of recurrent layers (preprocess)\n modeling_rnn_num_layer: the number of recurrent layers (modeling)\n predict_rnn_num_layer: the number of recurrent layers (predict)\n dropout: the dropout probability\n """\n\n def __init__(\n self,\n token_embedder,\n lang_code="en",\n aligned_query_embedding=False,\n answer_maxlen=17,\n rnn_dim=100,\n linear_dim=200,\n preprocess_rnn_num_layer=1,\n modeling_rnn_num_layer=2,\n predict_rnn_num_layer=1,\n dropout=0.2,\n weight_init=True,\n ):\n super(DocQA_No_Answer, self).__init__(token_embedder)\n\n self.lang_code = lang_code\n self.aligned_query_embedding = aligned_query_embedding\n self.answer_maxlen = answer_maxlen\n self.token_embedder = token_embedder\n self.dropout = nn.Dropout(p=dropout)\n\n context_embed_dim, query_embed_dim = token_embedder.get_embed_dim()\n if self.aligned_query_embedding:\n context_embed_dim += query_embed_dim\n\n if context_embed_dim != query_embed_dim:\n self.context_preprocess_rnn = nn.GRU(\n input_size=context_embed_dim,\n hidden_size=rnn_dim,\n bidirectional=True,\n num_layers=preprocess_rnn_num_layer,\n batch_first=True,\n )\n self.query_preprocess_rnn = nn.GRU(\n input_size=query_embed_dim,\n hidden_size=rnn_dim,\n bidirectional=True,\n num_layers=preprocess_rnn_num_layer,\n batch_first=True,\n )\n else:\n preprocess_rnn = nn.GRU(\n input_size=context_embed_dim,\n hidden_size=rnn_dim,\n bidirectional=True,\n num_layers=preprocess_rnn_num_layer,\n batch_first=True,\n )\n\n self.context_preprocess_rnn = preprocess_rnn\n self.query_preprocess_rnn = preprocess_rnn\n\n self.bi_attention = attention.DocQAAttention(rnn_dim, linear_dim)\n self.attn_linear = nn.Linear(rnn_dim * 8, linear_dim)\n\n self.modeling_rnn = nn.GRU(\n input_size=linear_dim,\n hidden_size=rnn_dim,\n num_layers=modeling_rnn_num_layer,\n bidirectional=True,\n batch_first=True,\n )\n self.self_attention = SelfAttention(rnn_dim, linear_dim, weight_init=weight_init)\n\n self.span_start_rnn = nn.GRU(\n input_size=linear_dim,\n hidden_size=rnn_dim,\n bidirectional=True,\n num_layers=predict_rnn_num_layer,\n batch_first=True,\n )\n self.span_start_linear = nn.Linear(rnn_dim * 2, 1)\n\n self.span_end_rnn = nn.GRU(\n input_size=linear_dim + rnn_dim * 2,\n hidden_size=rnn_dim,\n bidirectional=True,\n num_layers=predict_rnn_num_layer,\n batch_first=True,\n )\n self.span_end_linear = nn.Linear(rnn_dim * 2, 1)\n\n self.no_answer_op = NoAnswer(context_embed_dim, 80)\n\n self.activation_fn = F.relu\n self.criterion = nn.CrossEntropyLoss()\n\n if weight_init:\n modules = [\n self.context_preprocess_rnn,\n self.query_preprocess_rnn,\n self.modeling_rnn,\n self.attn_linear,\n self.span_start_rnn,\n self.span_start_linear,\n self.span_end_rnn,\n self.span_end_linear,\n ]\n initializer.weight(modules)\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {"feature_name1": {\n "token_name1": tensor,\n "toekn_name2": tensor},\n "feature_name2": ...}\n\n * Kwargs:\n label: label dictionary like below.\n {"label_name1": tensor,\n "label_name2": tensor}\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - start_logits: representing unnormalized log probabilities of the span start position.\n - end_logits: representing unnormalized log probabilities of the span end position.\n - best_span: the string from the original passage that the model thinks is the best answer to the question.\n - data_idx: the question id, mapping with answer\n - loss: A scalar loss to be optimised.\n """\n\n context = features["context"]\n question = features["question"]\n\n # Sorted Sequence config (seq_lengths, perm_idx, unperm_idx) for RNN pack_forward\n context_seq_config = f.get_sorted_seq_config(context)\n query_seq_config = f.get_sorted_seq_config(question)\n\n # Embedding\n query_params = {"frequent_word": {"frequent_tuning": True}}\n context_embed, query_embed = self.token_embedder(\n context, question, query_params=query_params, query_align=self.aligned_query_embedding\n )\n\n context_mask = f.get_mask_from_tokens(context).float() # B X C_L\n query_mask = f.get_mask_from_tokens(question).float() # B X Q_L\n\n # Pre-process\n context_embed = self.dropout(context_embed)\n context_encoded = f.forward_rnn_with_pack(\n self.context_preprocess_rnn, context_embed, context_seq_config\n )\n context_encoded = self.dropout(context_encoded)\n\n query_embed = self.dropout(query_embed)\n query_encoded = f.forward_rnn_with_pack(\n self.query_preprocess_rnn, query_embed, query_seq_config\n )\n query_encoded = self.dropout(query_encoded)\n\n # Attention -> Projection\n context_attnded = self.bi_attention(\n context_encoded, context_mask, query_encoded, query_mask\n )\n context_attnded = self.activation_fn(self.attn_linear(context_attnded)) # B X C_L X dim*2\n\n # Residual Self-Attention\n context_attnded = self.dropout(context_attnded)\n context_encoded = f.forward_rnn_with_pack(\n self.modeling_rnn, context_attnded, context_seq_config\n )\n context_encoded = self.dropout(context_encoded)\n\n context_self_attnded = self.self_attention(context_encoded, context_mask) # B X C_L X dim*2\n context_final = self.dropout(context_attnded + context_self_attnded) # B X C_L X dim*2\n\n # Prediction\n span_start_input = f.forward_rnn_with_pack(\n self.span_start_rnn, context_final, context_seq_config\n ) # B X C_L X dim*2\n span_start_input = self.dropout(span_start_input)\n span_start_logits = self.span_start_linear(span_start_input).squeeze(-1) # B X C_L\n\n span_end_input = torch.cat([span_start_input, context_final], dim=-1) # B X C_L X dim*4\n span_end_input = f.forward_rnn_with_pack(\n self.span_end_rnn, span_end_input, context_seq_config\n ) # B X C_L X dim*2\n span_end_input = self.dropout(span_end_input)\n span_end_logits = self.span_end_linear(span_end_input).squeeze(-1) # B X C_L\n\n # Masked Value\n span_start_logits = f.add_masked_value(span_start_logits, context_mask, value=-1e7)\n span_end_logits = f.add_masked_value(span_end_logits, context_mask, value=-1e7)\n\n # No_Asnwer Option\n bias = self.no_answer_op(context_embed, span_start_logits, span_end_logits)\n\n span_start_logits = torch.cat([span_start_logits, bias], dim=-1)\n span_end_logits = torch.cat([span_end_logits, bias], dim=-1)\n\n output_dict = {\n "start_logits": span_start_logits,\n "end_logits": span_end_logits,\n "best_span": self.get_best_span(\n span_start_logits[:, :-1],\n span_end_logits[:, :-1],\n answer_maxlen=self.answer_maxlen, # except no_answer bias\n ),\n }\n\n if labels:\n data_idx = labels["data_idx"]\n answer_start_idx = labels["answer_start_idx"]\n answer_end_idx = labels["answer_end_idx"]\n answerable = labels["answerable"]\n\n # No_Asnwer Case\n C_L = context_mask.size(1)\n answer_start_idx = answer_start_idx.masked_fill(answerable.eq(0), C_L)\n answer_end_idx = answer_end_idx.masked_fill(answerable.eq(0), C_L)\n\n output_dict["data_idx"] = data_idx\n\n # Loss\n loss = self.criterion(span_start_logits, answer_start_idx)\n loss += self.criterion(span_end_logits, answer_end_idx)\n output_dict["loss"] = loss.unsqueeze(0) # NOTE: DataParallel concat Error\n\n return output_dict\n\n\n[docs]class SelfAttention(nn.Module):\n """\n Same bi-attention mechanism, only now between the passage and itself.\n """\n\n def __init__(self, rnn_dim, linear_dim, dropout=0.2, weight_init=True):\n super(SelfAttention, self).__init__()\n\n self.self_attention = attention.DocQAAttention(\n rnn_dim, linear_dim, self_attn=True, weight_init=weight_init\n )\n self.self_attn_Linear = nn.Linear(rnn_dim * 6, linear_dim)\n self.dropout = nn.Dropout(p=dropout)\n self.activation_fn = F.relu\n\n if weight_init:\n initializer.weight(self.self_attn_Linear)\n\n[docs] def forward(self, context, context_mask):\n context_self_attnded = self.self_attention(context, context_mask, context, context_mask)\n context_self_attnded = self.activation_fn(self.self_attn_Linear(context_self_attnded))\n\n return context_self_attnded\n\n\n[docs]class NoAnswer(nn.Module):\n """\n No-Answer Option\n\n * Args:\n embed_dim: the number of passage embedding dimension\n bias_hidden_dim: bias use two layer mlp, the number of hidden_size\n """\n\n def __init__(self, embed_dim, bias_hidden_dim):\n super(NoAnswer, self).__init__()\n\n self.self_attn = nn.Linear(embed_dim, 1)\n self.bias_mlp = nn.Sequential(\n nn.Linear(embed_dim * 3, bias_hidden_dim), nn.ReLU(), nn.Linear(bias_hidden_dim, 1)\n )\n\n[docs] def forward(self, context_embed, span_start_logits, span_end_logits):\n p_1_h = F.softmax(span_start_logits, -1).unsqueeze(1) # B,1,T\n p_2_h = F.softmax(span_end_logits, -1).unsqueeze(1) # B,1,T\n p_3_h = self.self_attn(context_embed).transpose(1, 2) # B,1,T\n\n v_1 = torch.matmul(p_1_h, context_embed) # B,1,D\n v_2 = torch.matmul(p_2_h, context_embed) # B,1,D\n v_3 = torch.matmul(p_3_h, context_embed) # B,1,D\n\n return self.bias_mlp(torch.cat([v_1, v_2, v_3], -1)).squeeze(-1)\n\nfrom overrides import overrides\n\nimport torch.nn as nn\n\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithTokenEmbedder\nfrom claf.model.reading_comprehension.mixin import SQuADv1\nimport claf.modules.functional as f\nimport claf.modules.attention as attention\n\n\n[docs]@register("model:drqa")\nclass DrQA(SQuADv1, ModelWithTokenEmbedder):\n """\n Document Reader Model. `Span Detector`\n\n Implementation of model presented in\n Reading Wikipedia to Answer Open-Domain Questions\n (https://arxiv.org/abs/1704.00051)\n\n - Embedding + features\n - Align question embedding\n\n * Args:\n token_embedder: 'QATokenEmbedder', Used to embed the 'context' and 'question'.\n\n * Kwargs:\n lang_code: Dataset language code [en|ko]\n aligned_query_embedding: f_align(p_i) = sum(a_ij, E(qj), where the attention score a_ij\n captures the similarity between pi and each question words q_j.\n these features add soft alignments between similar but non-identical words (e.g., car and vehicle)\n it only apply to 'context_embed'.\n answer_maxlen: the most probable answer span of length less than or equal to {answer_maxlen}\n model_dim: the number of model dimension\n dropout: the dropout probability\n """\n\n def __init__(\n self,\n token_embedder,\n lang_code="en",\n aligned_query_embedding=False,\n answer_maxlen=None,\n model_dim=128,\n dropout=0.3,\n ):\n super(DrQA, self).__init__(token_embedder)\n\n self.lang_code = lang_code\n self.aligned_query_embedding = aligned_query_embedding\n self.answer_maxlen = answer_maxlen\n self.token_embedder = token_embedder\n self.dropout = nn.Dropout(p=dropout)\n\n context_embed_dim, query_embed_dim = token_embedder.get_embed_dim()\n if self.aligned_query_embedding:\n context_embed_dim += query_embed_dim\n\n self.paragraph_rnn = nn.LSTM(\n input_size=context_embed_dim,\n hidden_size=model_dim,\n num_layers=3,\n dropout=dropout,\n bidirectional=True,\n batch_first=True,\n )\n\n self.query_rnn = nn.LSTM(\n input_size=query_embed_dim,\n hidden_size=model_dim,\n num_layers=3,\n dropout=dropout,\n bidirectional=True,\n batch_first=True,\n )\n\n self.query_att = attention.LinearSeqAttn(model_dim * 2)\n\n self.start_attn = attention.BilinearSeqAttn(model_dim * 2, model_dim * 2)\n self.end_attn = attention.BilinearSeqAttn(model_dim * 2, model_dim * 2)\n\n self.criterion = nn.CrossEntropyLoss()\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {"feature_name1": {\n "token_name1": tensor,\n "toekn_name2": tensor},\n "feature_name2": ...}\n\n * Kwargs:\n label: label dictionary like below.\n {"label_name1": tensor,\n "label_name2": tensor}\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - start_logits: representing unnormalized log probabilities of the span start position.\n - end_logits: representing unnormalized log probabilities of the span end position.\n - best_span: the string from the original passage that the model thinks is the best answer to the question.\n - data_idx: the question id, mapping with answer\n - loss: A scalar loss to be optimised.\n """\n\n context = features["context"] # aka paragraph\n question = features["question"]\n\n # Sorted Sequence config (seq_lengths, perm_idx, unperm_idx) for RNN pack_forward\n context_seq_config = f.get_sorted_seq_config(context)\n query_seq_config = f.get_sorted_seq_config(question)\n\n # Embedding\n query_params = {"frequent_word": {"frequent_tuning": True}}\n context_embed, query_embed = self.token_embedder(\n context, question, query_params=query_params, query_align=self.aligned_query_embedding\n )\n\n context_mask = f.get_mask_from_tokens(context).float()\n query_mask = f.get_mask_from_tokens(question).float()\n\n context_embed = self.dropout(context_embed)\n query_embed = self.dropout(query_embed)\n\n # RNN (LSTM)\n context_encoded = f.forward_rnn_with_pack(\n self.paragraph_rnn, context_embed, context_seq_config\n )\n context_encoded = self.dropout(context_encoded)\n\n query_encoded = f.forward_rnn_with_pack(\n self.query_rnn, query_embed, query_seq_config\n ) # (B, Q_L, H*2)\n query_encoded = self.dropout(query_encoded)\n\n query_attention = self.query_att(query_encoded, query_mask) # (B, Q_L)\n query_att_sum = f.weighted_sum(query_attention, query_encoded) # (B, H*2)\n\n span_start_logits = self.start_attn(context_encoded, query_att_sum, context_mask)\n span_end_logits = self.end_attn(context_encoded, query_att_sum, context_mask)\n\n # Masked Value\n span_start_logits = f.add_masked_value(span_start_logits, context_mask, value=-1e7)\n span_end_logits = f.add_masked_value(span_end_logits, context_mask, value=-1e7)\n\n output_dict = {\n "start_logits": span_start_logits,\n "end_logits": span_end_logits,\n "best_span": self.get_best_span(\n span_start_logits, span_end_logits, answer_maxlen=self.answer_maxlen\n ),\n }\n\n if labels:\n data_idx = labels["data_idx"]\n answer_start_idx = labels["answer_start_idx"]\n answer_end_idx = labels["answer_end_idx"]\n\n output_dict["data_idx"] = data_idx\n\n loss = self.criterion(span_start_logits, answer_start_idx)\n loss += self.criterion(span_end_logits, answer_end_idx)\n output_dict["loss"] = loss.unsqueeze(0)\n\n return output_dict\n\n\nfrom collections import OrderedDict\n\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\n\nfrom claf.decorator import arguments_required\nfrom claf.metric import korquad_v1_official, squad_v1_official, squad_v2_official\nfrom claf.model.base import ModelBase\n\n\n[docs]class ReadingComprehension:\n """\n Reading Comprehension Mixin Class\n\n * Args:\n token_embedder: 'RCTokenEmbedder', Used to embed the 'context' and 'question'.\n\n """\n\n[docs] def get_best_span(self, span_start_logits, span_end_logits, answer_maxlen=None):\n """\n Take argmax of constrained score_s * score_e.\n\n * Args:\n span_start_logits: independent start logits\n span_end_logits: independent end logits\n\n * Kwargs:\n answer_maxlen: max span length to consider (default is None -> All)\n """\n\n B = span_start_logits.size(0)\n best_word_span = span_start_logits.new_zeros((B, 2), dtype=torch.long)\n\n score_starts = F.softmax(span_start_logits, dim=-1)\n score_ends = F.softmax(span_end_logits, dim=-1)\n\n max_len = answer_maxlen or score_starts.size(1)\n\n for i in range(score_starts.size(0)):\n # Outer product of scores to get full p_s * p_e matrix\n scores = torch.ger(score_starts[i], score_ends[i])\n\n # Zero out negative length and over-length span scores\n scores.triu_().tril_(max_len - 1)\n\n # Take argmax or top n\n scores = scores.detach().cpu().numpy()\n scores_flat = scores.flatten()\n\n idx_sort = [np.argmax(scores_flat)]\n\n s_idx, e_idx = np.unravel_index(idx_sort, scores.shape)\n best_word_span[i, 0] = int(s_idx[0])\n best_word_span[i, 1] = int(e_idx[0])\n\n return best_word_span\n\n def _make_span_metrics(self, predictions):\n """ span accuracy metrics """\n start_accuracy, end_accuracy, span_accuracy = 0, 0, 0\n\n for index, preds in predictions.items():\n _, _, (answer_start, answer_end) = self._dataset.get_ground_truths(index)\n\n start_acc = 1 if preds["pred_span_start"] == answer_start else 0\n end_acc = 1 if preds["pred_span_end"] == answer_end else 0\n span_acc = 1 if start_acc == 1 and end_acc == 1 else 0\n\n start_accuracy += start_acc\n end_accuracy += end_acc\n span_accuracy += span_acc\n\n start_accuracy = 100.0 * start_accuracy / len(self._dataset)\n end_accuracy = 100.0 * end_accuracy / len(self._dataset)\n span_accuracy = 100.0 * span_accuracy / len(self._dataset)\n\n return {"start_acc": start_accuracy, "end_acc": end_accuracy, "span_acc": span_accuracy}\n\n[docs] def make_predictions(self, output_dict):\n """\n Make predictions with model's output_dict\n\n * Args:\n output_dict: model's output dictionary consisting of\n - data_idx: question id\n - best_span: calculate the span_start_logits and span_end_logits to what is the best span\n - start_logits: span start logits\n - end_logits: span end logits\n\n * Returns:\n predictions: prediction dictionary consisting of\n - key: 'id' (question id)\n - value: consisting of dictionary\n predict_text, pred_span_start, pred_span_end, span_start_prob, span_end_prob\n """\n\n data_indices = output_dict["data_idx"]\n best_word_span = output_dict["best_span"]\n\n return OrderedDict(\n [\n (\n index.item(),\n {\n "predict_text": self._dataset.get_text_with_index(\n index.item(), best_span[0], best_span[1]\n ),\n "pred_span_start": best_span[0],\n "pred_span_end": best_span[1],\n "start_logits": start_logits,\n "end_logits": end_logits,\n },\n )\n for index, best_span, start_logits, end_logits in zip(\n list(data_indices.data),\n list(best_word_span.data),\n list(output_dict["start_logits"].data),\n list(output_dict["end_logits"].data),\n )\n ]\n )\n\n @arguments_required(["context", "question"])\n def predict(self, output_dict, arguments, helper):\n """\n Inference by raw_feature\n\n * Args:\n output_dict: model's output dictionary consisting of\n - data_idx: question id\n - best_span: calculate the span_start_logits and span_end_logits to what is the best span\n arguments: arguments dictionary consisting of user_input\n helper: dictionary for helping get answer\n\n * Returns:\n span: predict best_span\n """\n span_start, span_end = list(output_dict["best_span"][0].data)\n word_start = span_start.item()\n word_end = span_end.item()\n\n text_span = helper["text_span"]\n char_start = text_span[word_start][0]\n char_end = text_span[word_end][1]\n\n context_text = arguments["context"]\n answer_text = context_text[char_start:char_end]\n\n start_logit = output_dict["start_logits"][0]\n end_logit = output_dict["end_logits"][0]\n\n score = start_logit[span_start] + end_logit[span_end]\n score = score.item()\n\n return {"text": answer_text, "score": score}\n\n[docs] def print_examples(self, index, inputs, predictions):\n """\n Print evaluation examples\n\n * Args:\n index: data index\n inputs: mini-batch inputs\n predictions: prediction dictionary consisting of\n - key: 'id' (question id)\n - value: consisting of dictionary\n predict_text, pred_span_start, pred_span_end, span_start_prob, span_end_prob\n\n * Returns:\n print(Context, Question, Answers and Predict)\n """\n data_index = inputs["labels"]["data_idx"][index].item()\n qid = self._dataset.get_qid(data_index)\n if "#" in qid: # bert case (qid#index)\n qid = qid.split("#")[0]\n\n helper = self._dataset.helper\n\n context = helper["examples"][qid]["context"]\n question = helper["examples"][qid]["question"]\n answers = helper["examples"][qid]["answers"]\n\n predict_text = predictions[data_index]["predict_text"]\n\n print()\n print("- Context:", context)\n print("- Question:", question)\n print("- Answers:", answers)\n print("- Predict:", predict_text)\n print()\n\n\n # TODO: start and end logits (TypeError: Object of type 'Tensor' is not JSON serializable)\n # try:\n # super(ReadingComprehension, self).write_predictions(\n # predictions, file_path=file_path, is_dict=is_dict\n # )\n # except AttributeError:\n # # TODO: Need to Fix\n # model_base = ModelBase()\n # model_base._log_dir = self._log_dir\n # model_base._train_counter = self._train_counter\n # model_base.training = self.training\n # model_base.write_predictions(predictions, file_path=file_path, is_dict=is_dict)\n\n\n[docs]class SQuADv1(ReadingComprehension):\n """\n Reading Comprehension Mixin Class\n with SQuAD v1.1 evaluation\n\n * Args:\n token_embedder: 'QATokenEmbedder', Used to embed the 'context' and 'question'.\n\n """\n\n[docs] def make_metrics(self, predictions):\n """\n Make metrics with prediction dictionary\n\n * Args:\n predictions: prediction dictionary consisting of\n - key: 'id' (question id)\n - value: (predict_text, pred_span_start, pred_span_end)\n\n * Returns:\n metrics: metric dictionary consisting of\n - 'em': exact_match (SQuAD v1.1 official evaluation)\n - 'f1': f1 (SQuAD v1.1 official evaluation)\n - 'start_acc': span_start accuracy\n - 'end_acc': span_end accuracy\n - 'span_acc': span accuracy (start and end)\n """\n\n preds = {}\n for index, prediction in predictions.items():\n _, _, (answer_start, answer_end) = self._dataset.get_ground_truths(index)\n\n qid = self._dataset.get_qid(index)\n preds[qid] = prediction["predict_text"]\n\n self.write_predictions(preds)\n\n squad_offical_metrics = self._make_metrics_with_official(preds)\n\n metrics = self._make_span_metrics(predictions)\n metrics.update(squad_offical_metrics)\n return metrics\n\n def _make_metrics_with_official(self, preds):\n """ SQuAD v1.1 official evaluation """\n dataset = self._dataset.raw_dataset\n\n if self.lang_code.startswith("ko"):\n scores = korquad_v1_official.evaluate(dataset, preds)\n else:\n scores = squad_v1_official.evaluate(dataset, preds)\n return scores\n\n\n[docs]class SQuADv1ForBert(SQuADv1):\n """\n Reading Comprehension Mixin Class\n with SQuAD v1.1 evaluation\n\n * Args:\n token_embedder: 'QATokenEmbedder', Used to embed the 'context' and 'question'.\n\n """\n\n[docs] def make_metrics(self, predictions):\n """ BERT predictions need to get nbest result """\n\n best_predictions = {}\n for index, prediction in predictions.items():\n qid = self._dataset.get_qid(index)\n\n predict_text = prediction["predict_text"]\n\n start_logit = prediction["start_logits"][prediction["pred_span_start"]]\n end_logit = prediction["end_logits"][prediction["pred_span_end"]]\n predict_score = start_logit.item() + end_logit.item()\n\n if qid not in best_predictions:\n best_predictions[qid] = []\n best_predictions[qid].append((predict_text, predict_score))\n\n for qid, predictions in best_predictions.items():\n sorted_predictions = sorted(predictions, key=lambda x: x[1], reverse=True)\n best_predictions[qid] = sorted_predictions[0][0]\n\n self.write_predictions(best_predictions)\n return self._make_metrics_with_official(best_predictions)\n\n[docs] def predict(self, output_dict, arguments, helper):\n """\n Inference by raw_feature\n\n * Args:\n output_dict: model's output dictionary consisting of\n - data_idx: question id\n - best_span: calculate the span_start_logits and span_end_logits to what is the best span\n arguments: arguments dictionary consisting of user_input\n helper: dictionary for helping get answer\n\n * Returns:\n span: predict best_span\n """\n\n context_text = arguments["context"]\n bert_tokens = helper["bert_token"]\n predictions = [\n (best_span, start_logits, end_logits)\n for best_span, start_logits, end_logits in zip(\n list(output_dict["best_span"].data),\n list(output_dict["start_logits"].data),\n list(output_dict["end_logits"].data),\n )\n ]\n\n best_predictions = []\n for index, prediction in enumerate(predictions):\n bert_token = bert_tokens[index]\n best_span, start_logits, end_logits = prediction\n pred_start, pred_end = best_span\n\n predict_text = ""\n if (\n pred_start < len(bert_token)\n and pred_end < len(bert_token)\n and bert_token[pred_start].text_span is not None\n and bert_token[pred_end].text_span is not None\n ):\n char_start = bert_token[pred_start].text_span[0]\n char_end = bert_token[pred_end].text_span[1]\n predict_text = context_text[char_start:char_end]\n\n start_logit = start_logits[pred_start]\n end_logit = end_logits[pred_end]\n predict_score = start_logit.item() + end_logit.item()\n\n best_predictions.append((predict_text, predict_score))\n\n sorted_predictions = sorted(best_predictions, key=lambda x: x[1], reverse=True)\n return {"text": sorted_predictions[0][0], "score": sorted_predictions[0][1]}\n\n\n[docs]class SQuADv2(ReadingComprehension):\n """\n Reading Comprehension Mixin Class\n with SQuAD v2.0 evaluation\n\n * Args:\n token_embedder: 'RCTokenEmbedder', Used to embed the 'context' and 'question'.\n\n """\n\n[docs] def make_metrics(self, predictions):\n """\n Make metrics with prediction dictionary\n\n * Args:\n predictions: prediction dictionary consisting of\n - key: 'id' (question id)\n - value: consisting of dictionary\n predict_text, pred_span_start, pred_span_end, span_start_prob, span_end_prob\n\n * Returns:\n metrics: metric dictionary consisting of\n - 'start_acc': span_start accuracy\n - 'end_acc': span_end accuracy\n - 'span_acc': span accuracy (start and end)\n - 'em': exact_match (SQuAD v2.0 official evaluation)\n - 'f1': f1 (SQuAD v2.0 official evaluation)\n - 'HasAns_exact': has answer exact_match\n - 'HasAns_f1': has answer f1\n - 'NoAns_exact': no answer exact_match\n - 'NoAns_f1': no answer f1\n - 'best_exact': best exact_match score with best_exact_thresh\n - 'best_exact_thresh': best exact_match answerable threshold\n - 'best_f1': best f1 score with best_f1_thresh\n - 'best_f1_thresh': best f1 answerable threshold\n """\n\n preds, na_probs = {}, {}\n for index, prediction in predictions.items():\n _, _, (answer_start, answer_end) = self._dataset.get_ground_truths(index)\n\n # Metrics (SQuAD official metric)\n predict_text = prediction["predict_text"]\n if predict_text == "<noanswer>":\n predict_text = ""\n\n qid = self._dataset.get_qid(index)\n preds[qid] = predict_text\n\n span_start_probs = F.softmax(prediction["start_logits"], dim=-1)\n span_end_probs = F.softmax(prediction["end_logits"], dim=-1)\n\n start_no_prob = span_start_probs[-1].item()\n end_no_prob = span_end_probs[-1].item()\n no_answer_prob = start_no_prob * end_no_prob\n na_probs[qid] = no_answer_prob\n\n self.write_predictions(preds)\n\n model_type = "train" if self.training else "valid"\n self.write_predictions(\n na_probs, file_path=f"na_probs-{model_type}-{self._train_counter.get_display()}.json"\n )\n\n squad_offical_metrics = self._make_metrics_with_official(preds, na_probs)\n\n metrics = self._make_span_metrics(predictions)\n metrics.update(squad_offical_metrics)\n return metrics\n\n def _make_metrics_with_official(self, preds, na_probs, na_prob_thresh=1.0):\n """ SQuAD 2.0 official evaluation """\n dataset = self._dataset.raw_dataset\n\n squad_scores = squad_v2_official.evaluate(dataset, na_probs, preds)\n squad_scores["em"] = squad_scores["exact"]\n\n remove_keys = ["total", "exact", "HasAns_total", "NoAns_total"]\n for key in remove_keys:\n if key in squad_scores:\n del squad_scores[key]\n\n return squad_scores\n\nimport torch\nimport torch.nn as nn\n\nfrom overrides import overrides\n\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithTokenEmbedder\nfrom claf.model.reading_comprehension.mixin import SQuADv1\nimport claf.modules.functional as f\nimport claf.modules.attention as attention\nimport claf.modules.encoder as encoder\nimport claf.modules.conv as conv\nimport claf.modules.layer as layer\n\n\n[docs]@register("model:qanet")\nclass QANet(SQuADv1, ModelWithTokenEmbedder):\n """\n Document Reader Model. `Span Detector`\n\n Implementation of model presented in\n QANet:Combining Local Convolution with Global Self-Attention for Reading Comprehension\n (https://arxiv.org/abs/1804.09541)\n\n - Input Embedding Layer\n - Embedding Encoder Layer\n - Context-Query Attention Layer\n - Model Encoder Layer\n - Output Layer\n\n * Args:\n token_embedder: 'QATokenEmbedder', Used to embed the 'context' and 'question'.\n\n * Kwargs:\n lang_code: Dataset language code [en|ko]\n aligned_query_embedding: f_align(p_i) = sum(a_ij, E(qj), where the attention score a_ij\n captures the similarity between pi and each question words q_j.\n these features add soft alignments between similar but non-identical words (e.g., car and vehicle)\n it only apply to 'context_embed'.\n answer_maxlen: the most probable answer span of length less than or equal to {answer_maxlen}\n model_dim: the number of model dimension\n\n * Encoder Block Parameters (embedding, modeling)\n kernel_size: convolution kernel size in encoder block\n num_head: the number of multi-head attention's head\n num_conv_block: the number of convolution block in encoder block\n [Layernorm -> Conv (residual)]\n num_encoder_block: the number of the encoder block\n [position_encoding -> [n repeat conv block] -> Layernorm -> Self-attention (residual)\n -> Layernorm -> Feedforward (residual)]\n\n dropout: the dropout probability\n layer_dropout: the layer dropout probability\n (cf. Deep Networks with Stochastic Depth(https://arxiv.org/abs/1603.09382) )\n """\n\n def __init__(\n self,\n token_embedder,\n lang_code="en",\n aligned_query_embedding=False,\n answer_maxlen=None,\n model_dim=128,\n kernel_size_in_embedding=7,\n num_head_in_embedding=8,\n num_conv_block_in_embedding=4,\n num_embedding_encoder_block=1,\n kernel_size_in_modeling=5,\n num_head_in_modeling=8,\n num_conv_block_in_modeling=2,\n num_modeling_encoder_block=7,\n dropout=0.1,\n layer_dropout=0.9,\n ):\n super(QANet, self).__init__(token_embedder)\n\n self.lang_code = lang_code\n self.aligned_query_embedding = aligned_query_embedding\n self.answer_maxlen = answer_maxlen\n self.token_embedder = token_embedder\n\n context_embed_dim, query_embed_dim = token_embedder.get_embed_dim()\n\n if self.aligned_query_embedding:\n context_embed_dim += query_embed_dim\n\n if context_embed_dim != query_embed_dim:\n self.context_highway = layer.Highway(context_embed_dim)\n self.query_highway = layer.Highway(query_embed_dim)\n\n self.context_embed_pointwise_conv = conv.PointwiseConv(context_embed_dim, model_dim)\n self.query_embed_pointwise_conv = conv.PointwiseConv(query_embed_dim, model_dim)\n else:\n highway = layer.Highway(context_embed_dim)\n\n self.context_highway = highway\n self.query_highway = highway\n\n embed_pointwise_conv = conv.PointwiseConv(context_embed_dim, model_dim)\n\n self.context_embed_pointwise_conv = embed_pointwise_conv\n self.query_embed_pointwise_conv = embed_pointwise_conv\n\n self.embed_encoder_blocks = nn.ModuleList(\n [\n EncoderBlock(\n model_dim=model_dim,\n kernel_size=kernel_size_in_embedding,\n num_head=num_head_in_embedding,\n num_conv_block=num_conv_block_in_modeling,\n dropout=dropout,\n layer_dropout=layer_dropout,\n )\n for _ in range(num_embedding_encoder_block)\n ]\n )\n\n self.co_attention = attention.CoAttention(model_dim)\n\n self.pointwise_conv = conv.PointwiseConv(model_dim * 4, model_dim)\n self.model_encoder_blocks = nn.ModuleList(\n [\n EncoderBlock(\n model_dim=model_dim,\n kernel_size=kernel_size_in_modeling,\n num_head=num_head_in_modeling,\n num_conv_block=num_conv_block_in_modeling,\n dropout=dropout,\n layer_dropout=layer_dropout,\n )\n for _ in range(num_modeling_encoder_block)\n ]\n )\n\n self.span_start_linear = nn.Linear(model_dim * 2, 1, bias=False)\n self.span_end_linear = nn.Linear(model_dim * 2, 1, bias=False)\n\n self.dropout = nn.Dropout(p=dropout)\n\n self.criterion = nn.CrossEntropyLoss()\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {"feature_name1": {\n "token_name1": tensor,\n "toekn_name2": tensor},\n "feature_name2": ...}\n\n * Kwargs:\n label: label dictionary like below.\n {"label_name1": tensor,\n "label_name2": tensor}\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - start_logits: representing unnormalized log probabilities of the span start position.\n - end_logits: representing unnormalized log probabilities of the span end position.\n - best_span: the string from the original passage that the model thinks is the best answer to the question.\n - data_idx: the question id, mapping with answer\n - loss: A scalar loss to be optimised.\n """\n\n context = features["context"]\n question = features["question"]\n\n # 1. Input Embedding Layer\n query_params = {"frequent_word": {"frequent_tuning": True}}\n context_embed, query_embed = self.token_embedder(\n context, question, query_params=query_params, query_align=self.aligned_query_embedding\n )\n\n context_mask = f.get_mask_from_tokens(context).float()\n query_mask = f.get_mask_from_tokens(question).float()\n\n context_embed = self.context_highway(context_embed)\n context_embed = self.dropout(context_embed)\n context_embed = self.context_embed_pointwise_conv(context_embed)\n\n query_embed = self.query_highway(query_embed)\n query_embed = self.dropout(query_embed)\n query_embed = self.query_embed_pointwise_conv(query_embed)\n\n # 2. Embedding Encoder Layer\n for encoder_block in self.embed_encoder_blocks:\n context = encoder_block(context_embed)\n context_embed = context\n\n query = encoder_block(query_embed)\n query_embed = query\n\n # 3. Context-Query Attention Layer\n context_query_attention = self.co_attention(context, query, context_mask, query_mask)\n\n # Projection (memory issue)\n context_query_attention = self.pointwise_conv(context_query_attention)\n context_query_attention = self.dropout(context_query_attention)\n\n # 4. Model Encoder Layer\n model_encoder_block_inputs = context_query_attention\n\n # Stacked Model Encoder Block\n stacked_model_encoder_blocks = []\n for i in range(3):\n for _, model_encoder_block in enumerate(self.model_encoder_blocks):\n output = model_encoder_block(model_encoder_block_inputs, context_mask)\n model_encoder_block_inputs = output\n\n stacked_model_encoder_blocks.append(output)\n\n # 5. Output Layer\n span_start_inputs = torch.cat(\n [stacked_model_encoder_blocks[0], stacked_model_encoder_blocks[1]], dim=-1\n )\n span_start_inputs = self.dropout(span_start_inputs)\n span_start_logits = self.span_start_linear(span_start_inputs).squeeze(-1)\n\n span_end_inputs = torch.cat(\n [stacked_model_encoder_blocks[0], stacked_model_encoder_blocks[2]], dim=-1\n )\n span_end_inputs = self.dropout(span_end_inputs)\n span_end_logits = self.span_end_linear(span_end_inputs).squeeze(-1)\n\n # Masked Value\n span_start_logits = f.add_masked_value(span_start_logits, context_mask, value=-1e7)\n span_end_logits = f.add_masked_value(span_end_logits, context_mask, value=-1e7)\n\n output_dict = {\n "start_logits": span_start_logits,\n "end_logits": span_end_logits,\n "best_span": self.get_best_span(span_start_logits, span_end_logits),\n }\n\n if labels:\n data_idx = labels["data_idx"]\n answer_start_idx = labels["answer_start_idx"]\n answer_end_idx = labels["answer_end_idx"]\n\n output_dict["data_idx"] = data_idx\n\n # Loss\n loss = self.criterion(span_start_logits, answer_start_idx)\n loss += self.criterion(span_end_logits, answer_end_idx)\n output_dict["loss"] = loss.unsqueeze(0) # NOTE: DataParallel concat Error\n\n return output_dict\n\n\n[docs]class EncoderBlock(nn.Module):\n """\n Encoder Block\n\n []: residual\n position_encoding -> [convolution-layer] x # -> [self-attention-layer] -> [feed-forward-layer]\n\n - convolution-layer: depthwise separable convolutions\n - self-attention-layer: multi-head attention\n - feed-forward-layer: pointwise convolution\n\n * Args:\n model_dim: the number of model dimension\n num_heads: the number of head in multi-head attention\n kernel_size: convolution kernel size\n num_conv_block: the number of convolution block\n dropout: the dropout probability\n layer_dropout: the layer dropout probability\n (cf. Deep Networks with Stochastic Depth(https://arxiv.org/abs/1603.09382) )\n """\n\n def __init__(\n self,\n model_dim=128,\n num_head=8,\n kernel_size=5,\n num_conv_block=4,\n dropout=0.1,\n layer_dropout=0.9,\n ):\n super(EncoderBlock, self).__init__()\n\n self.position_encoding = encoder.PositionalEncoding(model_dim)\n self.dropout = nn.Dropout(dropout)\n\n self.num_conv_block = num_conv_block\n self.conv_blocks = nn.ModuleList(\n [conv.DepSepConv(model_dim, model_dim, kernel_size) for _ in range(num_conv_block)]\n )\n\n self.self_attention = attention.MultiHeadAttention(\n num_head=num_head, model_dim=model_dim, dropout=dropout\n )\n self.feedforward_layer = layer.PositionwiseFeedForward(\n model_dim, model_dim * 4, dropout=dropout\n )\n\n # survival probability for stochastic depth\n if layer_dropout < 1.0:\n L = (num_conv_block) + 2 - 1\n layer_dropout_prob = round(1 - (1 / L) * (1 - layer_dropout), 3)\n self.residuals = nn.ModuleList(\n layer.ResidualConnection(\n model_dim, layer_dropout=layer_dropout_prob, layernorm=True\n )\n for l in range(num_conv_block + 2)\n )\n else:\n self.residuals = nn.ModuleList(\n layer.ResidualConnection(model_dim, layernorm=True)\n for l in range(num_conv_block + 2)\n )\n\n[docs] def forward(self, x, mask=None):\n # Positional Encoding\n x = self.position_encoding(x)\n\n # Convolution Block (LayerNorm -> Conv)\n for i, conv_block in enumerate(self.conv_blocks):\n x = self.residuals[i](x, sub_layer_fn=conv_block)\n x = self.dropout(x)\n\n # LayerNorm -> Self-attention\n self_attention = lambda x: self.self_attention(q=x, k=x, v=x, mask=mask)\n x = self.residuals[self.num_conv_block](x, sub_layer_fn=self_attention)\n x = self.dropout(x)\n\n # LayerNorm -> Feedforward layer\n x = self.residuals[self.num_conv_block + 1](x, sub_layer_fn=self.feedforward_layer)\n x = self.dropout(x)\n return x\n\n\nfrom overrides import overrides\nfrom pytorch_transformers import RobertaModel\nimport torch.nn as nn\n\nfrom claf.data.data_handler import CachePath\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithoutTokenEmbedder\nfrom claf.model.reading_comprehension.mixin import SQuADv1ForBert\n\n\n[docs]@register("model:roberta_for_qa")\nclass RoBertaForQA(SQuADv1ForBert, ModelWithoutTokenEmbedder):\n """\n Document Reader Model. `Span Detector`\n\n Implementation of model presented in\n BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding\n (https://arxiv.org/abs/1810.04805)\n\n * Args:\n token_embedder: 'QATokenEmbedder', Used to embed the 'context' and 'question'.\n\n * Kwargs:\n lang_code: Dataset language code [en|ko]\n pretrained_model_name: the name of a pre-trained model\n answer_maxlen: the most probable answer span of length less than or equal to {answer_maxlen}\n """\n\n def __init__(self, token_makers, lang_code="en", pretrained_model_name=None, answer_maxlen=30):\n super(RoBertaForQA, self).__init__(token_makers)\n\n self.lang_code = lang_code\n self.use_pytorch_transformers = True # for optimizer's model parameters\n self.answer_maxlen = answer_maxlen\n\n self.model = RobertaModel.from_pretrained(\n pretrained_model_name, cache_dir=str(CachePath.ROOT)\n )\n self.qa_outputs = nn.Linear(self.model.config.hidden_size, self.model.config.num_labels)\n self.criterion = nn.CrossEntropyLoss()\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {"feature_name1": {\n "token_name1": tensor,\n "toekn_name2": tensor},\n "feature_name2": ...}\n\n * Kwargs:\n label: label dictionary like below.\n {"label_name1": tensor,\n "label_name2": tensor}\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - start_logits: representing unnormalized log probabilities of the span start position.\n - end_logits: representing unnormalized log probabilities of the span end position.\n - best_span: the string from the original passage that the model thinks is the best answer to the question.\n - data_idx: the question id, mapping with answer\n - loss: A scalar loss to be optimised.\n """\n\n bert_inputs = features["bert_input"]["feature"]\n attention_mask = (bert_inputs > 0).long()\n\n outputs = self.model(\n bert_inputs, token_type_ids=None, attention_mask=attention_mask\n )\n sequence_output = outputs[0]\n\n logits = self.qa_outputs(sequence_output)\n span_start_logits, span_end_logits = logits.split(1, dim=-1)\n\n span_start_logits = span_start_logits.squeeze(-1)\n span_end_logits = span_end_logits.squeeze(-1)\n\n output_dict = {\n "start_logits": span_start_logits,\n "end_logits": span_end_logits,\n "best_span": self.get_best_span(\n span_start_logits, span_end_logits, answer_maxlen=self.answer_maxlen\n ),\n }\n\n if labels:\n data_idx = labels["data_idx"]\n answer_start_idx = labels["answer_start_idx"]\n answer_end_idx = labels["answer_end_idx"]\n\n output_dict["data_idx"] = data_idx\n\n # If we are on multi-GPU, split add a dimension\n if len(answer_start_idx.size()) > 1:\n answer_start_idx = answer_start_idx.squeeze(-1)\n if len(answer_end_idx.size()) > 1:\n answer_end_idx = answer_end_idx.squeeze(-1)\n # sometimes the start/end positions are outside our model inputs, we ignore these terms\n ignored_index = span_start_logits.size(1)\n\n answer_start_idx.clamp_(0, ignored_index)\n answer_end_idx.clamp_(0, ignored_index)\n\n # Loss\n criterion = nn.CrossEntropyLoss(ignore_index=ignored_index)\n loss = criterion(span_start_logits, answer_start_idx)\n loss += criterion(span_end_logits, answer_end_idx)\n loss /= 2 # (start + end)\n output_dict["loss"] = loss\n\n return output_dict\n\n\nimport torch\n\nfrom claf.decorator import arguments_required\nfrom claf.metric import wikisql_official\nfrom claf.metric.wikisql_lib.dbengine import DBEngine\nfrom claf.metric.wikisql_lib.query import Query\n\n\n[docs]class WikiSQL:\n """\n WikiSQL Mixin Class\n with official evaluation\n\n * Args:\n token_embedder: 'TokenEmbedder'\n """\n\n AGG_OPS = ["None", "MAX", "MIN", "COUNT", "SUM", "AVG"]\n COND_OPS = ["EQL", "GT", "LT"]\n\n[docs] def make_metrics(self, predictions):\n """ aggregator, select_column, conditions accuracy """\n\n agg_accuracy, sel_accuracy, conds_accuracy = 0, 0, 0\n\n for index, pred in predictions.items():\n target = self._dataset.get_ground_truth(index)\n\n # Aggregator, Select_Column, Conditions\n agg_acc = 1 if pred["query"]["agg"] == target["agg_idx"] else 0\n sel_acc = 1 if pred["query"]["sel"] == target["sel_idx"] else 0\n\n pred_conds = pred["query"]["conds"]\n string_set_pred_conds = set(["#".join(map(str, cond)).lower() for cond in pred_conds])\n target_conds = [\n [target["conds_col"][i], target["conds_op"][i], target["conds_val_str"][i]]\n for i in range(target["conds_num"])\n ]\n string_set_target_conds = set(\n ["#".join(map(str, cond)).lower() for cond in target_conds]\n )\n\n conds_acc = (\n 1 if string_set_pred_conds == string_set_target_conds else 0\n ) # not matter in order\n\n agg_accuracy += agg_acc\n sel_accuracy += sel_acc\n conds_accuracy += conds_acc\n\n total_count = len(self._dataset)\n\n agg_accuracy = 100.0 * agg_accuracy / total_count\n sel_accuracy = 100.0 * sel_accuracy / total_count\n conds_accuracy = 100.0 * conds_accuracy / total_count\n\n metrics = {\n "agg_accuracy": agg_accuracy,\n "sel_accuracy": sel_accuracy,\n "conds_accuracy": conds_accuracy,\n }\n\n self.write_predictions(predictions)\n\n wikisql_official_metrics = self._make_metrics_with_official(predictions)\n metrics.update(wikisql_official_metrics)\n return metrics\n\n def _make_metrics_with_official(self, preds):\n """\n WikiSQL official evaluation\n\n lf_accuracy: Logical-form accuracy\n - Directly compare the synthesized SQL query with the ground truth to\n check whether they match each other.\n ex_accuracy: Execution accuracy\n - Execute both the synthesized query and the ground truth query and\n compare whether the results match to each other.\n """\n\n labels = self._dataset.labels\n db_path = self._dataset.helper["db_path"]\n\n return wikisql_official.evaluate(labels, preds, db_path)\n\n[docs] def make_predictions(self, output_dict):\n predictions = {}\n sql_quries = self.generate_queries(output_dict)\n\n for i in range(len(sql_quries)):\n query = sql_quries[i]\n\n prediction = {}\n prediction.update(query)\n\n data_id = self._dataset.get_id(output_dict["data_id"][i])\n predictions[data_id] = prediction\n return predictions\n\n[docs] def generate_queries(self, output_dict):\n preds_agg = torch.argmax(output_dict["agg_logits"], dim=-1)\n preds_sel = torch.argmax(output_dict["sel_logits"], dim=-1)\n\n conds_logits = output_dict["conds_logits"]\n conds_num_logits, conds_column_logits, conds_op_logits, conds_value_logits = conds_logits\n\n preds_conds_num = torch.argmax(conds_num_logits, dim=-1)\n preds_conds_op = torch.argmax(conds_op_logits, dim=-1)\n\n sql_quries = []\n B = output_dict["agg_logits"].size(0)\n\n for i in range(B):\n if "table_id" in output_dict:\n table_id = output_dict["table_id"]\n else:\n table_id = self._dataset.get_table_id(output_dict["data_id"][i])\n\n query = {\n "table_id": table_id,\n "query": {"agg": preds_agg[i].item(), "sel": preds_sel[i].item()},\n }\n\n pred_conds_num = preds_conds_num[i].item()\n conds_pred = []\n if pred_conds_num == 0:\n pass\n else:\n _, pred_conds_column_idx = torch.topk(conds_column_logits[i], pred_conds_num)\n\n if preds_conds_op.dim() == 1: # for one-example (TODO: fix hard-code)\n pred_conds_op = preds_conds_op\n conds_value_logits = conds_value_logits.squeeze(3)\n conds_value_logits = conds_value_logits.squeeze(0)\n else:\n pred_conds_op = preds_conds_op[i]\n\n if "tokenized_question" in output_dict:\n tokenized_question = output_dict["tokenized_question"]\n else:\n tokenized_question = self._dataset.get_tokenized_question(\n output_dict["data_id"][i]\n )\n\n conds_pred = [\n [\n pred_conds_column_idx[j].item(),\n pred_conds_op[j].item(),\n self.decode_pointer(tokenized_question, conds_value_logits[i][j]),\n ]\n for j in range(pred_conds_num)\n ]\n\n query["query"]["conds"] = conds_pred\n sql_quries.append(query)\n return sql_quries\n\n[docs] def decode_pointer(self, tokenized_question, cond_value_logits):\n question_text = " ".join(tokenized_question)\n tokenized_question = ["<BEG>"] + tokenized_question + ["<END>"]\n\n conds_value = []\n for value_logit in cond_value_logits:\n pred_value_pos = torch.argmax(value_logit[: len(tokenized_question)]).item()\n pred_value_token = tokenized_question[pred_value_pos]\n if pred_value_token == "<END>":\n break\n conds_value.append(pred_value_token)\n\n conds_value = self.merge_tokens(conds_value, question_text)\n return conds_value\n\n[docs] def merge_tokens(self, tok_list, raw_tok_str):\n lower_tok_str = raw_tok_str.lower()\n alphabet = set("abcdefghijklmnopqrstuvwxyz0123456789$(")\n special = {\n "-LRB-": "(",\n "-RRB-": ")",\n "-LSB-": "[",\n "-RSB-": "]",\n "``": '"',\n "''": '"',\n "--": "\\u2013",\n }\n ret = ""\n double_quote_appear = 0\n for raw_tok in tok_list:\n if not raw_tok:\n continue\n tok = special.get(raw_tok, raw_tok)\n lower_tok = tok.lower()\n if tok == '"':\n double_quote_appear = 1 - double_quote_appear\n\n if len(ret) == 0:\n pass\n elif len(ret) > 0 and ret + " " + lower_tok in lower_tok_str:\n ret = ret + " "\n elif len(ret) > 0 and ret + lower_tok in lower_tok_str:\n pass\n elif lower_tok == '"':\n if double_quote_appear:\n ret = ret + " "\n elif lower_tok[0] not in alphabet:\n pass\n elif (ret[-1] not in ["(", "/", "\\u2013", "#", "$", "&"]) and (\n ret[-1] != '"' or not double_quote_appear\n ):\n ret = ret + " "\n ret = ret + tok\n return ret.strip()\n\n @arguments_required(["db_path", "table_id"])\n def predict(self, output_dict, arguments, helper):\n """\n Inference by raw_feature\n\n * Args:\n output_dict: model's output dictionary consisting of\n arguments: arguments dictionary consisting of user_input\n helper: dictionary for helping get answer\n\n * Returns:\n query: Generated SQL Query\n execute_result: Execute result by generated query\n """\n output_dict["table_id"] = arguments["table_id"]\n output_dict["tokenized_question"] = helper["tokenized_question"]\n\n prediction = self.generate_queries(output_dict)[0]\n pred_query = Query.from_dict(prediction["query"], ordered=True)\n\n dbengine = DBEngine(arguments["db_path"])\n try:\n pred_execute_result = dbengine.execute_query(\n prediction["table_id"], pred_query, lower=True\n )\n except IndexError as e:\n pred_execute_result = str(e)\n\n return {"query": str(pred_query), "execute_result": pred_execute_result}\n\n[docs] def print_examples(self, index, inputs, predictions):\n """\n Print evaluation examples\n\n * Args:\n index: data index\n inputs: mini-batch inputs\n predictions: prediction dictionary consisting of\n - key: 'id' (question id)\n - value: consisting of dictionary\n table_id, query (agg, sel, conds)\n\n * Returns:\n print(Context, Question, Answers and Predict)\n """\n\n data_index = inputs["labels"]["data_idx"][index].item()\n data_id = self._dataset.get_id(data_index)\n\n helper = self._dataset.helper\n question = helper["examples"][data_id]["question"]\n\n label = self._dataset.get_ground_truth(data_id)\n\n dbengine = DBEngine(helper["db_path"])\n\n prediction = predictions[data_id]\n pred_query = Query.from_dict(prediction["query"], ordered=True)\n pred_execute_result = dbengine.execute_query(prediction["table_id"], pred_query, lower=True)\n\n print("- Question:", question)\n print("- Answers:")\n print(" SQL Query: ", label["sql_query"])\n print(" Execute Results:", label["execution_result"])\n print("- Predict:")\n print(" SQL Query: ", pred_query)\n print(" Execute Results:", pred_execute_result)\n print("-" * 30)\n\n\nfrom overrides import overrides\n\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithTokenEmbedder\nfrom claf.model.semantic_parsing import utils\nfrom claf.model.semantic_parsing.mixin import WikiSQL\nimport claf.modules.functional as f\nimport claf.modules.attention as attention\n\n\n[docs]@register("model:sqlnet")\nclass SQLNet(WikiSQL, ModelWithTokenEmbedder):\n """\n Nature Language to SQL Query Model. `Semantic Parsing`, `NL2SQL`\n\n Implementation of model presented in\n SQLNet: Generating Structured Queries From Natural Language\n Without Reinforcement Learning\n (https://arxiv.org/abs/1711.04436)\n\n * Args:\n token_embedder: 'WikiSQLTokenEmbedder', Used to embed the 'column' and 'question'.\n\n * Kwargs:\n column_attention: highlight that column attention is a special instance of\n the generic attention mechanism to compute the attention map on a question\n conditioned on the column names.\n model_dim: the number of model dimension\n rnn_num_layer: the number of recurrent layers (all of rnn)\n column_maxlen: an upper-bound N on the number of columns to choose\n token_maxlen: conds value slot - pointer network an upper-bound N on the number of token\n conds_column_loss_alpha: balance the positive data versus negative data\n """\n\n def __init__(\n self,\n token_embedder,\n column_attention=None,\n model_dim=100,\n rnn_num_layer=2,\n dropout=0.3,\n column_maxlen=4,\n token_maxlen=200,\n conds_column_loss_alpha=3,\n ):\n super(SQLNet, self).__init__(token_embedder)\n\n embed_dim = token_embedder.get_embed_dim() # NOTE: need to fix\n self.token_maxlen = token_maxlen\n self.column_maxlen = column_maxlen\n self.conds_column_loss_alpha = conds_column_loss_alpha\n\n # Predict aggregator\n self.agg_predictor = AggPredictor(\n embed_dim, model_dim, rnn_num_layer, dropout, len(self.AGG_OPS)\n )\n\n # Predict selected column\n self.sel_predictor = SelPredictor(\n embed_dim, model_dim, rnn_num_layer, dropout, column_attention=column_attention\n )\n\n # #Predict number of conditions\n self.conds_predictor = CondsPredictor(\n embed_dim,\n model_dim,\n rnn_num_layer,\n dropout,\n len(self.COND_OPS),\n column_maxlen,\n token_maxlen,\n column_attention=column_attention,\n )\n\n self.cross_entropy = nn.CrossEntropyLoss()\n self.bce_logit = nn.BCEWithLogitsLoss()\n\n[docs] @overrides\n def forward(self, features, labels=None):\n column = features["column"]\n question = features["question"]\n\n column_embed = self.token_embedder(column)\n question_embed = self.token_embedder(question)\n\n B, C_L = column_embed.size(0), column_embed.size(1)\n\n column_indexed = column[next(iter(column))]\n column_name_mask = column_indexed.gt(0).float() # NOTE: hard-code\n column_lengths = utils.get_column_lengths(column_embed, column_name_mask)\n column_mask = column_lengths.view(B, C_L).gt(0).float() # NOTE: hard-code\n question_mask = f.get_mask_from_tokens(question).float()\n\n agg_logits = self.agg_predictor(question_embed, question_mask)\n sel_logits = self.sel_predictor(\n question_embed, question_mask, column_embed, column_name_mask, column_mask\n )\n\n conds_col_idx, conds_val_pos = None, None\n if labels:\n data_idx = labels["data_idx"]\n ground_truths = self._dataset.get_ground_truths(data_idx)\n\n conds_col_idx = [ground_truth["conds_col"] for ground_truth in ground_truths]\n conds_val_pos = [ground_truth["conds_val_pos"] for ground_truth in ground_truths]\n\n conds_logits = self.conds_predictor(\n question_embed,\n question_mask,\n column_embed,\n column_name_mask,\n column_mask,\n conds_col_idx,\n conds_val_pos,\n )\n\n # Convert GPU to CPU\n agg_logits = agg_logits.cpu()\n sel_logits = sel_logits.cpu()\n conds_logits = [logits.cpu() for logits in conds_logits]\n\n output_dict = {\n "agg_logits": agg_logits,\n "sel_logits": sel_logits,\n "conds_logits": conds_logits,\n }\n\n if labels:\n data_idx = labels["data_idx"]\n output_dict["data_id"] = data_idx\n\n ground_truths = self._dataset.get_ground_truths(data_idx)\n\n # Aggregator, Select Column\n target_agg_idx = torch.LongTensor(\n [ground_truth["agg_idx"] for ground_truth in ground_truths]\n )\n target_sel_idx = torch.LongTensor(\n [ground_truth["sel_idx"] for ground_truth in ground_truths]\n )\n\n loss = 0\n loss += self.cross_entropy(agg_logits, target_agg_idx)\n loss += self.cross_entropy(sel_logits, target_sel_idx)\n\n conds_num_logits, conds_column_logits, conds_op_logits, conds_value_logits = (\n conds_logits\n )\n\n # Conditions\n # 1. The number of conditions\n target_conds_num = torch.LongTensor(\n [ground_truth["conds_num"] for ground_truth in ground_truths]\n )\n target_conds_column = [ground_truth["conds_col"] for ground_truth in ground_truths]\n\n loss += self.cross_entropy(conds_num_logits, target_conds_num)\n\n # 2. Columns of conditions\n B = conds_column_logits.size(0)\n\n target_conds_columns = np.zeros(list(conds_column_logits.size()), dtype=np.float32)\n for i in range(B):\n target_conds_column_idx = target_conds_column[i]\n if len(target_conds_column_idx) == 0:\n continue\n target_conds_columns[i][target_conds_column_idx] = 1\n target_conds_columns = torch.from_numpy(target_conds_columns)\n conds_column_probs = torch.sigmoid(conds_column_logits)\n\n bce_loss = -torch.mean(\n self.conds_column_loss_alpha\n * (target_conds_columns * torch.log(conds_column_probs + 1e-10))\n + (1 - target_conds_columns) * torch.log(1 - conds_column_probs + 1e-10)\n )\n loss += bce_loss\n\n # 3. Operator of conditions\n conds_op_loss = 0\n for i in range(B):\n target_conds_op = ground_truths[i]["conds_op"]\n if len(target_conds_op) == 0:\n continue\n\n target_conds_op = torch.from_numpy(np.array(target_conds_op))\n logits_conds_op = conds_op_logits[i, : len(target_conds_op)]\n\n target_op_count = len(target_conds_op)\n conds_op_loss += (\n self.cross_entropy(logits_conds_op, target_conds_op) / target_op_count\n )\n loss += conds_op_loss\n\n # 4. Value of conditions\n conds_val_pos = [ground_truth["conds_val_pos"] for ground_truth in ground_truths]\n\n conds_value_loss = 0\n for i in range(B):\n for j in range(len(conds_val_pos[i])):\n cond_val_pos = conds_val_pos[i][j]\n if len(cond_val_pos) == 1:\n continue\n\n target_cond_val_pos = torch.from_numpy(\n np.array(cond_val_pos[1:])\n ) # index 0: START_TOKEN\n logits_cond_val_pos = conds_value_logits[i, j, : len(cond_val_pos) - 1]\n\n conds_value_loss += self.cross_entropy(\n logits_cond_val_pos, target_cond_val_pos\n ) / len(conds_val_pos[i])\n\n loss += conds_value_loss / B\n\n output_dict["loss"] = loss.unsqueeze(0)\n\n return output_dict\n\n\n[docs]class AggPredictor(nn.Module):\n def __init__(self, embed_dim, model_dim, rnn_num_layer, dropout, agg_count):\n super(AggPredictor, self).__init__()\n\n self.question_rnn = nn.LSTM(\n input_size=embed_dim,\n hidden_size=model_dim // 2,\n num_layers=rnn_num_layer,\n batch_first=True,\n dropout=dropout,\n bidirectional=True,\n )\n self.seq_attn = attention.LinearSeqAttn(model_dim)\n self.mlp = nn.Sequential(\n nn.Linear(model_dim, model_dim), nn.Tanh(), nn.Linear(model_dim, agg_count)\n )\n\n[docs] def forward(self, question_embed, question_mask):\n encoded_question, _ = self.question_rnn(question_embed)\n attn_matrix = self.seq_attn(encoded_question, question_mask)\n attn_question = f.weighted_sum(attn_matrix, encoded_question)\n logits = self.mlp(attn_question)\n return logits\n\n\n[docs]class SelPredictor(nn.Module):\n def __init__(self, embed_dim, model_dim, rnn_num_layer, dropout, column_attention=None):\n super(SelPredictor, self).__init__()\n self.column_attention = column_attention\n\n self.question_rnn = nn.LSTM(\n input_size=embed_dim,\n hidden_size=model_dim // 2,\n num_layers=rnn_num_layer,\n batch_first=True,\n dropout=dropout,\n bidirectional=True,\n )\n\n if column_attention:\n self.linear_attn = nn.Linear(model_dim, model_dim)\n else:\n self.seq_attn = attention.LinearSeqAttn(model_dim)\n\n self.column_rnn = nn.LSTM(\n input_size=embed_dim,\n hidden_size=model_dim // 2,\n num_layers=rnn_num_layer,\n batch_first=True,\n dropout=dropout,\n bidirectional=True,\n )\n\n self.linear_question = nn.Linear(model_dim, model_dim)\n self.linear_column = nn.Linear(model_dim, model_dim)\n self.mlp = nn.Sequential(nn.Tanh(), nn.Linear(model_dim, 1))\n\n[docs] def forward(self, question_embed, question_mask, column_embed, column_name_mask, column_mask):\n\n B, C_L, N_L, embed_D = list(column_embed.size())\n\n encoded_column = utils.encode_column(column_embed, column_name_mask, self.column_rnn)\n encoded_question, _ = self.question_rnn(question_embed)\n\n if self.column_attention:\n attn_matrix = torch.bmm(\n encoded_column, self.linear_attn(encoded_question).transpose(1, 2)\n )\n attn_matrix = f.add_masked_value(attn_matrix, question_mask.unsqueeze(1), value=-1e7)\n attn_matrix = F.softmax(attn_matrix, dim=-1)\n attn_question = (encoded_question.unsqueeze(1) * attn_matrix.unsqueeze(3)).sum(2)\n else:\n attn_matrix = self.seq_attn(encoded_question, question_mask)\n attn_question = f.weighted_sum(attn_matrix, encoded_question)\n attn_question = attn_question.unsqueeze(1)\n\n logits = self.mlp(\n self.linear_question(attn_question) + self.linear_column(encoded_column)\n ).squeeze()\n logits = f.add_masked_value(logits, column_mask, value=-1e7)\n return logits\n\n\n[docs]class CondsPredictor(nn.Module):\n def __init__(\n self,\n embed_dim,\n model_dim,\n rnn_num_layer,\n dropout,\n conds_op_count,\n column_maxlen,\n token_maxlen,\n column_attention=None,\n ):\n super(CondsPredictor, self).__init__()\n\n self.num_predictor = CondsNumPredictor(\n embed_dim, model_dim, rnn_num_layer, dropout, column_maxlen\n )\n self.column_predictor = CondsColPredictor(\n embed_dim, model_dim, rnn_num_layer, dropout, column_attention=column_attention\n )\n self.op_predictor = CondsOpPredictor(\n embed_dim,\n model_dim,\n rnn_num_layer,\n dropout,\n conds_op_count,\n column_maxlen,\n column_attention=column_attention,\n )\n self.value_pointer = CondsValuePointer(\n embed_dim, model_dim, rnn_num_layer, dropout, column_maxlen, token_maxlen\n )\n\n[docs] def forward(\n self,\n question_embed,\n question_mask,\n column_embed,\n column_name_mask,\n column_mask,\n col_idx,\n conds_val_pos,\n ):\n num_logits = self.num_predictor(\n question_embed, question_mask, column_embed, column_name_mask, column_mask\n )\n column_logits = self.column_predictor(\n question_embed, question_mask, column_embed, column_name_mask, column_mask\n )\n\n if col_idx is None:\n col_idx = []\n preds_num = torch.argmax(num_logits, dim=-1)\n for i in range(column_logits.size(0)):\n _, pred_conds_column_idx = torch.topk(column_logits[i], preds_num[i])\n col_idx.append(pred_conds_column_idx.tolist())\n\n op_logits = self.op_predictor(\n question_embed, question_mask, column_embed, column_name_mask, col_idx\n )\n value_logits = self.value_pointer(\n question_embed, question_mask, column_embed, column_name_mask, col_idx, conds_val_pos\n )\n\n return (num_logits, column_logits, op_logits, value_logits)\n\n\n[docs]class CondsNumPredictor(nn.Module):\n def __init__(self, embed_dim, model_dim, rnn_num_layer, dropout, column_maxlen):\n super(CondsNumPredictor, self).__init__()\n\n self.model_dim = model_dim\n self.column_maxlen = column_maxlen\n\n self.column_rnn = nn.LSTM(\n input_size=embed_dim,\n hidden_size=model_dim // 2,\n num_layers=rnn_num_layer,\n batch_first=True,\n dropout=dropout,\n bidirectional=True,\n )\n self.column_seq_attn = attention.LinearSeqAttn(model_dim)\n self.column_to_hidden_state = nn.Linear(model_dim, 2 * model_dim)\n self.column_to_cell_state = nn.Linear(model_dim, 2 * model_dim)\n\n self.question_rnn = nn.LSTM(\n input_size=embed_dim,\n hidden_size=model_dim // 2,\n num_layers=rnn_num_layer,\n batch_first=True,\n dropout=dropout,\n bidirectional=True,\n )\n self.question_seq_attn = attention.LinearSeqAttn(model_dim)\n\n self.mlp = nn.Sequential(\n nn.Linear(model_dim, model_dim), nn.Tanh(), nn.Linear(model_dim, column_maxlen + 1)\n )\n\n[docs] def forward(self, question_embed, question_mask, column_embed, column_name_mask, column_mask):\n B, C_L, N_L, embed_D = list(column_embed.size())\n\n encoded_column = utils.encode_column(column_embed, column_name_mask, self.column_rnn)\n attn_column = self.column_seq_attn(encoded_column, column_mask)\n out_column = f.weighted_sum(attn_column, encoded_column)\n\n question_rnn_hidden_state = (\n self.column_to_hidden_state(out_column)\n .view(B, self.column_maxlen, self.model_dim // 2)\n .transpose(0, 1)\n .contiguous()\n )\n question_rnn_cell_state = (\n self.column_to_cell_state(out_column)\n .view(B, self.column_maxlen, self.model_dim // 2)\n .transpose(0, 1)\n .contiguous()\n )\n\n encoded_question, _ = self.question_rnn(\n question_embed, (question_rnn_hidden_state, question_rnn_cell_state)\n )\n attn_question = self.question_seq_attn(encoded_question, question_mask)\n out_question = f.weighted_sum(attn_question, encoded_question)\n return self.mlp(out_question)\n\n\n[docs]class CondsColPredictor(nn.Module):\n def __init__(self, embed_dim, model_dim, rnn_num_layer, dropout, column_attention=None):\n super(CondsColPredictor, self).__init__()\n self.column_attention = column_attention\n\n self.question_rnn = nn.LSTM(\n input_size=embed_dim,\n hidden_size=model_dim // 2,\n num_layers=rnn_num_layer,\n batch_first=True,\n dropout=dropout,\n bidirectional=True,\n )\n\n if column_attention:\n self.linear_attn = nn.Linear(model_dim, model_dim)\n else:\n self.seq_attn = attention.LinearSeqAttn(model_dim)\n\n self.column_rnn = nn.LSTM(\n input_size=embed_dim,\n hidden_size=model_dim // 2,\n num_layers=rnn_num_layer,\n batch_first=True,\n dropout=dropout,\n bidirectional=True,\n )\n\n self.linear_question = nn.Linear(model_dim, model_dim)\n self.linear_column = nn.Linear(model_dim, model_dim)\n self.mlp = nn.Sequential(nn.ReLU(), nn.Linear(model_dim, 1))\n\n[docs] def forward(self, question_embed, question_mask, column_embed, column_name_mask, column_mask):\n B, C_L, N_L, embed_D = list(column_embed.size())\n\n # Column Encoder\n encoded_column = utils.encode_column(column_embed, column_name_mask, self.column_rnn)\n encoded_question, _ = self.question_rnn(question_embed)\n\n if self.column_attention:\n attn_matrix = torch.bmm(\n encoded_column, self.linear_attn(encoded_question).transpose(1, 2)\n )\n attn_matrix = f.add_masked_value(attn_matrix, question_mask.unsqueeze(1), value=-1e7)\n attn_matrix = F.softmax(attn_matrix, dim=-1)\n attn_question = (encoded_question.unsqueeze(1) * attn_matrix.unsqueeze(3)).sum(2)\n else:\n attn_matrix = self.seq_attn(encoded_question, question_mask)\n attn_question = f.weighted_sum(attn_matrix, encoded_question)\n attn_question = attn_question.unsqueeze(1)\n\n logits = self.mlp(\n self.linear_question(attn_question) + self.linear_column(encoded_column)\n ).squeeze()\n logits = f.add_masked_value(logits, column_mask, value=-1e7)\n return logits\n\n\n[docs]class CondsOpPredictor(nn.Module):\n def __init__(\n self,\n embed_dim,\n model_dim,\n rnn_num_layer,\n dropout,\n op_count,\n column_maxlen,\n column_attention=None,\n ):\n super(CondsOpPredictor, self).__init__()\n self.column_attention = column_attention\n self.column_maxlen = column_maxlen\n\n self.question_rnn = nn.LSTM(\n input_size=embed_dim,\n hidden_size=model_dim // 2,\n num_layers=rnn_num_layer,\n batch_first=True,\n dropout=dropout,\n bidirectional=True,\n )\n\n if column_attention:\n self.linear_attn = nn.Linear(model_dim, model_dim)\n else:\n self.seq_attn = attention.LinearSeqAttn(model_dim)\n\n self.column_rnn = nn.LSTM(\n input_size=embed_dim,\n hidden_size=model_dim // 2,\n num_layers=rnn_num_layer,\n batch_first=True,\n dropout=dropout,\n bidirectional=True,\n )\n\n self.linear_question = nn.Linear(model_dim, model_dim)\n self.linear_column = nn.Linear(model_dim, model_dim)\n self.mlp = nn.Sequential(\n nn.Linear(model_dim, model_dim), nn.Tanh(), nn.Linear(model_dim, op_count)\n )\n\n[docs] def forward(self, question_embed, question_mask, column_embed, column_name_mask, col_idx):\n B, C_L, N_L, embed_D = list(column_embed.size())\n\n # Column Encoder\n encoded_column = utils.encode_column(column_embed, column_name_mask, self.column_rnn)\n encoded_used_column = utils.filter_used_column(\n encoded_column, col_idx, padding_count=self.column_maxlen\n )\n\n encoded_question, _ = self.question_rnn(question_embed)\n if self.column_attention:\n attn_matrix = torch.matmul(\n self.linear_attn(encoded_question).unsqueeze(1), encoded_used_column.unsqueeze(3)\n ).squeeze()\n attn_matrix = f.add_masked_value(attn_matrix, question_mask.unsqueeze(1), value=-1e7)\n attn_matrix = F.softmax(attn_matrix, dim=-1)\n attn_question = (encoded_question.unsqueeze(1) * attn_matrix.unsqueeze(3)).sum(2)\n else:\n attn_matrix = self.seq_attn(encoded_question, question_mask)\n attn_question = f.weighted_sum(attn_matrix, encoded_question)\n attn_question = attn_question.unsqueeze(1)\n\n return self.mlp(\n self.linear_question(attn_question) + self.linear_column(encoded_used_column)\n ).squeeze()\n\n\n[docs]class CondsValuePointer(nn.Module):\n def __init__(self, embed_dim, model_dim, rnn_num_layer, dropout, column_maxlen, token_maxlen):\n super(CondsValuePointer, self).__init__()\n\n self.model_dim = model_dim\n self.column_maxlen = column_maxlen\n self.token_maxlen = token_maxlen\n\n self.question_rnn = nn.LSTM(\n input_size=embed_dim,\n hidden_size=model_dim // 2,\n num_layers=rnn_num_layer,\n batch_first=True,\n dropout=dropout,\n bidirectional=True,\n )\n self.seq_attn = attention.LinearSeqAttn(model_dim)\n\n self.column_rnn = nn.LSTM(\n input_size=embed_dim,\n hidden_size=model_dim // 2,\n num_layers=rnn_num_layer,\n batch_first=True,\n dropout=dropout,\n bidirectional=True,\n )\n\n self.decoder = nn.LSTM(\n input_size=self.token_maxlen,\n hidden_size=model_dim,\n num_layers=rnn_num_layer,\n batch_first=True,\n dropout=dropout,\n )\n\n self.linear_column = nn.Linear(model_dim, model_dim)\n self.linear_conds = nn.Linear(model_dim, model_dim)\n self.linear_question = nn.Linear(model_dim, model_dim)\n self.mlp = nn.Sequential(nn.ReLU(), nn.Linear(model_dim, 1))\n\n[docs] def forward(\n self, question_embed, question_mask, column_embed, column_name_mask, col_idx, conds_val_pos\n ):\n B, C_L, N_L, embed_D = list(column_embed.size())\n\n question_embed, question_mask = self.concat_start_and_end_zero_padding(\n question_embed, question_mask\n )\n\n # Column Encoder\n encoded_column = utils.encode_column(column_embed, column_name_mask, self.column_rnn)\n encoded_used_column = utils.filter_used_column(\n encoded_column, col_idx, padding_count=self.column_maxlen\n )\n\n encoded_question, _ = self.question_rnn(question_embed)\n\n encoded_used_column = encoded_used_column.unsqueeze(2).unsqueeze(2)\n encoded_question = encoded_question.unsqueeze(1).unsqueeze(1)\n\n if conds_val_pos is None: # inference\n MAX_DECODER_STEP = 50\n\n decoder_input = torch.zeros(4 * B, 1, self.token_maxlen)\n decoder_input[:, 0, 0] = 2 # Set <s> Token\n if torch.cuda.is_available():\n decoder_input = decoder_input.cuda()\n decoder_hidden = None\n\n logits = []\n for _ in range(MAX_DECODER_STEP):\n step_logit, decoder_hidden = self.decode_then_output(\n encoded_used_column,\n encoded_question,\n question_mask,\n decoder_input,\n decoder_hidden=decoder_hidden,\n )\n step_logit = step_logit.unsqueeze(1)\n logits.append(step_logit)\n\n # To ont-hot\n _, decoder_idxs = step_logit.view(B * self.column_maxlen, -1).max(1)\n decoder_input = torch.zeros(B * self.column_maxlen, self.token_maxlen).scatter_(\n 1, decoder_idxs.cpu().unsqueeze(1), 1\n )\n if torch.cuda.is_available():\n decoder_input = decoder_input.cuda()\n\n logits = torch.stack(logits, 2)\n else:\n decoder_input, _ = utils.convert_position_to_decoder_input(\n conds_val_pos, token_maxlen=self.token_maxlen\n )\n logits, _ = self.decode_then_output(\n encoded_used_column, encoded_question, question_mask, decoder_input\n )\n return logits\n\n[docs] def concat_start_and_end_zero_padding(self, question_embed, mask):\n B, Q_L, embed_D = list(question_embed.size())\n\n zero_padding = torch.zeros(B, 1, embed_D)\n mask_with_start_end = torch.zeros(B, Q_L + 2)\n\n if torch.cuda.is_available():\n zero_padding = zero_padding.cuda(torch.cuda.current_device())\n mask_with_start_end = mask_with_start_end.cuda(torch.cuda.current_device())\n\n question_embed_with_start_end = torch.cat(\n [zero_padding, question_embed, zero_padding], dim=1\n ) # add <BEG> and <END>\n\n mask_with_start_end[:, 0] = 1 # <BEG>\n mask_with_start_end[:, 1 : Q_L + 1] = mask\n question_lengths = torch.sum(mask, dim=-1).byte()\n for i in range(B):\n mask_with_start_end[i, question_lengths[i].item() + 1] = 1 # <END>\n\n return question_embed_with_start_end, mask_with_start_end\n\n[docs] def decode_then_output(\n self,\n encoded_used_column,\n encoded_question,\n question_mask,\n decoder_input,\n decoder_hidden=None,\n ):\n B = encoded_used_column.size(0)\n\n decoder_output, decoder_hidden = self.decoder(\n decoder_input.view(B * self.column_maxlen, -1, self.token_maxlen), decoder_hidden\n )\n decoder_output = decoder_output.contiguous().view(B, self.column_maxlen, -1, self.model_dim)\n decoder_output = decoder_output.unsqueeze(3)\n\n logits = self.mlp(\n self.linear_column(encoded_used_column)\n + self.linear_conds(decoder_output)\n + self.linear_question(encoded_question)\n ).squeeze()\n logits = f.add_masked_value(logits, question_mask.unsqueeze(1).unsqueeze(1), value=-1e7)\n return logits, decoder_hidden\n\n\nimport numpy as np\nimport torch\n\n\n[docs]def encode_column(column_embed, column_name_mask, rnn_module):\n B, C_L, N_L, embed_D = list(column_embed.size())\n\n column_lengths = get_column_lengths(column_embed, column_name_mask)\n column_last_index = column_lengths - column_lengths.gt(0).long() # NOTE: hard-code\n\n column_reshape = [-1] + [N_L, embed_D]\n column_embed = column_embed.view(*column_reshape)\n\n encoded_column, _ = rnn_module(column_embed)\n encoded_D = encoded_column.size(-1)\n\n encoded_output_column = torch.cat(\n [\n torch.index_select(encoded_column[i], 0, column_last_index[i])\n for i in range(column_last_index.size(0))\n ],\n dim=0,\n )\n encoded_output_column = encoded_output_column.view([B, C_L, encoded_D])\n return encoded_output_column\n\n\n[docs]def get_column_lengths(column_embed, column_name_mask):\n _, _, N_L, embed_D = list(column_embed.size())\n column_reshape = [-1] + [N_L, embed_D]\n\n return torch.sum(column_name_mask.view(*column_reshape[:-1]), dim=-1).long()\n\n\n[docs]def filter_used_column(encoded_columns, col_idx, padding_count=4):\n B, C_L, D = list(encoded_columns.size())\n zero_padding = torch.zeros(D)\n if torch.cuda.is_available():\n zero_padding = zero_padding.cuda(torch.cuda.current_device())\n\n encoded_used_columns = []\n for i in range(B):\n encoded_used_column = torch.stack(\n [encoded_columns[i][j] for j in col_idx[i]]\n + [zero_padding] * (padding_count - len(col_idx[i]))\n )\n encoded_used_columns.append(encoded_used_column)\n return torch.stack(encoded_used_columns)\n\n\n[docs]def convert_position_to_decoder_input(conds_val_pos, token_maxlen=200):\n B = len(conds_val_pos)\n max_len = (\n max([max([len(tok) for tok in tok_seq] + [0]) for tok_seq in conds_val_pos]) - 1\n ) # The max seq len in the batch.\n if max_len < 1:\n max_len = 1\n ret_array = np.zeros((B, 4, max_len, token_maxlen), dtype=np.float32)\n ret_len = np.zeros((B, 4))\n for b, tok_seq in enumerate(conds_val_pos):\n idx = 0\n for idx, one_tok_seq in enumerate(tok_seq):\n out_one_tok_seq = one_tok_seq[:-1]\n ret_len[b, idx] = len(out_one_tok_seq)\n for t, tok_id in enumerate(out_one_tok_seq):\n ret_array[b, idx, t, tok_id] = 1\n if idx < 3:\n ret_array[b, idx + 1 :, 0, 1] = 1\n ret_len[b, idx + 1 :] = 1\n\n ret_inp = torch.from_numpy(ret_array)\n if torch.cuda.is_available():\n ret_inp = ret_inp.cuda(torch.cuda.current_device())\n\n return ret_inp, ret_len # [B, IDX, max_len, token_maxlen]\n\n\nfrom overrides import overrides\nfrom pytorch_transformers import BertModel\nimport torch.nn as nn\n\nfrom claf.data.data_handler import CachePath\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithoutTokenEmbedder\nfrom claf.model.sequence_classification.mixin import SequenceClassification\n\n\n[docs]@register("model:bert_for_seq_cls")\nclass BertForSeqCls(SequenceClassification, ModelWithoutTokenEmbedder):\n """\n Implementation of Sentence Classification model presented in\n BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding\n (https://arxiv.org/abs/1810.04805)\n\n * Args:\n token_embedder: used to embed the sequence\n num_classes: number of classified classes\n\n * Kwargs:\n pretrained_model_name: the name of a pre-trained model\n dropout: classification layer dropout\n """\n\n def __init__(self, token_makers, num_classes, pretrained_model_name=None, dropout=0.2):\n\n super(BertForSeqCls, self).__init__(token_makers)\n\n self.use_pytorch_transformers = True # for optimizer's model parameters\n\n self.num_classes = num_classes\n\n self._model = BertModel.from_pretrained(\n pretrained_model_name, cache_dir=str(CachePath.ROOT)\n )\n self.classifier = nn.Sequential(\n nn.Dropout(dropout), nn.Linear(self._model.config.hidden_size, num_classes)\n )\n self.classifier.apply(self._model.init_weights)\n\n self.criterion = nn.CrossEntropyLoss()\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {\n "bert_input": {\n "feature": [\n [3, 4, 1, 0, 0, 0, ...],\n ...,\n ]\n },\n "token_type": {\n "feature": [\n [0, 0, 0, 0, 0, 0, ...],\n ...,\n ],\n }\n }\n\n * Kwargs:\n label: label dictionary like below.\n {\n "class_idx": [2, 1, 0, 4, 5, ...]\n "data_idx": [2, 4, 5, 7, 2, 1, ...]\n }\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - sequence_embed: embedding vector of the sequence\n - logits: representing unnormalized log probabilities of the class.\n\n - class_idx: target class idx\n - data_idx: data idx\n - loss: a scalar loss to be optimized\n """\n\n bert_inputs = features["bert_input"]["feature"]\n token_type_ids = features["token_type"]["feature"]\n attention_mask = (bert_inputs > 0).long()\n\n outputs = self._model(\n bert_inputs, token_type_ids=token_type_ids, attention_mask=attention_mask\n )\n pooled_output = outputs[1]\n logits = self.classifier(pooled_output)\n\n output_dict = {"sequence_embed": pooled_output, "logits": logits}\n\n if labels:\n class_idx = labels["class_idx"]\n data_idx = labels["data_idx"]\n\n output_dict["class_idx"] = class_idx\n output_dict["data_idx"] = data_idx\n\n # Loss\n loss = self.criterion(\n logits.view(-1, self.num_classes), class_idx.view(-1)\n )\n output_dict["loss"] = loss.unsqueeze(0) # NOTE: DataParallel concat Error\n\n return output_dict\n\n[docs] @overrides\n def print_examples(self, index, inputs, predictions):\n """\n Print evaluation examples\n\n * Args:\n index: data index\n inputs: mini-batch inputs\n predictions: prediction dictionary consisting of\n - key: 'id' (sequence id)\n - value: dictionary consisting of\n - class_idx\n\n * Returns:\n print(Sequence, Sequence Tokens, Target Class, Predicted Class)\n """\n\n data_idx = inputs["labels"]["data_idx"][index].item()\n data_id = self._dataset.get_id(data_idx)\n\n helper = self._dataset.helper\n\n sequence_a = helper["examples"][data_id]["sequence_a"]\n sequence_a_tokens = helper["examples"][data_id]["sequence_a_tokens"]\n sequence_b = helper["examples"][data_id]["sequence_b"]\n sequence_b_tokens = helper["examples"][data_id]["sequence_b_tokens"]\n target_class_text = helper["examples"][data_id]["class_text"]\n\n pred_class_idx = predictions[data_id]["class_idx"]\n pred_class_text = self._dataset.get_class_text_with_idx(pred_class_idx)\n\n print()\n print("- Sequence a:", sequence_a)\n print("- Sequence a Tokens:", sequence_a_tokens)\n if sequence_b:\n print("- Sequence b:", sequence_b)\n print("- Sequence b Tokens:", sequence_b_tokens)\n print("- Target:")\n print(" Class:", target_class_text)\n print("- Predict:")\n print(" Class:", pred_class_text)\n print()\n\n\nfrom overrides import overrides\nfrom pytorch_transformers import BertModel\nimport torch.nn as nn\n\nfrom claf.data.data_handler import CachePath\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithoutTokenEmbedder\nfrom claf.model.sequence_classification.mixin import SequenceClassification\n\n\n[docs]@register("model:bert_for_seq_cls")\nclass BertForSeqCls(SequenceClassification, ModelWithoutTokenEmbedder):\n """\n Implementation of Single Sentence Classification model presented in\n BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding\n (https://arxiv.org/abs/1810.04805)\n\n * Args:\n token_embedder: used to embed the sequence\n num_classes: number of classified classes\n\n * Kwargs:\n pretrained_model_name: the name of a pre-trained model\n dropout: classification layer dropout\n """\n\n def __init__(self, token_makers, num_classes, pretrained_model_name=None, dropout=0.2):\n\n super(BertForSeqCls, self).__init__(token_makers)\n\n self.bert = True # for optimizer's model parameters\n\n self.num_classes = num_classes\n\n self._model = BertModel.from_pretrained(\n pretrained_model_name, cache_dir=str(CachePath.ROOT)\n )\n self.classifier = nn.Sequential(\n nn.Dropout(dropout), nn.Linear(self._model.config.hidden_size, num_classes)\n )\n self.classifier.apply(self._model.init_weights)\n\n self.criterion = nn.CrossEntropyLoss()\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {\n "bert_input": {\n "feature": [\n [3, 4, 1, 0, 0, 0, ...],\n ...,\n ]\n },\n "token_type": {\n "feature": [\n [0, 0, 0, 0, 0, 0, ...],\n ...,\n ],\n }\n }\n\n * Kwargs:\n label: label dictionary like below.\n {\n "class_idx": [2, 1, 0, 4, 5, ...]\n "data_idx": [2, 4, 5, 7, 2, 1, ...]\n }\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - sequence_embed: embedding vector of the sequence\n - class_logits: representing unnormalized log probabilities of the class.\n\n - class_idx: target class idx\n - data_idx: data idx\n - loss: a scalar loss to be optimized\n """\n\n bert_inputs = features["bert_input"]["feature"]\n token_type_ids = features["token_type"]["feature"]\n attention_mask = (bert_inputs > 0).long()\n\n outputs = self._model(\n bert_inputs, token_type_ids=token_type_ids, attention_mask=attention_mask\n )\n pooled_output = outputs[1]\n class_logits = self.classifier(pooled_output)\n\n output_dict = {"sequence_embed": pooled_output, "class_logits": class_logits}\n\n if labels:\n class_idx = labels["class_idx"]\n data_idx = labels["data_idx"]\n\n output_dict["class_idx"] = class_idx\n output_dict["data_idx"] = data_idx\n\n # Loss\n loss = self.criterion(class_logits, class_idx)\n output_dict["loss"] = loss.unsqueeze(0) # NOTE: DataParallel concat Error\n\n return output_dict\n\n[docs] @overrides\n def print_examples(self, index, inputs, predictions):\n """\n Print evaluation examples\n\n * Args:\n index: data index\n inputs: mini-batch inputs\n predictions: prediction dictionary consisting of\n - key: 'id' (sequence id)\n - value: dictionary consisting of\n - class_idx\n\n * Returns:\n print(Sequence, Sequence Tokens, Target Class, Predicted Class)\n """\n\n data_idx = inputs["labels"]["data_idx"][index].item()\n data_id = self._dataset.get_id(data_idx)\n\n helper = self._dataset.helper\n\n sequence_a = helper["examples"][data_id]["sequence_a"]\n sequence_a_tokens = helper["examples"][data_id]["sequence_a_sub_tokens"]\n sequence_b = helper["examples"][data_id]["sequence_b"]\n sequence_b_tokens = helper["examples"][data_id]["sequence_b_sub_tokens"]\n target_class_text = helper["examples"][data_id]["class_text"]\n\n pred_class_idx = predictions[data_id]["class_idx"]\n pred_class_text = self._dataset.get_class_text_with_idx(pred_class_idx)\n\n print()\n print("- Sequence a:", sequence_a)\n print("- Sequence a Tokens:", sequence_a_tokens)\n if sequence_b:\n print("- Sequence b:", sequence_b)\n print("- Sequence b Tokens:", sequence_b_tokens)\n print("- Target:")\n print(" Class:", target_class_text)\n print("- Predict:")\n print(" Class:", pred_class_text)\n print()\n\n\nfrom pathlib import Path\nimport logging\n\nimport torch\nimport pycm\nfrom pycm.pycm_obj import pycmVectorError\n\nfrom claf.model import cls_utils\nfrom claf.model.base import ModelBase\nfrom claf.metric.classification import macro_f1, macro_precision, macro_recall\nfrom claf.metric.glue import simple_accuracy, f1, matthews_corr\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]class SequenceClassification:\n """ Sequence Classification Mixin Class """\n\n[docs] def make_predictions(self, output_dict):\n """\n Make predictions with model's output_dict\n\n * Args:\n output_dict: model's output dictionary consisting of\n - sequence_embed: embedding vector of the sequence\n - logits: representing unnormalized log probabilities of the class\n\n - class_idx: target class idx\n - data_idx: data idx\n - loss: a scalar loss to be optimized\n\n * Returns:\n predictions: prediction dictionary consisting of\n - key: 'id' (sequence id)\n - value: dictionary consisting of\n - class_idx\n """\n\n data_indices = output_dict["data_idx"]\n pred_logits = output_dict["logits"]\n pred_class_idxs = torch.argmax(pred_logits, dim=-1)\n\n predictions = {\n self._dataset.get_id(data_idx.item()): {"class_idx": pred_class_idx.item()}\n for data_idx, pred_class_idx in zip(list(data_indices.data), list(pred_class_idxs.data))\n }\n\n return predictions\n\n[docs] def predict(self, output_dict, arguments, helper):\n """\n Inference by raw_feature\n\n * Args:\n output_dict: model's output dictionary consisting of\n - sequence_embed: embedding vector of the sequence\n - logits: representing unnormalized log probabilities of the class.\n arguments: arguments dictionary consisting of user_input\n helper: dictionary to get the classification result, consisting of\n - class_idx2text: dictionary converting class_idx to class_text\n\n * Returns: output dict (dict) consisting of\n - logits: representing unnormalized log probabilities of the class\n - class_idx: predicted class idx\n - class_text: predicted class text\n """\n\n logits = output_dict["logits"]\n class_idx = logits.argmax(dim=-1)\n\n return {\n "logits": logits,\n "class_idx": class_idx,\n "class_text": helper["class_idx2text"][class_idx.item()],\n }\n\n[docs] def make_metrics(self, predictions):\n """\n Make metrics with prediction dictionary\n\n * Args:\n predictions: prediction dictionary consisting of\n - key: 'id' (sequence id)\n - value: dictionary consisting of\n - class_idx\n\n * Returns:\n metrics: metric dictionary consisting of\n - 'macro_f1': class prediction macro(unweighted mean) f1\n - 'macro_precision': class prediction macro(unweighted mean) precision\n - 'macro_recall': class prediction macro(unweighted mean) recall\n - 'accuracy': class prediction accuracy\n """\n\n pred_idx = []\n pred_classes = []\n\n target_idx = []\n target_classes = []\n target_count = len(self._dataset.class_idx2text)\n\n for data_id, pred in predictions.items():\n target = self._dataset.get_ground_truth(data_id)\n\n pred_idx.append(pred["class_idx"])\n pred_classes.append(self._dataset.class_idx2text[pred["class_idx"]])\n\n target_idx.append(target["class_idx"])\n target_classes.append(target["class_text"])\n\n metrics = {\n "accuracy": simple_accuracy(pred_idx, target_idx),\n }\n\n if target_count == 2:\n # binary class\n f1_metric = f1(pred_idx, target_idx)\n metrics.update(f1_metric)\n\n matthews_corr_metric = matthews_corr(pred_idx, target_idx)\n metrics.update(matthews_corr_metric)\n return metrics\n\n[docs] def write_predictions(self, predictions, file_path=None, is_dict=True, pycm_obj=None):\n """\n Override write_predictions() in ModelBase to log confusion matrix\n """\n\n try:\n super(SequenceClassification, self).write_predictions(\n predictions, file_path=file_path, is_dict=is_dict\n )\n except AttributeError:\n # TODO: Need to Fix\n model_base = ModelBase()\n model_base._log_dir = self._log_dir\n model_base._train_counter = self._train_counter\n model_base.training = self.training\n model_base.write_predictions(predictions, file_path=file_path, is_dict=is_dict)\n\n data_type = "train" if self.training else "valid"\n\n if pycm_obj is not None:\n stats_file_path = f"predictions-{data_type}-{self._train_counter.get_display()}-stats"\n pycm_obj.save_csv(str(Path(self._log_dir) / "predictions" / stats_file_path))\n\n confusion_matrix_file_path = (\n f"predictions-{data_type}-{self._train_counter.get_display()}-confusion_matrix"\n )\n cls_utils.write_confusion_matrix_to_csv(\n str(Path(self._log_dir) / "predictions" / confusion_matrix_file_path), pycm_obj\n )\n\n[docs] def print_examples(self, index, inputs, predictions):\n """\n Print evaluation examples\n\n * Args:\n index: data index\n inputs: mini-batch inputs\n predictions: prediction dictionary consisting of\n - key: 'id' (sequence id)\n - value: dictionary consisting of\n - class_idx\n\n * Returns:\n print(Sequence, Target Class, Predicted Class)\n """\n\n data_idx = inputs["labels"]["data_idx"][index].item()\n data_id = self._dataset.get_id(data_idx)\n\n helper = self._dataset.helper\n sequence = helper["examples"][data_id]["sequence"]\n target_class_text = helper["examples"][data_id]["class_text"]\n\n pred_class_idx = predictions[data_id]["class_idx"]\n pred_class_text = self._dataset.get_class_text_with_idx(pred_class_idx)\n\n print()\n print("- Sequence:", sequence)\n print("- Target:")\n print(" Class:", target_class_text)\n print("- Predict:")\n print(" Class:", pred_class_text)\n print()\n\n\nfrom overrides import overrides\nfrom pytorch_transformers import RobertaModel\nimport torch.nn as nn\n\nfrom claf.data.data_handler import CachePath\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithoutTokenEmbedder\nfrom claf.model.sequence_classification.mixin import SequenceClassification\n\n\n[docs]@register("model:roberta_for_seq_cls")\nclass RobertaForSeqCls(SequenceClassification, ModelWithoutTokenEmbedder):\n """\n Implementation of Sentence Classification model presented in\n BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding\n (https://arxiv.org/abs/1810.04805)\n\n * Args:\n token_embedder: used to embed the sequence\n num_classes: number of classified classes\n\n * Kwargs:\n pretrained_model_name: the name of a pre-trained model\n dropout: classification layer dropout\n """\n\n def __init__(self, token_makers, num_classes, pretrained_model_name=None, dropout=0.2):\n\n super(RobertaForSeqCls, self).__init__(token_makers)\n\n self.use_pytorch_transformers = True # for optimizer's model parameters\n\n self.num_classes = num_classes\n\n self._model = RobertaModel.from_pretrained(\n pretrained_model_name, cache_dir=str(CachePath.ROOT)\n )\n self.classifier = nn.Sequential(\n nn.Linear(self._model.config.hidden_size, self._model.config.hidden_size),\n nn.Dropout(dropout),\n nn.Linear(self._model.config.hidden_size, num_classes)\n )\n self.classifier.apply(self._model.init_weights)\n\n self.criterion = nn.CrossEntropyLoss()\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {\n "bert_input": {\n "feature": [\n [3, 4, 1, 0, 0, 0, ...],\n ...,\n ]\n },\n }\n\n * Kwargs:\n label: label dictionary like below.\n {\n "class_idx": [2, 1, 0, 4, 5, ...]\n "data_idx": [2, 4, 5, 7, 2, 1, ...]\n }\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - sequence_embed: embedding vector of the sequence\n - logits: representing unnormalized log probabilities of the class.\n\n - class_idx: target class idx\n - data_idx: data idx\n - loss: a scalar loss to be optimized\n """\n\n bert_inputs = features["bert_input"]["feature"]\n attention_mask = (bert_inputs > 0).long()\n\n outputs = self._model(\n bert_inputs, token_type_ids=None, attention_mask=attention_mask\n )\n sequence_output = outputs[0]\n pooled_output = sequence_output[:, 0, :] # take <s> token (equiv. to [CLS])\n logits = self.classifier(pooled_output)\n\n output_dict = {"sequence_embed": pooled_output, "logits": logits}\n\n if labels:\n class_idx = labels["class_idx"]\n data_idx = labels["data_idx"]\n\n output_dict["class_idx"] = class_idx\n output_dict["data_idx"] = data_idx\n\n # Loss\n loss = self.criterion(\n logits.view(-1, self.num_classes), class_idx.view(-1)\n )\n output_dict["loss"] = loss.unsqueeze(0) # NOTE: DataParallel concat Error\n\n return output_dict\n\n[docs] @overrides\n def print_examples(self, index, inputs, predictions):\n """\n Print evaluation examples\n\n * Args:\n index: data index\n inputs: mini-batch inputs\n predictions: prediction dictionary consisting of\n - key: 'id' (sequence id)\n - value: dictionary consisting of\n - class_idx\n\n * Returns:\n print(Sequence, Sequence Tokens, Target Class, Predicted Class)\n """\n\n data_idx = inputs["labels"]["data_idx"][index].item()\n data_id = self._dataset.get_id(data_idx)\n\n helper = self._dataset.helper\n\n sequence_a = helper["examples"][data_id]["sequence_a"]\n sequence_a_tokens = helper["examples"][data_id]["sequence_a_tokens"]\n sequence_b = helper["examples"][data_id]["sequence_b"]\n sequence_b_tokens = helper["examples"][data_id]["sequence_b_tokens"]\n target_class_text = helper["examples"][data_id]["class_text"]\n\n pred_class_idx = predictions[data_id]["class_idx"]\n pred_class_text = self._dataset.get_class_text_with_idx(pred_class_idx)\n\n print()\n print("- Sequence a:", sequence_a)\n print("- Sequence a Tokens:", sequence_a_tokens)\n if sequence_b:\n print("- Sequence b:", sequence_b)\n print("- Sequence b Tokens:", sequence_b_tokens)\n print("- Target:")\n print(" Class:", target_class_text)\n print("- Predict:")\n print(" Class:", pred_class_text)\n print()\n\n\nfrom overrides import overrides\n\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithTokenEmbedder\nfrom claf.model.sequence_classification.mixin import SequenceClassification\nfrom claf.modules import functional as f\n\n\n[docs]@register("model:structured_self_attention")\nclass StructuredSelfAttention(SequenceClassification, ModelWithTokenEmbedder):\n """\n Implementation of model presented in\n A Structured Self-attentive Sentence Embedding\n (https://arxiv.org/abs/1703.03130)\n\n * Args:\n token_embedder: used to embed the sequence\n num_classes: number of classified classes\n\n * Kwargs:\n encoding_rnn_hidden_dim: hidden dimension of rnn (unidirectional)\n encoding_rnn_num_layer: the number of rnn layers\n encoding_rnn_dropout: rnn dropout probability\n attention_dim: attention dimension # d_a in the paper\n num_attention_heads: number of attention heads # r in the paper\n sequence_embed_dim: dimension of sequence embedding\n dropout: classification layer dropout\n penalization_coefficient: penalty coefficient for frobenius norm\n """\n\n def __init__(\n self,\n token_embedder,\n num_classes,\n encoding_rnn_hidden_dim=300,\n encoding_rnn_num_layer=2,\n encoding_rnn_dropout=0.,\n attention_dim=350,\n num_attention_heads=30,\n sequence_embed_dim=2000,\n dropout=0.5,\n penalization_coefficient=1.,\n ):\n super(StructuredSelfAttention, self).__init__(token_embedder)\n\n rnn_input_dim = token_embedder.get_embed_dim()\n\n self.num_classes = num_classes\n\n self.encoding_rnn_hidden_dim = encoding_rnn_hidden_dim * 2 # bidirectional\n self.attention_dim = attention_dim\n self.num_attention_heads = num_attention_heads\n self.project_dim = sequence_embed_dim\n self.dropout = dropout\n self.penalization_coefficient = penalization_coefficient\n\n self.encoder = nn.LSTM(\n input_size=rnn_input_dim,\n hidden_size=encoding_rnn_hidden_dim,\n num_layers=encoding_rnn_num_layer,\n dropout=encoding_rnn_dropout,\n bidirectional=True,\n batch_first=True,\n )\n\n self.A = nn.Sequential(\n nn.Linear(self.encoding_rnn_hidden_dim, attention_dim, bias=False),\n nn.Tanh(),\n nn.Linear(attention_dim, num_attention_heads, bias=False),\n )\n self.fully_connected = nn.Sequential(\n nn.Linear(self.encoding_rnn_hidden_dim * num_attention_heads, sequence_embed_dim),\n nn.ReLU(),\n nn.Dropout(dropout),\n )\n self.classifier = nn.Linear(sequence_embed_dim, num_classes)\n self.criterion = nn.CrossEntropyLoss()\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {"sequence": [0, 3, 4, 1]}\n\n * Kwargs:\n label: label dictionary like below.\n {"class_idx": 2, "data_idx": 0}\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - sequence_embed: embedding vector of the sequence\n - logits: representing unnormalized log probabilities of the class.\n\n - class_idx: target class idx\n - data_idx: data idx\n - loss: a scalar loss to be optimized\n """\n\n sequence = features["sequence"]\n\n # Sorted Sequence config (seq_lengths, perm_idx, unperm_idx) for RNN pack_forward\n sequence_config = f.get_sorted_seq_config(sequence)\n\n token_embed = self.token_embedder(sequence)\n\n token_encodings = f.forward_rnn_with_pack(\n self.encoder, token_embed, sequence_config\n ) # [B, L, encoding_rnn_hidden_dim]\n\n attention = self.A(token_encodings).transpose(1, 2) # [B, num_attention_heads, L]\n\n sequence_mask = f.get_mask_from_tokens(sequence).float() # [B, L]\n sequence_mask = sequence_mask.unsqueeze(1).expand_as(attention)\n attention = F.softmax(f.add_masked_value(attention, sequence_mask) + 1e-13, dim=2)\n\n attended_encodings = torch.bmm(\n attention, token_encodings\n ) # [B, num_attention_heads, sequence_embed_dim]\n sequence_embed = self.fully_connected(\n attended_encodings.view(attended_encodings.size(0), -1)\n ) # [B, sequence_embed_dim]\n\n logits = self.classifier(sequence_embed) # [B, num_classes]\n\n output_dict = {"sequence_embed": sequence_embed, "logits": logits}\n\n if labels:\n class_idx = labels["class_idx"]\n data_idx = labels["data_idx"]\n\n output_dict["class_idx"] = class_idx\n output_dict["data_idx"] = data_idx\n\n # Loss\n loss = self.criterion(logits, class_idx)\n loss += self.penalty(attention)\n output_dict["loss"] = loss.unsqueeze(0) # NOTE: DataParallel concat Error\n\n return output_dict\n\n[docs] def penalty(self, attention):\n aa = torch.bmm(\n attention, attention.transpose(1, 2)\n ) # [B, num_attention_heads, num_attention_heads]\n penalization_term = ((aa - aa.new_tensor(np.eye(aa.size(1)))) ** 2).sum() ** 0.5\n return penalization_term * self.penalization_coefficient\n\n\nfrom overrides import overrides\nfrom pytorch_transformers import BertModel\nimport torch.nn as nn\n\nfrom claf.data.data_handler import CachePath\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithoutTokenEmbedder\nfrom claf.model.token_classification.mixin import TokenClassification\n\nfrom claf.model import cls_utils\n\n\n[docs]@register("model:bert_for_tok_cls")\nclass BertForTokCls(TokenClassification, ModelWithoutTokenEmbedder):\n """\n Implementation of Single Sentence Tagging model presented in\n BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding\n (https://arxiv.org/abs/1810.04805)\n\n * Args:\n token_embedder: used to embed the sequence\n num_tags: number of classified tags\n ignore_tag_idx: index of the tag to ignore when calculating loss (tag pad value)\n\n * Kwargs:\n pretrained_model_name: the name of a pre-trained model\n dropout: classification layer dropout\n """\n\n def __init__(\n self, token_makers, num_tags, ignore_tag_idx, pretrained_model_name=None, dropout=0.2\n ):\n\n super(BertForTokCls, self).__init__(token_makers)\n\n self.use_pytorch_transformers = True # for optimizer's model parameters\n\n self.ignore_tag_idx = ignore_tag_idx\n self.num_tags = num_tags\n\n self._model = BertModel.from_pretrained(\n pretrained_model_name, cache_dir=str(CachePath.ROOT)\n )\n self.classifier = nn.Sequential(\n nn.Dropout(dropout), nn.Linear(self._model.config.hidden_size, num_tags)\n )\n self.classifier.apply(self._model.init_weights)\n\n self.criterion = nn.CrossEntropyLoss(ignore_index=ignore_tag_idx)\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {\n "bert_input": {\n "feature": [\n [100, 576, 21, 45, 7, 91, 101, 0, 0, ...],\n ...,\n ]\n }\n "token_type": {\n "feature": [\n [0, 0, 0, 0, 0, 0, 0, 0, 0, ...],\n ...,\n ]\n },\n "tagged_sub_token_idxs": {\n [\n [1, 3, 4, 0, 0, 0, 0, 0, 0, ...],\n ...,\n ]\n }\n }\n\n * Kwargs:\n label: label dictionary like below.\n {\n "class_idx": [2, 1, 0, 4, 5, ...]\n "data_idx": [2, 4, 5, 7, 2, 1, ...]\n }\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - sequence_embed: embedding vector of the sequence\n - tag_logits: representing unnormalized log probabilities of the tags.\n\n - tag_idxs: target class idx\n - data_idx: data idx\n - loss: a scalar loss to be optimized\n """\n\n bert_inputs = features["bert_input"]["feature"]\n token_type_ids = features["token_type"]["feature"]\n tagged_sub_token_idxs = features["tagged_sub_token_idxs"]["feature"]\n num_tokens = features["num_tokens"]["feature"]\n\n attention_mask = (bert_inputs > 0).long()\n\n outputs = self._model(\n bert_inputs, token_type_ids=token_type_ids, attention_mask=attention_mask\n )\n token_encodings = outputs[0]\n pooled_output = outputs[1]\n\n tag_logits = self.classifier(token_encodings) # [B, L, num_tags]\n\n # gather the logits of the tagged token positions.\n gather_token_pos_idxs = tagged_sub_token_idxs.unsqueeze(-1).repeat(1, 1, self.num_tags)\n token_tag_logits = tag_logits.gather(1, gather_token_pos_idxs) # [B, num_tokens, num_tags]\n\n sliced_token_tag_logits = [token_tag_logits[idx, :n, :] for idx, n in enumerate(num_tokens)]\n\n output_dict = {"sequence_embed": pooled_output, "tag_logits": sliced_token_tag_logits}\n\n if labels:\n tag_idxs = labels["tag_idxs"]\n data_idx = labels["data_idx"]\n\n output_dict["tag_idxs"] = tag_idxs\n output_dict["data_idx"] = data_idx\n\n # Loss\n loss = self.criterion(token_tag_logits.view(-1, self.num_tags), tag_idxs.view(-1))\n output_dict["loss"] = loss.unsqueeze(0) # NOTE: DataParallel concat Error\n\n return output_dict\n\n[docs] @overrides\n def print_examples(self, index, inputs, predictions):\n """\n Print evaluation examples\n\n * Args:\n index: data index\n inputs: mini-batch inputs\n predictions: prediction dictionary consisting of\n - key: 'id' (sequence id)\n - value: dictionary consisting of\n - class_idx\n\n * Returns:\n print(Sequence, Sequence Tokens, Target Tags, Target Slots, Predicted Tags, Predicted Slots)\n """\n\n data_idx = inputs["labels"]["data_idx"][index].item()\n data_id = self._dataset.get_id(data_idx)\n\n helper = self._dataset.helper\n sequence = helper["examples"][data_id]["sequence"]\n target_tag_texts = helper["examples"][data_id]["tag_texts"]\n\n pred_tag_idxs = predictions[data_id]["tag_idxs"]\n pred_tag_texts = self._dataset.get_tag_texts_with_idxs(pred_tag_idxs)\n\n sequence_tokens = helper["examples"][data_id]["sequence_sub_tokens"]\n\n print()\n print("- Sequence:", sequence)\n print("- Sequence Tokens:", sequence_tokens)\n print("- Target:")\n print(" Tags:", target_tag_texts)\n print(" (Slots)", cls_utils.get_tag_dict(sequence, target_tag_texts))\n print("- Predict:")\n print(" Tags:", pred_tag_texts)\n print(" (Slots)", cls_utils.get_tag_dict(sequence, pred_tag_texts))\n print()\n\n\nfrom overrides import overrides\nfrom pytorch_transformers import BertModel\nimport torch.nn as nn\n\nfrom claf.data.data_handler import CachePath\nfrom claf.decorator import register\nfrom claf.model.base import ModelWithoutTokenEmbedder\nfrom claf.model.token_classification.mixin import TokenClassification\n\nfrom claf.model import cls_utils\n\n\n[docs]@register("model:bert_for_tok_cls")\nclass BertForTokCls(TokenClassification, ModelWithoutTokenEmbedder):\n """\n Implementation of Single Sentence Tagging model presented in\n BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding\n (https://arxiv.org/abs/1810.04805)\n\n * Args:\n token_embedder: used to embed the sequence\n num_tags: number of classified tags\n ignore_tag_idx: index of the tag to ignore when calculating loss (tag pad value)\n\n * Kwargs:\n pretrained_model_name: the name of a pre-trained model\n dropout: classification layer dropout\n """\n\n def __init__(\n self, token_makers, num_tags, ignore_tag_idx, pretrained_model_name=None, dropout=0.2\n ):\n\n super(BertForTokCls, self).__init__(token_makers)\n\n self.bert = True # for optimizer's model parameters\n\n self.ignore_tag_idx = ignore_tag_idx\n self.num_tags = num_tags\n\n self._model = BertModel.from_pretrained(\n pretrained_model_name, cache_dir=str(CachePath.ROOT)\n )\n self.classifier = nn.Sequential(\n nn.Dropout(dropout), nn.Linear(self._model.config.hidden_size, num_tags)\n )\n self.classifier.apply(self._model.init_weights)\n\n self.criterion = nn.CrossEntropyLoss(ignore_index=ignore_tag_idx)\n\n[docs] @overrides\n def forward(self, features, labels=None):\n """\n * Args:\n features: feature dictionary like below.\n {\n "bert_input": {\n "feature": [\n [100, 576, 21, 45, 7, 91, 101, 0, 0, ...],\n ...,\n ]\n }\n "token_type": {\n "feature": [\n [0, 0, 0, 0, 0, 0, 0, 0, 0, ...],\n ...,\n ]\n },\n "tagged_sub_token_idxs": {\n [\n [1, 3, 4, 0, 0, 0, 0, 0, 0, ...],\n ...,\n ]\n }\n }\n\n * Kwargs:\n label: label dictionary like below.\n {\n "class_idx": [2, 1, 0, 4, 5, ...]\n "data_idx": [2, 4, 5, 7, 2, 1, ...]\n }\n Do not calculate loss when there is no label. (inference/predict mode)\n\n * Returns: output_dict (dict) consisting of\n - sequence_embed: embedding vector of the sequence\n - tag_logits: representing unnormalized log probabilities of the tags.\n\n - tag_idxs: target class idx\n - data_idx: data idx\n - loss: a scalar loss to be optimized\n """\n\n bert_inputs = features["bert_input"]["feature"]\n token_type_ids = features["token_type"]["feature"]\n tagged_sub_token_idxs = features["tagged_sub_token_idxs"]["feature"]\n num_tokens = features["num_tokens"]["feature"]\n\n attention_mask = (bert_inputs > 0).long()\n\n outputs = self._model(\n bert_inputs, token_type_ids=token_type_ids, attention_mask=attention_mask\n )\n token_encodings = outputs[0]\n pooled_output = outputs[1]\n\n tag_logits = self.classifier(token_encodings) # [B, L, num_tags]\n\n # gather the logits of the tagged token positions.\n gather_token_pos_idxs = tagged_sub_token_idxs.unsqueeze(-1).repeat(1, 1, self.num_tags)\n token_tag_logits = tag_logits.gather(1, gather_token_pos_idxs) # [B, num_tokens, num_tags]\n\n sliced_token_tag_logits = [token_tag_logits[idx, :n, :] for idx, n in enumerate(num_tokens)]\n\n output_dict = {"sequence_embed": pooled_output, "tag_logits": sliced_token_tag_logits}\n\n if labels:\n tag_idxs = labels["tag_idxs"]\n data_idx = labels["data_idx"]\n\n output_dict["tag_idxs"] = tag_idxs\n output_dict["data_idx"] = data_idx\n\n # Loss\n loss = self.criterion(token_tag_logits.view(-1, self.num_tags), tag_idxs.view(-1))\n output_dict["loss"] = loss.unsqueeze(0) # NOTE: DataParallel concat Error\n\n return output_dict\n\n[docs] @overrides\n def print_examples(self, index, inputs, predictions):\n """\n Print evaluation examples\n\n * Args:\n index: data index\n inputs: mini-batch inputs\n predictions: prediction dictionary consisting of\n - key: 'id' (sequence id)\n - value: dictionary consisting of\n - class_idx\n\n * Returns:\n print(Sequence, Sequence Tokens, Target Tags, Target Slots, Predicted Tags, Predicted Slots)\n """\n\n data_idx = inputs["labels"]["data_idx"][index].item()\n data_id = self._dataset.get_id(data_idx)\n\n helper = self._dataset.helper\n sequence = helper["examples"][data_id]["sequence"]\n target_tag_texts = helper["examples"][data_id]["tag_texts"]\n\n pred_tag_idxs = predictions[data_id]["tag_idxs"]\n pred_tag_texts = self._dataset.get_tag_texts_with_idxs(pred_tag_idxs)\n\n sequence_tokens = helper["examples"][data_id]["sequence_sub_tokens"]\n\n print()\n print("- Sequence:", sequence)\n print("- Sequence Tokens:", sequence_tokens)\n print("- Target:")\n print(" Tags:", target_tag_texts)\n print(" (Slots)", cls_utils.get_tag_dict(sequence, target_tag_texts))\n print("- Predict:")\n print(" Tags:", pred_tag_texts)\n print(" (Slots)", cls_utils.get_tag_dict(sequence, pred_tag_texts))\n print()\n\n\nfrom pathlib import Path\nimport logging\n\nimport numpy as np\nimport torch\nimport pycm\nfrom pycm.pycm_obj import pycmVectorError\n\nfrom claf.decorator import arguments_required\nimport claf.utils as common_utils\nfrom claf.model import cls_utils\nfrom claf.metric.classification import macro_f1, macro_precision, macro_recall\nfrom seqeval.metrics import accuracy_score as conlleval_accuracy\nfrom seqeval.metrics import f1_score as conlleval_f1\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]class TokenClassification:\n """ Token Classification Mixin Class """\n\n[docs] def make_predictions(self, output_dict):\n """\n Make predictions with model's output_dict\n\n * Args:\n output_dict: model's output dictionary consisting of\n - sequence_embed: embedding vector of the sequence\n - tag_logits: representing unnormalized log probabilities of the tag\n\n - tag_idxs: target tag idxs\n - data_idx: data idx\n - loss: a scalar loss to be optimized\n\n * Returns:\n predictions: prediction dictionary consisting of\n - key: 'id' (sequence id)\n - value: dictionary consisting of\n - tag_idxs\n """\n\n data_indices = output_dict["data_idx"]\n pred_tag_logits = output_dict["tag_logits"]\n pred_tag_idxs = [\n torch.argmax(pred_tag_logit, dim=-1).tolist() for pred_tag_logit in pred_tag_logits\n ]\n\n predictions = {\n self._dataset.get_id(data_idx.item()): {"tag_idxs": pred_tag_idx}\n for data_idx, pred_tag_idx in zip(list(data_indices.data), pred_tag_idxs)\n }\n\n return predictions\n\n @arguments_required(["sequence"])\n def predict(self, output_dict, arguments, helper):\n """\n Inference by raw_feature\n\n * Args:\n output_dict: model's output dictionary consisting of\n - sequence_embed: embedding vector of the sequence\n - tag_logits: representing unnormalized log probabilities of the tags.\n arguments: arguments dictionary consisting of user_input\n helper: dictionary to get the classification result, consisting of\n - tag_idx2text: dictionary converting tag_idx to tag_text\n\n * Returns: output dict (dict) consisting of\n - tag_logits: representing unnormalized log probabilities of the tags\n - tag_idxs: predicted tag idxs\n - tag_texts: predicted tag texts\n - tag_slots: predicted tag slots\n """\n\n sequence = arguments["sequence"]\n tag_logits = output_dict["tag_logits"][0]\n tag_idxs = [tag_logit.argmax(dim=-1) for tag_logit in tag_logits]\n tag_texts = [helper["tag_idx2text"][tag_idx.item()] for tag_idx in tag_idxs]\n\n return {\n "tag_logits": tag_logits,\n "tag_idxs": tag_idxs,\n "tag_texts": tag_texts,\n "tag_dict": cls_utils.get_tag_dict(sequence, tag_texts),\n }\n\n[docs] def make_metrics(self, predictions):\n """\n Make metrics with prediction dictionary\n\n * Args:\n predictions: prediction dictionary consisting of\n - key: 'id' (sequence id)\n - value: dictionary consisting of\n - tag_idxs\n\n * Returns:\n metrics: metric dictionary consisting of\n - 'accuracy': sequence level accuracy\n - 'tag_accuracy': tag level accuracy\n - 'macro_f1': tag prediction macro(unweighted mean) f1\n - 'macro_precision': tag prediction macro(unweighted mean) precision\n - 'macro_recall': tag prediction macro(unweighted mean) recall\n """\n\n pred_tag_idxs_list = []\n target_tag_idxs_list = []\n\n accurate_sequence = []\n\n for data_idx, pred in predictions.items():\n target = self._dataset.get_ground_truth(data_idx)\n\n pred_tag_idxs_list.append(pred["tag_idxs"])\n target_tag_idxs_list.append(target["tag_idxs"])\n\n accurate_sequence.append(\n 1 if (np.asarray(target["tag_idxs"]) == np.asarray(pred["tag_idxs"])).all() else 0\n )\n\n pred_tags = [\n [self._dataset.tag_idx2text[tag_idx] for tag_idx in tag_idxs] for tag_idxs in pred_tag_idxs_list\n ]\n target_tags = [\n [self._dataset.tag_idx2text[tag_idx] for tag_idx in tag_idxs] for tag_idxs in target_tag_idxs_list\n ]\n\n flat_pred_tags = list(common_utils.flatten(pred_tags))\n flat_target_tags = list(common_utils.flatten(target_tags))\n\n # confusion matrix\n try:\n pycm_obj = pycm.ConfusionMatrix(actual_vector=flat_target_tags, predict_vector=flat_pred_tags)\n except pycmVectorError as e:\n if str(e) == "Number of the classes is lower than 2":\n logger.warning("Number of tags in the batch is 1. Sanity check is highly recommended.")\n return {\n "accuracy": 1.,\n "tag_accuracy": 1.,\n\n "macro_f1": 1.,\n "macro_precision": 1.,\n "macro_recall": 1.,\n\n "conlleval_accuracy": 1.,\n "conlleval_f1": 1.,\n }\n raise\n\n self.write_predictions(\n {"target": flat_target_tags, "predict": flat_pred_tags}, pycm_obj=pycm_obj\n )\n\n sequence_accuracy = sum(accurate_sequence) / len(accurate_sequence)\n\n metrics = {\n "accuracy": sequence_accuracy,\n "tag_accuracy": pycm_obj.Overall_ACC,\n\n "macro_f1": macro_f1(pycm_obj),\n "macro_precision": macro_precision(pycm_obj),\n "macro_recall": macro_recall(pycm_obj),\n\n "conlleval_accuracy": conlleval_accuracy(target_tags, pred_tags),\n "conlleval_f1": conlleval_f1(target_tags, pred_tags),\n }\n\n return metrics\n\n[docs] def write_predictions(self, predictions, file_path=None, is_dict=True, pycm_obj=None):\n """\n Override write_predictions() in ModelBase to log confusion matrix\n """\n\n super(TokenClassification, self).write_predictions(\n predictions, file_path=file_path, is_dict=is_dict\n )\n\n data_type = "train" if self.training else "valid"\n\n if pycm_obj is not None:\n stats_file_path = f"predictions-{data_type}-{self._train_counter.get_display()}-stats"\n pycm_obj.save_csv(str(Path(self._log_dir) / "predictions" / stats_file_path))\n\n confusion_matrix_file_path = (\n f"predictions-{data_type}-{self._train_counter.get_display()}-confusion_matrix"\n )\n cls_utils.write_confusion_matrix_to_csv(\n str(Path(self._log_dir) / "predictions" / confusion_matrix_file_path), pycm_obj\n )\n\n[docs] def print_examples(self, index, inputs, predictions):\n """\n Print evaluation examples\n\n * Args:\n index: data index\n inputs: mini-batch inputs\n predictions: prediction dictionary consisting of\n - key: 'id' (sequence id)\n - value: dictionary consisting of\n - class_idx\n\n * Returns:\n print(Sequence, Target Tags, Target Slots, Predicted Tags, Predicted Slots)\n """\n\n data_idx = inputs["labels"]["data_idx"][index].item()\n data_id = self._dataset.get_id(data_idx)\n\n helper = self._dataset.helper\n sequence = helper["examples"][data_id]["sequence"]\n target_tag_texts = helper["examples"][data_id]["tag_texts"]\n\n pred_tag_idxs = predictions[data_id]["tag_idxs"]\n pred_tag_texts = self._dataset.get_tag_texts_with_idxs(pred_tag_idxs)\n\n print()\n print("- Sequence:", sequence)\n print("- Target:")\n print(" Tags:", target_tag_texts)\n print(" (Slots)", cls_utils.get_tag_dict(sequence, target_tag_texts))\n print("- Predict:")\n print(" Tags:", pred_tag_texts)\n print(" (Slots)", cls_utils.get_tag_dict(sequence, pred_tag_texts))\n print()\n\n\nimport torch.nn as nn\n\n\n[docs]def get_activation_fn(name):\n """ PyTorch built-in activation functions """\n\n activation_functions = {\n "linear": lambda: lambda x: x,\n "relu": nn.ReLU,\n "relu6": nn.ReLU6,\n "elu": nn.ELU,\n "prelu": nn.PReLU,\n "leaky_relu": nn.LeakyReLU,\n "threshold": nn.Threshold,\n "hardtanh": nn.Hardtanh,\n "sigmoid": nn.Sigmoid,\n "tanh": nn.Tanh,\n "log_sigmoid": nn.LogSigmoid,\n "softplus": nn.Softplus,\n "softshrink": nn.Softshrink,\n "softsign": nn.Softsign,\n "tanhshrink": nn.Tanhshrink,\n }\n\n if name not in activation_functions:\n raise ValueError(\n f"'{name}' is not included in activation_functions. use below one. \\n {activation_functions.keys()}"\n )\n\n return activation_functions[name]\n\n\nimport torch\nimport torch.nn as nn\n\nimport claf.modules.functional as f\n\n\n[docs]class BiAttention(nn.Module):\n """\n Attention Flow Layer\n in BiDAF (https://arxiv.org/pdf/1611.01603.pdf)\n\n The Similarity matrix\n Context-to-query Attention (C2Q)\n Query-to-context Attention (Q2C)\n\n * Args:\n model_dim: The number of module dimension\n """\n\n def __init__(self, model_dim):\n super(BiAttention, self).__init__()\n self.model_dim = model_dim\n self.W = nn.Linear(6 * model_dim, 1, bias=False)\n\n[docs] def forward(self, context, context_mask, query, query_mask):\n c, c_mask, q, q_mask = context, context_mask, query, query_mask\n\n S = self._make_similiarity_matrix(c, q) # (B, C_L, Q_L)\n masked_S = f.add_masked_value(S, query_mask.unsqueeze(1), value=-1e7)\n\n c2q = self._context2query(S, q, q_mask)\n q2c = self._query2context(masked_S.max(dim=-1)[0], c, c_mask)\n\n # [h; u˜; h◦u˜; h◦h˜] ~ (B, C_L, 8d)\n G = torch.cat((c, c2q, c * c2q, c * q2c), dim=-1)\n return G\n\n def _make_similiarity_matrix(self, c, q):\n # B: batch_size, C_L: context_maxlen, Q_L: query_maxlen\n B, C_L, Q_L = c.size(0), c.size(1), q.size(1)\n\n matrix_shape = (B, C_L, Q_L, self.model_dim * 2)\n\n c_aug = c.unsqueeze(2).expand(matrix_shape) # (B, C_L, Q_L, 2d)\n q_aug = q.unsqueeze(1).expand(matrix_shape) # (B, C_L, Q_L, 2d)\n\n c_q = torch.mul(c_aug, q_aug) # element-wise multiplication\n\n concated_vector = torch.cat((c_aug, q_aug, c_q), dim=3) # [h; u; h◦u]\n return self.W(concated_vector).view(c.size(0), C_L, Q_L)\n\n def _context2query(self, S, q, q_mask):\n attention = f.last_dim_masked_softmax(S, q_mask) # (B, C_L, Q_L)\n c2q = f.weighted_sum(attention=attention, matrix=q) # (B, C_L, 2d)\n\n return c2q\n\n def _query2context(self, S, c, c_mask):\n attention = f.masked_softmax(S, c_mask) # (B, C_L)\n q2c = f.weighted_sum(attention=attention, matrix=c)\n\n return q2c.unsqueeze(1).expand(c.size()) # (B, C_L, 2d)\n\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport claf.modules.functional as f\n\n\n[docs]class CoAttention(nn.Module):\n """\n CoAttention encoder\n in Dynamic Coattention Networks For Question Answering (https://arxiv.org/abs/1611.01604)\n\n check the Figure 2 in paper\n\n * Args:\n embed_dim: the number of input embedding dimension\n """\n\n def __init__(self, embed_dim):\n super(CoAttention, self).__init__()\n\n self.W_0 = nn.Linear(embed_dim * 3, 1, bias=False)\n\n[docs] def forward(self, context_embed, question_embed, context_mask=None, question_mask=None):\n C, Q = context_embed, question_embed\n B, C_L, Q_L, D = C.size(0), C.size(1), Q.size(1), Q.size(2)\n\n similarity_matrix_shape = torch.zeros(B, C_L, Q_L, D) # (B, C_L, Q_L, D)\n\n C_ = C.unsqueeze(2).expand_as(similarity_matrix_shape)\n Q_ = Q.unsqueeze(1).expand_as(similarity_matrix_shape)\n C_Q = torch.mul(C_, Q_)\n\n S = self.W_0(torch.cat([C_, Q_, C_Q], 3)).squeeze(3) # (B, C_L, Q_L)\n\n S_question = S\n if question_mask is not None:\n S_question = f.add_masked_value(S_question, question_mask.unsqueeze(1), value=-1e7)\n S_q = F.softmax(S_question, 2) # (B, C_L, Q_L)\n\n S_context = S.transpose(1, 2)\n if context_mask is not None:\n S_context = f.add_masked_value(S_context, context_mask.unsqueeze(1), value=-1e7)\n S_c = F.softmax(S_context, 2) # (B, Q_L, C_L)\n\n A = torch.bmm(S_q, Q) # context2query (B, C_L, D)\n B = torch.bmm(S_q, S_c).bmm(C) # query2context (B, Q_L, D)\n out = torch.cat([C, A, C * A, C * B], dim=-1)\n return out\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom claf.modules import initializer\nimport claf.modules.functional as f\n\n\n[docs]class DocQAAttention(nn.Module):\n """\n Bi-Attention Layer + (Self-Attention)\n in DocumentQA (https://arxiv.org/abs/1710.10723)\n\n * Args:\n rnn_dim: the number of GRU cell hidden size\n linear_dim: the number of linear hidden size\n\n * Kwargs:\n self_attn: (bool) self-attention\n weight_init: (bool) weight initialization\n\n """\n\n def __init__(self, rnn_dim, linear_dim, self_attn=False, weight_init=True):\n super(DocQAAttention, self).__init__()\n self.self_attn = self_attn\n\n self.input_w = nn.Linear(2 * rnn_dim, 1, bias=False)\n self.key_w = nn.Linear(2 * rnn_dim, 1, bias=False)\n\n self.dot_w = nn.Parameter(torch.randn(1, 1, rnn_dim * 2))\n torch.nn.init.xavier_uniform_(self.dot_w)\n\n self.bias = nn.Parameter(torch.FloatTensor([[1]]))\n self.diag_mask = nn.Parameter(torch.eye(5000)) # NOTE: (hard-code) max_sequence_length\n\n if weight_init:\n initializer.weight(self.input_w)\n initializer.weight(self.key_w)\n\n[docs] def forward(self, x, x_mask, key, key_mask):\n S = self._trilinear(x, key)\n\n if self.self_attn:\n seq_length = x.size(1)\n diag_mask = self.diag_mask.narrow(0, 0, seq_length).narrow(1, 0, seq_length)\n joint_mask = 1 - self._compute_attention_mask(x_mask, key_mask)\n mask = torch.clamp(diag_mask + joint_mask, 0, 1)\n masked_S = S + mask * (-1e7)\n x2key = self._x2key(masked_S, key, key_mask)\n return torch.cat((x, x2key, x * x2key), dim=-1)\n else:\n joint_mask = 1 - self._compute_attention_mask(x_mask, key_mask)\n masked_S = S + joint_mask * (-1e7)\n x2key = self._x2key(masked_S, key, key_mask)\n\n masked_S = f.add_masked_value(S, key_mask.unsqueeze(1), value=-1e7)\n key2x = self._key2x(masked_S.max(dim=-1)[0], x, x_mask)\n return torch.cat((x, x2key, x * x2key, x * key2x), dim=-1)\n\n def _compute_attention_mask(self, x_mask, key_mask):\n x_mask = x_mask.unsqueeze(2)\n key_mask = key_mask.unsqueeze(1)\n joint_mask = torch.mul(x_mask, key_mask)\n return joint_mask\n\n def _trilinear(self, x, key):\n B, X_L, K_L = x.size(0), x.size(1), key.size(1)\n\n matrix_shape = (B, X_L, K_L)\n x_logits = self.input_w(x).expand(matrix_shape)\n key_logits = self.key_w(key).transpose(1, 2).expand(matrix_shape)\n\n x_dots = torch.mul(x, self.dot_w)\n x_key = torch.matmul(x_dots, key.transpose(1, 2))\n\n return x_logits + key_logits + x_key\n\n def _x2key(self, S, key, key_mask):\n if self.self_attn:\n bias = torch.exp(self.bias)\n S = torch.exp(S)\n attention = S / (S.sum(dim=-1, keepdim=True).expand(S.size()) + bias.expand(S.size()))\n else:\n attention = F.softmax(S, dim=-1) # (B, C_L, Q_L)\n\n x2key = f.weighted_sum(attention=attention, matrix=key) # (B, C_L, 2d)\n return x2key\n\n def _key2x(self, S, x, x_mask):\n attention = f.masked_softmax(S, x_mask) # (B, C_L)\n key2x = f.weighted_sum(attention=attention, matrix=x)\n return key2x.unsqueeze(1).expand(x.size()) # (B, C_L, 2d)\n\n\nimport math\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport claf.modules.functional as f\n\n\n[docs]class MultiHeadAttention(nn.Module):\n """\n Transformer's Multi-Head Attention\n in "Attention is All You Need" (https://arxiv.org/abs/1706.03762)\n\n * Kwargs:\n num_head: the number of Head\n model_dim: the number of model dimension\n linear_key_dim: the number of linear key dimemsion\n linear_value_dim: the number of linear value dimension\n """\n\n def __init__(\n self, num_head=8, model_dim=100, dropout=0.1, linear_key_dim=None, linear_value_dim=None\n ):\n super(MultiHeadAttention, self).__init__()\n if linear_key_dim is None:\n linear_key_dim = model_dim\n if linear_value_dim is None:\n linear_value_dim = model_dim\n\n assert linear_key_dim % num_head == 0\n assert linear_value_dim % num_head == 0\n\n self.model_dim = model_dim\n self.num_head = num_head\n self.projection = nn.ModuleList(\n [\n nn.Linear(model_dim, linear_key_dim, bias=False), # query\n nn.Linear(model_dim, linear_key_dim, bias=False), # key\n nn.Linear(model_dim, linear_value_dim, bias=False), # value\n ]\n )\n self.out_linear = nn.Linear(linear_value_dim, model_dim)\n\n if dropout > 0:\n self.dropout = nn.Dropout(dropout)\n else:\n self.dropout = lambda x: x\n\n[docs] def forward(self, q, k, v, mask=None):\n q, k, v = self._linear_projection(q, k, v)\n qs, ks, vs = self._split_heads(q, k, v)\n outputs = self._scaled_dot_product(qs, ks, vs, mask=mask)\n output = self._concat_heads(outputs)\n return self.out_linear(output)\n\n def _linear_projection(self, query, key, value):\n q = self.projection[0](query)\n k = self.projection[1](key)\n v = self.projection[2](value)\n return q, k, v\n\n def _split_heads(self, query, key, value):\n B = query.size(0)\n qs, ks, vs = [\n x.view(B, -1, self.num_head, x.size(-1) // self.num_head).transpose(1, 2)\n for x in [query, key, value]\n ]\n return qs, ks, vs\n\n def _scaled_dot_product(self, query, key, value, mask=None):\n K_D = query.size(-1)\n\n scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(K_D)\n\n if mask is not None:\n mask = mask.unsqueeze(1).unsqueeze(1) # [B, #H, C_L, D]\n scores = f.add_masked_value(scores, mask, value=-1e7)\n\n attn = F.softmax(scores, dim=-1)\n attn = self.dropout(attn)\n return torch.matmul(attn, value)\n\n def _concat_heads(self, outputs):\n B = outputs.size(0)\n num_head, dim = outputs.size()[-2:]\n\n return outputs.transpose(1, 2).contiguous().view(B, -1, self.num_head * dim)\n\n#!/usr/bin/env python3\n# Copyright 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the license found in the\n# LICENSE file in the root directory of this source tree.\n\n"""\noriginal code from: https://github.com/facebookresearch/DrQA/blob/master/drqa/reader/layers.py\n"""\n\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\n[docs]class SeqAttnMatch(nn.Module):\n """\n Given sequences X and Y, match sequence Y to each element in X.\n * o_i = sum(alpha_j * y_j) for i in X\n * alpha_j = softmax(y_j * x_i)\n """\n\n def __init__(self, embed_dim, identity=False):\n super(SeqAttnMatch, self).__init__()\n if not identity:\n self.linear = nn.Linear(embed_dim, embed_dim)\n else:\n self.linear = None\n\n[docs] def forward(self, x, y, y_mask):\n if self.linear:\n x_proj = self.linear(x.view(-1, x.size(2))).view(x.size())\n x_proj = F.relu(x_proj)\n y_proj = self.linear(y.view(-1, y.size(2))).view(y.size())\n y_proj = F.relu(y_proj)\n else:\n x_proj = x\n y_proj = y\n\n scores = x_proj.bmm(y_proj.transpose(2, 1))\n\n y_mask = y_mask.unsqueeze(1).expand(scores.size())\n scores = scores.masked_fill((y_mask == 0), -1e30)\n\n alpha_flat = F.softmax(scores.view(-1, y.size(1)), -1)\n alpha = alpha_flat.view(-1, x.size(1), y.size(1))\n\n matched_seq = alpha.bmm(y)\n return matched_seq\n\n\n[docs]class LinearSeqAttn(nn.Module):\n """\n Self attention over a sequence:\n * o_i = softmax(Wx_i) for x_i in X.\n """\n\n def __init__(self, input_size):\n super(LinearSeqAttn, self).__init__()\n self.linear = nn.Linear(input_size, 1)\n\n[docs] def forward(self, x, x_mask):\n x_flat = x.contiguous().view(-1, x.size(-1))\n scores = self.linear(x_flat).view(x.size(0), x.size(1))\n scores.data.masked_fill_((x_mask == 0), -1e30)\n alpha = F.softmax(scores, dim=-1)\n return alpha\n\n\n[docs]class BilinearSeqAttn(nn.Module):\n """\n A bilinear attention layer over a sequence X w.r.t y:\n * o_i = softmax(x_i'Wy) for x_i in X.\n Optionally don't normalize output weights.\n """\n\n def __init__(self, x_size, y_size, identity=False, normalize=True):\n super(BilinearSeqAttn, self).__init__()\n self.normalize = normalize\n\n if not identity:\n self.linear = nn.Linear(y_size, x_size)\n else:\n self.linear = None\n\n[docs] def forward(self, x, y, x_mask):\n Wy = self.linear(y) if self.linear is not None else y\n xWy = x.bmm(Wy.unsqueeze(2)).squeeze(2)\n xWy.data.masked_fill_((x_mask == 0), -1e30)\n if self.normalize:\n if self.training:\n alpha = F.log_softmax(xWy, dim=-1)\n else:\n alpha = F.softmax(xWy, dim=-1)\n else:\n alpha = xWy.exp()\n return alpha\n\n\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom .pointwise_conv import PointwiseConv\n\n\n[docs]class DepSepConv(nn.Module):\n """\n Depthwise Separable Convolutions\n in Xception: Deep Learning with Depthwise Separable Convolutions (https://arxiv.org/abs/1610.02357)\n\n depthwise -> pointwise (1x1 conv)\n\n * Args:\n input_size: the number of input tensor's dimension\n num_filters: the number of convolution filter\n kernel_size: the number of convolution kernel size\n """\n\n def __init__(self, input_size=None, num_filters=None, kernel_size=None):\n super(DepSepConv, self).__init__()\n\n self.depthwise = nn.Conv1d(\n in_channels=input_size,\n out_channels=input_size,\n kernel_size=kernel_size,\n groups=input_size,\n padding=kernel_size // 2,\n )\n nn.init.kaiming_normal_(self.depthwise.weight)\n self.pointwise = PointwiseConv(input_size=input_size, num_filters=num_filters)\n self.activation_fn = F.relu\n\n[docs] def forward(self, x):\n x = self.depthwise(x.transpose(1, 2))\n x = self.pointwise(x.transpose(1, 2))\n x = self.activation_fn(x)\n return x\n\n\nimport torch\nimport torch.nn as nn\n\n\n[docs]class PointwiseConv(nn.Module):\n """\n Pointwise Convolution (1x1 Conv)\n\n Convolution 1 Dimension (Faster version)\n (cf. https://github.com/huggingface/pytorch-openai-transformer-lm/blob/\\\n eafc28abdfadfa0732f03a0fc65805c5bfb2ffe7/model_pytorch.py#L45)\n\n * Args:\n input_size: the number of input tensor's dimension\n num_filters: the number of convolution filter\n """\n\n # nf: num_filters, rf: kernel_size, nx: in_channels\n def __init__(self, input_size, num_filters):\n super(PointwiseConv, self).__init__()\n\n self.kernel_size = 1\n self.num_filters = num_filters\n\n weight = torch.empty(input_size, num_filters)\n nn.init.normal_(weight, std=0.02)\n self.weight = nn.Parameter(weight)\n self.bias = nn.Parameter(torch.zeros(num_filters))\n\n[docs] def forward(self, x):\n size_out = x.size()[:-1] + (self.num_filters,)\n x = torch.addmm(self.bias, x.contiguous().view(-1, x.size(-1)), self.weight)\n x = x.view(*size_out)\n return x\n\n"""\nThis code is from allenai/allennlp\n(https://github.com/allenai/allennlp/blob/master/allennlp/modules/lstm_cell_with_projection.py)\n"""\n\nimport itertools\n\nfrom typing import Callable, List, Tuple, Union, Optional\nimport torch\nfrom torch.nn.utils.rnn import pack_padded_sequence, PackedSequence\n\n\n[docs]class LstmCellWithProjection(torch.nn.Module): # pragma: no cover\n """\n An LSTM with Recurrent Dropout and a projected and clipped hidden state and\n memory. Note: this implementation is slower than the native Pytorch LSTM because\n it cannot make use of CUDNN optimizations for stacked RNNs due to and\n variational dropout and the custom nature of the cell state.\n Parameters\n ----------\n input_size : ``int``, required.\n The dimension of the inputs to the LSTM.\n hidden_size : ``int``, required.\n The dimension of the outputs of the LSTM.\n cell_size : ``int``, required.\n The dimension of the memory cell used for the LSTM.\n go_forward: ``bool``, optional (default = True)\n The direction in which the LSTM is applied to the sequence.\n Forwards by default, or backwards if False.\n recurrent_dropout_probability: ``float``, optional (default = 0.0)\n The dropout probability to be used in a dropout scheme as stated in\n `A Theoretically Grounded Application of Dropout in Recurrent Neural Networks\n <https://arxiv.org/abs/1512.05287>`_ . Implementation wise, this simply\n applies a fixed dropout mask per sequence to the recurrent connection of the\n LSTM.\n state_projection_clip_value: ``float``, optional, (default = None)\n The magnitude with which to clip the hidden_state after projecting it.\n memory_cell_clip_value: ``float``, optional, (default = None)\n The magnitude with which to clip the memory cell.\n Returns\n -------\n output_accumulator : ``torch.FloatTensor``\n The outputs of the LSTM for each timestep. A tensor of shape\n (batch_size, max_timesteps, hidden_size) where for a given batch\n element, all outputs past the sequence length for that batch are\n zero tensors.\n final_state: ``Tuple[torch.FloatTensor, torch.FloatTensor]``\n The final (state, memory) states of the LSTM, with shape\n (1, batch_size, hidden_size) and (1, batch_size, cell_size)\n respectively. The first dimension is 1 in order to match the Pytorch\n API for returning stacked LSTM states.\n """\n\n def __init__(\n self,\n input_size: int,\n hidden_size: int,\n cell_size: int,\n go_forward: bool = True,\n recurrent_dropout_probability: float = 0.0,\n memory_cell_clip_value: Optional[float] = None,\n state_projection_clip_value: Optional[float] = None,\n ) -> None:\n super(LstmCellWithProjection, self).__init__()\n # Required to be wrapped with a :class:`PytorchSeq2SeqWrapper`.\n self.input_size = input_size\n self.hidden_size = hidden_size\n self.cell_size = cell_size\n\n self.go_forward = go_forward\n self.state_projection_clip_value = state_projection_clip_value\n self.memory_cell_clip_value = memory_cell_clip_value\n self.recurrent_dropout_probability = recurrent_dropout_probability\n\n # We do the projections for all the gates all at once.\n self.input_linearity = torch.nn.Linear(input_size, 4 * cell_size, bias=False)\n self.state_linearity = torch.nn.Linear(hidden_size, 4 * cell_size, bias=True)\n\n # Additional projection matrix for making the hidden state smaller.\n self.state_projection = torch.nn.Linear(cell_size, hidden_size, bias=False)\n self.reset_parameters()\n\n[docs] def reset_parameters(self):\n # Use sensible default initializations for parameters.\n block_orthogonal(self.input_linearity.weight.data, [self.cell_size, self.input_size])\n block_orthogonal(self.state_linearity.weight.data, [self.cell_size, self.hidden_size])\n\n self.state_linearity.bias.data.fill_(0.0)\n # Initialize forget gate biases to 1.0 as per An Empirical\n # Exploration of Recurrent Network Architectures, (Jozefowicz, 2015).\n self.state_linearity.bias.data[self.cell_size : 2 * self.cell_size].fill_(1.0)\n\n[docs] def forward(\n self, # pylint: disable=arguments-differ\n inputs: torch.FloatTensor,\n batch_lengths: List[int],\n initial_state: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,\n ):\n """\n Parameters\n ----------\n inputs : ``torch.FloatTensor``, required.\n A tensor of shape (batch_size, num_timesteps, input_size)\n to apply the LSTM over.\n batch_lengths : ``List[int]``, required.\n A list of length batch_size containing the lengths of the sequences in batch.\n initial_state : ``Tuple[torch.Tensor, torch.Tensor]``, optional, (default = None)\n A tuple (state, memory) representing the initial hidden state and memory\n of the LSTM. The ``state`` has shape (1, batch_size, hidden_size) and the\n ``memory`` has shape (1, batch_size, cell_size).\n Returns\n -------\n output_accumulator : ``torch.FloatTensor``\n The outputs of the LSTM for each timestep. A tensor of shape\n (batch_size, max_timesteps, hidden_size) where for a given batch\n element, all outputs past the sequence length for that batch are\n zero tensors.\n final_state : ``Tuple[``torch.FloatTensor, torch.FloatTensor]``\n A tuple (state, memory) representing the initial hidden state and memory\n of the LSTM. The ``state`` has shape (1, batch_size, hidden_size) and the\n ``memory`` has shape (1, batch_size, cell_size).\n """\n batch_size = inputs.size()[0]\n total_timesteps = inputs.size()[1]\n\n output_accumulator = inputs.new_zeros(batch_size, total_timesteps, self.hidden_size)\n\n if initial_state is None:\n full_batch_previous_memory = inputs.new_zeros(batch_size, self.cell_size)\n full_batch_previous_state = inputs.new_zeros(batch_size, self.hidden_size)\n else:\n full_batch_previous_state = initial_state[0].squeeze(0)\n full_batch_previous_memory = initial_state[1].squeeze(0)\n\n current_length_index = batch_size - 1 if self.go_forward else 0\n if self.recurrent_dropout_probability > 0.0 and self.training:\n dropout_mask = get_dropout_mask(\n self.recurrent_dropout_probability, full_batch_previous_state\n )\n else:\n dropout_mask = None\n\n for timestep in range(total_timesteps):\n # The index depends on which end we start.\n index = timestep if self.go_forward else total_timesteps - timestep - 1\n\n # What we are doing here is finding the index into the batch dimension\n # which we need to use for this timestep, because the sequences have\n # variable length, so once the index is greater than the length of this\n # particular batch sequence, we no longer need to do the computation for\n # this sequence. The key thing to recognise here is that the batch inputs\n # must be _ordered_ by length from longest (first in batch) to shortest\n # (last) so initially, we are going forwards with every sequence and as we\n # pass the index at which the shortest elements of the batch finish,\n # we stop picking them up for the computation.\n if self.go_forward:\n while batch_lengths[current_length_index] <= index:\n current_length_index -= 1\n # If we're going backwards, we are _picking up_ more indices.\n else:\n # First conditional: Are we already at the maximum number of elements in the batch?\n # Second conditional: Does the next shortest sequence beyond the current batch\n # index require computation use this timestep?\n while (\n current_length_index < (len(batch_lengths) - 1)\n and batch_lengths[current_length_index + 1] > index\n ):\n current_length_index += 1\n\n # Actually get the slices of the batch which we\n # need for the computation at this timestep.\n # shape (batch_size, cell_size)\n previous_memory = full_batch_previous_memory[0 : current_length_index + 1].clone()\n # Shape (batch_size, hidden_size)\n previous_state = full_batch_previous_state[0 : current_length_index + 1].clone()\n # Shape (batch_size, input_size)\n timestep_input = inputs[0 : current_length_index + 1, index]\n\n # Do the projections for all the gates all at once.\n # Both have shape (batch_size, 4 * cell_size)\n projected_input = self.input_linearity(timestep_input)\n projected_state = self.state_linearity(previous_state)\n\n # Main LSTM equations using relevant chunks of the big linear\n # projections of the hidden state and inputs.\n input_gate = torch.sigmoid(\n projected_input[:, (0 * self.cell_size) : (1 * self.cell_size)]\n + projected_state[:, (0 * self.cell_size) : (1 * self.cell_size)]\n )\n forget_gate = torch.sigmoid(\n projected_input[:, (1 * self.cell_size) : (2 * self.cell_size)]\n + projected_state[:, (1 * self.cell_size) : (2 * self.cell_size)]\n )\n memory_init = torch.tanh(\n projected_input[:, (2 * self.cell_size) : (3 * self.cell_size)]\n + projected_state[:, (2 * self.cell_size) : (3 * self.cell_size)]\n )\n output_gate = torch.sigmoid(\n projected_input[:, (3 * self.cell_size) : (4 * self.cell_size)]\n + projected_state[:, (3 * self.cell_size) : (4 * self.cell_size)]\n )\n memory = input_gate * memory_init + forget_gate * previous_memory\n\n # Here is the non-standard part of this LSTM cell; first, we clip the\n # memory cell, then we project the output of the timestep to a smaller size\n # and again clip it.\n\n if self.memory_cell_clip_value:\n # pylint: disable=invalid-unary-operand-type\n memory = torch.clamp(\n memory, -self.memory_cell_clip_value, self.memory_cell_clip_value\n )\n\n # shape (current_length_index, cell_size)\n pre_projection_timestep_output = output_gate * torch.tanh(memory)\n\n # shape (current_length_index, hidden_size)\n timestep_output = self.state_projection(pre_projection_timestep_output)\n if self.state_projection_clip_value:\n # pylint: disable=invalid-unary-operand-type\n timestep_output = torch.clamp(\n timestep_output,\n -self.state_projection_clip_value,\n self.state_projection_clip_value,\n )\n\n # Only do dropout if the dropout prob is > 0.0 and we are in training mode.\n if dropout_mask is not None:\n timestep_output = timestep_output * dropout_mask[0 : current_length_index + 1]\n\n # We've been doing computation with less than the full batch, so here we create a new\n # variable for the the whole batch at this timestep and insert the result for the\n # relevant elements of the batch into it.\n full_batch_previous_memory = full_batch_previous_memory.clone()\n full_batch_previous_state = full_batch_previous_state.clone()\n full_batch_previous_memory[0 : current_length_index + 1] = memory\n full_batch_previous_state[0 : current_length_index + 1] = timestep_output\n output_accumulator[0 : current_length_index + 1, index] = timestep_output\n\n # Mimic the pytorch API by returning state in the following shape:\n # (num_layers * num_directions, batch_size, ...). As this\n # LSTM cell cannot be stacked, the first dimension here is just 1.\n final_state = (\n full_batch_previous_state.unsqueeze(0),\n full_batch_previous_memory.unsqueeze(0),\n )\n\n return output_accumulator, final_state\n\n\n[docs]def get_dropout_mask(\n dropout_probability: float, tensor_for_masking: torch.Tensor\n): # pragma: no cover\n """\n Computes and returns an element-wise dropout mask for a given tensor, where\n each element in the mask is dropped out with probability dropout_probability.\n Note that the mask is NOT applied to the tensor - the tensor is passed to retain\n the correct CUDA tensor type for the mask.\n Parameters\n ----------\n dropout_probability : float, required.\n Probability of dropping a dimension of the input.\n tensor_for_masking : torch.Tensor, required.\n Returns\n -------\n A torch.FloatTensor consisting of the binary mask scaled by 1/ (1 - dropout_probability).\n This scaling ensures expected values and variances of the output of applying this mask\n and the original tensor are the same.\n """\n binary_mask = tensor_for_masking.new_tensor(\n torch.rand(tensor_for_masking.size()) > dropout_probability\n )\n # Scale mask by 1/keep_prob to preserve output statistics.\n dropout_mask = binary_mask.float().div(1.0 - dropout_probability)\n return dropout_mask\n\n\n[docs]def block_orthogonal(\n tensor: torch.Tensor, split_sizes: List[int], gain: float = 1.0\n) -> None: # pragma: no cover\n """\n An initializer which allows initializing model parameters in "blocks". This is helpful\n in the case of recurrent models which use multiple gates applied to linear projections,\n which can be computed efficiently if they are concatenated together. However, they are\n separate parameters which should be initialized independently.\n Parameters\n ----------\n tensor : ``torch.Tensor``, required.\n A tensor to initialize.\n split_sizes : List[int], required.\n A list of length ``tensor.ndim()`` specifying the size of the\n blocks along that particular dimension. E.g. ``[10, 20]`` would\n result in the tensor being split into chunks of size 10 along the\n first dimension and 20 along the second.\n gain : float, optional (default = 1.0)\n The gain (scaling) applied to the orthogonal initialization.\n """\n data = tensor.data\n sizes = list(tensor.size())\n if any([a % b != 0 for a, b in zip(sizes, split_sizes)]):\n raise ValueError(\n "tensor dimensions must be divisible by their respective "\n "split_sizes. Found size: {} and split_sizes: {}".format(sizes, split_sizes)\n )\n indexes = [list(range(0, max_size, split)) for max_size, split in zip(sizes, split_sizes)]\n # Iterate over all possible blocks within the tensor.\n for block_start_indices in itertools.product(*indexes):\n # A list of tuples containing the index to start at for this block\n # and the appropriate step size (i.e split_size[i] for dimension i).\n index_and_step_tuples = zip(block_start_indices, split_sizes)\n # This is a tuple of slices corresponding to:\n # tensor[index: index + step_size, ...]. This is\n # required because we could have an arbitrary number\n # of dimensions. The actual slices we need are the\n # start_index: start_index + step for each dimension in the tensor.\n block_slice = tuple(\n [slice(start_index, start_index + step) for start_index, step in index_and_step_tuples]\n )\n data[block_slice] = torch.nn.init.orthogonal_(tensor[block_slice].contiguous(), gain=gain)\n\n\n[docs]def sort_batch_by_length(tensor: torch.Tensor, sequence_lengths: torch.Tensor): # pragma: no cover\n """\n Sort a batch first tensor by some specified lengths.\n Parameters\n ----------\n tensor : torch.FloatTensor, required.\n A batch first Pytorch tensor.\n sequence_lengths : torch.LongTensor, required.\n A tensor representing the lengths of some dimension of the tensor which\n we want to sort by.\n Returns\n -------\n sorted_tensor : torch.FloatTensor\n The original tensor sorted along the batch dimension with respect to sequence_lengths.\n sorted_sequence_lengths : torch.LongTensor\n The original sequence_lengths sorted by decreasing size.\n restoration_indices : torch.LongTensor\n Indices into the sorted_tensor such that\n ``sorted_tensor.index_select(0, restoration_indices) == original_tensor``\n permuation_index : torch.LongTensor\n The indices used to sort the tensor. This is useful if you want to sort many\n tensors using the same ordering.\n """\n\n if not isinstance(tensor, torch.Tensor) or not isinstance(sequence_lengths, torch.Tensor):\n raise ValueError("Both the tensor and sequence lengths must be torch.Tensors.")\n\n sorted_sequence_lengths, permutation_index = sequence_lengths.sort(0, descending=True)\n sorted_tensor = tensor.index_select(0, permutation_index)\n\n index_range = sequence_lengths.new_tensor(torch.arange(0, len(sequence_lengths)))\n # This is the equivalent of zipping with index, sorting by the original\n # sequence lengths and returning the now sorted indices.\n _, reverse_mapping = permutation_index.sort(0, descending=False)\n restoration_indices = index_range.index_select(0, reverse_mapping)\n return sorted_tensor, sorted_sequence_lengths, restoration_indices, permutation_index\n\n\n# We have two types here for the state, because storing the state in something\n# which is Iterable (like a tuple, below), is helpful for internal manipulation\n# - however, the states are consumed as either Tensors or a Tuple of Tensors, so\n# returning them in this format is unhelpful.\nRnnState = Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] # pylint: disable=invalid-name\nRnnStateStorage = Tuple[torch.Tensor, ...] # pylint: disable=invalid-name\n\n\nclass _EncoderBase(torch.nn.Module): # pragma: no cover\n # pylint: disable=abstract-method\n """\n This abstract class serves as a base for the 3 ``Encoder`` abstractions in AllenNLP.\n - :class:`~allennlp.modules.seq2seq_encoders.Seq2SeqEncoders`\n - :class:`~allennlp.modules.seq2vec_encoders.Seq2VecEncoders`\n Additionally, this class provides functionality for sorting sequences by length\n so they can be consumed by Pytorch RNN classes, which require their inputs to be\n sorted by length. Finally, it also provides optional statefulness to all of it's\n subclasses by allowing the caching and retrieving of the hidden states of RNNs.\n """\n\n def __init__(self, stateful: bool = False) -> None:\n super(_EncoderBase, self).__init__()\n self.stateful = stateful\n self._states: Optional[RnnStateStorage] = None\n\n def sort_and_run_forward(\n self,\n module: Callable[\n [PackedSequence, Optional[RnnState]],\n Tuple[Union[PackedSequence, torch.Tensor], RnnState],\n ],\n inputs: torch.Tensor,\n mask: torch.Tensor,\n hidden_state: Optional[RnnState] = None,\n ):\n """\n This function exists because Pytorch RNNs require that their inputs be sorted\n before being passed as input. As all of our Seq2xxxEncoders use this functionality,\n it is provided in a base class. This method can be called on any module which\n takes as input a ``PackedSequence`` and some ``hidden_state``, which can either be a\n tuple of tensors or a tensor.\n As all of our Seq2xxxEncoders have different return types, we return `sorted`\n outputs from the module, which is called directly. Additionally, we return the\n indices into the batch dimension required to restore the tensor to it's correct,\n unsorted order and the number of valid batch elements (i.e the number of elements\n in the batch which are not completely masked). This un-sorting and re-padding\n of the module outputs is left to the subclasses because their outputs have different\n types and handling them smoothly here is difficult.\n Parameters\n ----------\n module : ``Callable[[PackedSequence, Optional[RnnState]],\n Tuple[Union[PackedSequence, torch.Tensor], RnnState]]``, required.\n A function to run on the inputs. In most cases, this is a ``torch.nn.Module``.\n inputs : ``torch.Tensor``, required.\n A tensor of shape ``(batch_size, sequence_length, embedding_size)`` representing\n the inputs to the Encoder.\n mask : ``torch.Tensor``, required.\n A tensor of shape ``(batch_size, sequence_length)``, representing masked and\n non-masked elements of the sequence for each element in the batch.\n hidden_state : ``Optional[RnnState]``, (default = None).\n A single tensor of shape (num_layers, batch_size, hidden_size) representing the\n state of an RNN with or a tuple of\n tensors of shapes (num_layers, batch_size, hidden_size) and\n (num_layers, batch_size, memory_size), representing the hidden state and memory\n state of an LSTM-like RNN.\n Returns\n -------\n module_output : ``Union[torch.Tensor, PackedSequence]``.\n A Tensor or PackedSequence representing the output of the Pytorch Module.\n The batch size dimension will be equal to ``num_valid``, as sequences of zero\n length are clipped off before the module is called, as Pytorch cannot handle\n zero length sequences.\n final_states : ``Optional[RnnState]``\n A Tensor representing the hidden state of the Pytorch Module. This can either\n be a single tensor of shape (num_layers, num_valid, hidden_size), for instance in\n the case of a GRU, or a tuple of tensors, such as those required for an LSTM.\n restoration_indices : ``torch.LongTensor``\n A tensor of shape ``(batch_size,)``, describing the re-indexing required to transform\n the outputs back to their original batch order.\n """\n # In some circumstances you may have sequences of zero length. ``pack_padded_sequence``\n # requires all sequence lengths to be > 0, so remove sequences of zero length before\n # calling self._module, then fill with zeros.\n\n # First count how many sequences are empty.\n batch_size = mask.size(0)\n num_valid = torch.sum(mask[:, 0]).int().item()\n\n sequence_lengths = mask.long().sum(-1)\n sorted_inputs, sorted_sequence_lengths, restoration_indices, sorting_indices = sort_batch_by_length(\n inputs, sequence_lengths\n )\n\n # Now create a PackedSequence with only the non-empty, sorted sequences.\n packed_sequence_input = pack_padded_sequence(\n sorted_inputs[:num_valid, :, :],\n sorted_sequence_lengths[:num_valid].data.tolist(),\n batch_first=True,\n )\n # Prepare the initial states.\n if not self.stateful:\n if hidden_state is None:\n initial_states = hidden_state\n elif isinstance(hidden_state, tuple):\n initial_states = [\n state.index_select(1, sorting_indices)[:, :num_valid, :].contiguous()\n for state in hidden_state\n ]\n else:\n initial_states = hidden_state.index_select(1, sorting_indices)[\n :, :num_valid, :\n ].contiguous()\n\n else:\n initial_states = self._get_initial_states(batch_size, num_valid, sorting_indices)\n\n # Actually call the module on the sorted PackedSequence.\n module_output, final_states = module(packed_sequence_input, initial_states)\n\n return module_output, final_states, restoration_indices\n\n def _get_initial_states(\n self, batch_size: int, num_valid: int, sorting_indices: torch.LongTensor\n ) -> Optional[RnnState]:\n """\n Returns an initial state for use in an RNN. Additionally, this method handles\n the batch size changing across calls by mutating the state to append initial states\n for new elements in the batch. Finally, it also handles sorting the states\n with respect to the sequence lengths of elements in the batch and removing rows\n which are completely padded. Importantly, this `mutates` the state if the\n current batch size is larger than when it was previously called.\n Parameters\n ----------\n batch_size : ``int``, required.\n The batch size can change size across calls to stateful RNNs, so we need\n to know if we need to expand or shrink the states before returning them.\n Expanded states will be set to zero.\n num_valid : ``int``, required.\n The batch may contain completely padded sequences which get removed before\n the sequence is passed through the encoder. We also need to clip these off\n of the state too.\n sorting_indices ``torch.LongTensor``, required.\n Pytorch RNNs take sequences sorted by length. When we return the states to be\n used for a given call to ``module.forward``, we need the states to match up to\n the sorted sequences, so before returning them, we sort the states using the\n same indices used to sort the sequences.\n Returns\n -------\n This method has a complex return type because it has to deal with the first time it\n is called, when it has no state, and the fact that types of RNN have heterogeneous\n states.\n If it is the first time the module has been called, it returns ``None``, regardless\n of the type of the ``Module``.\n Otherwise, for LSTMs, it returns a tuple of ``torch.Tensors`` with shape\n ``(num_layers, num_valid, state_size)`` and ``(num_layers, num_valid, memory_size)``\n respectively, or for GRUs, it returns a single ``torch.Tensor`` of shape\n ``(num_layers, num_valid, state_size)``.\n """\n # We don't know the state sizes the first time calling forward,\n # so we let the module define what it's initial hidden state looks like.\n if self._states is None:\n return None\n\n # Otherwise, we have some previous states.\n if batch_size > self._states[0].size(1):\n # This batch is larger than the all previous states.\n # If so, resize the states.\n num_states_to_concat = batch_size - self._states[0].size(1)\n resized_states = []\n # state has shape (num_layers, batch_size, hidden_size)\n for state in self._states:\n # This _must_ be inside the loop because some\n # RNNs have states with different last dimension sizes.\n zeros = state.new_zeros(state.size(0), num_states_to_concat, state.size(2))\n resized_states.append(torch.cat([state, zeros], 1))\n self._states = tuple(resized_states)\n correctly_shaped_states = self._states\n\n elif batch_size < self._states[0].size(1):\n # This batch is smaller than the previous one.\n correctly_shaped_states = tuple(state[:, :batch_size, :] for state in self._states)\n else:\n correctly_shaped_states = self._states\n\n # At this point, our states are of shape (num_layers, batch_size, hidden_size).\n # However, the encoder uses sorted sequences and additionally removes elements\n # of the batch which are fully padded. We need the states to match up to these\n # sorted and filtered sequences, so we do that in the next two blocks before\n # returning the state/s.\n if len(self._states) == 1:\n # GRUs only have a single state. This `unpacks` it from the\n # tuple and returns the tensor directly.\n correctly_shaped_state = correctly_shaped_states[0]\n sorted_state = correctly_shaped_state.index_select(1, sorting_indices)\n return sorted_state[:, :num_valid, :]\n else:\n # LSTMs have a state tuple of (state, memory).\n sorted_states = [\n state.index_select(1, sorting_indices) for state in correctly_shaped_states\n ]\n return tuple(state[:, :num_valid, :] for state in sorted_states)\n\n def _update_states(\n self, final_states: RnnStateStorage, restoration_indices: torch.LongTensor\n ) -> None:\n """\n After the RNN has run forward, the states need to be updated.\n This method just sets the state to the updated new state, performing\n several pieces of book-keeping along the way - namely, unsorting the\n states and ensuring that the states of completely padded sequences are\n not updated. Finally, it also detaches the state variable from the\n computational graph, such that the graph can be garbage collected after\n each batch iteration.\n Parameters\n ----------\n final_states : ``RnnStateStorage``, required.\n The hidden states returned as output from the RNN.\n restoration_indices : ``torch.LongTensor``, required.\n The indices that invert the sorting used in ``sort_and_run_forward``\n to order the states with respect to the lengths of the sequences in\n the batch.\n """\n # TODO(Mark): seems weird to sort here, but append zeros in the subclasses.\n # which way around is best?\n new_unsorted_states = [state.index_select(1, restoration_indices) for state in final_states]\n\n if self._states is None:\n # We don't already have states, so just set the\n # ones we receive to be the current state.\n self._states = tuple(state.data for state in new_unsorted_states)\n else:\n # Now we've sorted the states back so that they correspond to the original\n # indices, we need to figure out what states we need to update, because if we\n # didn't use a state for a particular row, we want to preserve its state.\n # Thankfully, the rows which are all zero in the state correspond exactly\n # to those which aren't used, so we create masks of shape (new_batch_size,),\n # denoting which states were used in the RNN computation.\n current_state_batch_size = self._states[0].size(1)\n new_state_batch_size = final_states[0].size(1)\n # Masks for the unused states of shape (1, new_batch_size, 1)\n used_new_rows_mask = [\n (state[0, :, :].sum(-1) != 0.0).float().view(1, new_state_batch_size, 1)\n for state in new_unsorted_states\n ]\n new_states = []\n if current_state_batch_size > new_state_batch_size:\n # The new state is smaller than the old one,\n # so just update the indices which we used.\n for old_state, new_state, used_mask in zip(\n self._states, new_unsorted_states, used_new_rows_mask\n ):\n # zero out all rows in the previous state\n # which _were_ used in the current state.\n masked_old_state = old_state[:, :new_state_batch_size, :] * (1 - used_mask)\n # The old state is larger, so update the relevant parts of it.\n old_state[:, :new_state_batch_size, :] = new_state + masked_old_state\n new_states.append(old_state.detach())\n else:\n # The states are the same size, so we just have to\n # deal with the possibility that some rows weren't used.\n new_states = []\n for old_state, new_state, used_mask in zip(\n self._states, new_unsorted_states, used_new_rows_mask\n ):\n # zero out all rows which _were_ used in the current state.\n masked_old_state = old_state * (1 - used_mask)\n # The old state is larger, so update the relevant parts of it.\n new_state += masked_old_state\n new_states.append(new_state.detach())\n\n # It looks like there should be another case handled here - when\n # the current_state_batch_size < new_state_batch_size. However,\n # this never happens, because the states themeselves are mutated\n # by appending zeros when calling _get_inital_states, meaning that\n # the new states are either of equal size, or smaller, in the case\n # that there are some unused elements (zero-length) for the RNN computation.\n self._states = tuple(new_states)\n\n def reset_states(self):\n self._states = None\n\n\nimport math\nimport numpy as np\nimport torch\nimport torch.nn as nn\n\n\n[docs]class PositionalEncoding(nn.Module):\n """\n Positional Encoding\n in "Attention is All You Need" (https://arxiv.org/abs/1706.03762)\n\n The use of relative position is possible because sin(x+y) and cos(x+y) can be\n expressed in terms of y, sin(x) and cos(x).\n\n (cf. https://github.com/tensorflow/tensor2tensor/blob/42c3f377f441e5a0f431127d63e71414ead291c4/\\\n tensor2tensor/layers/common_attention.py#L388)\n\n * Args:\n embed_dim: the number of embedding dimension\n\n * Kwargs:\n max_len: the number of maximum sequence length\n """\n\n def __init__(self, embed_dim, max_length=2000):\n super(PositionalEncoding, self).__init__()\n signal_sinusoid = self._get_timing_signal(max_length, embed_dim)\n\n self.register_buffer("position_encoding", signal_sinusoid)\n\n def _get_timing_signal(self, length, channels, min_timescale=1.0, max_timescale=1.0e4):\n position = np.arange(length)\n num_timescales = channels // 2\n log_timescale_increment = math.log(\n float(max_timescale) / float(min_timescale) / (float(num_timescales) - 1)\n )\n inv_timescales = min_timescale * np.exp(\n np.arange(num_timescales).astype(np.float) * -log_timescale_increment\n )\n scaled_time = np.expand_dims(position, 1) * np.expand_dims(inv_timescales, 0)\n\n signal = np.concatenate([np.sin(scaled_time), np.cos(scaled_time)], axis=1)\n signal = np.pad(signal, [[0, 0], [0, channels % 2]], "constant", constant_values=[0.0, 0.0])\n signal = signal.reshape([1, length, channels])\n\n return torch.from_numpy(signal).type(torch.FloatTensor)\n\n\n\n"""\n some functional codes from allennlp: https://github.com/allenai/allennlp\n\n - add_masked_value : replace_masked_values (allennlp)\n - get_mask_from_tokens : get_mask_from_tokens (allennlp)\n - last_dim_masked_softmax : last_dim_masked_softmax (allennlp)\n - masked_softmax : masked_softmax (allennlp)\n - weighted_sum : weighted_sum (allennlp)\n"""\n\nimport torch\nimport torch.nn.functional as F\nfrom torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n\n\n[docs]def add_masked_value(tensor, mask, value=-1e7):\n mask = mask.float()\n one_minus_mask = 1.0 - mask\n values_to_add = value * one_minus_mask\n return tensor * mask + values_to_add\n\n\n[docs]def get_mask_from_tokens(tokens):\n tensor_dims = [(tensor.dim(), tensor) for tensor in tokens.values()]\n tensor_dims.sort(key=lambda x: x[0])\n\n smallest_dim = tensor_dims[0][0]\n if smallest_dim == 2:\n token_tensor = tensor_dims[0][1]\n return (token_tensor != 0).long()\n elif smallest_dim == 3:\n character_tensor = tensor_dims[0][1]\n return ((character_tensor > 0).long().sum(dim=-1) > 0).long()\n else:\n raise ValueError("Expected a tensor with dimension 2 or 3, found {}".format(smallest_dim))\n\n\n[docs]def last_dim_masked_softmax(x, mask):\n x_shape = x.size()\n reshaped_x = x.view(-1, x.size()[-1])\n\n while mask.dim() < x.dim():\n mask = mask.unsqueeze(1)\n mask = mask.expand_as(x).contiguous().float()\n mask = mask.view(-1, mask.size()[-1])\n\n reshaped_result = masked_softmax(reshaped_x, mask)\n return reshaped_result.view(*x_shape)\n\n\n[docs]def masked_softmax(x, mask):\n if mask is None:\n raise ValueError("mask can't be None.")\n\n output = F.softmax(x * mask, dim=-1)\n output = output * mask\n output = output / (output.sum(dim=1, keepdim=True) + 1e-13)\n return output\n\n\n[docs]def weighted_sum(attention, matrix): # pragma: no cover\n if attention.dim() == 2 and matrix.dim() == 3:\n return attention.unsqueeze(1).bmm(matrix).squeeze(1)\n elif attention.dim() == 3 and matrix.dim() == 3:\n return attention.bmm(matrix)\n else:\n raise ValueError(\n f"attention dim {attention.dim()} and matrix dim {matrix.dim()} operation not support. (2, 3) and (3, 3) are available dimemsion."\n )\n\n\n[docs]def masked_zero(tensor, mask):\n """ Tensor masking operation """\n while mask.dim() < tensor.dim():\n mask = mask.unsqueeze(-1)\n\n if isinstance(tensor, torch.FloatTensor):\n mask = mask.float()\n elif isinstance(tensor, torch.ByteTensor):\n mask = mask.byte()\n elif isinstance(tensor, torch.LongTensor):\n mask = mask.long()\n\n return tensor * mask\n\n\n[docs]def masked_log_softmax(vector, mask): # pragma: no cover\n if mask is not None:\n vector = vector + mask.float().log()\n return torch.nn.functional.log_softmax(vector, dim=1)\n\n\n[docs]def get_sorted_seq_config(features, pad_index=0):\n tensor_dims = [(tensor.dim(), tensor) for tensor in features.values()]\n tensor_dims.sort(key=lambda x: x[0])\n\n smallest_dim = tensor_dims[0][0]\n if smallest_dim == 2:\n token_tensor = tensor_dims[0][1]\n else:\n raise ValueError("features smallest_dim must be `2` ([B, S_L]) ")\n\n seq_lengths = torch.sum(token_tensor > pad_index, dim=-1)\n seq_lengths, perm_idx = seq_lengths.sort(0, descending=True)\n _, unperm_idx = perm_idx.sort(0)\n\n return {"seq_lengths": seq_lengths, "perm_idx": perm_idx, "unperm_idx": unperm_idx}\n\n\n[docs]def forward_rnn_with_pack(rnn_module, tensor, seq_config):\n sorted_tensor = tensor[seq_config["perm_idx"]]\n packed_input = pack_padded_sequence(sorted_tensor, seq_config["seq_lengths"], batch_first=True)\n packed_output, _ = rnn_module(packed_input)\n output, _ = pad_packed_sequence(packed_output, batch_first=True)\n output = output[seq_config["unperm_idx"]] # restore origin order\n return output\n\n\nimport logging\n\nimport torch\nimport torch.nn as nn\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]def weight(module):\n """\n weight initialization (according to module type)\n\n * Args:\n module: torch.nn.Module\n """\n\n if type(module) == list:\n for m in module:\n weight(m)\n\n if isinstance(module, nn.Conv2d):\n logger.info("initializing Conv Layer")\n torch.nn.init.uniform_(module.weight)\n\n elif isinstance(module, nn.Linear):\n torch.nn.init.xavier_uniform_(module.weight)\n logger.info("Initializing Linear Layer")\n\n elif isinstance(module, nn.GRU):\n torch.nn.init.normal_(module.weight_hh_l0, std=0.05)\n logger.info("Initializing GRU Layer")\n\n\nimport torch\nimport torch.nn as nn\n\nfrom claf.modules.activation import get_activation_fn\n\n\n[docs]class Highway(nn.Module):\n """\n Highway Networks (https://arxiv.org/abs/1505.00387)\n https://github.com/allenai/allennlp/blob/master/allennlp/modules/highway.py\n\n * Args:\n input_size: The number of expected features in the input `x`\n num_layers: The number of Highway layers.\n activation: Activation Function (ReLU is default)\n """\n\n def __init__(self, input_size, num_layers=2, activation="relu"):\n super(Highway, self).__init__()\n self.activation_fn = activation\n if type(activation) == str:\n self.activation_fn = get_activation_fn(activation)()\n self._layers = torch.nn.ModuleList(\n [nn.Linear(input_size, input_size * 2) for _ in range(num_layers)]\n )\n\n for layer in self._layers:\n layer.bias[input_size:].data.fill_(\n 1\n ) # should bias the highway layer to just carry its input forward.\n\n[docs] def forward(self, x):\n current_input = x\n for layer in self._layers:\n projected_input = layer(current_input)\n linear_part = current_input\n\n nonlinear_part, gate = projected_input.chunk(2, dim=-1)\n nonlinear_part = self.activation_fn(nonlinear_part)\n gate = torch.sigmoid(gate)\n\n current_input = gate * linear_part + (1 - gate) * nonlinear_part\n return current_input\n\n\nimport torch\nimport torch.nn as nn\n\n\n[docs]class LayerNorm(nn.Module):\n """\n Layer Normalization\n (https://arxiv.org/abs/1607.06450)\n """\n\n def __init__(self, normalized_shape, eps=1e-5):\n super(LayerNorm, self).__init__()\n self.gamma = nn.Parameter(torch.ones(normalized_shape))\n self.beta = nn.Parameter(torch.zeros(normalized_shape))\n self.eps = eps\n\n[docs] def forward(self, x):\n mean = x.mean(-1, keepdim=True)\n std = x.std(-1, keepdim=True)\n return self.gamma * (x - mean) / (std + self.eps) + self.beta\n\n\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom claf.modules.conv import PointwiseConv\n\n\n[docs]class PositionwiseFeedForward(nn.Module):\n """\n Pointwise Feed-Forward Layer\n\n * Args:\n input_size: the number of input size\n hidden_size: the number of hidden size\n\n * Kwargs:\n dropout: the probability of dropout\n """\n\n def __init__(self, input_size, hidden_size, dropout=0.1):\n super(PositionwiseFeedForward, self).__init__()\n self.pointwise_conv1 = PointwiseConv(input_size=input_size, num_filters=hidden_size)\n self.pointwise_conv2 = PointwiseConv(input_size=hidden_size, num_filters=input_size)\n self.activation_fn = F.relu\n self.dropout = nn.Dropout(p=dropout)\n\n[docs] def forward(self, x):\n x = self.pointwise_conv1(x)\n x = self.activation_fn(x)\n x = self.pointwise_conv2(x)\n x = self.dropout(x)\n return x\n\nimport torch\nimport torch.nn as nn\n\nfrom claf.modules.layer.normalization import LayerNorm\n\n\n[docs]class ResidualConnection(nn.Module):\n """\n ResidualConnection\n in Deep Residual Learning for Image Recognition (https://arxiv.org/abs/1512.03385)\n\n => f(x) + x\n\n * Args:\n dim: the number of dimension\n\n * Kwargs:\n layer_dropout: layer dropout probability (stochastic depth)\n dropout: dropout probability\n """\n\n def __init__(self, dim, layer_dropout=None, layernorm=False):\n super(ResidualConnection, self).__init__()\n\n self.survival = None\n if layer_dropout < 1:\n self.survival = torch.FloatTensor([layer_dropout])\n if layernorm:\n self.norm = LayerNorm(dim)\n else:\n self.norm = lambda x: x\n\n[docs] def forward(self, x, sub_layer_fn):\n # implementation of stochastic depth\n if self.training and self.survival is not None:\n survival_prob = torch.bernoulli(self.survival).item()\n if survival_prob == 1:\n return x + sub_layer_fn(self.norm(x))\n else:\n return x\n else:\n return x + sub_layer_fn(self.norm(x))\n\n"""\nThis code is from allenai/allennlp\n(https://github.com/allenai/allennlp/blob/master/allennlp/modules/scalar_mix.py)\n"""\n\nfrom typing import List\n\nimport torch\nfrom torch.nn import ParameterList, Parameter\n\n\n[docs]class ScalarMix(torch.nn.Module): # pragma: no cover\n """\n Computes a parameterised scalar mixture of N tensors, ``mixture = gamma * sum(s_k * tensor_k)``\n where ``s = softmax(w)``, with ``w`` and ``gamma`` scalar parameters.\n In addition, if ``do_layer_norm=True`` then apply layer normalization to each tensor\n before weighting.\n """\n\n def __init__(\n self,\n mixture_size: int,\n do_layer_norm: bool = False,\n initial_scalar_parameters: List[float] = None,\n trainable: bool = True,\n ) -> None:\n super(ScalarMix, self).__init__()\n self.mixture_size = mixture_size\n self.do_layer_norm = do_layer_norm\n\n if initial_scalar_parameters is None:\n initial_scalar_parameters = [0.0] * mixture_size\n elif len(initial_scalar_parameters) != mixture_size:\n raise ValueError(\n "Length of initial_scalar_parameters {} differs "\n "from mixture_size {}".format(initial_scalar_parameters, mixture_size)\n )\n\n self.scalar_parameters = ParameterList(\n [\n Parameter(\n torch.FloatTensor([initial_scalar_parameters[i]]), requires_grad=trainable\n )\n for i in range(mixture_size)\n ]\n )\n self.gamma = Parameter(torch.FloatTensor([1.0]), requires_grad=trainable)\n\n[docs] def forward(\n self,\n tensors: List[torch.Tensor], # pylint: disable=arguments-differ\n mask: torch.Tensor = None,\n ) -> torch.Tensor:\n """\n Compute a weighted average of the ``tensors``. The input tensors an be any shape\n with at least two dimensions, but must all be the same shape.\n When ``do_layer_norm=True``, the ``mask`` is required input. If the ``tensors`` are\n dimensioned ``(dim_0, ..., dim_{n-1}, dim_n)``, then the ``mask`` is dimensioned\n ``(dim_0, ..., dim_{n-1})``, as in the typical case with ``tensors`` of shape\n ``(batch_size, timesteps, dim)`` and ``mask`` of shape ``(batch_size, timesteps)``.\n When ``do_layer_norm=False`` the ``mask`` is ignored.\n """\n if len(tensors) != self.mixture_size:\n raise ValueError(\n "{} tensors were passed, but the module was initialized to "\n "mix {} tensors.".format(len(tensors), self.mixture_size)\n )\n\n def _do_layer_norm(tensor, broadcast_mask, num_elements_not_masked):\n tensor_masked = tensor * broadcast_mask\n mean = torch.sum(tensor_masked) / num_elements_not_masked\n variance = (\n torch.sum(((tensor_masked - mean) * broadcast_mask) ** 2) / num_elements_not_masked\n )\n return (tensor - mean) / torch.sqrt(variance + 1E-12)\n\n normed_weights = torch.nn.functional.softmax(\n torch.cat([parameter for parameter in self.scalar_parameters]), dim=0\n )\n normed_weights = torch.split(normed_weights, split_size_or_sections=1)\n\n if not self.do_layer_norm:\n pieces = []\n for weight, tensor in zip(normed_weights, tensors):\n pieces.append(weight * tensor)\n return self.gamma * sum(pieces)\n\n else:\n mask_float = mask.float()\n broadcast_mask = mask_float.unsqueeze(-1)\n input_dim = tensors[0].size(-1)\n num_elements_not_masked = torch.sum(mask_float) * input_dim\n\n pieces = []\n for weight, tensor in zip(normed_weights, tensors):\n pieces.append(\n weight * _do_layer_norm(tensor, broadcast_mask, num_elements_not_masked)\n )\n return self.gamma * sum(pieces)\n\n"""\nThis code is from salesforce/cove\n(https://github.com/salesforce/cove/blob/master/cove/encoder.py)\n"""\n\nimport torch\nfrom torch import nn\n\nfrom claf.data.data_handler import CachePath, DataHandler\n\n\n[docs]class MTLSTM(nn.Module):\n def __init__(\n self, word_embedding, pretrained_path=None, requires_grad=False, residual_embeddings=False\n ):\n """Initialize an MTLSTM.\n\n Arguments:\n n_vocab (bool): If not None, initialize MTLSTM with an embedding matrix with n_vocab vectors\n vectors (Float Tensor): If not None, initiapize embedding matrix with specified vectors\n residual_embedding (bool): If True, concatenate the input embeddings with MTLSTM outputs during forward\n """\n super(MTLSTM, self).__init__()\n self.word_embedding = word_embedding\n self.rnn = nn.LSTM(300, 300, num_layers=2, bidirectional=True, batch_first=True)\n\n data_handler = DataHandler(cache_path=CachePath.PRETRAINED_VECTOR)\n cove_weight_path = data_handler.read(pretrained_path, return_path=True)\n\n if torch.cuda.is_available():\n checkpoint = torch.load(cove_weight_path)\n else:\n checkpoint = torch.load(cove_weight_path, map_location="cpu")\n\n self.rnn.load_state_dict(checkpoint)\n self.residual_embeddings = residual_embeddings\n self.requires_grad = requires_grad\n\n[docs] def forward(self, inputs):\n """A pretrained MT-LSTM (McCann et. al. 2017).\n This LSTM was trained with 300d 840B GloVe on the WMT 2017 machine translation dataset.\n\n Arguments:\n inputs (Tensor): If MTLSTM handles embedding, a Long Tensor of size (batch_size, timesteps).\n Otherwise, a Float Tensor of size (batch_size, timesteps, features).\n lengths (Long Tensor): (batch_size, lengths) lenghts of each sequence for handling padding\n hidden (Float Tensor): initial hidden state of the LSTM\n """\n embedded_inputs = self.word_embedding(inputs)\n encoded_inputs, _ = self.rnn(embedded_inputs)\n if not self.requires_grad:\n encoded_inputs.detach()\n\n outputs = encoded_inputs\n if self.residual_embeddings:\n outputs = torch.cat([embedded_inputs, encoded_inputs], 2)\n\n return outputs\n\n"""\nThis code is from allenai/allennlp\n(https://github.com/allenai/allennlp/blob/master/allennlp/modules/elmo.py)\n"""\n\nimport json\nimport logging\nfrom typing import Union, List, Dict, Any, Optional, Tuple\nimport warnings\n\nimport numpy\nfrom overrides import overrides\nimport torch\nfrom torch.nn.utils.rnn import PackedSequence, pad_packed_sequence\nfrom torch.nn.modules import Dropout\n\n\nwith warnings.catch_warnings(): # pragma: no cover\n warnings.filterwarnings("ignore", category=FutureWarning)\n import h5py\n\nfrom claf.modules.layer import Highway, ScalarMix\nfrom claf.modules.encoder import _EncoderBase, LstmCellWithProjection\n\n\nlogger = logging.getLogger(__name__) # pylint: disable=invalid-name\n\n# pylint: disable=attribute-defined-outside-init\n\n\n[docs]class Elmo(torch.nn.Module): # pragma: no cover\n """\n Compute ELMo representations using a pre-trained bidirectional language model.\n See "Deep contextualized word representations", Peters et al. for details.\n This module takes character id input and computes ``num_output_representations`` different layers\n of ELMo representations. Typically ``num_output_representations`` is 1 or 2. For example, in\n the case of the SRL model in the above paper, ``num_output_representations=1`` where ELMo was included at\n the input token representation layer. In the case of the SQuAD model, ``num_output_representations=2``\n as ELMo was also included at the GRU output layer.\n In the implementation below, we learn separate scalar weights for each output layer,\n but only run the biLM once on each input sequence for efficiency.\n Parameters\n ----------\n options_file : ``str``, required.\n ELMo JSON options file\n weight_file : ``str``, required.\n ELMo hdf5 weight file\n num_output_representations: ``int``, required.\n The number of ELMo representation layers to output.\n requires_grad: ``bool``, optional\n If True, compute gradient of ELMo parameters for fine tuning.\n do_layer_norm : ``bool``, optional, (default=False).\n Should we apply layer normalization (passed to ``ScalarMix``)?\n dropout : ``float``, optional, (default = 0.5).\n The dropout to be applied to the ELMo representations.\n vocab_to_cache : ``List[str]``, optional, (default = 0.5).\n A list of words to pre-compute and cache character convolutions\n for. If you use this option, Elmo expects that you pass word\n indices of shape (batch_size, timesteps) to forward, instead\n of character indices. If you use this option and pass a word which\n wasn't pre-cached, this will break.\n module : ``torch.nn.Module``, optional, (default = None).\n If provided, then use this module instead of the pre-trained ELMo biLM.\n If using this option, then pass ``None`` for both ``options_file``\n and ``weight_file``. The module must provide a public attribute\n ``num_layers`` with the number of internal layers and its ``forward``\n method must return a ``dict`` with ``activations`` and ``mask`` keys\n (see `_ElmoBilm`` for an example). Note that ``requires_grad`` is also\n ignored with this option.\n """\n\n def __init__(\n self,\n options_file: str,\n weight_file: str,\n num_output_representations: int,\n requires_grad: bool = False,\n do_layer_norm: bool = False,\n dropout: float = 0.5,\n vocab_to_cache: List[str] = None,\n module: torch.nn.Module = None,\n ) -> None:\n super(Elmo, self).__init__()\n\n logging.info("Initializing ELMo")\n if module is not None:\n if options_file is not None or weight_file is not None:\n raise ValueError("Don't provide options_file or weight_file with module")\n self._elmo_lstm = module\n else:\n self._elmo_lstm = _ElmoBiLm(\n options_file,\n weight_file,\n requires_grad=requires_grad,\n vocab_to_cache=vocab_to_cache,\n )\n self._has_cached_vocab = vocab_to_cache is not None\n self._dropout = Dropout(p=dropout)\n self._scalar_mixes: Any = []\n for k in range(num_output_representations):\n scalar_mix = ScalarMix(self._elmo_lstm.num_layers, do_layer_norm=do_layer_norm)\n self.add_module("scalar_mix_{}".format(k), scalar_mix)\n self._scalar_mixes.append(scalar_mix)\n\n\n\n[docs] def forward(\n self,\n inputs: torch.Tensor,\n word_inputs: torch.Tensor = None, # pylint: disable=arguments-differ\n ) -> Dict[str, Union[torch.Tensor, List[torch.Tensor]]]:\n """\n Parameters\n ----------\n inputs: ``torch.Tensor``, required.\n Shape ``(batch_size, timesteps, 50)`` of character ids representing the current batch.\n word_inputs : ``torch.Tensor``, required.\n If you passed a cached vocab, you can in addition pass a tensor of shape\n ``(batch_size, timesteps)``, which represent word ids which have been pre-cached.\n Returns\n -------\n Dict with keys:\n ``'elmo_representations'``: ``List[torch.Tensor]``\n A ``num_output_representations`` list of ELMo representations for the input sequence.\n Each representation is shape ``(batch_size, timesteps, embedding_dim)``\n ``'mask'``: ``torch.Tensor``\n Shape ``(batch_size, timesteps)`` long tensor with sequence mask.\n """\n # reshape the input if needed\n original_shape = inputs.size()\n if len(original_shape) > 3:\n timesteps, num_characters = original_shape[-2:]\n reshaped_inputs = inputs.view(-1, timesteps, num_characters)\n else:\n reshaped_inputs = inputs\n\n if word_inputs is not None:\n original_word_size = word_inputs.size()\n if self._has_cached_vocab and len(original_word_size) > 2:\n reshaped_word_inputs = word_inputs.view(-1, original_word_size[-1])\n logger.warning(\n "Word inputs were passed to ELMo but it does not have a cached vocab."\n )\n reshaped_word_inputs = None\n else:\n reshaped_word_inputs = word_inputs\n else:\n reshaped_word_inputs = word_inputs\n\n # run the biLM\n bilm_output = self._elmo_lstm(reshaped_inputs, reshaped_word_inputs)\n layer_activations = bilm_output["activations"]\n mask_with_bos_eos = bilm_output["mask"]\n\n # compute the elmo representations\n representations = []\n for i in range(len(self._scalar_mixes)):\n scalar_mix = getattr(self, "scalar_mix_{}".format(i))\n representation_with_bos_eos = scalar_mix(layer_activations, mask_with_bos_eos)\n representation_without_bos_eos, mask_without_bos_eos = remove_sentence_boundaries(\n representation_with_bos_eos, mask_with_bos_eos\n )\n representations.append(self._dropout(representation_without_bos_eos))\n\n # reshape if necessary\n if word_inputs is not None and len(original_word_size) > 2:\n mask = mask_without_bos_eos.view(original_word_size)\n elmo_representations = [\n representation.view(original_word_size + (-1,))\n for representation in representations\n ]\n elif len(original_shape) > 3:\n mask = mask_without_bos_eos.view(original_shape[:-1])\n elmo_representations = [\n representation.view(original_shape[:-1] + (-1,))\n for representation in representations\n ]\n else:\n mask = mask_without_bos_eos\n elmo_representations = representations\n\n return {"elmo_representations": elmo_representations, "mask": mask}\n\n[docs] @classmethod\n def from_params(cls, params) -> "Elmo":\n # Add files to archive\n params.add_file_to_archive("options_file")\n params.add_file_to_archive("weight_file")\n\n options_file = params.pop("options_file")\n weight_file = params.pop("weight_file")\n requires_grad = params.pop("requires_grad", False)\n num_output_representations = params.pop("num_output_representations")\n do_layer_norm = params.pop_bool("do_layer_norm", False)\n dropout = params.pop_float("dropout", 0.5)\n params.assert_empty(cls.__name__)\n\n return cls(\n options_file=options_file,\n weight_file=weight_file,\n num_output_representations=num_output_representations,\n requires_grad=requires_grad,\n do_layer_norm=do_layer_norm,\n dropout=dropout,\n )\n\n\n[docs]def remove_sentence_boundaries(\n tensor: torch.Tensor, mask: torch.Tensor\n) -> Tuple[torch.Tensor, torch.Tensor]: # pragma: no cover\n """\n Remove begin/end of sentence embeddings from the batch of sentences.\n Given a batch of sentences with size ``(batch_size, timesteps, dim)``\n this returns a tensor of shape ``(batch_size, timesteps - 2, dim)`` after removing\n the beginning and end sentence markers. The sentences are assumed to be padded on the right,\n with the beginning of each sentence assumed to occur at index 0 (i.e., ``mask[:, 0]`` is assumed\n to be 1).\n Returns both the new tensor and updated mask.\n This function is the inverse of ``add_sentence_boundary_token_ids``.\n Parameters\n ----------\n tensor : ``torch.Tensor``\n A tensor of shape ``(batch_size, timesteps, dim)``\n mask : ``torch.Tensor``\n A tensor of shape ``(batch_size, timesteps)``\n Returns\n -------\n tensor_without_boundary_tokens : ``torch.Tensor``\n The tensor after removing the boundary tokens of shape ``(batch_size, timesteps - 2, dim)``\n new_mask : ``torch.Tensor``\n The new mask for the tensor of shape ``(batch_size, timesteps - 2)``.\n """\n # TODO: matthewp, profile this transfer\n sequence_lengths = mask.sum(dim=1).detach().cpu().numpy()\n tensor_shape = list(tensor.data.shape)\n new_shape = list(tensor_shape)\n new_shape[1] = tensor_shape[1] - 2\n tensor_without_boundary_tokens = tensor.new_zeros(*new_shape)\n new_mask = tensor.new_zeros((new_shape[0], new_shape[1]), dtype=torch.long)\n for i, j in enumerate(sequence_lengths):\n if j > 2:\n tensor_without_boundary_tokens[i, : (j - 2), :] = tensor[i, 1 : (j - 1), :]\n new_mask[i, : (j - 2)] = 1\n\n return tensor_without_boundary_tokens, new_mask\n\n\nclass _ElmoBiLm(torch.nn.Module): # pragma: no cover\n """\n Run a pre-trained bidirectional language model, outputing the activations at each\n layer for weighting together into an ELMo representation (with\n ``allennlp.modules.seq2seq_encoders.Elmo``). This is a lower level class, useful\n for advanced uses, but most users should use ``allennlp.modules.seq2seq_encoders.Elmo``\n directly.\n Parameters\n ----------\n options_file : ``str``\n ELMo JSON options file\n weight_file : ``str``\n ELMo hdf5 weight file\n requires_grad: ``bool``, optional\n If True, compute gradient of ELMo parameters for fine tuning.\n vocab_to_cache : ``List[str]``, optional, (default = 0.5).\n A list of words to pre-compute and cache character convolutions\n for. If you use this option, _ElmoBiLm expects that you pass word\n indices of shape (batch_size, timesteps) to forward, instead\n of character indices. If you use this option and pass a word which\n wasn't pre-cached, this will break.\n """\n\n def __init__(\n self,\n options_file: str,\n weight_file: str,\n requires_grad: bool = False,\n vocab_to_cache: List[str] = None,\n ) -> None:\n super(_ElmoBiLm, self).__init__()\n\n self._token_embedder = _ElmoCharacterEncoder(\n options_file, weight_file, requires_grad=requires_grad\n )\n\n self._requires_grad = requires_grad\n if requires_grad and vocab_to_cache:\n logging.warning(\n "You are fine tuning ELMo and caching char CNN word vectors. "\n "This behaviour is not guaranteed to be well defined, particularly. "\n "if not all of your inputs will occur in the vocabulary cache."\n )\n # This is an embedding, used to look up cached\n # word vectors built from character level cnn embeddings.\n self._word_embedding = None\n self._bos_embedding: torch.Tensor = None\n self._eos_embedding: torch.Tensor = None\n\n with open(options_file, "r") as fin:\n options = json.load(fin)\n if not options["lstm"].get("use_skip_connections"):\n raise ValueError("We only support pretrained biLMs with residual connections")\n self._elmo_lstm = ElmoLstm(\n input_size=options["lstm"]["projection_dim"],\n hidden_size=options["lstm"]["projection_dim"],\n cell_size=options["lstm"]["dim"],\n num_layers=options["lstm"]["n_layers"],\n memory_cell_clip_value=options["lstm"]["cell_clip"],\n state_projection_clip_value=options["lstm"]["proj_clip"],\n requires_grad=requires_grad,\n )\n self._elmo_lstm.load_weights(weight_file)\n # Number of representation layers including context independent layer\n self.num_layers = options["lstm"]["n_layers"] + 1\n\n def get_output_dim(self):\n return 2 * self._token_embedder.get_output_dim()\n\n def forward(\n self,\n inputs: torch.Tensor,\n word_inputs: torch.Tensor = None, # pylint: disable=arguments-differ\n ) -> Dict[str, Union[torch.Tensor, List[torch.Tensor]]]:\n """\n Parameters\n ----------\n inputs: ``torch.Tensor``, required.\n Shape ``(batch_size, timesteps, 50)`` of character ids representing the current batch.\n word_inputs : ``torch.Tensor``, required.\n If you passed a cached vocab, you can in addition pass a tensor of shape ``(batch_size, timesteps)``,\n which represent word ids which have been pre-cached.\n Returns\n -------\n Dict with keys:\n ``'activations'``: ``List[torch.Tensor]``\n A list of activations at each layer of the network, each of shape\n ``(batch_size, timesteps + 2, embedding_dim)``\n ``'mask'``: ``torch.Tensor``\n Shape ``(batch_size, timesteps + 2)`` long tensor with sequence mask.\n Note that the output tensors all include additional special begin and end of sequence\n markers.\n """\n if self._word_embedding is not None and word_inputs is not None:\n try:\n mask_without_bos_eos = (word_inputs > 0).long()\n # The character cnn part is cached - just look it up.\n embedded_inputs = self._word_embedding(word_inputs) # type: ignore\n # shape (batch_size, timesteps + 2, embedding_dim)\n type_representation, mask = add_sentence_boundary_token_ids(\n embedded_inputs, mask_without_bos_eos, self._bos_embedding, self._eos_embedding\n )\n except RuntimeError:\n # Back off to running the character convolutions,\n # as we might not have the words in the cache.\n token_embedding = self._token_embedder(inputs)\n mask = token_embedding["mask"]\n type_representation = token_embedding["token_embedding"]\n else:\n token_embedding = self._token_embedder(inputs)\n mask = token_embedding["mask"]\n type_representation = token_embedding["token_embedding"]\n lstm_outputs = self._elmo_lstm(type_representation, mask)\n\n # Prepare the output. The first layer is duplicated.\n # Because of minor differences in how masking is applied depending\n # on whether the char cnn layers are cached, we'll be defensive and\n # multiply by the mask here. It's not strictly necessary, as the\n # mask passed on is correct, but the values in the padded areas\n # of the char cnn representations can change.\n output_tensors = [\n torch.cat([type_representation, type_representation], dim=-1)\n * mask.float().unsqueeze(-1)\n ]\n for layer_activations in torch.chunk(lstm_outputs, lstm_outputs.size(0), dim=0):\n output_tensors.append(layer_activations.squeeze(0))\n\n return {"activations": output_tensors, "mask": mask}\n\n\n[docs]def add_sentence_boundary_token_ids(\n tensor: torch.Tensor, mask: torch.Tensor, sentence_begin_token: Any, sentence_end_token: Any\n) -> Tuple[torch.Tensor, torch.Tensor]: # pragma: no cover\n """\n Add begin/end of sentence tokens to the batch of sentences.\n Given a batch of sentences with size ``(batch_size, timesteps)`` or\n ``(batch_size, timesteps, dim)`` this returns a tensor of shape\n ``(batch_size, timesteps + 2)`` or ``(batch_size, timesteps + 2, dim)`` respectively.\n Returns both the new tensor and updated mask.\n Parameters\n ----------\n tensor : ``torch.Tensor``\n A tensor of shape ``(batch_size, timesteps)`` or ``(batch_size, timesteps, dim)``\n mask : ``torch.Tensor``\n A tensor of shape ``(batch_size, timesteps)``\n sentence_begin_token: Any (anything that can be broadcast in torch for assignment)\n For 2D input, a scalar with the <S> id. For 3D input, a tensor with length dim.\n sentence_end_token: Any (anything that can be broadcast in torch for assignment)\n For 2D input, a scalar with the </S> id. For 3D input, a tensor with length dim.\n Returns\n -------\n tensor_with_boundary_tokens : ``torch.Tensor``\n The tensor with the appended and prepended boundary tokens. If the input was 2D,\n it has shape (batch_size, timesteps + 2) and if the input was 3D, it has shape\n (batch_size, timesteps + 2, dim).\n new_mask : ``torch.Tensor``\n The new mask for the tensor, taking into account the appended tokens\n marking the beginning and end of the sentence.\n """\n # TODO: matthewp, profile this transfer\n sequence_lengths = mask.sum(dim=1).detach().cpu().numpy()\n tensor_shape = list(tensor.data.shape)\n new_shape = list(tensor_shape)\n new_shape[1] = tensor_shape[1] + 2\n tensor_with_boundary_tokens = tensor.new_zeros(*new_shape)\n if len(tensor_shape) == 2:\n tensor_with_boundary_tokens[:, 1:-1] = tensor\n tensor_with_boundary_tokens[:, 0] = sentence_begin_token\n for i, j in enumerate(sequence_lengths):\n tensor_with_boundary_tokens[i, j + 1] = sentence_end_token\n new_mask = (tensor_with_boundary_tokens != 0).long()\n elif len(tensor_shape) == 3:\n tensor_with_boundary_tokens[:, 1:-1, :] = tensor\n for i, j in enumerate(sequence_lengths):\n tensor_with_boundary_tokens[i, 0, :] = sentence_begin_token\n tensor_with_boundary_tokens[i, j + 1, :] = sentence_end_token\n new_mask = ((tensor_with_boundary_tokens > 0).long().sum(dim=-1) > 0).long()\n else:\n raise ValueError("add_sentence_boundary_token_ids only accepts 2D and 3D input")\n\n return tensor_with_boundary_tokens, new_mask\n\n\ndef _make_bos_eos(\n character: int,\n padding_character: int,\n beginning_of_word_character: int,\n end_of_word_character: int,\n max_word_length: int,\n): # pragma: no cover\n char_ids = [padding_character] * max_word_length\n char_ids[0] = beginning_of_word_character\n char_ids[1] = character\n char_ids[2] = end_of_word_character\n return char_ids\n\n\nclass _ElmoCharacterEncoder(torch.nn.Module): # pragma: no cover\n """\n Compute context sensitive token representation using pretrained biLM.\n This embedder has input character ids of size (batch_size, sequence_length, 50)\n and returns (batch_size, sequence_length + 2, embedding_dim), where embedding_dim\n is specified in the options file (typically 512).\n We add special entries at the beginning and end of each sequence corresponding\n to <S> and </S>, the beginning and end of sentence tokens.\n Note: this is a lower level class useful for advanced usage. Most users should\n use ``ElmoTokenEmbedder`` or ``allennlp.modules.Elmo`` instead.\n Parameters\n ----------\n options_file : ``str``\n ELMo JSON options file\n weight_file : ``str``\n ELMo hdf5 weight file\n requires_grad: ``bool``, optional\n If True, compute gradient of ELMo parameters for fine tuning.\n The relevant section of the options file is something like:\n .. example-code::\n .. code-block:: python\n {'char_cnn': {\n 'activation': 'relu',\n 'embedding': {'dim': 4},\n 'filters': [[1, 4], [2, 8], [3, 16], [4, 32], [5, 64]],\n 'max_characters_per_token': 50,\n 'n_characters': 262,\n 'n_highway': 2\n }\n }\n """\n\n def __init__(self, options_file: str, weight_file: str, requires_grad: bool = False) -> None:\n super(_ElmoCharacterEncoder, self).__init__()\n\n with open(options_file, "r") as fin:\n self._options = json.load(fin)\n self._weight_file = weight_file\n\n self.output_dim = self._options["lstm"]["projection_dim"]\n self.requires_grad = requires_grad\n\n self._load_weights()\n\n max_word_length = 50\n\n # char ids 0-255 come from utf-8 encoding bytes\n # assign 256-300 to special chars\n beginning_of_sentence_character = 256 # <begin sentence>\n end_of_sentence_character = 257 # <end sentence>\n beginning_of_word_character = 258 # <begin word>\n end_of_word_character = 259 # <end word>\n padding_character = 260 # <padding>\n\n beginning_of_sentence_characters = _make_bos_eos(\n beginning_of_sentence_character,\n padding_character,\n beginning_of_word_character,\n end_of_word_character,\n max_word_length,\n )\n end_of_sentence_characters = _make_bos_eos(\n end_of_sentence_character,\n padding_character,\n beginning_of_word_character,\n end_of_word_character,\n max_word_length,\n )\n\n # Cache the arrays for use in forward -- +1 due to masking.\n self._beginning_of_sentence_characters = torch.from_numpy(\n numpy.array(beginning_of_sentence_characters) + 1\n )\n self._end_of_sentence_characters = torch.from_numpy(\n numpy.array(end_of_sentence_characters) + 1\n )\n\n def get_output_dim(self):\n return self.output_dim\n\n @overrides\n def forward(\n self, inputs: torch.Tensor\n ) -> Dict[str, torch.Tensor]: # pylint: disable=arguments-differ\n """\n Compute context insensitive token embeddings for ELMo representations.\n Parameters\n ----------\n inputs: ``torch.Tensor``\n Shape ``(batch_size, sequence_length, 50)`` of character ids representing the\n current batch.\n Returns\n -------\n Dict with keys:\n ``'token_embedding'``: ``torch.Tensor``\n Shape ``(batch_size, sequence_length + 2, embedding_dim)`` tensor with context\n insensitive token representations.\n ``'mask'``: ``torch.Tensor``\n Shape ``(batch_size, sequence_length + 2)`` long tensor with sequence mask.\n """\n # Add BOS/EOS\n mask = ((inputs > 0).long().sum(dim=-1) > 0).long()\n character_ids_with_bos_eos, mask_with_bos_eos = add_sentence_boundary_token_ids(\n inputs, mask, self._beginning_of_sentence_characters, self._end_of_sentence_characters\n )\n\n # the character id embedding\n max_chars_per_token = self._options["char_cnn"]["max_characters_per_token"]\n # (batch_size * sequence_length, max_chars_per_token, embed_dim)\n character_embedding = torch.nn.functional.embedding(\n character_ids_with_bos_eos.view(-1, max_chars_per_token), self._char_embedding_weights\n )\n\n # run convolutions\n cnn_options = self._options["char_cnn"]\n if cnn_options["activation"] == "tanh":\n activation = torch.nn.functional.tanh\n elif cnn_options["activation"] == "relu":\n activation = torch.nn.functional.relu\n else:\n raise ValueError("Unknown activation")\n\n # (batch_size * sequence_length, embed_dim, max_chars_per_token)\n character_embedding = torch.transpose(character_embedding, 1, 2)\n convs = []\n for i in range(len(self._convolutions)):\n conv = getattr(self, "char_conv_{}".format(i))\n convolved = conv(character_embedding)\n # (batch_size * sequence_length, n_filters for this width)\n convolved, _ = torch.max(convolved, dim=-1)\n convolved = activation(convolved)\n convs.append(convolved)\n\n # (batch_size * sequence_length, n_filters)\n token_embedding = torch.cat(convs, dim=-1)\n\n # apply the highway layers (batch_size * sequence_length, n_filters)\n token_embedding = self._highways(token_embedding)\n\n # final projection (batch_size * sequence_length, embedding_dim)\n token_embedding = self._projection(token_embedding)\n\n # reshape to (batch_size, sequence_length, embedding_dim)\n batch_size, sequence_length, _ = character_ids_with_bos_eos.size()\n\n return {\n "mask": mask_with_bos_eos,\n "token_embedding": token_embedding.view(batch_size, sequence_length, -1),\n }\n\n def _load_weights(self):\n self._load_char_embedding()\n self._load_cnn_weights()\n self._load_highway()\n self._load_projection()\n\n def _load_char_embedding(self):\n with h5py.File(self._weight_file, "r") as fin:\n char_embed_weights = fin["char_embed"][...]\n\n weights = numpy.zeros(\n (char_embed_weights.shape[0] + 1, char_embed_weights.shape[1]), dtype="float32"\n )\n weights[1:, :] = char_embed_weights\n\n self._char_embedding_weights = torch.nn.Parameter(\n torch.FloatTensor(weights), requires_grad=self.requires_grad\n )\n\n def _load_cnn_weights(self):\n cnn_options = self._options["char_cnn"]\n filters = cnn_options["filters"]\n char_embed_dim = cnn_options["embedding"]["dim"]\n\n convolutions = []\n for i, (width, num) in enumerate(filters):\n conv = torch.nn.Conv1d(\n in_channels=char_embed_dim, out_channels=num, kernel_size=width, bias=True\n )\n # load the weights\n with h5py.File(self._weight_file, "r") as fin:\n weight = fin["CNN"]["W_cnn_{}".format(i)][...]\n bias = fin["CNN"]["b_cnn_{}".format(i)][...]\n\n w_reshaped = numpy.transpose(weight.squeeze(axis=0), axes=(2, 1, 0))\n if w_reshaped.shape != tuple(conv.weight.data.shape):\n raise ValueError("Invalid weight file")\n conv.weight.data.copy_(torch.FloatTensor(w_reshaped))\n conv.bias.data.copy_(torch.FloatTensor(bias))\n\n conv.weight.requires_grad = self.requires_grad\n conv.bias.requires_grad = self.requires_grad\n\n convolutions.append(conv)\n self.add_module("char_conv_{}".format(i), conv)\n\n self._convolutions = convolutions\n\n def _load_highway(self):\n # pylint: disable=protected-access\n # the highway layers have same dimensionality as the number of cnn filters\n cnn_options = self._options["char_cnn"]\n filters = cnn_options["filters"]\n n_filters = sum(f[1] for f in filters)\n n_highway = cnn_options["n_highway"]\n\n # create the layers, and load the weights\n self._highways = Highway(n_filters, n_highway, activation=torch.nn.functional.relu)\n for k in range(n_highway):\n # The AllenNLP highway is one matrix multplication with concatenation of\n # transform and carry weights.\n with h5py.File(self._weight_file, "r") as fin:\n # The weights are transposed due to multiplication order assumptions in tf\n # vs pytorch (tf.matmul(X, W) vs pytorch.matmul(W, X))\n w_transform = numpy.transpose(fin["CNN_high_{}".format(k)]["W_transform"][...])\n # -1.0 since AllenNLP is g * x + (1 - g) * f(x) but tf is (1 - g) * x + g * f(x)\n w_carry = -1.0 * numpy.transpose(fin["CNN_high_{}".format(k)]["W_carry"][...])\n weight = numpy.concatenate([w_transform, w_carry], axis=0)\n self._highways._layers[k].weight.data.copy_(torch.FloatTensor(weight))\n self._highways._layers[k].weight.requires_grad = self.requires_grad\n\n b_transform = fin["CNN_high_{}".format(k)]["b_transform"][...]\n b_carry = -1.0 * fin["CNN_high_{}".format(k)]["b_carry"][...]\n bias = numpy.concatenate([b_transform, b_carry], axis=0)\n self._highways._layers[k].bias.data.copy_(torch.FloatTensor(bias))\n self._highways._layers[k].bias.requires_grad = self.requires_grad\n\n def _load_projection(self):\n cnn_options = self._options["char_cnn"]\n filters = cnn_options["filters"]\n n_filters = sum(f[1] for f in filters)\n\n self._projection = torch.nn.Linear(n_filters, self.output_dim, bias=True)\n with h5py.File(self._weight_file, "r") as fin:\n weight = fin["CNN_proj"]["W_proj"][...]\n bias = fin["CNN_proj"]["b_proj"][...]\n self._projection.weight.data.copy_(torch.FloatTensor(numpy.transpose(weight)))\n self._projection.bias.data.copy_(torch.FloatTensor(bias))\n\n self._projection.weight.requires_grad = self.requires_grad\n self._projection.bias.requires_grad = self.requires_grad\n\n\n[docs]class ElmoLstm(_EncoderBase): # pragma: no cover\n """\n A stacked, bidirectional LSTM which uses\n :class:`~allennlp.modules.lstm_cell_with_projection.LstmCellWithProjection`'s\n with highway layers between the inputs to layers.\n The inputs to the forward and backward directions are independent - forward and backward\n states are not concatenated between layers.\n Additionally, this LSTM maintains its `own` state, which is updated every time\n ``forward`` is called. It is dynamically resized for different batch sizes and is\n designed for use with non-continuous inputs (i.e inputs which aren't formatted as a stream,\n such as text used for a language modelling task, which is how stateful RNNs are typically used).\n This is non-standard, but can be thought of as having an "end of sentence" state, which is\n carried across different sentences.\n Parameters\n ----------\n input_size : ``int``, required\n The dimension of the inputs to the LSTM.\n hidden_size : ``int``, required\n The dimension of the outputs of the LSTM.\n cell_size : ``int``, required.\n The dimension of the memory cell of the\n :class:`~allennlp.modules.lstm_cell_with_projection.LstmCellWithProjection`.\n num_layers : ``int``, required\n The number of bidirectional LSTMs to use.\n requires_grad: ``bool``, optional\n If True, compute gradient of ELMo parameters for fine tuning.\n recurrent_dropout_probability: ``float``, optional (default = 0.0)\n The dropout probability to be used in a dropout scheme as stated in\n `A Theoretically Grounded Application of Dropout in Recurrent Neural Networks\n <https://arxiv.org/abs/1512.05287>`_ .\n state_projection_clip_value: ``float``, optional, (default = None)\n The magnitude with which to clip the hidden_state after projecting it.\n memory_cell_clip_value: ``float``, optional, (default = None)\n The magnitude with which to clip the memory cell.\n """\n\n def __init__(\n self,\n input_size: int,\n hidden_size: int,\n cell_size: int,\n num_layers: int,\n requires_grad: bool = False,\n recurrent_dropout_probability: float = 0.0,\n memory_cell_clip_value: Optional[float] = None,\n state_projection_clip_value: Optional[float] = None,\n ) -> None:\n super(ElmoLstm, self).__init__(stateful=True)\n\n # Required to be wrapped with a :class:`PytorchSeq2SeqWrapper`.\n self.input_size = input_size\n self.hidden_size = hidden_size\n self.num_layers = num_layers\n self.cell_size = cell_size\n self.requires_grad = requires_grad\n\n forward_layers = []\n backward_layers = []\n\n lstm_input_size = input_size\n go_forward = True\n for layer_index in range(num_layers):\n forward_layer = LstmCellWithProjection(\n lstm_input_size,\n hidden_size,\n cell_size,\n go_forward,\n recurrent_dropout_probability,\n memory_cell_clip_value,\n state_projection_clip_value,\n )\n backward_layer = LstmCellWithProjection(\n lstm_input_size,\n hidden_size,\n cell_size,\n not go_forward,\n recurrent_dropout_probability,\n memory_cell_clip_value,\n state_projection_clip_value,\n )\n lstm_input_size = hidden_size\n\n self.add_module("forward_layer_{}".format(layer_index), forward_layer)\n self.add_module("backward_layer_{}".format(layer_index), backward_layer)\n forward_layers.append(forward_layer)\n backward_layers.append(backward_layer)\n self.forward_layers = forward_layers\n self.backward_layers = backward_layers\n\n[docs] def forward(\n self, inputs: torch.Tensor, mask: torch.LongTensor # pylint: disable=arguments-differ\n ) -> torch.Tensor:\n """\n Parameters\n ----------\n inputs : ``torch.Tensor``, required.\n A Tensor of shape ``(batch_size, sequence_length, hidden_size)``.\n mask : ``torch.LongTensor``, required.\n A binary mask of shape ``(batch_size, sequence_length)`` representing the\n non-padded elements in each sequence in the batch.\n Returns\n -------\n A ``torch.Tensor`` of shape (num_layers, batch_size, sequence_length, hidden_size),\n where the num_layers dimension represents the LSTM output from that layer.\n """\n batch_size, total_sequence_length = mask.size()\n stacked_sequence_output, final_states, restoration_indices = self.sort_and_run_forward(\n self._lstm_forward, inputs, mask\n )\n\n num_layers, num_valid, returned_timesteps, encoder_dim = stacked_sequence_output.size()\n # Add back invalid rows which were removed in the call to sort_and_run_forward.\n if num_valid < batch_size:\n zeros = stacked_sequence_output.new_zeros(\n num_layers, batch_size - num_valid, returned_timesteps, encoder_dim\n )\n stacked_sequence_output = torch.cat([stacked_sequence_output, zeros], 1)\n\n # The states also need to have invalid rows added back.\n new_states = []\n for state in final_states:\n state_dim = state.size(-1)\n zeros = state.new_zeros(num_layers, batch_size - num_valid, state_dim)\n new_states.append(torch.cat([state, zeros], 1))\n final_states = new_states\n\n # It's possible to need to pass sequences which are padded to longer than the\n # max length of the sequence to a Seq2StackEncoder. However, packing and unpacking\n # the sequences mean that the returned tensor won't include these dimensions, because\n # the RNN did not need to process them. We add them back on in the form of zeros here.\n sequence_length_difference = total_sequence_length - returned_timesteps\n if sequence_length_difference > 0:\n zeros = stacked_sequence_output.new_zeros(\n num_layers,\n batch_size,\n sequence_length_difference,\n stacked_sequence_output[0].size(-1),\n )\n stacked_sequence_output = torch.cat([stacked_sequence_output, zeros], 2)\n\n self._update_states(final_states, restoration_indices)\n\n # Restore the original indices and return the sequence.\n # Has shape (num_layers, batch_size, sequence_length, hidden_size)\n return stacked_sequence_output.index_select(1, restoration_indices)\n\n def _lstm_forward(\n self,\n inputs: PackedSequence,\n initial_state: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,\n ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:\n """\n Parameters\n ----------\n inputs : ``PackedSequence``, required.\n A batch first ``PackedSequence`` to run the stacked LSTM over.\n initial_state : ``Tuple[torch.Tensor, torch.Tensor]``, optional, (default = None)\n A tuple (state, memory) representing the initial hidden state and memory\n of the LSTM, with shape (num_layers, batch_size, 2 * hidden_size) and\n (num_layers, batch_size, 2 * cell_size) respectively.\n Returns\n -------\n output_sequence : ``torch.FloatTensor``\n The encoded sequence of shape (num_layers, batch_size, sequence_length, hidden_size)\n final_states: ``Tuple[torch.FloatTensor, torch.FloatTensor]``\n The per-layer final (state, memory) states of the LSTM, with shape\n (num_layers, batch_size, 2 * hidden_size) and (num_layers, batch_size, 2 * cell_size)\n respectively. The last dimension is duplicated because it contains the state/memory\n for both the forward and backward layers.\n """\n if initial_state is None:\n hidden_states: List[Optional[Tuple[torch.Tensor, torch.Tensor]]] = [None] * len(\n self.forward_layers\n )\n elif initial_state[0].size()[0] != len(self.forward_layers):\n raise ValueError(\n "Initial states were passed to forward() but the number of "\n "initial states does not match the number of layers."\n )\n else:\n hidden_states = list(zip(initial_state[0].split(1, 0), initial_state[1].split(1, 0)))\n\n inputs, batch_lengths = pad_packed_sequence(inputs, batch_first=True)\n forward_output_sequence = inputs\n backward_output_sequence = inputs\n\n final_states = []\n sequence_outputs = []\n for layer_index, state in enumerate(hidden_states):\n forward_layer = getattr(self, "forward_layer_{}".format(layer_index))\n backward_layer = getattr(self, "backward_layer_{}".format(layer_index))\n\n forward_cache = forward_output_sequence\n backward_cache = backward_output_sequence\n\n if state is not None:\n forward_hidden_state, backward_hidden_state = state[0].split(self.hidden_size, 2)\n forward_memory_state, backward_memory_state = state[1].split(self.cell_size, 2)\n forward_state = (forward_hidden_state, forward_memory_state)\n backward_state = (backward_hidden_state, backward_memory_state)\n else:\n forward_state = None\n backward_state = None\n\n forward_output_sequence, forward_state = forward_layer(\n forward_output_sequence, batch_lengths, forward_state\n )\n backward_output_sequence, backward_state = backward_layer(\n backward_output_sequence, batch_lengths, backward_state\n )\n # Skip connections, just adding the input to the output.\n if layer_index != 0:\n forward_output_sequence += forward_cache\n backward_output_sequence += backward_cache\n\n sequence_outputs.append(\n torch.cat([forward_output_sequence, backward_output_sequence], -1)\n )\n # Append the state tuples in a list, so that we can return\n # the final states for all the layers.\n final_states.append(\n (\n torch.cat([forward_state[0], backward_state[0]], -1),\n torch.cat([forward_state[1], backward_state[1]], -1),\n )\n )\n\n stacked_sequence_outputs: torch.FloatTensor = torch.stack(sequence_outputs)\n # Stack the hidden state and memory for each layer into 2 tensors of shape\n # (num_layers, batch_size, hidden_size) and (num_layers, batch_size, cell_size)\n # respectively.\n final_hidden_states, final_memory_states = zip(*final_states)\n final_state_tuple: Tuple[torch.FloatTensor, torch.FloatTensor] = (\n torch.cat(final_hidden_states, 0),\n torch.cat(final_memory_states, 0),\n )\n return stacked_sequence_outputs, final_state_tuple\n\n[docs] def load_weights(self, weight_file: str) -> None:\n """\n Load the pre-trained weights from the file.\n """\n requires_grad = self.requires_grad\n\n with h5py.File(weight_file, "r") as fin:\n for i_layer, lstms in enumerate(zip(self.forward_layers, self.backward_layers)):\n for j_direction, lstm in enumerate(lstms):\n # lstm is an instance of LSTMCellWithProjection\n cell_size = lstm.cell_size\n\n dataset = fin["RNN_%s" % j_direction]["RNN"]["MultiRNNCell"][\n "Cell%s" % i_layer\n ]["LSTMCell"]\n\n # tensorflow packs together both W and U matrices into one matrix,\n # but pytorch maintains individual matrices. In addition, tensorflow\n # packs the gates as input, memory, forget, output but pytorch\n # uses input, forget, memory, output. So we need to modify the weights.\n tf_weights = numpy.transpose(dataset["W_0"][...])\n torch_weights = tf_weights.copy()\n\n # split the W from U matrices\n input_size = lstm.input_size\n input_weights = torch_weights[:, :input_size]\n recurrent_weights = torch_weights[:, input_size:]\n tf_input_weights = tf_weights[:, :input_size]\n tf_recurrent_weights = tf_weights[:, input_size:]\n\n # handle the different gate order convention\n for torch_w, tf_w in [\n [input_weights, tf_input_weights],\n [recurrent_weights, tf_recurrent_weights],\n ]:\n torch_w[(1 * cell_size) : (2 * cell_size), :] = tf_w[\n (2 * cell_size) : (3 * cell_size), :\n ]\n torch_w[(2 * cell_size) : (3 * cell_size), :] = tf_w[\n (1 * cell_size) : (2 * cell_size), :\n ]\n\n lstm.input_linearity.weight.data.copy_(torch.FloatTensor(input_weights))\n lstm.state_linearity.weight.data.copy_(torch.FloatTensor(recurrent_weights))\n lstm.input_linearity.weight.requires_grad = requires_grad\n lstm.state_linearity.weight.requires_grad = requires_grad\n\n # the bias weights\n tf_bias = dataset["B"][...]\n # tensorflow adds 1.0 to forget gate bias instead of modifying the\n # parameters...\n tf_bias[(2 * cell_size) : (3 * cell_size)] += 1\n torch_bias = tf_bias.copy()\n torch_bias[(1 * cell_size) : (2 * cell_size)] = tf_bias[\n (2 * cell_size) : (3 * cell_size)\n ]\n torch_bias[(2 * cell_size) : (3 * cell_size)] = tf_bias[\n (1 * cell_size) : (2 * cell_size)\n ]\n lstm.state_linearity.bias.data.copy_(torch.FloatTensor(torch_bias))\n lstm.state_linearity.bias.requires_grad = requires_grad\n\n # the projection weights\n proj_weights = numpy.transpose(dataset["W_P_0"][...])\n lstm.state_projection.weight.data.copy_(torch.FloatTensor(proj_weights))\n lstm.state_projection.weight.requires_grad = requires_grad\n\n\nimport torch\n\n\n[docs]class TokenEmbedding(torch.nn.Module):\n """\n Token Embedding\n\n It can be embedding matrix, language model (ELMo), neural machine translation model (CoVe) and features.\n\n * Args:\n vocab: Vocab (rqa.tokens.vocab)\n """\n\n def __init__(self, vocab):\n super(TokenEmbedding, self).__init__()\n\n self.vocab = vocab\n\n\n\n\n\n\n\n\nfrom overrides import overrides\n\nfrom pytorch_transformers import BertModel\n\nimport claf.modules.functional as f\n\nfrom .base import TokenEmbedding\n\n\n[docs]class BertEmbedding(TokenEmbedding):\n """\n BERT Embedding(Encoder)\n\n BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding\n (https://arxiv.org/abs/1810.04805)\n\n * Args:\n vocab: Vocab (claf.tokens.vocab)\n\n * Kwargs:\n pretrained_model_name: ...\n use_as_embedding: ...\n trainable: Finetune or fixed\n """\n\n def __init__(self, vocab, pretrained_model_name=None, trainable=False, unit="subword"):\n super(BertEmbedding, self).__init__(vocab)\n self.trainable = trainable\n\n self.pad_index = vocab.get_index(vocab.pad_token)\n self.sep_index = vocab.get_index(vocab.sep_token)\n\n if unit != "subword":\n raise NotImplementedError("BertEmbedding is only available 'subword' unit, right now.")\n\n self.bert_model = BertModel.from_pretrained(pretrained_model_name) # BertModel with config\n\n[docs] @overrides\n def forward(self, inputs):\n if inputs.size(1) > self.bert_model.config.max_position_embeddings:\n raise ValueError(\n f"max_seq_length in this bert_model is '{self.bert_model.config.max_position_embeddings}'. (input seq_length: {inputs.size(1)})"\n )\n\n # TODO: add text_unit option\n # current: sub_word (default) / later: sub_words --(average)--> word\n attention_mask = (inputs != self.pad_index).long()\n sequence_output, pooled_output = self.bert_model(\n inputs, attention_mask=attention_mask, output_all_encoded_layers=False\n )\n sequence_output = f.masked_zero(sequence_output, attention_mask)\n\n if not self.trainable:\n sequence_output = sequence_output.detach()\n pooled_output = pooled_output.detach()\n\n sequence_output = self.remove_cls_sep_token(inputs, sequence_output)\n return sequence_output\n\n\n\n[docs] def remove_cls_sep_token(self, inputs, outputs):\n seq_mask = inputs.eq(self.sep_index).eq(0)\n outputs = f.masked_zero(outputs, seq_mask)\n return outputs[:, 1:-1, :] # B, S_L, D\n\n\nfrom overrides import overrides\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom claf.modules.activation import get_activation_fn\n\nfrom .base import TokenEmbedding\n\n\n[docs]class CharEmbedding(TokenEmbedding):\n """\n Character Embedding (CharCNN)\n (https://arxiv.org/abs/1509.01626)\n\n * Args:\n vocab: Vocab (claf.tokens.vocab)\n\n * Kwargs:\n dropout: The number of dropout probability\n embed_dim: The number of embedding dimension\n kernel_sizes: The list of kernel size (n-gram)\n num_filter: The number of cnn filter\n activation: Activation Function (eg. ReLU)\n """\n\n def __init__(\n self, vocab, dropout=0.2, embed_dim=16, kernel_sizes=[5], num_filter=100, activation="relu"\n ):\n super(CharEmbedding, self).__init__(vocab)\n\n self.embed_dim = embed_dim\n self.num_filter = num_filter\n\n self.weight = self._init_weight(trainable=True)\n self.convs = [\n nn.Conv1d(\n in_channels=1,\n out_channels=num_filter,\n kernel_size=embed_dim * kernel_size,\n stride=embed_dim,\n )\n for kernel_size in kernel_sizes\n ] # kernel_size = n-gram\n for i, conv in enumerate(self.convs):\n self.add_module(f"conv_{i}", conv)\n\n self.activation_fn = get_activation_fn(activation)()\n self.dropout = nn.Dropout(p=dropout)\n\n self.projection = None\n if len(kernel_sizes) > 1:\n maxpool_output_dim = len(kernel_sizes) * num_filter\n self.projection = nn.Linear(maxpool_output_dim, num_filter)\n\n def _init_weight(self, trainable=False):\n weight = torch.FloatTensor(self.get_vocab_size(), self.embed_dim)\n weight = torch.nn.Parameter(weight, requires_grad=trainable)\n torch.nn.init.xavier_uniform_(weight)\n return weight\n\n[docs] @overrides\n def forward(self, chars):\n mask_chars = (chars != 0).long()\n\n B, W_L, C_L = chars.size() # (batch_size, word_maxlen, char_maxlen)\n chars = chars.view(B, W_L * C_L)\n\n char_embedds = F.embedding(chars, self.weight)\n char_embedds = char_embedds.view(B, W_L, C_L, -1)\n\n # Masking\n char_embedds = char_embedds * mask_chars.unsqueeze(-1).float()\n char_embedds = char_embedds.view(B * W_L, 1, -1)\n\n conv_outputs = []\n for i in range(len(self.convs)):\n conv = getattr(self, f"conv_{i}")\n output = self.activation_fn(conv(char_embedds))\n pooled = F.max_pool1d(output, output.size(2)).squeeze(2)\n\n conv_outputs.append(pooled)\n\n encoded = conv_outputs[0]\n if len(conv_outputs) > 1:\n encoded = torch.cat(conv_outputs, dim=1)\n encoded = encoded.view(B, W_L, -1)\n\n if self.projection:\n encoded = self.projection(encoded)\n return self.dropout(encoded)\n\n\n\n\n\nfrom overrides import overrides\n\nimport torch.nn as nn\n\nfrom claf.tokens.cove import MTLSTM\n\nfrom .base import TokenEmbedding\nfrom .word_embedding import WordEmbedding\n\n\n[docs]class CoveEmbedding(TokenEmbedding):\n """\n Cove Embedding\n\n Learned in Translation: Contextualized Word Vectors\n (http://papers.nips.cc/paper/7209-learned-in-translation-contextualized-word-vectors.pdf)\n\n * Args:\n vocab: Vocab (claf.tokens.vocab)\n\n * Kwargs:\n dropout: The number of dropout probability\n pretrained_path: pretrained vector path (eg. GloVe)\n trainable: finetune or fixed\n project_dim: The number of project (linear) dimension\n """\n\n def __init__(\n self,\n vocab,\n glove_pretrained_path=None,\n model_pretrained_path=None,\n dropout=0.2,\n trainable=False,\n project_dim=None,\n ):\n super(CoveEmbedding, self).__init__(vocab)\n\n self.embed_dim = 600 # MTLSTM (hidden_size=300 + bidirectional => 600)\n word_embedding = WordEmbedding(\n vocab, dropout=0, embed_dim=300, pretrained_path=glove_pretrained_path\n )\n self.cove = MTLSTM(\n word_embedding, pretrained_path=model_pretrained_path, requires_grad=trainable\n )\n\n if dropout and dropout > 0:\n self.dropout = nn.Dropout(p=dropout)\n else:\n self.dropout = lambda x: x\n\n self.project_dim = project_dim\n self.project_linear = None\n if project_dim:\n self.project_linear = nn.Linear(self.elmo.get_output_dim(), project_dim)\n\n[docs] @overrides\n def forward(self, words):\n embedded_words = self.cove(words)\n return self.dropout(embedded_words)\n\n[docs] @overrides\n def get_output_dim(self):\n if self.project_linear:\n return self.project_dim\n return self.embed_dim\n\n\nfrom overrides import overrides\n\nimport torch.nn as nn\n\nfrom claf.data.data_handler import CachePath, DataHandler\nfrom claf.tokens.elmo import Elmo\n\nfrom .base import TokenEmbedding\n\n\nDEFAULT_OPTIONS_FILE = "elmo_2x4096_512_2048cnn_2xhighway_options.json"\nDEFAULT_WEIGHT_FILE = "elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5"\nHIDDEN_SIZE = 1024\n\n\n[docs]class ELMoEmbedding(TokenEmbedding):\n """\n ELMo Embedding\n Embedding From Language Model\n\n Deep contextualized word representations\n (https://arxiv.org/abs/1802.0536)\n\n * Args:\n vocab: Vocab (claf.tokens.vocab)\n\n * Kwargs:\n options_file: ELMo model config file path\n weight_file: ELMo model weight file path\n do_layer_norm: Should we apply layer normalization (passed to ``ScalarMix``)?\n default is False\n dropout: The number of dropout probability\n trainable: Finetune or fixed\n project_dim: The number of project (linear) dimension\n """\n\n def __init__(\n self,\n vocab,\n options_file=DEFAULT_OPTIONS_FILE,\n weight_file=DEFAULT_WEIGHT_FILE,\n do_layer_norm=False,\n dropout=0.5,\n trainable=False,\n project_dim=None,\n ):\n super(ELMoEmbedding, self).__init__(vocab)\n data_handler = DataHandler(cache_path=CachePath.PRETRAINED_VECTOR)\n option_path = data_handler.read(options_file, return_path=True)\n weight_path = data_handler.read(weight_file, return_path=True)\n\n self.elmo = Elmo(option_path, weight_path, 1, requires_grad=trainable, dropout=dropout)\n\n self.project_dim = project_dim\n self.project_linear = None\n if project_dim:\n self.project_linear = nn.Linear(self.elmo.get_output_dim(), project_dim)\n\n[docs] @overrides\n def forward(self, chars):\n elmo_output = self.elmo(chars)\n elmo_representations = elmo_output["elmo_representations"][0]\n\n if self.project_linear:\n elmo_representations = self.project_linear(elmo_representations)\n return elmo_representations\n\n[docs] @overrides\n def get_output_dim(self):\n if self.project_linear:\n return self.project_dim\n return self.elmo.get_output_dim()\n\n\nfrom overrides import overrides\nimport torch\nimport torch.nn as nn\n\nimport claf.modules.functional as f\n\nfrom .base import TokenEmbedding\nfrom .word_embedding import WordEmbedding\n\n\n[docs]class FrequentTuningWordEmbedding(TokenEmbedding):\n """\n Frequent Word Finetuning Embedding\n Finetuning embedding matrix, according to 'threshold_index'\n\n * Args:\n vocab: Vocab (claf.tokens.vocab)\n\n * Kwargs:\n dropout: The number of dropout probability\n embed_dim: The number of embedding dimension\n padding_idx: If given, pads the output with the embedding vector at padding_idx\n (initialized to zeros) whenever it encounters the index.\n max_norm: If given, will renormalize the embedding vectors to have a norm lesser\n than this before extracting. Note: this will modify weight in-place.\n norm_type: The p of the p-norm to compute for the max_norm option. Default 2.\n scale_grad_by_freq: if given, this will scale gradients by the inverse of\n frequency of the words in the mini-batch. Default False.\n sparse: if True, gradient w.r.t. weight will be a sparse tensor.\n See Notes under torch.nn.Embedding for more details regarding sparse gradients.\n pretrained_path: pretrained vector path (eg. GloVe)\n trainable: finetune or fixed\n """\n\n def __init__(\n self,\n vocab,\n dropout=0.2,\n embed_dim=100,\n padding_idx=None,\n max_norm=None,\n norm_type=2,\n scale_grad_by_freq=False,\n sparse=False,\n pretrained_path=None,\n ):\n super(FrequentTuningWordEmbedding, self).__init__(vocab)\n\n self.threshold_index = vocab.threshold_index\n\n self.embed_dim = embed_dim\n self.fine_tune_word_embedding = WordEmbedding(\n vocab,\n dropout=0,\n embed_dim=embed_dim,\n padding_idx=padding_idx,\n max_norm=max_norm,\n norm_type=norm_type,\n scale_grad_by_freq=scale_grad_by_freq,\n sparse=sparse,\n pretrained_path=pretrained_path,\n )\n self.fixed_word_embedding = WordEmbedding(\n vocab,\n dropout=0,\n embed_dim=embed_dim,\n padding_idx=padding_idx,\n max_norm=max_norm,\n norm_type=norm_type,\n scale_grad_by_freq=scale_grad_by_freq,\n sparse=sparse,\n pretrained_path=pretrained_path,\n )\n\n if dropout > 0:\n self.dropout = nn.Dropout(p=dropout)\n else:\n self.dropout = lambda x: x\n\n[docs] @overrides\n def forward(self, words, frequent_tuning=False):\n if frequent_tuning and self.training:\n\n padding_mask = words.eq(0).long()\n\n # Fine-tuning - N the most frequent\n fine_tune_mask = torch.lt(words, self.threshold_index) * padding_mask.eq(\n 0\n ) # < threshold_index\n fine_tune_words = words * fine_tune_mask.long()\n\n fine_tune_embedded = self.fine_tune_word_embedding(fine_tune_words)\n fine_tune_embedded = f.masked_zero(fine_tune_embedded, fine_tune_mask)\n\n # Fixed - under N frequent\n fixed_mask = torch.ge(words, self.threshold_index) # >= threshold_index\n\n fixed_embedeed = self.fixed_word_embedding(words).detach() # Fixed\n fixed_embedeed = f.masked_zero(fixed_embedeed, fixed_mask)\n\n embedded_words = fine_tune_embedded + fixed_embedeed\n else:\n embedded_words = self.fixed_word_embedding(words)\n\n return self.dropout(embedded_words)\n\n\n\n\nfrom overrides import overrides\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom claf.tokens.vocabulary import Vocab\n\nfrom .base import TokenEmbedding\nfrom .word_embedding import WordEmbedding\n\n\n[docs]class SparseFeature(TokenEmbedding):\n """\n Sparse Feature\n\n 1. Sparse to Embedding\n 2. One Hot Encoding\n\n * Args:\n vocab: Vocab (claf.tokens.vocab)\n embed_type: The type of embedding [one_hot|embedding]\n feature_count: The number of feature count\n\n * Kwargs:\n params: additional parameters for embedding module\n """\n\n def __init__(self, vocab, embed_type, feature_count, params={}):\n super(SparseFeature, self).__init__(vocab)\n\n self.feature_count = feature_count\n\n if embed_type == "embedding":\n embed_module = SparseToEmbedding\n else:\n embed_module = OneHotEncoding\n\n self.embed_modules = nn.ModuleList(\n [embed_module(i, vocab.token_name, **params) for i in range(feature_count)]\n )\n\n indexs = torch.arange(feature_count).long()\n indexs = indexs.view(feature_count, 1)\n self.indexs = nn.Parameter(indexs, requires_grad=False)\n\n[docs] @overrides\n def forward(self, inputs):\n embedded_inputs = []\n\n for i in range(len(self.embed_modules)):\n tensors = torch.index_select(inputs, -1, self.indexs[i]).squeeze(-1)\n embedded = self.embed_modules[i](tensors)\n\n embedded_inputs.append(embedded)\n return torch.cat(embedded_inputs, dim=-1)\n\n[docs] @overrides\n def get_output_dim(self):\n return sum(e.get_output_dim() for e in self.embed_modules)\n\n\n[docs]class SparseToEmbedding(nn.Module):\n """\n Sparse to Embedding\n\n * Args:\n token_name: token_name\n\n * Kwargs:\n dropout: The number of dropout probability\n embed_dim: The number of embedding dimension\n padding_idx: If given, pads the output with the embedding vector at padding_idx\n (initialized to zeros) whenever it encounters the index.\n max_norm: If given, will renormalize the embedding vectors to have a norm lesser\n than this before extracting. Note: this will modify weight in-place.\n norm_type: The p of the p-norm to compute for the max_norm option. Default 2.\n scale_grad_by_freq: if given, this will scale gradients by the inverse of\n frequency of the words in the mini-batch. Default False.\n sparse: if True, gradient w.r.t. weight will be a sparse tensor.\n See Notes under torch.nn.Embedding for more details regarding sparse gradients.\n pretrained_path: pretrained vector path (eg. GloVe)\n trainable: finetune or fixed\n """\n\n def __init__(\n self,\n index,\n token_name,\n classes,\n dropout=0,\n embed_dim=15,\n trainable=True,\n padding_idx=None,\n max_norm=None,\n norm_type=2,\n scale_grad_by_freq=False,\n sparse=False,\n ):\n super(SparseToEmbedding, self).__init__()\n\n self.embed_dim = embed_dim\n\n vocab = Vocab(token_name)\n vocab.init()\n for c in classes[index]:\n vocab.add(c)\n\n embedding_params = {\n "vocab": vocab,\n "dropout": dropout,\n "embed_dim": embed_dim,\n "trainable": trainable,\n "padding_idx": padding_idx,\n "max_norm": max_norm,\n "norm_type": norm_type,\n "scale_grad_by_freq": scale_grad_by_freq,\n "sparse": sparse,\n }\n\n self.embedding = WordEmbedding(**embedding_params)\n\n\n\n\n\n\n[docs]class OneHotEncoding(nn.Module):\n """\n Sparse to one-hot encoding\n\n * Args:\n vocab: Vocab (claf.tokens.vocab)\n\n """\n\n def __init__(self, index, token_name, classes):\n super(OneHotEncoding, self).__init__()\n\n vocab = Vocab(token_name)\n vocab.init()\n for c in classes[index]:\n vocab.add(c)\n\n num_class = len(vocab)\n self.num_class = num_class\n\n one_hot_encoding = torch.eye(num_class)\n self.one_hots = nn.Parameter(one_hot_encoding, requires_grad=False)\n\n[docs] @overrides\n def forward(self, inputs):\n if self.num_class == 4:\n inputs = inputs - 2 # make 0, 1 binary_feature\n return inputs.float().unsqueeze(-1)\n\n return F.embedding(inputs, self.one_hots)\n\n[docs] def get_output_dim(self):\n if self.num_class == 4: # binary_feature\n return 1 # 0 or 1\n\n return self.num_class\n\n\nimport logging\nfrom overrides import overrides\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom claf.data.data_handler import CachePath, DataHandler\n\nfrom .base import TokenEmbedding\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]class WordEmbedding(TokenEmbedding):\n """\n Word Embedding\n Default Token Embedding\n\n * Args:\n vocab: Vocab (claf.tokens.vocab)\n\n * Kwargs:\n dropout: The number of dropout probability\n embed_dim: The number of embedding dimension\n padding_idx: If given, pads the output with the embedding vector at padding_idx\n (initialized to zeros) whenever it encounters the index.\n max_norm: If given, will renormalize the embedding vectors to have a norm lesser\n than this before extracting. Note: this will modify weight in-place.\n norm_type: The p of the p-norm to compute for the max_norm option. Default 2.\n scale_grad_by_freq: if given, this will scale gradients by the inverse of\n frequency of the words in the mini-batch. Default False.\n sparse: if True, gradient w.r.t. weight will be a sparse tensor.\n See Notes under torch.nn.Embedding for more details regarding sparse gradients.\n pretrained_path: pretrained vector path (eg. GloVe)\n trainable: finetune or fixed\n """\n\n def __init__(\n self,\n vocab,\n dropout=0.2,\n embed_dim=100,\n padding_idx=None,\n max_norm=None,\n norm_type=2,\n scale_grad_by_freq=False,\n sparse=False,\n pretrained_path=None,\n trainable=True,\n ):\n super(WordEmbedding, self).__init__(vocab)\n self.data_handler = DataHandler(cache_path=CachePath.PRETRAINED_VECTOR)\n\n self.embed_dim = embed_dim\n if dropout and dropout > 0:\n self.dropout = nn.Dropout(p=dropout)\n else:\n self.dropout = lambda x: x\n\n if pretrained_path:\n weight = self._read_pretrained_file(pretrained_path)\n self.weight = torch.nn.Parameter(weight, requires_grad=trainable)\n else:\n self.weight = self._init_weight(trainable=trainable)\n\n # nn.functional.embedding = optional paramters\n # (padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)\n # check - https://pytorch.org/docs/master/nn.html#torch.nn.functional.embeddin\\\n # ://pytorch.org/docs/master/nn.html#torch.nn.functional.embedding\n self.padding_idx = padding_idx\n self.max_norm = max_norm\n self.norm_type = norm_type\n self.scale_grad_by_freq = scale_grad_by_freq\n self.sparse = sparse\n\n def _init_weight(self, trainable=True):\n weight = torch.FloatTensor(self.get_vocab_size(), self.embed_dim)\n weight = torch.nn.Parameter(weight, requires_grad=trainable)\n torch.nn.init.xavier_uniform_(weight)\n return weight\n\n[docs] @overrides\n def forward(self, words):\n input_size = words.size()\n if len(input_size) > 2:\n words = words.view(-1, input_size[-1])\n\n embedded_words = F.embedding(\n words,\n self.weight,\n padding_idx=self.padding_idx,\n max_norm=self.max_norm,\n norm_type=self.norm_type,\n scale_grad_by_freq=self.scale_grad_by_freq,\n sparse=self.sparse,\n )\n\n if len(input_size) > 2:\n embedded_size = list(input_size) + [embedded_words.size(-1)]\n embedded_words = embedded_words.view(*embedded_size)\n return self.dropout(embedded_words)\n\n def _read_pretrained_file(self, file_path):\n words_to_keep = set(self.vocab.get_all_tokens())\n vocab_size = self.get_vocab_size()\n embeddings = {}\n\n # First we read the embeddings from the file, only keeping vectors for the words we need.\n logger.info("Reading embeddings from file")\n file_path = self.data_handler.read(file_path, return_path=True)\n with open(file_path, "rb") as embeddings_file:\n for line in embeddings_file:\n fields = line.decode("utf-8").rstrip().split(" ")\n\n if len(fields) - 1 != self.embed_dim:\n logger.info(\n f"Found line with wrong number of dimensions (expected {self.embed_dim}, was {len(fields)}): {line}"\n )\n continue\n\n word = fields[0]\n if word in words_to_keep:\n vector = np.asarray(fields[1:], dtype="float32")\n embeddings[word] = vector\n\n if not embeddings:\n raise ValueError(\n "No embeddings of correct dimension found. check input dimension value"\n )\n\n all_embeddings = np.asarray(list(embeddings.values()))\n embeddings_mean = float(np.mean(all_embeddings))\n embeddings_std = float(np.std(all_embeddings))\n # Now we initialize the weight matrix for an embedding layer, starting with random vectors,\n # then filling in the word vectors we just read.\n logger.info("Initializing pre-trained embedding layer")\n embedding_matrix = torch.FloatTensor(vocab_size, self.embed_dim).normal_(\n embeddings_mean, embeddings_std\n )\n\n match_count = 0\n for i in range(0, vocab_size):\n word = self.vocab.get_token(i)\n if word in embeddings:\n embedding_matrix[i] = torch.FloatTensor(embeddings[word])\n match_count += 1\n else:\n # f"Word {word} was not found in the embedding file. Initialising randomly."\n pass\n logger.info(f"Match embedding vocab size: {match_count}. [{match_count}/{vocab_size}]")\n return embedding_matrix\n\n\n\n#!/usr/bin/env python\n# encoding: utf-8\n\n"""\nHangulpy.py\nCopyright (C) 2012 Ryan Rho, Hyunwoo Cho\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the "Software"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies\nof the Software, and to permit persons to whom the Software is furnished to do\nso, subject to the following conditions:\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\nTHE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"""\n\nimport string\nimport re\n\n################################################################################\n# Hangul Unicode Variables\n################################################################################\n\n# Code = 0xAC00 + (Chosung_index * NUM_JOONGSUNGS * NUM_JONGSUNGS) + (Joongsung_index * NUM_JONGSUNGS) + (Jongsung_index)\nCHOSUNGS = [\n "ㄱ",\n "ㄲ",\n "ㄴ",\n "ㄷ",\n "ㄸ",\n "ㄹ",\n "ㅁ",\n "ㅂ",\n "ㅃ",\n "ㅅ",\n "ㅆ",\n "ㅇ",\n "ㅈ",\n "ㅉ",\n "ㅊ",\n "ㅋ",\n "ㅌ",\n "ㅍ",\n "ㅎ",\n]\nJOONGSUNGS = [\n "ㅏ",\n "ㅐ",\n "ㅑ",\n "ㅒ",\n "ㅓ",\n "ㅔ",\n "ㅕ",\n "ㅖ",\n "ㅗ",\n "ㅘ",\n "ㅙ",\n "ㅚ",\n "ㅛ",\n "ㅜ",\n "ㅝ",\n "ㅞ",\n "ㅟ",\n "ㅠ",\n "ㅡ",\n "ㅢ",\n "ㅣ",\n]\nJONGSUNGS = [\n "",\n "ㄱ",\n "ㄲ",\n "ㄳ",\n "ㄴ",\n "ㄵ",\n "ㄶ",\n "ㄷ",\n "ㄹ",\n "ㄺ",\n "ㄻ",\n "ㄼ",\n "ㄽ",\n "ㄾ",\n "ㄿ",\n "ㅀ",\n "ㅁ",\n "ㅂ",\n "ㅄ",\n "ㅅ",\n "ㅆ",\n "ㅇ",\n "ㅈ",\n "ㅊ",\n "ㅋ",\n "ㅌ",\n "ㅍ",\n "ㅎ",\n]\n\nNUM_CHOSUNGS = 19\nNUM_JOONGSUNGS = 21\nNUM_JONGSUNGS = 28\n\nFIRST_HANGUL_UNICODE = 0xAC00 # '가'\nLAST_HANGUL_UNICODE = 0xD7A3 # '힣'\n\n################################################################################\n# Boolean Hangul functions\n################################################################################\n\n\n[docs]def is_hangul(phrase): # pragma: no cover\n """Check whether the phrase is Hangul.\n This method ignores white spaces, punctuations, and numbers.\n @param phrase a target string\n @return True if the phrase is Hangul. False otherwise."""\n\n # If the input is only one character, test whether the character is Hangul.\n if len(phrase) == 1:\n return is_all_hangul(phrase)\n\n # Remove all white spaces, punctuations, numbers.\n exclude = set(string.whitespace + string.punctuation + "0123456789")\n phrase = "".join(ch for ch in phrase if ch not in exclude)\n\n return is_all_hangul(phrase)\n\n\n[docs]def is_all_hangul(phrase): # pragma: no cover\n """Check whether the phrase contains all Hangul letters\n @param phrase a target string\n @return True if the phrase only consists of Hangul. False otherwise."""\n\n for unicode_value in map(lambda letter: ord(letter), phrase):\n if unicode_value < FIRST_HANGUL_UNICODE or unicode_value > LAST_HANGUL_UNICODE:\n # Check whether the letter is chosungs, joongsungs, or jongsungs.\n if unicode_value not in map(lambda v: ord(v), CHOSUNGS + JOONGSUNGS + JONGSUNGS[1:]):\n return False\n return True\n\n\n[docs]def has_jongsung(letter): # pragma: no cover\n """Check whether this letter contains Jongsung"""\n if len(letter) != 1:\n raise Exception("The target string must be one letter.")\n if not is_hangul(letter):\n raise NotHangulException("The target string must be Hangul")\n\n unicode_value = ord(letter)\n return (unicode_value - FIRST_HANGUL_UNICODE) % NUM_JONGSUNGS > 0\n\n\n[docs]def has_batchim(letter): # pragma: no cover\n """This method is the same as has_jongsung()"""\n return has_jongsung(letter)\n\n\n[docs]def has_approximant(letter): # pragma: no cover\n """Approximant makes complex vowels, such as ones starting with y or w.\n In Korean there is a unique approximant euㅡ making uiㅢ, but ㅢ does not make many irregularities."""\n if len(letter) != 1:\n raise Exception("The target string must be one letter.")\n if not is_hangul(letter):\n raise NotHangulException("The target string must be Hangul")\n\n jaso = decompose(letter)\n diphthong = (2, 3, 6, 7, 9, 10, 12, 14, 15, 17)\n # [u'ㅑ',u'ㅒ',',u'ㅕ',u'ㅖ',u'ㅘ',u'ㅙ',u'ㅛ',u'ㅝ',u'ㅞ',u'ㅠ']\n # excluded 'ㅢ' because y- and w-based complex vowels are irregular.\n # vowels with umlauts (ㅐ, ㅔ, ㅚ, ㅟ) are not considered complex vowels.\n return jaso[1] in diphthong\n\n\n################################################################################\n# Decomposition & Combination\n################################################################################\n\n\n[docs]def compose(chosung, joongsung, jongsung=""): # pragma: no cover\n """This function returns a Hangul letter by composing the specified chosung, joongsung, and jongsung.\n @param chosung\n @param joongsung\n @param jongsung the terminal Hangul letter. This is optional if you do not need a jongsung."""\n\n if jongsung is None:\n jongsung = ""\n\n try:\n chosung_index = CHOSUNGS.index(chosung)\n joongsung_index = JOONGSUNGS.index(joongsung)\n jongsung_index = JONGSUNGS.index(jongsung)\n except Exception as e:\n raise NotHangulException(\n "No valid Hangul character can be generated using given combination of chosung, joongsung, and jongsung."\n )\n\n return chr(\n 0xAC00\n + chosung_index * NUM_JOONGSUNGS * NUM_JONGSUNGS\n + joongsung_index * NUM_JONGSUNGS\n + jongsung_index\n )\n\n\n[docs]def decompose(hangul_letter): # pragma: no cover\n """This function returns letters by decomposing the specified Hangul letter."""\n\n if len(hangul_letter) < 1:\n raise NotLetterException("")\n elif not is_hangul(hangul_letter):\n raise NotHangulException("")\n\n code = ord(hangul_letter) - FIRST_HANGUL_UNICODE\n jongsung_index = int(code % NUM_JONGSUNGS)\n code /= NUM_JONGSUNGS\n joongsung_index = int(code % NUM_JOONGSUNGS)\n code /= NUM_JOONGSUNGS\n chosung_index = int(code)\n\n return (CHOSUNGS[chosung_index], JOONGSUNGS[joongsung_index], JONGSUNGS[jongsung_index])\n\n\n################################################################################\n# Josa functions\n################################################################################\n\n\n[docs]def josa_en(word): # pragma: no cover\n """add josa either '은' or '는' at the end of this word"""\n word = word.strip()\n if not is_hangul(word):\n raise NotHangulException("")\n\n last_letter = word[-1]\n josa = "은" if has_jongsung(last_letter) else "는"\n return word + josa\n\n\n[docs]def josa_eg(word): # pragma: no cover\n """add josa either '이' or '가' at the end of this word"""\n word = word.strip()\n if not is_hangul(word):\n raise NotHangulException("")\n\n last_letter = word[-1]\n josa = "이" if has_jongsung(last_letter) else "가"\n return word + josa\n\n\n[docs]def josa_el(word): # pragma: no cover\n """add josa either '을' or '를' at the end of this word"""\n word = word.strip()\n if not is_hangul(word):\n raise NotHangulException("")\n\n last_letter = word[-1]\n josa = "을" if has_jongsung(last_letter) else "를"\n return word + josa\n\n\n[docs]def josa_ro(word): # pragma: no cover\n """add josa either '으로' or '로' at the end of this word"""\n word = word.strip()\n if not is_hangul(word):\n raise NotHangulException("")\n\n last_letter = word[-1]\n if not has_jongsung(last_letter):\n josa = "로"\n elif (ord(last_letter) - FIRST_HANGUL_UNICODE) % NUM_JONGSUNGS == 9: # ㄹ\n josa = "로"\n else:\n josa = "으로"\n\n return word + josa\n\n\n[docs]def josa_gwa(word): # pragma: no cover\n """add josa either '과' or '와' at the end of this word"""\n word = word.strip()\n if not is_hangul(word):\n raise NotHangulException("")\n\n last_letter = word[-1]\n josa = "과" if has_jongsung(last_letter) else "와"\n return word + josa\n\n\n[docs]def josa_ida(word): # pragma: no cover\n """add josa either '이다' or '다' at the end of this word"""\n word = word.strip()\n if not is_hangul(word):\n raise NotHangulException("")\n\n last_letter = word[-1]\n josa = "이다" if has_jongsung(last_letter) else "다"\n return word + josa\n\n\n################################################################################\n# Prefixes and suffixes\n# Practice area; need more organization\n################################################################################\n\n\n[docs]def add_ryul(word): # pragma: no cover\n """add suffix either '률' or '율' at the end of this word"""\n word = word.strip()\n if not is_hangul(word):\n raise NotHangulException("")\n\n last_letter = word[-1]\n if not has_jongsung(last_letter):\n ryul = "율"\n elif (ord(last_letter) - FIRST_HANGUL_UNICODE) % NUM_JONGSUNGS == 4: # ㄴ\n ryul = "율"\n else:\n ryul = "률"\n\n return word + ryul\n\n\n################################################################################\n# The formatter, or ultimately, a template system\n# Practice area; need more organization\n################################################################################\n\n\n[docs]def ili(word): # pragma: no cover\n """convert {가} or {이} to their correct respective particles automagically."""\n word = word.strip()\n if not is_hangul(word):\n raise NotHangulException("")\n\n last_letter = word[word.find("{가}") - 1]\n word = word.replace("{가}", ("이" if has_jongsung(last_letter) else "가"))\n\n last_letter = word[word.find("{이}") - 1]\n word = word.replace("{이}", ("이" if has_jongsung(last_letter) else "가"))\n return word\n\n\n################################################################################\n# Exceptions\n################################################################################\n\n\n\n\n\n\n\n\n\n\n[docs]class TokenIndexer:\n """\n Token Indexer\n\n indexing tokens (eg. 'hi' -> 4)\n """\n\n def __init__(self, tokenizer):\n self.param_key = None\n self.tokenizer = tokenizer\n\n\n\n\n\n\nfrom overrides import overrides\n\nfrom .base import TokenIndexer\n\n\n[docs]class BertIndexer(TokenIndexer):\n """\n Bert Token Indexer\n\n * Property\n vocab: Vocab (claf.tokens.vocabulary)\n\n * Args:\n tokenizer: SubwordTokenizer\n\n * Kwargs:\n lowercase: word token to lowercase\n insert_start: insert start_token to first\n insert_end: append end_token\n """\n\n def __init__(self, tokenizer, do_tokenize=True):\n super(BertIndexer, self).__init__(tokenizer)\n self.do_tokenize = do_tokenize\n\n[docs] @overrides\n def index(self, text):\n input_type = type(text)\n if input_type == str:\n return self._index_text(text)\n elif input_type == list:\n texts = text # List of text case\n return [self._index_text(text) for text in texts]\n else:\n raise ValueError(f"Not supported type: {type(text)}")\n\n def _index_text(self, text):\n if self.do_tokenize:\n tokens = self.tokenizer.tokenize(text)\n else:\n tokens = [text]\n\n indexed_tokens = [self.vocab.get_index(token) for token in tokens]\n\n # Insert CLS_TOKEN ans SEP_TOKEN\n insert_start = self.vocab.get_index(self.vocab.cls_token)\n indexed_tokens.insert(0, insert_start)\n\n insert_end = self.vocab.get_index(self.vocab.sep_token)\n indexed_tokens.append(insert_end)\n return indexed_tokens\n\n\nfrom overrides import overrides\n\nfrom .base import TokenIndexer\n\n\n[docs]class CharIndexer(TokenIndexer):\n """\n Character Token Indexer\n\n * Property\n vocab: Vocab (claf.tokens.vocabulary)\n\n * Args:\n tokenizer: CharTokenizer\n\n * Kwargs:\n insert_char_start: insert start index (eg. ['h', 'i'] -> ['<s>', 'h', 'i'] )\n default is None\n insert_char_end: insert end index (eg. ['h', 'i'] -> ['h', 'i', '</s>'] )\n default is None\n """\n\n def __init__(self, tokenizer, insert_char_start=None, insert_char_end=None):\n super(CharIndexer, self).__init__(tokenizer)\n\n self.insert_char_start = insert_char_start\n self.insert_char_end = insert_char_end\n\n[docs] @overrides\n def index(self, text):\n indexed_tokens = [self.index_token(token) for token in self.tokenizer.tokenize(text)]\n return indexed_tokens\n\n[docs] def index_token(self, chars):\n char_ids = [self.vocab.get_index(char) for char in chars]\n\n if self.insert_char_start is not None:\n char_ids.insert(0, self.vocab.get_index(self.vocab.start_token))\n if self.insert_char_end is not None:\n char_ids.append(self.vocab.get_index(self.vocab.end_token))\n return char_ids\n\n"""\nThis code is from allenai/allennlp\n(https://github.com/allenai/allennlp/blob/master/allennlp/data/token_indexers/elmo_indexer.py)\n"""\n\nfrom overrides import overrides\n\nfrom .base import TokenIndexer\n\n\ndef _make_bos_eos(\n character: int,\n padding_character: int,\n beginning_of_word_character: int,\n end_of_word_character: int,\n max_word_length: int,\n):\n char_ids = [padding_character] * max_word_length\n char_ids[0] = beginning_of_word_character\n char_ids[1] = character\n char_ids[2] = end_of_word_character\n return char_ids\n\n\n[docs]class ELMoIndexer(TokenIndexer):\n """\n Maps individual tokens to sequences of character ids, compatible with ELMo.\n To be consistent with previously trained models, we include it here as special of existing\n character indexers.\n """\n\n max_word_length = 50\n\n # char ids 0-255 come from utf-8 encoding bytes\n # assign 256-300 to special chars\n beginning_of_sentence_character = 256 # <begin sentence>\n end_of_sentence_character = 257 # <end sentence>\n beginning_of_word_character = 258 # <begin word>\n end_of_word_character = 259 # <end word>\n padding_character = 260 # <padding><Paste>\n\n beginning_of_sentence_characters = _make_bos_eos(\n beginning_of_sentence_character,\n padding_character,\n beginning_of_word_character,\n end_of_word_character,\n max_word_length,\n )\n\n end_of_sentence_characters = _make_bos_eos(\n end_of_sentence_character,\n padding_character,\n beginning_of_word_character,\n end_of_word_character,\n max_word_length,\n )\n\n BOS_TOKEN = "<S>"\n EOS_TOKEN = "</S>"\n\n def __init__(self, tokenizer):\n super(ELMoIndexer, self).__init__(tokenizer)\n\n[docs] @overrides\n def index(self, text):\n indexed_tokens = [self.index_token(token) for token in self.tokenizer.tokenize(text)]\n return indexed_tokens\n\n[docs] def index_token(self, word):\n if word == self.BOS_TOKEN:\n char_ids = self.beginning_of_sentence_characters\n elif word == self.EOS_TOKEN:\n char_ids = self.end_of_sentence_characters\n else:\n word_encodeds = word.encode("utf-8", "ignore")[: (self.max_word_length - 2)]\n char_ids = [char_id for char_id in word_encodeds]\n char_ids = [self.beginning_of_word_character] + char_ids + [self.end_of_word_character]\n return [c + 1 for c in char_ids]\n\n\n\nfrom overrides import overrides\nfrom nltk.stem import WordNetLemmatizer\n\nfrom .base import TokenIndexer\n\n\n[docs]class ExactMatchIndexer(TokenIndexer):\n """\n Exact Match Token Indexer\n\n * Property\n vocab: Vocab (claf.tokens.vocabulary)\n\n * Args:\n tokenizer: WordTokenizer\n\n * Kwargs:\n lower: add lower feature. default is True (0 or 1)\n lemma: add lemma case feature. feature is True (0 or 1)\n """\n\n def __init__(self, tokenizer, lower=True, lemma=True):\n super(ExactMatchIndexer, self).__init__(tokenizer)\n\n self.param_key = "question"\n self.lemmatizer = WordNetLemmatizer()\n\n self.lower = lower\n self.lemma = lemma\n\n[docs] @overrides\n def index(self, text, query_text):\n tokenized_query_text = self.tokenizer.tokenize(query_text)\n query_tokens = {\n "origin": set(tokenized_query_text),\n "lower": set([token.lower() for token in tokenized_query_text]),\n "lemma": set(\n [self.lemmatizer.lemmatize(token.lower()) for token in tokenized_query_text]\n ),\n }\n\n indexed_tokens = [\n self.index_token(token, query_tokens) for token in self.tokenizer.tokenize(text)\n ]\n return indexed_tokens\n\n[docs] def index_token(self, token, query_tokens):\n em_feature = []\n\n # 1. origin\n origin_case = 1 if token in query_tokens["origin"] else 0\n em_feature.append(origin_case + 2)\n\n # 2. lower\n if self.lower:\n lower_case = 1 if token.lower() in query_tokens["lower"] else 0\n em_feature.append(lower_case + 2)\n\n # 3. lemma\n if self.lemma:\n lemma_case = (\n 1 if self.lemmatizer.lemmatize(token.lower()) in query_tokens["lemma"] else 0\n )\n em_feature.append(lemma_case + 2)\n return em_feature\n\n\nfrom overrides import overrides\nimport spacy\n\nfrom claf.tokens.linguistic import POSTag, NER\n\nfrom .base import TokenIndexer\n\n\n[docs]class LinguisticIndexer(TokenIndexer):\n """\n Linguistic Token Indexer\n\n * Property\n vocab: Vocab (claf.tokens.vocabulary)\n\n * Args:\n tokenizer: WordTokenizer\n\n * Kwargs:\n pos_tag: POS Tagging\n ner: Named Entity Recognition\n dep: Dependency Parser\n """\n\n def __init__(self, tokenizer, pos_tag=None, ner=None, dep=None):\n super(LinguisticIndexer, self).__init__(tokenizer)\n\n self.spacy_model = None\n\n # Features\n self.use_pos_tag = pos_tag\n self.pos_to_index = {t: i for i, t in enumerate(POSTag.classes)}\n\n self.use_ner = ner\n self.ner_to_index = {t: i for i, t in enumerate(NER.classes)}\n\n self.use_dep = dep\n if dep:\n raise NotImplementedError("Dependency Parser feature")\n\n[docs] @overrides\n def index(self, text):\n package = self.tokenizer.name\n return getattr(self, f"_{package}")(text)\n\n """ Need to match with Tokenizer's package """\n\n def _mecab_ko(self, text):\n raise NotImplementedError("Linguistic Feature with mecab package")\n\n def _nltk_en(self, text):\n raise NotImplementedError("Linguistic Feature with nltk package")\n\n def _spacy_en(self, text):\n if self.spacy_model is None:\n from claf.tokens.tokenizer.utils import load_spacy_model_for_tokenizer\n\n disables = ["vectors", "textcat", "parser"]\n if not self.use_pos_tag:\n disables.apppend("tagger")\n if not self.use_ner:\n disables.apppend("ner")\n\n self.spacy_model = spacy.load("en_core_web_sm", disable=disables)\n self.spacy_model.tokenizer = load_spacy_model_for_tokenizer(\n self.tokenizer.extra_split_chars_re\n )\n\n sent_tokenizer = self.tokenizer.sent_tokenizer\n sentences = sent_tokenizer.tokenize(text)\n\n ner_entities = {}\n docs = []\n for sentence in sentences:\n doc = self.spacy_model(sentence)\n docs.append(doc)\n\n if self.use_ner:\n for e in doc.ents:\n ner_entities[e.text] = e.label_\n\n linguistic_features = []\n for doc in docs:\n for token in doc:\n if token.is_space:\n continue\n\n feature = []\n if self.use_pos_tag:\n feature.append(self.pos_to_index[token.pos_])\n if self.use_ner:\n feature.append(self.ner_to_index[ner_entities.get(token.text, "NONE")])\n\n linguistic_features.append(feature)\n return linguistic_features\n\n\nfrom overrides import overrides\n\nfrom .base import TokenIndexer\n\n\n[docs]class WordIndexer(TokenIndexer):\n """\n Word Token Indexer\n\n * Property\n vocab: Vocab (claf.tokens.vocabulary)\n\n * Args:\n tokenizer: WordTokenizer\n\n * Kwargs:\n lowercase: word token to lowercase\n insert_start: insert start_token to first\n insert_end: append end_token\n """\n\n def __init__(\n self, tokenizer, do_tokenize=True, lowercase=False, insert_start=None, insert_end=None\n ):\n super(WordIndexer, self).__init__(tokenizer)\n\n self.do_tokenize = do_tokenize\n self.lowercase = lowercase\n\n self.insert_start = insert_start\n self.insert_end = insert_end\n\n[docs] @overrides\n def index(self, text):\n input_type = type(text)\n if input_type == str:\n indexed_tokens = self._index_text(text)\n elif input_type == list:\n indexed_tokens = self._index_list_of_text(text)\n else:\n raise ValueError(f"Not supported type: {type(text)}")\n\n if self.insert_start is not None:\n insert_start = self.vocab.get_index(self.vocab.start_token)\n indexed_tokens.insert(0, insert_start)\n if self.insert_end is not None:\n insert_end = self.vocab.get_index(self.vocab.end_token)\n indexed_tokens.append(insert_end)\n return indexed_tokens\n\n def _index_text(self, text):\n if not self.do_tokenize:\n raise ValueError("input text type is 'str'. 'do_tokenize' is required.")\n\n return [self._index_token(token) for token in self.tokenizer.tokenize(text)]\n\n def _index_list_of_text(self, list_of_text):\n if self.do_tokenize:\n indexed_tokens = [\n [self._index_token(token) for token in self.tokenizer.tokenize(text)]\n for text in list_of_text\n ]\n else:\n indexed_tokens = [self._index_token(text) for text in list_of_text]\n return indexed_tokens\n\n def _index_token(self, token):\n if self.lowercase:\n token = token.lower()\n\n return self.vocab.get_index(token)\n\n[docs]class POSTag:\n """\n Universal POS tags expends by spacy\n (https://spacy.io/api/annotation#section-pos-tagging)\n """\n\n classes = [\n "ADJ", # adjectives\n "ADP", # adpositions (prepositions and postpositions)\n "ADV", # adverbs\n "AUX", # auxiliary (spacy)\n "CONJ", # conjunctions\n "CCONJ", # coordinating conjunction (spacy)\n "DET", # determiners\n "INTJ", # interjection (spacy)\n "NOUN", # nouns (common and proper)\n "NUM", # cardinal numbers\n "PART", # particles or other function words (spacy)\n "PRON", # pronouns\n "PROPN", # proper noun\n "PUNCT", # punctuation\n "SCONJ", # subordinating conjunction\n "SYM", # symbol\n "VERB", # verbs (all tenses and modes)\n "X", # other: foreign words, typos, abbreviations\n "SPACE", # space\n ]\n\n\n[docs]class NER:\n """\n Named Entity Recognition\n\n Models trained on the OntoNotes 5 corpus support\n the following entity types:\n (https://spacy.io/api/annotation#section-dependency-parsing)\n """\n\n classes = [\n "NONE", # None\n "PERSON", # People, including fictional.\n "NORP", # Nationalities or religious or political groups.\n "FAC", # Buildings, airports, highways, bridges, etc.\n "ORG", # Companies, agencies, institutions, etc.\n "GPE", # Countries, cities, states.\n "LOC", # Non-GPE locations, mountain ranges, bodies of water.\n "PRODUCT", # Objects, vehicles, foods, etc. (Not services.)\n "EVENT", # Named hurricanes, battles, wars, sports events, etc.\n "WORK_OF_ART", # Titles of books, songs, etc.\n "LAW", # Named documents made into laws.\n "LANGUAGE", # Any named language.\n "DATE", # Absolute or relative dates or periods.\n "TIME", # Times smaller than a day.\n "PERCENT", # Percentage, including "%".\n "MONEY", # Monetary values, including unit.\n "QUANTITY", # Measurements, as of weight or distance.\n "ORDINAL", # "first", "second", etc.\n "CARDINAL", # Numerals that do not fall under another type.\n ]\n\n\nfrom collections import Counter\nimport logging\nimport time\n\nfrom tqdm import tqdm\n\nfrom claf.data.data_handler import CachePath, DataHandler\nfrom claf.data.utils import padding_tokens, transpose\nfrom claf.tokens.token_maker import TokenMaker\nfrom claf.tokens.vocabulary import Vocab\nfrom claf import utils as common_utils\n\nlogger = logging.getLogger(__name__)\n\n\n[docs]class TextHandler:\n """\n Text Handler\n\n - voacb and token_counter\n - raw_features -> indexed_features\n - raw_features -> tensor\n\n * Args:\n token_makers: Dictionary consisting of\n - key: token_name\n - value: TokenMaker (claf.tokens.token_maker)\n\n * Kwargs:\n lazy_indexing: Apply `Lazy Evaluation` to text indexing\n """\n\n def __init__(self, token_makers, lazy_indexing=True):\n self.token_makers = token_makers\n self.lazy_indexing = lazy_indexing\n\n self.data_handler = DataHandler(cache_path=CachePath.TOKEN_COUNTER)\n\n[docs] def build_vocabs(self, token_counters):\n logger.info("Start build vocab")\n vocab_start_time = time.time()\n\n vocabs = {}\n for token_name, token_maker in self.token_makers.items():\n is_defined_config = type(token_maker.vocab_config) == dict\n if is_defined_config:\n token_counter = token_counters[token_name]\n vocab = self._build_vocab_with_config(token_name, token_maker, token_counter)\n else:\n vocab = Vocab(token_name)\n vocab.init()\n\n vocabs[token_name] = vocab\n logger.info(\n f" => {token_name} vocab size: {len(vocab)} (use predefine vocab: {vocab.pretrained_path is not None})"\n )\n\n vocab_elapased_time = time.time() - vocab_start_time\n logger.info(f"Complete build vocab... elapsed_time: {vocab_elapased_time}\\n")\n\n # Setting Indexer (vocab)\n for token_name, token_maker in self.token_makers.items():\n token_maker.set_vocab(vocabs[token_name])\n return vocabs\n\n def _build_vocab_with_config(self, token_name, token_maker, token_counter):\n token_maker.vocab_config["token_name"] = token_name\n vocab = Vocab(**token_maker.vocab_config)\n\n if vocab.pretrained_path is not None:\n vocab.build_with_pretrained_file(token_counter)\n else:\n vocab.build(token_counter)\n return vocab\n\n[docs] def is_all_vocab_use_pretrained(self):\n for token_name, token_maker in self.token_makers.items():\n if token_maker.vocab_config.get("pretrained_path", None) is None:\n return False\n if token_maker.vocab_config.get("pretrained_token", "") != Vocab.PRETRAINED_ALL:\n return False\n return True\n\n[docs] def make_token_counters(self, texts, config=None):\n token_counters = {}\n for token_name, token_maker in self.token_makers.items():\n token_vocab_config = token_maker.vocab_config\n if type(token_vocab_config) == dict:\n if token_vocab_config.get("pretrained_token", None) == Vocab.PRETRAINED_ALL:\n texts = [\n ""\n ] # do not use token_counter from dataset -> make empty token_counter\n\n token_counter = self._make_token_counter(\n texts, token_maker.tokenizer, config=config, desc=f"{token_name}-vocab"\n )\n logger.info(f" * {token_name} token_counter size: {len(token_counter)}")\n\n token_counters[token_name] = token_counter\n return token_counters\n\n def _make_token_counter(self, texts, tokenizer, config=None, desc=None):\n tokenizer_name = tokenizer.name\n\n cache_token_counter = None\n if config is not None:\n data_reader_config = config.data_reader\n cache_token_counter = self.data_handler.cache_token_counter(\n data_reader_config, tokenizer_name\n )\n\n if cache_token_counter:\n return cache_token_counter\n else:\n tokens = [\n token for text in tqdm(texts, desc=desc) for token in tokenizer.tokenize(text)\n ]\n flatten_list = list(common_utils.flatten(tokens))\n token_counter = Counter(flatten_list)\n\n if config is not None: # Cache TokenCounter\n self.data_handler.cache_token_counter(\n data_reader_config, tokenizer_name, obj=token_counter\n )\n return token_counter\n\n[docs] def index(self, datas, text_columns):\n logger.info(f"Start token indexing, Lazy: {self.lazy_indexing}")\n indexing_start_time = time.time()\n\n for data_type, data in datas.items():\n if type(data) == list:\n # Multi-Data Indexing\n for d in data:\n self._index_features(\n d.features, text_columns, desc=f"indexing features ({data_type})"\n )\n else:\n self._index_features(\n data.features, text_columns, desc=f"indexing features ({data_type})"\n )\n\n indexing_elapased_time = time.time() - indexing_start_time\n logger.info(f"Complete token indexing... elapsed_time: {indexing_elapased_time} \\n")\n\n def _index_features(self, features, text_columns, desc=None, suppress_tqdm=False):\n for feature in tqdm(features, desc=desc, disable=suppress_tqdm):\n for key, text in feature.items():\n if key not in text_columns:\n continue\n\n # Set data_type (text => {"text": ..., "token1": ..., ...})\n if type(feature[key]) != dict:\n feature[key] = {"text": text}\n if type(text) == dict:\n text = text["text"]\n\n for token_name, token_maker in self.token_makers.items():\n param_key = token_maker.indexer.param_key\n if param_key == key:\n continue\n\n feature[key][token_name] = self._index_token(token_maker, text, feature)\n\n def _index_token(self, token_maker, text, data):\n def index():\n indexer = token_maker.indexer\n params = {}\n if token_maker.type_name == TokenMaker.EXACT_MATCH_TYPE:\n param_text = data[indexer.param_key]\n if type(param_text) == dict:\n param_text = param_text["text"]\n params["query_text"] = param_text\n return indexer.index(text, **params)\n\n if self.lazy_indexing:\n return index\n else:\n return index()\n\n[docs] def raw_to_tensor_fn(self, data_reader, cuda_device=None, helper={}):\n def raw_to_tensor(inputs):\n is_one = True # batch_size 1 flag\n feature, _helper = data_reader.read_one_example(inputs)\n\n nonlocal helper\n helper.update(_helper)\n\n if type(feature) == list:\n is_one = False\n features = feature\n else:\n features = [feature]\n\n self._index_features(features, data_reader.text_columns, suppress_tqdm=True)\n\n if is_one:\n indexed_features = features[0]\n else: # when features > 1, need to transpose (dict_of_list -> list_of_dict)\n indexed_features = {}\n for key in features[0]:\n feature_with_key = [feature[key] for feature in features]\n indexed_features[key] = transpose(feature_with_key, skip_keys=["text"])\n\n for key in indexed_features:\n for token_name in self.token_makers:\n if token_name not in indexed_features[key]:\n continue\n\n indexed_values = indexed_features[key][token_name]\n if is_one:\n indexed_values = [indexed_values]\n\n tensor = padding_tokens(indexed_values, token_name=token_name)\n if cuda_device is not None and type(tensor) != list:\n tensor = tensor.cuda(cuda_device)\n indexed_features[key][token_name] = tensor\n\n for key in indexed_features:\n if "text" in indexed_features[key]:\n del indexed_features[key]["text"]\n\n return indexed_features, helper\n\n return raw_to_tensor\n\n\n\nimport torch\n\n\n[docs]class TokenEmbedder(torch.nn.Module):\n """\n Token Embedder\n\n Take a tensor(indexed token) look up Embedding modules.\n\n * Args:\n token_makers: dictionary of TokenMaker (claf.token_makers.token)\n """\n\n def __init__(self, token_makers):\n super(TokenEmbedder, self).__init__()\n\n self.embed_dims = {}\n\n self.vocabs = {\n token_name: token_maker.vocab for token_name, token_maker in token_makers.items()\n }\n self.add_embedding_modules(token_makers)\n\n[docs] def add_embedding_modules(self, token_makers):\n """ add embedding module to TokenEmbedder """\n self.token_names = []\n for token_name, token_maker in token_makers.items():\n self.token_names.append(token_name)\n\n vocab = token_maker.vocab\n embedding = token_maker.embedding_fn(vocab)\n self.add_module(token_name, embedding)\n\n self.embed_dims[token_name] = embedding.get_output_dim()\n\n\n\n\n\n\nfrom overrides import overrides\n\nimport torch\n\nfrom .base import TokenEmbedder\n\n\n[docs]class BasicTokenEmbedder(TokenEmbedder):\n """\n Basic Token Embedder\n\n Take a tensor(indexed token) look up Embedding modules.\n Output is concatenating all embedded tensors.\n\n * Args:\n token_makers: dictionary of TokenMaker (claf.tokens.token_maker)\n """\n\n def __init__(self, token_makers):\n super(BasicTokenEmbedder, self).__init__(token_makers)\n\n\n\n[docs] @overrides\n def forward(self, inputs, except_keys=[], params={}):\n token_names = [name for name in self.token_names if name not in except_keys]\n if set(token_names) != set(inputs.keys()):\n raise ValueError(\n f"Mismatch token_names inputs: {inputs.keys()}, embeddings: {self.token_names}"\n )\n\n embedded_tokens = []\n for token_name, tensors in inputs.items():\n embedding = getattr(self, token_name)\n\n embedded_token = embedding(tensors, **params)\n embedded_tokens.append(embedded_token)\n\n output = torch.cat(embedded_tokens, dim=-1)\n return output\n\n\nfrom overrides import overrides\nimport torch\n\nimport claf.modules.functional as f\nimport claf.modules.attention as attention\n\nfrom .base import TokenEmbedder\n\n\n[docs]class RCTokenEmbedder(TokenEmbedder):\n """\n Reading Comprehension Token Embedder\n\n Take a tensor(indexed token) look up Embedding modules.\n Inputs are seperated context and query for individual token setting.\n\n * Args:\n token_makers: dictionary of TokenMaker (claf.tokens.token_maker)\n vocabs: dictionary of vocab\n {"token_name": Vocab (claf.token_makers.vocaburary), ...}\n """\n\n EXCLUSIVE_TOKENS = ["exact_match"] # only context\n\n def __init__(self, token_makers):\n super(RCTokenEmbedder, self).__init__(token_makers)\n\n self.context_embed_dim = sum(self.embed_dims.values())\n self.query_embed_dim = sum(self._filter(self.embed_dims, exclusive=False).values())\n\n self.align_attention = attention.SeqAttnMatch(self.query_embed_dim)\n\n\n\n[docs] @overrides\n def forward(self, context, query, context_params={}, query_params={}, query_align=False):\n """\n * Args:\n context: context inputs (eg. {"token_name1": tensor, "token_name2": tensor, ...})\n query: query inputs (eg. {"token_name1": tensor, "token_name2": tensor, ...})\n\n * Kwargs:\n context_params: custom context parameters\n query_params: query context parameters\n query_align: f_align(p_i) = sum(a_ij, E(qj), where the attention score a_ij\n captures the similarity between pi and each question words q_j.\n these features add soft alignments between similar but non-identical words (e.g., car and vehicle)\n it only apply to 'context_embed'.\n """\n\n if set(self.token_names) != set(context.keys()):\n raise ValueError(\n f"Mismatch token_names inputs: {context.keys()}, embeddings: {self.token_names}"\n )\n\n context_tokens, query_tokens = {}, {}\n for token_name, context_tensors in context.items():\n embedding = getattr(self, token_name)\n\n context_tokens[token_name] = embedding(\n context_tensors, **context_params.get(token_name, {})\n )\n if token_name in query:\n query_tokens[token_name] = embedding(\n query[token_name], **query_params.get(token_name, {})\n )\n\n # query_align_embedding\n if query_align:\n common_context = self._filter(context_tokens, exclusive=False)\n embedded_common_context = torch.cat(list(common_context.values()), dim=-1)\n exclusive_context = self._filter(context_tokens, exclusive=True)\n\n embedded_exclusive_context = None\n if exclusive_context != {}:\n embedded_exclusive_context = torch.cat(list(exclusive_context.values()), dim=-1)\n\n query_mask = f.get_mask_from_tokens(query_tokens)\n embedded_query = torch.cat(list(query_tokens.values()), dim=-1)\n\n embedded_aligned_query = self.align_attention(\n embedded_common_context, embedded_query, query_mask\n )\n\n # Merge context embedded\n embedded_context = [embedded_common_context, embedded_aligned_query]\n if embedded_exclusive_context is not None:\n embedded_context.append(embedded_exclusive_context)\n\n context_output = torch.cat(embedded_context, dim=-1)\n query_output = embedded_query\n else:\n context_output = torch.cat(list(context_tokens.values()), dim=-1)\n query_output = torch.cat(list(query_tokens.values()), dim=-1)\n\n return context_output, query_output\n\n def _filter(self, token_data, exclusive=False):\n if exclusive:\n return {k: v for k, v in token_data.items() if k in self.EXCLUSIVE_TOKENS}\n else:\n return {k: v for k, v in token_data.items() if k not in self.EXCLUSIVE_TOKENS}\n\n[docs]class TokenMaker:\n """\n Token Maker (Data Transfer Object)\n\n Token Maker consists of Tokenizer, Indexer, Embedding and Vocab\n\n * Kwargs:\n tokenizer: Tokenizer (claf.tokens.tokenizer.base)\n indexer: TokenIndexer (claf.tokens.indexer.base)\n embedding_fn: wrapper function of TokenEmbedding (claf.tokens.embedding.base)\n vocab_config: config dict of Vocab (claf.tokens.vocaburary)\n """\n\n # Token Type List\n FEATURE_TYPE = "feature" # Do not use embedding, pass indexed_feature\n\n BERT_TYPE = "bert"\n CHAR_TYPE = "char"\n COVE_TYPE = "cove"\n ELMO_TYPE = "elmo"\n EXACT_MATCH_TYPE = "exact_match"\n WORD_TYPE = "word"\n FREQUENT_WORD_TYPE = "frequent_word"\n LINGUISTIC_TYPE = "linguistic"\n\n def __init__(\n self, token_type, tokenizer=None, indexer=None, embedding_fn=None, vocab_config=None\n ):\n self.type_name = token_type\n self._tokenizer = tokenizer\n self._indexer = indexer\n self._embedding_fn = embedding_fn\n self._vocab_config = vocab_config\n\n @property\n def tokenizer(self):\n return self._tokenizer\n\n @tokenizer.setter\n def tokenizer(self, tokenizer):\n self._tokenizer = tokenizer\n\n @property\n def indexer(self):\n return self._indexer\n\n @indexer.setter\n def indexer(self, indexer):\n self._indexer = indexer\n\n @property\n def embedding_fn(self):\n return self._embedding_fn\n\n @embedding_fn.setter\n def embedding_fn(self, embedding_fn):\n self._embedding_fn = embedding_fn\n\n @property\n def vocab_config(self):\n return self._vocab_config\n\n @vocab_config.setter\n def vocab_config(self, vocab_config):\n self._vocab_config = vocab_config\n\n @property\n def vocab(self):\n return self._vocab\n\n @vocab.setter\n def vocab(self, vocab):\n self._vocab = vocab\n\n\n\n[docs]class Tokenizer:\n """\n Tokenizer Base Class\n """\n\n MAX_TO_KEEP_CACHE = 3\n\n def __init__(self, name, cache_name):\n self.cache = {} # dict: {text: tokenized_tokens}\n self.name = name\n self.cache_name = cache_name\n\n[docs] def tokenize(self, text, unit="text"):\n if type(text) == str and text in self.cache:\n return self.cache[text]\n\n tokenized_tokens = self._tokenize(text, unit="text")\n\n # Cache\n if len(self.cache) <= self.MAX_TO_KEEP_CACHE:\n self.cache[text] = tokenized_tokens\n else:\n first_key = next(iter(self.cache.keys()))\n del self.cache[first_key]\n\n return tokenized_tokens\n\n def _tokenize(self, text, unit="text"):\n """ splitting text into tokens. """\n if type(text) != str:\n raise ValueError(f"text type is must be str. not {type(text)}")\n\n return getattr(self, f"_{self.name}")(text, unit=unit)\n\n\nfrom pytorch_transformers import RobertaTokenizer\n\nfrom claf.data.data_handler import CachePath, DataHandler\n\nfrom .base import Tokenizer\n\n\n[docs]class BPETokenizer(Tokenizer):\n """\n BPTE(Byte-Pair Encoding) Tokenizer\n text -> ...\n * Args:\n name: tokenizer name [roberta]\n """\n\n def __init__(self, name, config={}):\n super(BPETokenizer, self).__init__(name, f"bpe-{name}")\n self.data_handler = DataHandler(CachePath.VOCAB)\n self.config = config\n\n self.bpe_tokenizer = None\n\n """ Tokenizers """\n\n def _roberta(self, text, unit="text"):\n """\n ex)\n """\n if self.bpe_tokenizer is None:\n vocab_path = self.data_handler.read(self.config["vocab_path"], return_path=True)\n merges_path = self.data_handler.read(self.config["merges_path"], return_path=True)\n del self.config["vocab_path"]\n del self.config["merges_path"]\n\n self.bpe_tokenizer = RobertaTokenizer(vocab_path, merges_path, **self.config)\n\n return self.bpe_tokenizer._tokenize(text)\n\n\n\nfrom claf.tokens import hangul as hg\n\nfrom .base import Tokenizer\n\n\n[docs]class CharTokenizer(Tokenizer):\n """\n Character Tokenizer\n\n text -> word tokens -> [char tokens]\n\n * Args:\n name: tokenizer name [character|decompose_ko]\n word_tokenizer: word tokenizer object\n """\n\n def __init__(self, name, word_tokenizer, config={}):\n super(CharTokenizer, self).__init__(name, f"char-{name}+{word_tokenizer.cache_name}")\n self.config = config\n self.word_tokenizer = word_tokenizer\n\n """ Tokenizers """\n\n def _character(self, text, unit="text"):\n """\n ex) Hello World -> ['Hello', 'World'] -> [['H', 'e', 'l', 'l', 'o'], ['W', 'o', 'r', 'l', 'd']]\n """\n if unit == "word":\n return [char for char in text]\n else:\n return [[char for char in word] for word in self.word_tokenizer.tokenize(text)]\n\n def _jamo_ko(self, text, unit="text"):\n """\n ex) 안녕 세상 -> ['안녕', '세상'] -> [['ㅇ', 'ㅏ', 'ㄴ', 'ㄴ', 'ㅕ', 'ㅇ'], ['ㅅ', 'ㅔ', 'ㅅ', 'ㅏ', 'ㅇ']]\n """\n\n def decompose(char):\n if hg.is_hangul(char):\n try:\n return [c for c in hg.decompose(char) if c != ""]\n except IndexError: # Case: ㅋㅋㅋㅋ\n return [char]\n else:\n return [char]\n\n tokens = []\n if unit == "word":\n chars = []\n for char in text:\n chars.extend(decompose(char))\n tokens.append(chars)\n else:\n for word in self.word_tokenizer.tokenize(text):\n chars = []\n for char in word:\n chars.extend(decompose(char))\n tokens.append(chars)\n return tokens\n\n\nimport nltk.data\n\nfrom .base import Tokenizer\n\n\n[docs]class SentTokenizer(Tokenizer):\n """\n Sentence Tokenizer\n\n text -> [sent tokens]\n\n * Args:\n name: tokenizer name [punkt]\n """\n\n def __init__(self, name, config={}):\n super(SentTokenizer, self).__init__(name, f"sent-{name}")\n self.config = config\n\n """ Tokenizers """\n\n def _punkt(self, text, unit="text"):\n """\n ex) Hello World. This is punkt tokenizer -> ['Hello World', 'This is punkt tokenizer']\n """\n sent_tokenizer = nltk.data.load("tokenizers/punkt/english.pickle")\n return sent_tokenizer.tokenize(text)\n\n\nfrom pytorch_transformers import WordpieceTokenizer\nfrom pytorch_transformers.tokenization_bert import load_vocab\n\n\nfrom claf.data.data_handler import CachePath, DataHandler\n\nfrom .base import Tokenizer\n\n\n[docs]class SubwordTokenizer(Tokenizer):\n """\n Subword Tokenizer\n\n text -> [word tokens] -> [[sub word tokens], ...]\n\n * Args:\n name: tokenizer name [wordpiece]\n """\n\n def __init__(self, name, word_tokenizer, config={}):\n super(SubwordTokenizer, self).__init__(name, f"subword-{name}+{word_tokenizer.cache_name}")\n self.data_handler = DataHandler(CachePath.VOCAB)\n self.config = config\n self.word_tokenizer = word_tokenizer\n self.subword_tokenizer = None\n\n """ Tokenizers """\n\n def _wordpiece(self, text, unit="text"):\n """\n ex) Hello World -> ['Hello', 'World'] -> ['He', '##llo', 'Wo', '##rld']\n """\n if self.subword_tokenizer is None:\n vocab_path = self.data_handler.read(self.config["vocab_path"], return_path=True)\n vocab = load_vocab(vocab_path)\n self.subword_tokenizer = WordpieceTokenizer(\n vocab, unk_token=self.config.get("unk_token", "[UNK]"))\n\n tokens = []\n\n if unit == "word":\n for sub_token in self.subword_tokenizer.tokenize(text):\n tokens.append(sub_token)\n else:\n for token in self.word_tokenizer.tokenize(text):\n for sub_token in self.subword_tokenizer.tokenize(token):\n tokens.append(sub_token)\n\n return tokens\n\n\nimport spacy\n\n\n[docs]def create_tokenizer_with_regex(nlp, split_regex):\n prefixes_re = spacy.util.compile_prefix_regex(nlp.Defaults.prefixes)\n infix_re = split_regex\n suffix_re = spacy.util.compile_suffix_regex(nlp.Defaults.suffixes)\n\n return spacy.tokenizer.Tokenizer(\n nlp.vocab,\n nlp.Defaults.tokenizer_exceptions,\n prefix_search=prefixes_re.search,\n infix_finditer=infix_re.finditer,\n suffix_search=suffix_re.search,\n token_match=None,\n )\n\n\n[docs]def load_spacy_model_for_tokenizer(split_regex):\n model = spacy.load("en_core_web_sm", disable=["vectors", "textcat", "tagger", "parser", "ner"])\n\n if split_regex is not None:\n spacy_tokenizer = create_tokenizer_with_regex(model, split_regex)\n model.tokenizer = spacy_tokenizer\n return model\n\n\n\nimport re\n\nfrom overrides import overrides\n\nfrom claf import utils as common_utils\n\nfrom .base import Tokenizer\n\n\n[docs]class WordTokenizer(Tokenizer):\n """\n Word Tokenizer\n\n * Args:\n name: tokenizer name [treebank_en|spacy_en|mecab_ko|bert_basic]\n\n * Kwargs:\n flatten: return type as flatten list\n split_with_regex: post split action. Split tokens that the tokenizer cannot split.\n """\n\n def __init__(self, name, sent_tokenizer, config={}, split_with_regex=True):\n super(WordTokenizer, self).__init__(name, f"word-{name}+{sent_tokenizer.cache_name}")\n self.config = config\n self.sent_tokenizer = sent_tokenizer\n self.word_tokenizer = None\n\n self.split_with_regex = split_with_regex\n if split_with_regex:\n self.extra_split_chars_re = self.make_split_regex_expression()\n\n[docs] def make_split_regex_expression(self):\n """\n Apply a small amount of extra splitting to the given tokens, this is in particular to avoid UNK tokens\n due to contraction, quotation, or other forms of puncutation. I haven't really done tests to see\n if/how much difference this makes, but it does avoid some common UNKs I noticed in SQuAD/TriviaQA\n """\n extra_split_chars = (\n "-",\n "£",\n "€",\n "¥",\n "¢",\n "₹",\n "*",\n "\\u2212",\n "\\u2014",\n "\\u2013",\n "/",\n "~",\n '"',\n "'",\n "\\ud01C",\n "\\u2019",\n "\\u201D",\n "\\u2018",\n "\\u00B0",\n ".",\n ":",\n )\n extra_split_tokens = (\n "``",\n "(?<=[^_])_(?=[^_])", # dashes w/o a preceeding or following dash, so __wow___ -> ___ wow ___\n "''",\n "[" + "".join(extra_split_chars) + "]",\n )\n return re.compile("(" + "|".join(extra_split_tokens) + ")")\n\n @overrides\n def _tokenize(self, text, unit="text"):\n """ Text -> word tokens """\n if type(text) != str:\n raise ValueError(f"text type is must be str. not {type(text)}")\n\n if unit == "sentence":\n tokens = getattr(self, f"_{self.name}")(text)\n else:\n sentences = self.sent_tokenizer.tokenize(text)\n tokens = [getattr(self, f"_{self.name}")(sentence) for sentence in sentences]\n\n if self.split_with_regex and self.name != "spacy_en":\n tokens = self._split_with_regex(tokens)\n\n return list(common_utils.flatten(tokens))\n\n def _split_with_regex(self, sentences):\n for i, sentence in enumerate(sentences):\n sentences[i] = [token for token in self._post_split_tokens(sentence)]\n return sentences\n\n def _post_split_tokens(self, tokens):\n return [[x for x in self.extra_split_chars_re.split(token) if x != ""] for token in tokens]\n\n """ Tokenizers """\n\n def _space_all(self, text):\n def is_whitespace(c):\n if c == " " or c == "\\t" or c == "\\r" or c == "\\n" or ord(c) == 0x202F:\n return True\n return False\n\n prev_is_whitespace = True\n tokens = []\n for char in text:\n if is_whitespace(char):\n prev_is_whitespace = True\n else:\n if prev_is_whitespace:\n tokens.append(char)\n else:\n tokens[-1] += char\n prev_is_whitespace = False\n return tokens\n\n def _treebank_en(self, text):\n if self.word_tokenizer is None:\n import nltk\n\n self.word_tokenizer = nltk.TreebankWordTokenizer()\n\n return [\n token.replace("''", '"').replace("``", '"')\n for token in self.word_tokenizer.tokenize(text)\n ]\n\n def _spacy_en(self, text):\n if self.word_tokenizer is None:\n from claf.tokens.tokenizer.utils import load_spacy_model_for_tokenizer\n\n self.word_tokenizer = load_spacy_model_for_tokenizer(self.extra_split_chars_re)\n\n def _remove_spaces(tokens):\n return [token.text for token in tokens if not token.is_space]\n\n return _remove_spaces(self.word_tokenizer(text))\n\n def _bert_basic(self, text):\n if self.word_tokenizer is None:\n from pytorch_transformers import BasicTokenizer\n\n self.word_tokenizer = BasicTokenizer(**self.config)\n\n return self.word_tokenizer.tokenize(text)\n\n def _mecab_ko(self, text):\n if self.word_tokenizer is None:\n from konlpy.tag import Mecab\n\n self.word_tokenizer = Mecab()\n\n return self.word_tokenizer.morphs(text)\n\n\nfrom collections import defaultdict\nimport json\n\nfrom claf.data.data_handler import CachePath, DataHandler\n\n\n[docs]class VocabDict(defaultdict):\n """\n Vocab DefaultDict Class\n\n * Kwargs:\n oov_value: out-of-vocaburary token value (eg. <unk>)\n """\n\n def __init__(self, oov_value):\n self.oov_value = oov_value\n\n def __missing__(self, key):\n return self.oov_value\n\n\n[docs]class Vocab:\n """\n Vocaburary Class\n\n Vocab consists of token_to_index and index_to_token.\n\n * Args:\n token_name: Token name (Token and Vocab is one-to-one relationship)\n\n * Kwargs:\n pad_token: padding token value (eg. <pad>)\n oov_token: out-of-vocaburary token value (eg. <unk>)\n start_token: start token value (eg. <s>, <bos>)\n end_token: end token value (eg. </s>, <eos>)\n cls_token: CLS token value for BERT (eg. [CLS])\n sep_token: SEP token value for BERT (eg. [SEP])\n min_count: token's minimal frequent count.\n when you define min_count, tokens remain that bigger than min_count.\n max_vocab_size: vocaburary's maximun size.\n when you define max_vocab_size, tokens are selected according to frequent count.\n frequent_count: get frequent_count threshold_index.\n (eg. frequent_count = 1000, threshold_index is the tokens that frequent_count is 999 index number.)\n pretrained_path: pretrained vocab file path\n (format: A\\nB\\nC\\nD\\n...)\n """\n\n DEFAULT_PAD_INDEX, DEFAULT_PAD_TOKEN = 0, "[PAD]"\n DEFAULT_OOV_INDEX, DEFAULT_OOV_TOKEN = 1, "[UNK]"\n\n # pretrained_vocab handle methods\n PRETRAINED_ALL = "all" # Case. embedding matrix - predefine_vocab fixed\n PRETRAINED_INTERSECT = "intersect" # add token that included in predefine_vocab, else UNK_token\n\n def __init__(\n self,\n token_name,\n pad_token=None,\n oov_token=None,\n start_token=None,\n end_token=None,\n cls_token=None,\n sep_token=None,\n min_count=None,\n max_vocab_size=None,\n frequent_count=None,\n pretrained_path=None,\n pretrained_token=None,\n ):\n self.token_name = token_name\n\n # basic token (pad and oov)\n self.pad_index = self.DEFAULT_PAD_INDEX\n self.pad_token = pad_token\n if pad_token is None:\n self.pad_token = self.DEFAULT_PAD_TOKEN\n\n self.oov_index = self.DEFAULT_OOV_INDEX\n self.oov_token = oov_token\n if oov_token is None:\n self.oov_token = self.DEFAULT_OOV_TOKEN\n\n # special_tokens\n self.start_token = start_token\n self.end_token = end_token\n self.cls_token = cls_token\n self.sep_token = sep_token\n\n self.min_count = min_count\n self.max_vocab_size = max_vocab_size\n\n self.token_counter = None\n self.frequent_count = frequent_count\n self.threshold_index = None\n\n self.pretrained_path = pretrained_path\n self.pretrained_token = pretrained_token\n self.pretrained_token_methods = [self.PRETRAINED_ALL, self.PRETRAINED_INTERSECT]\n\n[docs] def init(self):\n self.token_to_index = VocabDict(self.oov_index)\n self.index_to_token = VocabDict(self.oov_token)\n\n # add default token (pad, oov)\n self.add(self.pad_token)\n self.add(self.oov_token)\n\n special_tokens = [self.start_token, self.end_token, self.cls_token, self.sep_token]\n for token in special_tokens:\n if token is not None:\n self.add(token)\n\n[docs] def build(self, token_counter, predefine_vocab=None):\n """\n build token with token_counter\n\n * Args:\n token_counter: (collections.Counter) token's frequent_count Counter.\n """\n\n if predefine_vocab is not None:\n if (\n self.pretrained_token is None\n or self.pretrained_token not in self.pretrained_token_methods\n ):\n raise ValueError(\n f"When use 'predefine_vocab', need to set 'pretrained_token' {self.pretrained_token_methods}"\n )\n\n if predefine_vocab:\n if self.pretrained_token == self.PRETRAINED_ALL:\n self.from_texts(predefine_vocab)\n return\n else:\n predefine_vocab = set(predefine_vocab)\n\n self.token_counter = token_counter\n self.init()\n\n token_counts = list(token_counter.items())\n token_counts.sort(key=lambda x: x[1], reverse=True) # order: DESC\n\n if self.max_vocab_size is not None:\n token_counts = token_counts[: self.max_vocab_size]\n\n for token, count in token_counts:\n if self.min_count is not None:\n if count >= self.min_count:\n self.add(token, predefine_vocab=predefine_vocab)\n else:\n self.add(token, predefine_vocab=predefine_vocab)\n\n if self.threshold_index is None and self.frequent_count is not None:\n if count < self.frequent_count:\n self.threshold_index = len(self.token_to_index)\n\n[docs] def build_with_pretrained_file(self, token_counter):\n data_handler = DataHandler(CachePath.VOCAB)\n vocab_texts = data_handler.read(self.pretrained_path)\n\n if self.pretrained_path.endswith(".txt"):\n predefine_vocab = vocab_texts.split("\\n")\n elif self.pretrained_path.endswith(".json"):\n vocab_texts = json.loads(vocab_texts) # {token: id}\n predefine_vocab = [item[0] for item in\n sorted(vocab_texts.items(), key=lambda x: x[1])]\n else:\n raise ValueError(f"support vocab extention. .txt or .json")\n\n self.build(token_counter, predefine_vocab=predefine_vocab)\n\n def __len__(self):\n return len(self.token_to_index)\n\n[docs] def add(self, token, predefine_vocab=None):\n if token in self.token_to_index:\n return # already added\n if predefine_vocab:\n if self.pretrained_token == self.PRETRAINED_INTERSECT and token not in predefine_vocab:\n return\n\n index = len(self.token_to_index)\n\n self.token_to_index[token] = index\n self.index_to_token[index] = token\n\n\n\n\n\n\n\n[docs] def dump(self, path):\n with open(path, "w", encoding="utf-8") as out_file:\n out_file.write(self.to_text())\n\n[docs] def load(self, path):\n with open(path, "r", encoding="utf-8") as in_file:\n texts = in_file.read()\n\n self.from_texts(texts)\n\n\n\n[docs] def from_texts(self, texts):\n if type(texts) == list:\n tokens = texts\n else:\n tokens = [token for token in texts.split("\\n")]\n tokens = [token for token in tokens if token] # filtering empty string\n\n # basic token (pad and oov)\n if self.pad_token in tokens:\n self.pad_index = tokens.index(self.pad_token)\n else:\n self.pad_index = len(tokens)\n tokens.append(self.pad_token)\n\n if self.oov_token in tokens:\n self.oov_index = tokens.index(self.oov_token)\n else:\n self.oov_index = len(tokens)\n tokens.append(self.oov_token)\n\n self.token_to_index = VocabDict(self.oov_index)\n self.index_to_token = VocabDict(self.oov_token)\n\n for token in tokens:\n self.add(token)\n return self\n\n\nfrom claf.decorator import register\nfrom claf.tokens import indexer, embedding\nfrom claf.tokens.linguistic import POSTag, NER\nfrom claf.tokens.token_maker import TokenMaker\nfrom claf.tokens.tokenizer import PassText\n\n\n[docs]def basic_embedding_fn(embedding_config, module):\n def wrapper(vocab):\n embedding_config["vocab"] = vocab\n return module(**embedding_config)\n\n return wrapper\n\n\n[docs]@register(f"token:{TokenMaker.FEATURE_TYPE}")\nclass FeatureTokenMaker(TokenMaker):\n """\n Feature Token\n\n Do not use Embedding function.\n Just pass indexed_feature\n\n example.\n hello -> ['hello', 'world'] -> [3, 5] -> tensor\n\n consisting of\n - tokenizer: Tokenizer (need to define unit)\n - indexer: WordIndexer\n - embedding: None\n - vocab: Vocab\n """\n\n def __init__(self, tokenizers, indexer_config, embedding_config, vocab_config):\n tokenizer = PassText()\n do_tokenize = indexer_config.get("do_tokenize", False)\n if do_tokenize:\n text_unit = indexer_config.get("unit", None)\n if text_unit is None:\n raise ValueError("When use 'do_tokenize', 'unit' is required. ")\n\n del indexer_config["unit"]\n tokenizer = tokenizers[text_unit]\n\n super(FeatureTokenMaker, self).__init__(\n TokenMaker.FEATURE_TYPE,\n tokenizer=tokenizer,\n indexer=indexer.WordIndexer(tokenizer, **indexer_config),\n embedding_fn=None,\n vocab_config=vocab_config,\n )\n\n\n[docs]@register(f"token:{TokenMaker.BERT_TYPE}")\nclass BertTokenMaker(TokenMaker):\n """\n BERT Token\n Pre-training of Deep Bidirectional Transformers for Language Understanding\n\n example.\n hello -> ['[CLS]', 'he', '##llo', [SEP]] -> [1, 4, 7, 2] -> BERT -> tensor\n\n consisting of\n - tokenizer: WordTokenizer\n - indexer: WordIndexer\n - embedding: ELMoEmbedding (Language Modeling BiLSTM)\n - vocab: Vocab\n """\n\n def __init__(self, tokenizers, indexer_config, embedding_config, vocab_config):\n tokenizer = tokenizers["subword"]\n super(BertTokenMaker, self).__init__(\n TokenMaker.BERT_TYPE,\n tokenizer=tokenizer,\n indexer=indexer.BertIndexer(tokenizer, **indexer_config),\n embedding_fn=basic_embedding_fn(embedding_config, embedding.BertEmbedding),\n vocab_config=vocab_config,\n )\n\n\n[docs]@register(f"token:{TokenMaker.CHAR_TYPE}")\nclass CharTokenMaker(TokenMaker):\n """\n Character Token\n\n Character-level Convolutional Networks for Text Classification\n (https://arxiv.org/abs/1509.01626)\n\n example.\n hello -> ['h', 'e', 'l', 'l', 'o'] -> [2, 3, 4, 4, 5] -> CharCNN -> tensor\n\n consisting of\n - tokenizer: CharTokenizer\n - indexer: CharIndexer\n - embedding: CharEmbedding (CharCNN)\n - vocab: Vocab\n """\n\n def __init__(self, tokenizers, indexer_config, embedding_config, vocab_config):\n super(CharTokenMaker, self).__init__(\n TokenMaker.CHAR_TYPE,\n tokenizer=tokenizers["char"],\n indexer=indexer.CharIndexer(tokenizers["char"], **indexer_config),\n embedding_fn=basic_embedding_fn(embedding_config, embedding.CharEmbedding),\n vocab_config=vocab_config,\n )\n\n\n[docs]@register(f"token:{TokenMaker.COVE_TYPE}")\nclass CoveTokenMaker(TokenMaker):\n """\n CoVe Token\n\n Learned in Translation: Contextualized Word Vectors (McCann et. al. 2017)\n (https://github.com/salesforce/cove)\n\n example.\n hello -> ['hello'] -> [2] -> CoVe -> tensor\n\n consisting of\n - tokenizer: WordTokenizer\n - indexer: WordIndexer\n - embedding: CoveEmbedding (Machine Translation LSTM)\n - vocab: Vocab\n """\n\n def __init__(self, tokenizers, indexer_config, embedding_config, vocab_config):\n super(CoveTokenMaker, self).__init__(\n TokenMaker.CHAR_TYPE,\n tokenizer=tokenizers["word"],\n indexer=indexer.WordIndexer(tokenizers["word"], **indexer_config),\n embedding_fn=basic_embedding_fn(embedding_config, embedding.CoveEmbedding),\n vocab_config=vocab_config,\n )\n\n\n[docs]@register(f"token:{TokenMaker.ELMO_TYPE}")\nclass ElmoTokenMaker(TokenMaker):\n """\n ELMo Token\n Embedding from Language Modeling\n\n Deep contextualized word representations\n (https://github.com/allenai/allennlp/blob/master/allennlp/modules/elmo.py)\n\n example.\n hello -> ['h', 'e', 'l', 'l', 'o'] -> [2, 3, 4, 4, 5] -> ELMo -> tensor\n\n consisting of\n - tokenizer: WordTokenizer\n - indexer: WordIndexer\n - embedding: ELMoEmbedding (Language Modeling BiLSTM)\n - vocab: Vocab\n """\n\n def __init__(self, tokenizers, indexer_config, embedding_config, vocab_config):\n super(ElmoTokenMaker, self).__init__(\n TokenMaker.WORD_TYPE,\n tokenizer=tokenizers["word"],\n indexer=indexer.ELMoIndexer(tokenizers["word"], **indexer_config),\n embedding_fn=basic_embedding_fn(embedding_config, embedding.ELMoEmbedding),\n vocab_config="elmo",\n )\n\n\n[docs]@register(f"token:{TokenMaker.EXACT_MATCH_TYPE}")\nclass ExactMatchTokenMaker(TokenMaker):\n """\n Exact Match Token (Sparse Feature)\n\n Three simple binary features, indicating whether p_i can be exactly matched\n to one question word in q, either in its original, lowercase or lemma form.\n\n example.\n c: i do, q: i -> ['i', 'do'] -> [1, 0] -> tensor\n\n consisting of\n - tokenizer: WordTokenizer\n - indexer: WordIndexer\n - embedding: SparseFeature\n - vocab: Vocab\n """\n\n def __init__(self, tokenizers, indexer_config, embedding_config, vocab_config):\n super(ExactMatchTokenMaker, self).__init__(\n TokenMaker.EXACT_MATCH_TYPE,\n tokenizer=tokenizers["word"],\n indexer=indexer.ExactMatchIndexer(tokenizers["word"], **indexer_config),\n embedding_fn=self._embedding_fn(embedding_config, indexer_config),\n vocab_config=vocab_config,\n )\n\n def _embedding_fn(self, embedding_config, indexer_config):\n def wrapper(vocab):\n embed_type = embedding_config.get("type", "sparse")\n if "type" in embedding_config:\n del embedding_config["type"]\n\n binary_classes = ["False", "True"]\n\n feature_count = 1 # origin\n embedding_config["classes"] = [binary_classes]\n\n if indexer_config.get("lower", False):\n feature_count += 1\n embedding_config["classes"].append(binary_classes)\n if indexer_config.get("lemma", False):\n feature_count += 1\n embedding_config["classes"].append(binary_classes)\n\n return embedding.SparseFeature(\n vocab, embed_type, feature_count, params=embedding_config\n )\n\n return wrapper\n\n\n[docs]@register(f"token:{TokenMaker.WORD_TYPE}")\nclass WordTokenMaker(TokenMaker):\n """\n Word Token (default)\n\n i do -> ['i', 'do'] -> [1, 2] -> Embedding Matrix -> tensor\n\n consisting of\n - tokenizer: WordTokenizer\n - indexer: WordIndexer\n - embedding: WordEmbedding\n - vocab: Vocab\n """\n\n def __init__(self, tokenizers, indexer_config, embedding_config, vocab_config):\n super(WordTokenMaker, self).__init__(\n TokenMaker.WORD_TYPE,\n tokenizer=tokenizers["word"],\n indexer=indexer.WordIndexer(tokenizers["word"], **indexer_config),\n embedding_fn=basic_embedding_fn(embedding_config, embedding.WordEmbedding),\n vocab_config=vocab_config,\n )\n\n\n[docs]@register(f"token:{TokenMaker.FREQUENT_WORD_TYPE}")\nclass FrequentWordTokenMaker(TokenMaker):\n """\n Frequent-Tuning Word Token\n\n word token + pre-trained word embeddings fixed and only fine-tune the N most frequent\n\n example.\n i do -> ['i', 'do'] -> [1, 2] -> Embedding Matrix -> tensor\n finetuning only 'do'\n\n consisting of\n - tokenizer: WordTokenizer\n - indexer: WordIndexer\n - embedding: FrequentTuningWordEmbedding\n - vocab: Vocab\n """\n\n def __init__(self, tokenizers, indexer_config, embedding_config, vocab_config):\n super(FrequentWordTokenMaker, self).__init__(\n TokenMaker.FREQUENT_WORD_TYPE,\n tokenizer=tokenizers["word"],\n indexer=indexer.WordIndexer(tokenizers["word"], **indexer_config),\n embedding_fn=basic_embedding_fn(\n embedding_config, embedding.FrequentTuningWordEmbedding\n ),\n vocab_config=vocab_config,\n )\n\n\n[docs]@register(f"token:{TokenMaker.LINGUISTIC_TYPE}")\nclass LinguisticTokenMaker(TokenMaker):\n """\n Exact Match Token (Sparse Feature)\n\n Three simple binary features, indicating whether p_i can be exactly matched\n to one question word in q, either in its original, lowercase or lemma form.\n\n example.\n c: i do, q: i -> ['i', 'do'] -> [1, 0] -> tensor\n\n consisting of\n - tokenizer: WordTokenizer\n - indexer: WordIndexer\n - embedding: SparseFeature\n - vocab: Vocab\n """\n\n def __init__(self, tokenizers, indexer_config, embedding_config, vocab_config):\n super(LinguisticTokenMaker, self).__init__(\n TokenMaker.LINGUISTIC_TYPE,\n tokenizer=tokenizers["word"],\n indexer=indexer.LinguisticIndexer(tokenizers["word"], **indexer_config),\n embedding_fn=self._embedding_fn(embedding_config, indexer_config),\n vocab_config=vocab_config,\n )\n\n def _embedding_fn(self, embedding_config, indexer_config):\n def wrapper(vocab):\n embed_type = embedding_config.get("type", "sparse")\n if "type" in embedding_config:\n del embedding_config["type"]\n\n feature_count = 0\n embedding_config["classes"] = []\n\n if indexer_config.get("pos_tag", False):\n feature_count += 1\n embedding_config["classes"].append(POSTag.classes)\n if indexer_config.get("ner", False):\n feature_count += 1\n embedding_config["classes"].append(NER.classes)\n return embedding.SparseFeature(\n vocab, embed_type, feature_count, params=embedding_config\n )\n\n return wrapper\n\n\nimport logging\nimport os\nimport sys\n\nfrom claf.learn.mode import Mode\n\n\n""" Interface """\n\n\n[docs]def get_user_input(category):\n print(f"{category.capitalize()} > ", end="")\n sys.stdout.flush()\n\n user_input = sys.stdin.readline()\n try:\n return eval(user_input)\n except BaseException:\n return str(user_input)\n\n\n[docs]def flatten(l):\n for item in l:\n if isinstance(item, list):\n for in_item in flatten(item):\n yield in_item\n else:\n yield item\n\n\n""" Logging """\n\n\n[docs]def set_logging_config(mode, config):\n stdout_handler = logging.StreamHandler(sys.stdout)\n\n logging_handlers = [stdout_handler]\n logging_level = logging.INFO\n\n if mode == Mode.TRAIN:\n log_path = os.path.join(\n config.trainer.log_dir, f"{config.data_reader.dataset}_{config.model.name}.log"\n )\n os.makedirs(os.path.dirname(log_path), exist_ok=True)\n\n file_handler = logging.FileHandler(log_path)\n logging_handlers.append(file_handler)\n elif mode == Mode.PREDICT:\n logging_level = logging.WARNING\n\n logging.basicConfig(\n format="%(asctime)s (%(filename)s:%(lineno)d): [%(levelname)s] - %(message)s",\n handlers=logging_handlers,\n level=logging_level,\n )\n\n# Copyright 2001-2016 by Vinay Sajip. All Rights Reserved.\n#\n# Permission to use, copy, modify, and distribute this software and its\n# documentation for any purpose and without fee is hereby granted,\n# provided that the above copyright notice appear in all copies and that\n# both that copyright notice and this permission notice appear in\n# supporting documentation, and that the name of Vinay Sajip\n# not be used in advertising or publicity pertaining to distribution\n# of the software without specific, written prior permission.\n# VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING\n# ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL\n# VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR\n# ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER\n# IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT\n# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\n\n"""\nLogging package for Python. Based on PEP 282 and comments thereto in\ncomp.lang.python.\n\nCopyright (C) 2001-2016 Vinay Sajip. All Rights Reserved.\n\nTo use, simply 'import logging' and log away!\n"""\n\nimport sys, os, time, io, traceback, warnings, weakref, collections\n\nfrom string import Template\n\n__all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR',\n 'FATAL', 'FileHandler', 'Filter', 'Formatter', 'Handler', 'INFO',\n 'LogRecord', 'Logger', 'LoggerAdapter', 'NOTSET', 'NullHandler',\n 'StreamHandler', 'WARN', 'WARNING', 'addLevelName', 'basicConfig',\n 'captureWarnings', 'critical', 'debug', 'disable', 'error',\n 'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass',\n 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown',\n 'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory',\n 'lastResort', 'raiseExceptions']\n\ntry:\n import threading\nexcept ImportError: #pragma: no cover\n threading = None\n\n__author__ = "Vinay Sajip <vinay_sajip@red-dove.com>"\n__status__ = "production"\n# The following module attributes are no longer updated.\n__version__ = "0.5.1.2"\n__date__ = "07 February 2010"\n\n#---------------------------------------------------------------------------\n# Miscellaneous module data\n#---------------------------------------------------------------------------\n\n#\n#_startTime is used as the base when calculating the relative time of events\n#\n_startTime = time.time()\n\n#\n#raiseExceptions is used to see if exceptions during handling should be\n#propagated\n#\nraiseExceptions = True\n\n#\n# If you don't want threading information in the log, set this to zero\n#\nlogThreads = True\n\n#\n# If you don't want multiprocessing information in the log, set this to zero\n#\nlogMultiprocessing = True\n\n#\n# If you don't want process information in the log, set this to zero\n#\nlogProcesses = True\n\n#---------------------------------------------------------------------------\n# Level related stuff\n#---------------------------------------------------------------------------\n#\n# Default levels and level names, these can be replaced with any positive set\n# of values having corresponding names. There is a pseudo-level, NOTSET, which\n# is only really there as a lower limit for user-defined levels. Handlers and\n# loggers are initialized with NOTSET so that they will log all messages, even\n# at user-defined levels.\n#\n\nCRITICAL = 50\nFATAL = CRITICAL\nERROR = 40\nWARNING = 30\nWARN = WARNING\nINFO = 20\nDEBUG = 10\nNOTSET = 0\n\n_levelToName = {\n CRITICAL: 'CRITICAL',\n ERROR: 'ERROR',\n WARNING: 'WARNING',\n INFO: 'INFO',\n DEBUG: 'DEBUG',\n NOTSET: 'NOTSET',\n}\n_nameToLevel = {\n 'CRITICAL': CRITICAL,\n 'FATAL': FATAL,\n 'ERROR': ERROR,\n 'WARN': WARNING,\n 'WARNING': WARNING,\n 'INFO': INFO,\n 'DEBUG': DEBUG,\n 'NOTSET': NOTSET,\n}\n\ndef getLevelName(level):\n """\n Return the textual representation of logging level 'level'.\n\n If the level is one of the predefined levels (CRITICAL, ERROR, WARNING,\n INFO, DEBUG) then you get the corresponding string. If you have\n associated levels with names using addLevelName then the name you have\n associated with 'level' is returned.\n\n If a numeric value corresponding to one of the defined levels is passed\n in, the corresponding string representation is returned.\n\n Otherwise, the string "Level %s" % level is returned.\n """\n # See Issues #22386, #27937 and #29220 for why it's this way\n result = _levelToName.get(level)\n if result is not None:\n return result\n result = _nameToLevel.get(level)\n if result is not None:\n return result\n return "Level %s" % level\n\ndef addLevelName(level, levelName):\n """\n Associate 'levelName' with 'level'.\n\n This is used when converting levels to text during message formatting.\n """\n _acquireLock()\n try: #unlikely to cause an exception, but you never know...\n _levelToName[level] = levelName\n _nameToLevel[levelName] = level\n finally:\n _releaseLock()\n\nif hasattr(sys, '_getframe'):\n currentframe = lambda: sys._getframe(3)\nelse: #pragma: no cover\n def currentframe():\n """Return the frame object for the caller's stack frame."""\n try:\n raise Exception\n except Exception:\n return sys.exc_info()[2].tb_frame.f_back\n\n#\n# _srcfile is used when walking the stack to check when we've got the first\n# caller stack frame, by skipping frames whose filename is that of this\n# module's source. It therefore should contain the filename of this module's\n# source file.\n#\n# Ordinarily we would use __file__ for this, but frozen modules don't always\n# have __file__ set, for some reason (see Issue #21736). Thus, we get the\n# filename from a handy code object from a function defined in this module.\n# (There's no particular reason for picking addLevelName.)\n#\n\n_srcfile = os.path.normcase(addLevelName.__code__.co_filename)\n\n# _srcfile is only used in conjunction with sys._getframe().\n# To provide compatibility with older versions of Python, set _srcfile\n# to None if _getframe() is not available; this value will prevent\n# findCaller() from being called. You can also do this if you want to avoid\n# the overhead of fetching caller information, even when _getframe() is\n# available.\n#if not hasattr(sys, '_getframe'):\n# _srcfile = None\n\n\ndef _checkLevel(level):\n if isinstance(level, int):\n rv = level\n elif str(level) == level:\n if level not in _nameToLevel:\n raise ValueError("Unknown level: %r" % level)\n rv = _nameToLevel[level]\n else:\n raise TypeError("Level not an integer or a valid string: %r" % level)\n return rv\n\n#---------------------------------------------------------------------------\n# Thread-related stuff\n#---------------------------------------------------------------------------\n\n#\n#_lock is used to serialize access to shared data structures in this module.\n#This needs to be an RLock because fileConfig() creates and configures\n#Handlers, and so might arbitrary user threads. Since Handler code updates the\n#shared dictionary _handlers, it needs to acquire the lock. But if configuring,\n#the lock would already have been acquired - so we need an RLock.\n#The same argument applies to Loggers and Manager.loggerDict.\n#\nif threading:\n _lock = threading.RLock()\nelse: #pragma: no cover\n _lock = None\n\n\ndef _acquireLock():\n """\n Acquire the module-level lock for serializing access to shared data.\n\n This should be released with _releaseLock().\n """\n if _lock:\n _lock.acquire()\n\ndef _releaseLock():\n """\n Release the module-level lock acquired by calling _acquireLock().\n """\n if _lock:\n _lock.release()\n\n#---------------------------------------------------------------------------\n# The logging record\n#---------------------------------------------------------------------------\n\nclass LogRecord(object):\n """\n A LogRecord instance represents an event being logged.\n\n LogRecord instances are created every time something is logged. They\n contain all the information pertinent to the event being logged. The\n main information passed in is in msg and args, which are combined\n using str(msg) % args to create the message field of the record. The\n record also includes information such as when the record was created,\n the source line where the logging call was made, and any exception\n information to be logged.\n """\n def __init__(self, name, level, pathname, lineno,\n msg, args, exc_info, func=None, sinfo=None, **kwargs):\n """\n Initialize a logging record with interesting information.\n """\n ct = time.time()\n self.name = name\n self.msg = msg\n #\n # The following statement allows passing of a dictionary as a sole\n # argument, so that you can do something like\n # logging.debug("a %(a)d b %(b)s", {'a':1, 'b':2})\n # Suggested by Stefan Behnel.\n # Note that without the test for args[0], we get a problem because\n # during formatting, we test to see if the arg is present using\n # 'if self.args:'. If the event being logged is e.g. 'Value is %d'\n # and if the passed arg fails 'if self.args:' then no formatting\n # is done. For example, logger.warning('Value is %d', 0) would log\n # 'Value is %d' instead of 'Value is 0'.\n # For the use case of passing a dictionary, this should not be a\n # problem.\n # Issue #21172: a request was made to relax the isinstance check\n # to hasattr(args[0], '__getitem__'). However, the docs on string\n # formatting still seem to suggest a mapping object is required.\n # Thus, while not removing the isinstance check, it does now look\n # for collections.Mapping rather than, as before, dict.\n if (args and len(args) == 1 and isinstance(args[0], collections.Mapping)\n and args[0]):\n args = args[0]\n self.args = args\n self.levelname = getLevelName(level)\n self.levelno = level\n self.pathname = pathname\n try:\n self.filename = os.path.basename(pathname)\n self.module = os.path.splitext(self.filename)[0]\n except (TypeError, ValueError, AttributeError):\n self.filename = pathname\n self.module = "Unknown module"\n self.exc_info = exc_info\n self.exc_text = None # used to cache the traceback text\n self.stack_info = sinfo\n self.lineno = lineno\n self.funcName = func\n self.created = ct\n self.msecs = (ct - int(ct)) * 1000\n self.relativeCreated = (self.created - _startTime) * 1000\n if logThreads and threading:\n self.thread = threading.get_ident()\n self.threadName = threading.current_thread().name\n else: # pragma: no cover\n self.thread = None\n self.threadName = None\n if not logMultiprocessing: # pragma: no cover\n self.processName = None\n else:\n self.processName = 'MainProcess'\n mp = sys.modules.get('multiprocessing')\n if mp is not None:\n # Errors may occur if multiprocessing has not finished loading\n # yet - e.g. if a custom import hook causes third-party code\n # to run when multiprocessing calls import. See issue 8200\n # for an example\n try:\n self.processName = mp.current_process().name\n except Exception: #pragma: no cover\n pass\n if logProcesses and hasattr(os, 'getpid'):\n self.process = os.getpid()\n else:\n self.process = None\n\n def __str__(self):\n return '<LogRecord: %s, %s, %s, %s, "%s">'%(self.name, self.levelno,\n self.pathname, self.lineno, self.msg)\n\n __repr__ = __str__\n\n def getMessage(self):\n """\n Return the message for this LogRecord.\n\n Return the message for this LogRecord after merging any user-supplied\n arguments with the message.\n """\n msg = str(self.msg)\n if self.args:\n msg = msg % self.args\n return msg\n\n#\n# Determine which class to use when instantiating log records.\n#\n_logRecordFactory = LogRecord\n\ndef setLogRecordFactory(factory):\n """\n Set the factory to be used when instantiating a log record.\n\n :param factory: A callable which will be called to instantiate\n a log record.\n """\n global _logRecordFactory\n _logRecordFactory = factory\n\ndef getLogRecordFactory():\n """\n Return the factory to be used when instantiating a log record.\n """\n\n return _logRecordFactory\n\ndef makeLogRecord(dict):\n """\n Make a LogRecord whose attributes are defined by the specified dictionary,\n This function is useful for converting a logging event received over\n a socket connection (which is sent as a dictionary) into a LogRecord\n instance.\n """\n rv = _logRecordFactory(None, None, "", 0, "", (), None, None)\n rv.__dict__.update(dict)\n return rv\n\n#---------------------------------------------------------------------------\n# Formatter classes and functions\n#---------------------------------------------------------------------------\n\nclass PercentStyle(object):\n\n default_format = '%(message)s'\n asctime_format = '%(asctime)s'\n asctime_search = '%(asctime)'\n\n def __init__(self, fmt):\n self._fmt = fmt or self.default_format\n\n def usesTime(self):\n return self._fmt.find(self.asctime_search) >= 0\n\n def format(self, record):\n return self._fmt % record.__dict__\n\nclass StrFormatStyle(PercentStyle):\n default_format = '{message}'\n asctime_format = '{asctime}'\n asctime_search = '{asctime'\n\n def format(self, record):\n return self._fmt.format(**record.__dict__)\n\n\nclass StringTemplateStyle(PercentStyle):\n default_format = '${message}'\n asctime_format = '${asctime}'\n asctime_search = '${asctime}'\n\n def __init__(self, fmt):\n self._fmt = fmt or self.default_format\n self._tpl = Template(self._fmt)\n\n def usesTime(self):\n fmt = self._fmt\n return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_format) >= 0\n\n def format(self, record):\n return self._tpl.substitute(**record.__dict__)\n\nBASIC_FORMAT = "%(levelname)s:%(name)s:%(message)s"\n\n_STYLES = {\n '%': (PercentStyle, BASIC_FORMAT),\n '{': (StrFormatStyle, '{levelname}:{name}:{message}'),\n '$': (StringTemplateStyle, '${levelname}:${name}:${message}'),\n}\n\nclass Formatter(object):\n """\n Formatter instances are used to convert a LogRecord to text.\n\n Formatters need to know how a LogRecord is constructed. They are\n responsible for converting a LogRecord to (usually) a string which can\n be interpreted by either a human or an external system. The base Formatter\n allows a formatting string to be specified. If none is supplied, the\n the style-dependent default value, "%(message)s", "{message}", or\n "${message}", is used.\n\n The Formatter can be initialized with a format string which makes use of\n knowledge of the LogRecord attributes - e.g. the default value mentioned\n above makes use of the fact that the user's message and arguments are pre-\n formatted into a LogRecord's message attribute. Currently, the useful\n attributes in a LogRecord are described by:\n\n %(name)s Name of the logger (logging channel)\n %(levelno)s Numeric logging level for the message (DEBUG, INFO,\n WARNING, ERROR, CRITICAL)\n %(levelname)s Text logging level for the message ("DEBUG", "INFO",\n "WARNING", "ERROR", "CRITICAL")\n %(pathname)s Full pathname of the source file where the logging\n call was issued (if available)\n %(filename)s Filename portion of pathname\n %(module)s Module (name portion of filename)\n %(lineno)d Source line number where the logging call was issued\n (if available)\n %(funcName)s Function name\n %(created)f Time when the LogRecord was created (time.time()\n return value)\n %(asctime)s Textual time when the LogRecord was created\n %(msecs)d Millisecond portion of the creation time\n %(relativeCreated)d Time in milliseconds when the LogRecord was created,\n relative to the time the logging module was loaded\n (typically at application startup time)\n %(thread)d Thread ID (if available)\n %(threadName)s Thread name (if available)\n %(process)d Process ID (if available)\n %(message)s The result of record.getMessage(), computed just as\n the record is emitted\n """\n\n converter = time.localtime\n\n def __init__(self, fmt=None, datefmt=None, style='%'):\n """\n Initialize the formatter with specified format strings.\n\n Initialize the formatter either with the specified format string, or a\n default as described above. Allow for specialized date formatting with\n the optional datefmt argument. If datefmt is omitted, you get an\n ISO8601-like (or RFC 3339-like) format.\n\n Use a style parameter of '%', '{' or '$' to specify that you want to\n use one of %-formatting, :meth:`str.format` (``{}``) formatting or\n :class:`string.Template` formatting in your format string.\n\n .. versionchanged:: 3.2\n Added the ``style`` parameter.\n """\n if style not in _STYLES:\n raise ValueError('Style must be one of: %s' % ','.join(\n _STYLES.keys()))\n self._style = _STYLES[style][0](fmt)\n self._fmt = self._style._fmt\n self.datefmt = datefmt\n\n default_time_format = '%Y-%m-%d %H:%M:%S'\n default_msec_format = '%s,%03d'\n\n def formatTime(self, record, datefmt=None):\n """\n Return the creation time of the specified LogRecord as formatted text.\n\n This method should be called from format() by a formatter which\n wants to make use of a formatted time. This method can be overridden\n in formatters to provide for any specific requirement, but the\n basic behaviour is as follows: if datefmt (a string) is specified,\n it is used with time.strftime() to format the creation time of the\n record. Otherwise, an ISO8601-like (or RFC 3339-like) format is used.\n The resulting string is returned. This function uses a user-configurable\n function to convert the creation time to a tuple. By default,\n time.localtime() is used; to change this for a particular formatter\n instance, set the 'converter' attribute to a function with the same\n signature as time.localtime() or time.gmtime(). To change it for all\n formatters, for example if you want all logging times to be shown in GMT,\n set the 'converter' attribute in the Formatter class.\n """\n ct = self.converter(record.created)\n if datefmt:\n s = time.strftime(datefmt, ct)\n else:\n t = time.strftime(self.default_time_format, ct)\n s = self.default_msec_format % (t, record.msecs)\n return s\n\n def formatException(self, ei):\n """\n Format and return the specified exception information as a string.\n\n This default implementation just uses\n traceback.print_exception()\n """\n sio = io.StringIO()\n tb = ei[2]\n # See issues #9427, #1553375. Commented out for now.\n #if getattr(self, 'fullstack', False):\n # traceback.print_stack(tb.tb_frame.f_back, file=sio)\n traceback.print_exception(ei[0], ei[1], tb, None, sio)\n s = sio.getvalue()\n sio.close()\n if s[-1:] == "\\n":\n s = s[:-1]\n return s\n\n def usesTime(self):\n """\n Check if the format uses the creation time of the record.\n """\n return self._style.usesTime()\n\n def formatMessage(self, record):\n return self._style.format(record)\n\n def formatStack(self, stack_info):\n """\n This method is provided as an extension point for specialized\n formatting of stack information.\n\n The input data is a string as returned from a call to\n :func:`traceback.print_stack`, but with the last trailing newline\n removed.\n\n The base implementation just returns the value passed in.\n """\n return stack_info\n\n def format(self, record):\n """\n Format the specified record as text.\n\n The record's attribute dictionary is used as the operand to a\n string formatting operation which yields the returned string.\n Before formatting the dictionary, a couple of preparatory steps\n are carried out. The message attribute of the record is computed\n using LogRecord.getMessage(). If the formatting string uses the\n time (as determined by a call to usesTime(), formatTime() is\n called to format the event time. If there is exception information,\n it is formatted using formatException() and appended to the message.\n """\n record.message = record.getMessage()\n if self.usesTime():\n record.asctime = self.formatTime(record, self.datefmt)\n s = self.formatMessage(record)\n if record.exc_info:\n # Cache the traceback text to avoid converting it multiple times\n # (it's constant anyway)\n if not record.exc_text:\n record.exc_text = self.formatException(record.exc_info)\n if record.exc_text:\n if s[-1:] != "\\n":\n s = s + "\\n"\n s = s + record.exc_text\n if record.stack_info:\n if s[-1:] != "\\n":\n s = s + "\\n"\n s = s + self.formatStack(record.stack_info)\n return s\n\n#\n# The default formatter to use when no other is specified\n#\n_defaultFormatter = Formatter()\n\nclass BufferingFormatter(object):\n """\n A formatter suitable for formatting a number of records.\n """\n def __init__(self, linefmt=None):\n """\n Optionally specify a formatter which will be used to format each\n individual record.\n """\n if linefmt:\n self.linefmt = linefmt\n else:\n self.linefmt = _defaultFormatter\n\n def formatHeader(self, records):\n """\n Return the header string for the specified records.\n """\n return ""\n\n def formatFooter(self, records):\n """\n Return the footer string for the specified records.\n """\n return ""\n\n def format(self, records):\n """\n Format the specified records and return the result as a string.\n """\n rv = ""\n if len(records) > 0:\n rv = rv + self.formatHeader(records)\n for record in records:\n rv = rv + self.linefmt.format(record)\n rv = rv + self.formatFooter(records)\n return rv\n\n#---------------------------------------------------------------------------\n# Filter classes and functions\n#---------------------------------------------------------------------------\n\nclass Filter(object):\n """\n Filter instances are used to perform arbitrary filtering of LogRecords.\n\n Loggers and Handlers can optionally use Filter instances to filter\n records as desired. The base filter class only allows events which are\n below a certain point in the logger hierarchy. For example, a filter\n initialized with "A.B" will allow events logged by loggers "A.B",\n "A.B.C", "A.B.C.D", "A.B.D" etc. but not "A.BB", "B.A.B" etc. If\n initialized with the empty string, all events are passed.\n """\n def __init__(self, name=''):\n """\n Initialize a filter.\n\n Initialize with the name of the logger which, together with its\n children, will have its events allowed through the filter. If no\n name is specified, allow every event.\n """\n self.name = name\n self.nlen = len(name)\n\n def filter(self, record):\n """\n Determine if the specified record is to be logged.\n\n Is the specified record to be logged? Returns 0 for no, nonzero for\n yes. If deemed appropriate, the record may be modified in-place.\n """\n if self.nlen == 0:\n return True\n elif self.name == record.name:\n return True\n elif record.name.find(self.name, 0, self.nlen) != 0:\n return False\n return (record.name[self.nlen] == ".")\n\nclass Filterer(object):\n """\n A base class for loggers and handlers which allows them to share\n common code.\n """\n def __init__(self):\n """\n Initialize the list of filters to be an empty list.\n """\n self.filters = []\n\n def addFilter(self, filter):\n """\n Add the specified filter to this handler.\n """\n if not (filter in self.filters):\n self.filters.append(filter)\n\n def removeFilter(self, filter):\n """\n Remove the specified filter from this handler.\n """\n if filter in self.filters:\n self.filters.remove(filter)\n\n def filter(self, record):\n """\n Determine if a record is loggable by consulting all the filters.\n\n The default is to allow the record to be logged; any filter can veto\n this and the record is then dropped. Returns a zero value if a record\n is to be dropped, else non-zero.\n\n .. versionchanged:: 3.2\n\n Allow filters to be just callables.\n """\n rv = True\n for f in self.filters:\n if hasattr(f, 'filter'):\n result = f.filter(record)\n else:\n result = f(record) # assume callable - will raise if not\n if not result:\n rv = False\n break\n return rv\n\n#---------------------------------------------------------------------------\n# Handler classes and functions\n#---------------------------------------------------------------------------\n\n_handlers = weakref.WeakValueDictionary() #map of handler names to handlers\n_handlerList = [] # added to allow handlers to be removed in reverse of order initialized\n\ndef _removeHandlerRef(wr):\n """\n Remove a handler reference from the internal cleanup list.\n """\n # This function can be called during module teardown, when globals are\n # set to None. It can also be called from another thread. So we need to\n # pre-emptively grab the necessary globals and check if they're None,\n # to prevent race conditions and failures during interpreter shutdown.\n acquire, release, handlers = _acquireLock, _releaseLock, _handlerList\n if acquire and release and handlers:\n acquire()\n try:\n if wr in handlers:\n handlers.remove(wr)\n finally:\n release()\n\ndef _addHandlerRef(handler):\n """\n Add a handler to the internal cleanup list using a weak reference.\n """\n _acquireLock()\n try:\n _handlerList.append(weakref.ref(handler, _removeHandlerRef))\n finally:\n _releaseLock()\n\nclass Handler(Filterer):\n """\n Handler instances dispatch logging events to specific destinations.\n\n The base handler class. Acts as a placeholder which defines the Handler\n interface. Handlers can optionally use Formatter instances to format\n records as desired. By default, no formatter is specified; in this case,\n the 'raw' message as determined by record.message is logged.\n """\n def __init__(self, level=NOTSET):\n """\n Initializes the instance - basically setting the formatter to None\n and the filter list to empty.\n """\n Filterer.__init__(self)\n self._name = None\n self.level = _checkLevel(level)\n self.formatter = None\n # Add the handler to the global _handlerList (for cleanup on shutdown)\n _addHandlerRef(self)\n self.createLock()\n\n def get_name(self):\n return self._name\n\n def set_name(self, name):\n _acquireLock()\n try:\n if self._name in _handlers:\n del _handlers[self._name]\n self._name = name\n if name:\n _handlers[name] = self\n finally:\n _releaseLock()\n\n name = property(get_name, set_name)\n\n def createLock(self):\n """\n Acquire a thread lock for serializing access to the underlying I/O.\n """\n if threading:\n self.lock = threading.RLock()\n else: #pragma: no cover\n self.lock = None\n\n def acquire(self):\n """\n Acquire the I/O thread lock.\n """\n if self.lock:\n self.lock.acquire()\n\n def release(self):\n """\n Release the I/O thread lock.\n """\n if self.lock:\n self.lock.release()\n\n def setLevel(self, level):\n """\n Set the logging level of this handler. level must be an int or a str.\n """\n self.level = _checkLevel(level)\n\n def format(self, record):\n """\n Format the specified record.\n\n If a formatter is set, use it. Otherwise, use the default formatter\n for the module.\n """\n if self.formatter:\n fmt = self.formatter\n else:\n fmt = _defaultFormatter\n return fmt.format(record)\n\n def emit(self, record):\n """\n Do whatever it takes to actually log the specified logging record.\n\n This version is intended to be implemented by subclasses and so\n raises a NotImplementedError.\n """\n raise NotImplementedError('emit must be implemented '\n 'by Handler subclasses')\n\n def handle(self, record):\n """\n Conditionally emit the specified logging record.\n\n Emission depends on filters which may have been added to the handler.\n Wrap the actual emission of the record with acquisition/release of\n the I/O thread lock. Returns whether the filter passed the record for\n emission.\n """\n rv = self.filter(record)\n if rv:\n self.acquire()\n try:\n self.emit(record)\n finally:\n self.release()\n return rv\n\n def setFormatter(self, fmt):\n """\n Set the formatter for this handler.\n """\n self.formatter = fmt\n\n def flush(self):\n """\n Ensure all logging output has been flushed.\n\n This version does nothing and is intended to be implemented by\n subclasses.\n """\n pass\n\n def close(self):\n """\n Tidy up any resources used by the handler.\n\n This version removes the handler from an internal map of handlers,\n _handlers, which is used for handler lookup by name. Subclasses\n should ensure that this gets called from overridden close()\n methods.\n """\n #get the module data lock, as we're updating a shared structure.\n _acquireLock()\n try: #unlikely to raise an exception, but you never know...\n if self._name and self._name in _handlers:\n del _handlers[self._name]\n finally:\n _releaseLock()\n\n def handleError(self, record):\n """\n Handle errors which occur during an emit() call.\n\n This method should be called from handlers when an exception is\n encountered during an emit() call. If raiseExceptions is false,\n exceptions get silently ignored. This is what is mostly wanted\n for a logging system - most users will not care about errors in\n the logging system, they are more interested in application errors.\n You could, however, replace this with a custom handler if you wish.\n The record which was being processed is passed in to this method.\n """\n if raiseExceptions and sys.stderr: # see issue 13807\n t, v, tb = sys.exc_info()\n try:\n sys.stderr.write('--- Logging error ---\\n')\n traceback.print_exception(t, v, tb, None, sys.stderr)\n sys.stderr.write('Call stack:\\n')\n # Walk the stack frame up until we're out of logging,\n # so as to print the calling context.\n frame = tb.tb_frame\n while (frame and os.path.dirname(frame.f_code.co_filename) ==\n __path__[0]):\n frame = frame.f_back\n if frame:\n traceback.print_stack(frame, file=sys.stderr)\n else:\n # couldn't find the right stack frame, for some reason\n sys.stderr.write('Logged from file %s, line %s\\n' % (\n record.filename, record.lineno))\n # Issue 18671: output logging message and arguments\n try:\n sys.stderr.write('Message: %r\\n'\n 'Arguments: %s\\n' % (record.msg,\n record.args))\n except Exception:\n sys.stderr.write('Unable to print the message and arguments'\n ' - possible formatting error.\\nUse the'\n ' traceback above to help find the error.\\n'\n )\n except OSError: #pragma: no cover\n pass # see issue 5971\n finally:\n del t, v, tb\n\n def __repr__(self):\n level = getLevelName(self.level)\n return '<%s (%s)>' % (self.__class__.__name__, level)\n\nclass StreamHandler(Handler):\n """\n A handler class which writes logging records, appropriately formatted,\n to a stream. Note that this class does not close the stream, as\n sys.stdout or sys.stderr may be used.\n """\n\n terminator = '\\n'\n\n def __init__(self, stream=None):\n """\n Initialize the handler.\n\n If stream is not specified, sys.stderr is used.\n """\n Handler.__init__(self)\n if stream is None:\n stream = sys.stderr\n self.stream = stream\n\n def flush(self):\n """\n Flushes the stream.\n """\n self.acquire()\n try:\n if self.stream and hasattr(self.stream, "flush"):\n self.stream.flush()\n finally:\n self.release()\n\n def emit(self, record):\n """\n Emit a record.\n\n If a formatter is specified, it is used to format the record.\n The record is then written to the stream with a trailing newline. If\n exception information is present, it is formatted using\n traceback.print_exception and appended to the stream. If the stream\n has an 'encoding' attribute, it is used to determine how to do the\n output to the stream.\n """\n try:\n msg = self.format(record)\n stream = self.stream\n stream.write(msg)\n stream.write(self.terminator)\n self.flush()\n except Exception:\n self.handleError(record)\n\n def __repr__(self):\n level = getLevelName(self.level)\n name = getattr(self.stream, 'name', '')\n if name:\n name += ' '\n return '<%s %s(%s)>' % (self.__class__.__name__, name, level)\n\n\nclass FileHandler(StreamHandler):\n """\n A handler class which writes formatted logging records to disk files.\n """\n def __init__(self, filename, mode='a', encoding=None, delay=False):\n """\n Open the specified file and use it as the stream for logging.\n """\n # Issue #27493: add support for Path objects to be passed in\n filename = os.fspath(filename)\n #keep the absolute path, otherwise derived classes which use this\n #may come a cropper when the current directory changes\n self.baseFilename = os.path.abspath(filename)\n self.mode = mode\n self.encoding = encoding\n self.delay = delay\n if delay:\n #We don't open the stream, but we still need to call the\n #Handler constructor to set level, formatter, lock etc.\n Handler.__init__(self)\n self.stream = None\n else:\n StreamHandler.__init__(self, self._open())\n\n def close(self):\n """\n Closes the stream.\n """\n self.acquire()\n try:\n try:\n if self.stream:\n try:\n self.flush()\n finally:\n stream = self.stream\n self.stream = None\n if hasattr(stream, "close"):\n stream.close()\n finally:\n # Issue #19523: call unconditionally to\n # prevent a handler leak when delay is set\n StreamHandler.close(self)\n finally:\n self.release()\n\n def _open(self):\n """\n Open the current base file with the (original) mode and encoding.\n Return the resulting stream.\n """\n return open(self.baseFilename, self.mode, encoding=self.encoding)\n\n def emit(self, record):\n """\n Emit a record.\n\n If the stream was not opened because 'delay' was specified in the\n constructor, open it before calling the superclass's emit.\n """\n if self.stream is None:\n self.stream = self._open()\n StreamHandler.emit(self, record)\n\n def __repr__(self):\n level = getLevelName(self.level)\n return '<%s %s (%s)>' % (self.__class__.__name__, self.baseFilename, level)\n\n\nclass _StderrHandler(StreamHandler):\n """\n This class is like a StreamHandler using sys.stderr, but always uses\n whatever sys.stderr is currently set to rather than the value of\n sys.stderr at handler construction time.\n """\n def __init__(self, level=NOTSET):\n """\n Initialize the handler.\n """\n Handler.__init__(self, level)\n\n @property\n def stream(self):\n return sys.stderr\n\n\n_defaultLastResort = _StderrHandler(WARNING)\nlastResort = _defaultLastResort\n\n#---------------------------------------------------------------------------\n# Manager classes and functions\n#---------------------------------------------------------------------------\n\nclass PlaceHolder(object):\n """\n PlaceHolder instances are used in the Manager logger hierarchy to take\n the place of nodes for which no loggers have been defined. This class is\n intended for internal use only and not as part of the public API.\n """\n def __init__(self, alogger):\n """\n Initialize with the specified logger being a child of this placeholder.\n """\n self.loggerMap = { alogger : None }\n\n def append(self, alogger):\n """\n Add the specified logger as a child of this placeholder.\n """\n if alogger not in self.loggerMap:\n self.loggerMap[alogger] = None\n\n#\n# Determine which class to use when instantiating loggers.\n#\n\ndef setLoggerClass(klass):\n """\n Set the class to be used when instantiating a logger. The class should\n define __init__() such that only a name argument is required, and the\n __init__() should call Logger.__init__()\n """\n if klass != Logger:\n if not issubclass(klass, Logger):\n raise TypeError("logger not derived from logging.Logger: "\n + klass.__name__)\n global _loggerClass\n _loggerClass = klass\n\ndef getLoggerClass():\n """\n Return the class to be used when instantiating a logger.\n """\n return _loggerClass\n\nclass Manager(object):\n """\n There is [under normal circumstances] just one Manager instance, which\n holds the hierarchy of loggers.\n """\n def __init__(self, rootnode):\n """\n Initialize the manager with the root node of the logger hierarchy.\n """\n self.root = rootnode\n self.disable = 0\n self.emittedNoHandlerWarning = False\n self.loggerDict = {}\n self.loggerClass = None\n self.logRecordFactory = None\n\n def getLogger(self, name):\n """\n Get a logger with the specified name (channel name), creating it\n if it doesn't yet exist. This name is a dot-separated hierarchical\n name, such as "a", "a.b", "a.b.c" or similar.\n\n If a PlaceHolder existed for the specified name [i.e. the logger\n didn't exist but a child of it did], replace it with the created\n logger and fix up the parent/child references which pointed to the\n placeholder to now point to the logger.\n """\n rv = None\n if not isinstance(name, str):\n raise TypeError('A logger name must be a string')\n _acquireLock()\n try:\n if name in self.loggerDict:\n rv = self.loggerDict[name]\n if isinstance(rv, PlaceHolder):\n ph = rv\n rv = (self.loggerClass or _loggerClass)(name)\n rv.manager = self\n self.loggerDict[name] = rv\n self._fixupChildren(ph, rv)\n self._fixupParents(rv)\n else:\n rv = (self.loggerClass or _loggerClass)(name)\n rv.manager = self\n self.loggerDict[name] = rv\n self._fixupParents(rv)\n finally:\n _releaseLock()\n return rv\n\n def setLoggerClass(self, klass):\n """\n Set the class to be used when instantiating a logger with this Manager.\n """\n if klass != Logger:\n if not issubclass(klass, Logger):\n raise TypeError("logger not derived from logging.Logger: "\n + klass.__name__)\n self.loggerClass = klass\n\n def setLogRecordFactory(self, factory):\n """\n Set the factory to be used when instantiating a log record with this\n Manager.\n """\n self.logRecordFactory = factory\n\n def _fixupParents(self, alogger):\n """\n Ensure that there are either loggers or placeholders all the way\n from the specified logger to the root of the logger hierarchy.\n """\n name = alogger.name\n i = name.rfind(".")\n rv = None\n while (i > 0) and not rv:\n substr = name[:i]\n if substr not in self.loggerDict:\n self.loggerDict[substr] = PlaceHolder(alogger)\n else:\n obj = self.loggerDict[substr]\n if isinstance(obj, Logger):\n rv = obj\n else:\n assert isinstance(obj, PlaceHolder)\n obj.append(alogger)\n i = name.rfind(".", 0, i - 1)\n if not rv:\n rv = self.root\n alogger.parent = rv\n\n def _fixupChildren(self, ph, alogger):\n """\n Ensure that children of the placeholder ph are connected to the\n specified logger.\n """\n name = alogger.name\n namelen = len(name)\n for c in ph.loggerMap.keys():\n #The if means ... if not c.parent.name.startswith(nm)\n if c.parent.name[:namelen] != name:\n alogger.parent = c.parent\n c.parent = alogger\n\n#---------------------------------------------------------------------------\n# Logger classes and functions\n#---------------------------------------------------------------------------\n\nclass Logger(Filterer):\n """\n Instances of the Logger class represent a single logging channel. A\n "logging channel" indicates an area of an application. Exactly how an\n "area" is defined is up to the application developer. Since an\n application can have any number of areas, logging channels are identified\n by a unique string. Application areas can be nested (e.g. an area\n of "input processing" might include sub-areas "read CSV files", "read\n XLS files" and "read Gnumeric files"). To cater for this natural nesting,\n channel names are organized into a namespace hierarchy where levels are\n separated by periods, much like the Java or Python package namespace. So\n in the instance given above, channel names might be "input" for the upper\n level, and "input.csv", "input.xls" and "input.gnu" for the sub-levels.\n There is no arbitrary limit to the depth of nesting.\n """\n def __init__(self, name, level=NOTSET):\n """\n Initialize the logger with a name and an optional level.\n """\n Filterer.__init__(self)\n self.name = name\n self.level = _checkLevel(level)\n self.parent = None\n self.propagate = True\n self.handlers = []\n self.disabled = False\n\n def setLevel(self, level):\n """\n Set the logging level of this logger. level must be an int or a str.\n """\n self.level = _checkLevel(level)\n\n def debug(self, msg, *args, **kwargs):\n """\n Log 'msg % args' with severity 'DEBUG'.\n\n To pass exception information, use the keyword argument exc_info with\n a true value, e.g.\n\n logger.debug("Houston, we have a %s", "thorny problem", exc_info=1)\n """\n if self.isEnabledFor(DEBUG):\n self._log(DEBUG, msg, args, **kwargs)\n\n def info(self, msg, *args, **kwargs):\n """\n Log 'msg % args' with severity 'INFO'.\n\n To pass exception information, use the keyword argument exc_info with\n a true value, e.g.\n\n logger.info("Houston, we have a %s", "interesting problem", exc_info=1)\n """\n if self.isEnabledFor(INFO):\n self._log(INFO, msg, args, **kwargs)\n\n def warning(self, msg, *args, **kwargs):\n """\n Log 'msg % args' with severity 'WARNING'.\n\n To pass exception information, use the keyword argument exc_info with\n a true value, e.g.\n\n logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1)\n """\n if self.isEnabledFor(WARNING):\n self._log(WARNING, msg, args, **kwargs)\n\n def warn(self, msg, *args, **kwargs):\n warnings.warn("The 'warn' method is deprecated, "\n "use 'warning' instead", DeprecationWarning, 2)\n self.warning(msg, *args, **kwargs)\n\n def error(self, msg, *args, **kwargs):\n """\n Log 'msg % args' with severity 'ERROR'.\n\n To pass exception information, use the keyword argument exc_info with\n a true value, e.g.\n\n logger.error("Houston, we have a %s", "major problem", exc_info=1)\n """\n if self.isEnabledFor(ERROR):\n self._log(ERROR, msg, args, **kwargs)\n\n def exception(self, msg, *args, exc_info=True, **kwargs):\n """\n Convenience method for logging an ERROR with exception information.\n """\n self.error(msg, *args, exc_info=exc_info, **kwargs)\n\n def critical(self, msg, *args, **kwargs):\n """\n Log 'msg % args' with severity 'CRITICAL'.\n\n To pass exception information, use the keyword argument exc_info with\n a true value, e.g.\n\n logger.critical("Houston, we have a %s", "major disaster", exc_info=1)\n """\n if self.isEnabledFor(CRITICAL):\n self._log(CRITICAL, msg, args, **kwargs)\n\n fatal = critical\n\n def log(self, level, msg, *args, **kwargs):\n """\n Log 'msg % args' with the integer severity 'level'.\n\n To pass exception information, use the keyword argument exc_info with\n a true value, e.g.\n\n logger.log(level, "We have a %s", "mysterious problem", exc_info=1)\n """\n if not isinstance(level, int):\n if raiseExceptions:\n raise TypeError("level must be an integer")\n else:\n return\n if self.isEnabledFor(level):\n self._log(level, msg, args, **kwargs)\n\n def findCaller(self, stack_info=False):\n """\n Find the stack frame of the caller so that we can note the source\n file name, line number and function name.\n """\n f = currentframe()\n #On some versions of IronPython, currentframe() returns None if\n #IronPython isn't run with -X:Frames.\n if f is not None:\n f = f.f_back\n rv = "(unknown file)", 0, "(unknown function)", None\n while hasattr(f, "f_code"):\n co = f.f_code\n filename = os.path.normcase(co.co_filename)\n if filename == _srcfile:\n f = f.f_back\n continue\n sinfo = None\n if stack_info:\n sio = io.StringIO()\n sio.write('Stack (most recent call last):\\n')\n traceback.print_stack(f, file=sio)\n sinfo = sio.getvalue()\n if sinfo[-1] == '\\n':\n sinfo = sinfo[:-1]\n sio.close()\n rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)\n break\n return rv\n\n def makeRecord(self, name, level, fn, lno, msg, args, exc_info,\n func=None, extra=None, sinfo=None):\n """\n A factory method which can be overridden in subclasses to create\n specialized LogRecords.\n """\n rv = _logRecordFactory(name, level, fn, lno, msg, args, exc_info, func,\n sinfo)\n if extra is not None:\n for key in extra:\n if (key in ["message", "asctime"]) or (key in rv.__dict__):\n raise KeyError("Attempt to overwrite %r in LogRecord" % key)\n rv.__dict__[key] = extra[key]\n return rv\n\n def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False):\n """\n Low-level logging routine which creates a LogRecord and then calls\n all the handlers of this logger to handle the record.\n """\n sinfo = None\n if _srcfile:\n #IronPython doesn't track Python frames, so findCaller raises an\n #exception on some versions of IronPython. We trap it here so that\n #IronPython can use logging.\n try:\n fn, lno, func, sinfo = self.findCaller(stack_info)\n except ValueError: # pragma: no cover\n fn, lno, func = "(unknown file)", 0, "(unknown function)"\n else: # pragma: no cover\n fn, lno, func = "(unknown file)", 0, "(unknown function)"\n if exc_info:\n if isinstance(exc_info, BaseException):\n exc_info = (type(exc_info), exc_info, exc_info.__traceback__)\n elif not isinstance(exc_info, tuple):\n exc_info = sys.exc_info()\n record = self.makeRecord(self.name, level, fn, lno, msg, args,\n exc_info, func, extra, sinfo)\n self.handle(record)\n\n def handle(self, record):\n """\n Call the handlers for the specified record.\n\n This method is used for unpickled records received from a socket, as\n well as those created locally. Logger-level filtering is applied.\n """\n if (not self.disabled) and self.filter(record):\n self.callHandlers(record)\n\n def addHandler(self, hdlr):\n """\n Add the specified handler to this logger.\n """\n _acquireLock()\n try:\n if not (hdlr in self.handlers):\n self.handlers.append(hdlr)\n finally:\n _releaseLock()\n\n def removeHandler(self, hdlr):\n """\n Remove the specified handler from this logger.\n """\n _acquireLock()\n try:\n if hdlr in self.handlers:\n self.handlers.remove(hdlr)\n finally:\n _releaseLock()\n\n def hasHandlers(self):\n """\n See if this logger has any handlers configured.\n\n Loop through all handlers for this logger and its parents in the\n logger hierarchy. Return True if a handler was found, else False.\n Stop searching up the hierarchy whenever a logger with the "propagate"\n attribute set to zero is found - that will be the last logger which\n is checked for the existence of handlers.\n """\n c = self\n rv = False\n while c:\n if c.handlers:\n rv = True\n break\n if not c.propagate:\n break\n else:\n c = c.parent\n return rv\n\n def callHandlers(self, record):\n """\n Pass a record to all relevant handlers.\n\n Loop through all handlers for this logger and its parents in the\n logger hierarchy. If no handler was found, output a one-off error\n message to sys.stderr. Stop searching up the hierarchy whenever a\n logger with the "propagate" attribute set to zero is found - that\n will be the last logger whose handlers are called.\n """\n c = self\n found = 0\n while c:\n for hdlr in c.handlers:\n found = found + 1\n if record.levelno >= hdlr.level:\n hdlr.handle(record)\n if not c.propagate:\n c = None #break out\n else:\n c = c.parent\n if (found == 0):\n if lastResort:\n if record.levelno >= lastResort.level:\n lastResort.handle(record)\n elif raiseExceptions and not self.manager.emittedNoHandlerWarning:\n sys.stderr.write("No handlers could be found for logger"\n " \\"%s\\"\\n" % self.name)\n self.manager.emittedNoHandlerWarning = True\n\n def getEffectiveLevel(self):\n """\n Get the effective level for this logger.\n\n Loop through this logger and its parents in the logger hierarchy,\n looking for a non-zero logging level. Return the first one found.\n """\n logger = self\n while logger:\n if logger.level:\n return logger.level\n logger = logger.parent\n return NOTSET\n\n def isEnabledFor(self, level):\n """\n Is this logger enabled for level 'level'?\n """\n if self.manager.disable >= level:\n return False\n return level >= self.getEffectiveLevel()\n\n def getChild(self, suffix):\n """\n Get a logger which is a descendant to this one.\n\n This is a convenience method, such that\n\n logging.getLogger('abc').getChild('def.ghi')\n\n is the same as\n\n logging.getLogger('abc.def.ghi')\n\n It's useful, for example, when the parent logger is named using\n __name__ rather than a literal string.\n """\n if self.root is not self:\n suffix = '.'.join((self.name, suffix))\n return self.manager.getLogger(suffix)\n\n def __repr__(self):\n level = getLevelName(self.getEffectiveLevel())\n return '<%s %s (%s)>' % (self.__class__.__name__, self.name, level)\n\n\nclass RootLogger(Logger):\n """\n A root logger is not that different to any other logger, except that\n it must have a logging level and there is only one instance of it in\n the hierarchy.\n """\n def __init__(self, level):\n """\n Initialize the logger with the name "root".\n """\n Logger.__init__(self, "root", level)\n\n_loggerClass = Logger\n\nclass LoggerAdapter(object):\n """\n An adapter for loggers which makes it easier to specify contextual\n information in logging output.\n """\n\n def __init__(self, logger, extra):\n """\n Initialize the adapter with a logger and a dict-like object which\n provides contextual information. This constructor signature allows\n easy stacking of LoggerAdapters, if so desired.\n\n You can effectively pass keyword arguments as shown in the\n following example:\n\n adapter = LoggerAdapter(someLogger, dict(p1=v1, p2="v2"))\n """\n self.logger = logger\n self.extra = extra\n\n def process(self, msg, kwargs):\n """\n Process the logging message and keyword arguments passed in to\n a logging call to insert contextual information. You can either\n manipulate the message itself, the keyword args or both. Return\n the message and kwargs modified (or not) to suit your needs.\n\n Normally, you'll only need to override this one method in a\n LoggerAdapter subclass for your specific needs.\n """\n kwargs["extra"] = self.extra\n return msg, kwargs\n\n #\n # Boilerplate convenience methods\n #\n def debug(self, msg, *args, **kwargs):\n """\n Delegate a debug call to the underlying logger.\n """\n self.log(DEBUG, msg, *args, **kwargs)\n\n def info(self, msg, *args, **kwargs):\n """\n Delegate an info call to the underlying logger.\n """\n self.log(INFO, msg, *args, **kwargs)\n\n def warning(self, msg, *args, **kwargs):\n """\n Delegate a warning call to the underlying logger.\n """\n self.log(WARNING, msg, *args, **kwargs)\n\n def warn(self, msg, *args, **kwargs):\n warnings.warn("The 'warn' method is deprecated, "\n "use 'warning' instead", DeprecationWarning, 2)\n self.warning(msg, *args, **kwargs)\n\n def error(self, msg, *args, **kwargs):\n """\n Delegate an error call to the underlying logger.\n """\n self.log(ERROR, msg, *args, **kwargs)\n\n def exception(self, msg, *args, exc_info=True, **kwargs):\n """\n Delegate an exception call to the underlying logger.\n """\n self.log(ERROR, msg, *args, exc_info=exc_info, **kwargs)\n\n def critical(self, msg, *args, **kwargs):\n """\n Delegate a critical call to the underlying logger.\n """\n self.log(CRITICAL, msg, *args, **kwargs)\n\n def log(self, level, msg, *args, **kwargs):\n """\n Delegate a log call to the underlying logger, after adding\n contextual information from this adapter instance.\n """\n if self.isEnabledFor(level):\n msg, kwargs = self.process(msg, kwargs)\n self.logger.log(level, msg, *args, **kwargs)\n\n def isEnabledFor(self, level):\n """\n Is this logger enabled for level 'level'?\n """\n if self.logger.manager.disable >= level:\n return False\n return level >= self.getEffectiveLevel()\n\n def setLevel(self, level):\n """\n Set the specified level on the underlying logger.\n """\n self.logger.setLevel(level)\n\n def getEffectiveLevel(self):\n """\n Get the effective level for the underlying logger.\n """\n return self.logger.getEffectiveLevel()\n\n def hasHandlers(self):\n """\n See if the underlying logger has any handlers.\n """\n return self.logger.hasHandlers()\n\n def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False):\n """\n Low-level log implementation, proxied to allow nested logger adapters.\n """\n return self.logger._log(\n level,\n msg,\n args,\n exc_info=exc_info,\n extra=extra,\n stack_info=stack_info,\n )\n\n @property\n def manager(self):\n return self.logger.manager\n\n @manager.setter\n def manager(self, value):\n self.logger.manager = value\n\n @property\n def name(self):\n return self.logger.name\n\n def __repr__(self):\n logger = self.logger\n level = getLevelName(logger.getEffectiveLevel())\n return '<%s %s (%s)>' % (self.__class__.__name__, logger.name, level)\n\nroot = RootLogger(WARNING)\nLogger.root = root\nLogger.manager = Manager(Logger.root)\n\n#---------------------------------------------------------------------------\n# Configuration classes and functions\n#---------------------------------------------------------------------------\n\ndef basicConfig(**kwargs):\n """\n Do basic configuration for the logging system.\n\n This function does nothing if the root logger already has handlers\n configured. It is a convenience method intended for use by simple scripts\n to do one-shot configuration of the logging package.\n\n The default behaviour is to create a StreamHandler which writes to\n sys.stderr, set a formatter using the BASIC_FORMAT format string, and\n add the handler to the root logger.\n\n A number of optional keyword arguments may be specified, which can alter\n the default behaviour.\n\n filename Specifies that a FileHandler be created, using the specified\n filename, rather than a StreamHandler.\n filemode Specifies the mode to open the file, if filename is specified\n (if filemode is unspecified, it defaults to 'a').\n format Use the specified format string for the handler.\n datefmt Use the specified date/time format.\n style If a format string is specified, use this to specify the\n type of format string (possible values '%', '{', '$', for\n %-formatting, :meth:`str.format` and :class:`string.Template`\n - defaults to '%').\n level Set the root logger level to the specified level.\n stream Use the specified stream to initialize the StreamHandler. Note\n that this argument is incompatible with 'filename' - if both\n are present, 'stream' is ignored.\n handlers If specified, this should be an iterable of already created\n handlers, which will be added to the root handler. Any handler\n in the list which does not have a formatter assigned will be\n assigned the formatter created in this function.\n\n Note that you could specify a stream created using open(filename, mode)\n rather than passing the filename and mode in. However, it should be\n remembered that StreamHandler does not close its stream (since it may be\n using sys.stdout or sys.stderr), whereas FileHandler closes its stream\n when the handler is closed.\n\n .. versionchanged:: 3.2\n Added the ``style`` parameter.\n\n .. versionchanged:: 3.3\n Added the ``handlers`` parameter. A ``ValueError`` is now thrown for\n incompatible arguments (e.g. ``handlers`` specified together with\n ``filename``/``filemode``, or ``filename``/``filemode`` specified\n together with ``stream``, or ``handlers`` specified together with\n ``stream``.\n """\n # Add thread safety in case someone mistakenly calls\n # basicConfig() from multiple threads\n _acquireLock()\n try:\n if len(root.handlers) == 0:\n handlers = kwargs.pop("handlers", None)\n if handlers is None:\n if "stream" in kwargs and "filename" in kwargs:\n raise ValueError("'stream' and 'filename' should not be "\n "specified together")\n else:\n if "stream" in kwargs or "filename" in kwargs:\n raise ValueError("'stream' or 'filename' should not be "\n "specified together with 'handlers'")\n if handlers is None:\n filename = kwargs.pop("filename", None)\n mode = kwargs.pop("filemode", 'a')\n if filename:\n h = FileHandler(filename, mode)\n else:\n stream = kwargs.pop("stream", None)\n h = StreamHandler(stream)\n handlers = [h]\n dfs = kwargs.pop("datefmt", None)\n style = kwargs.pop("style", '%')\n if style not in _STYLES:\n raise ValueError('Style must be one of: %s' % ','.join(\n _STYLES.keys()))\n fs = kwargs.pop("format", _STYLES[style][1])\n fmt = Formatter(fs, dfs, style)\n for h in handlers:\n if h.formatter is None:\n h.setFormatter(fmt)\n root.addHandler(h)\n level = kwargs.pop("level", None)\n if level is not None:\n root.setLevel(level)\n if kwargs:\n keys = ', '.join(kwargs.keys())\n raise ValueError('Unrecognised argument(s): %s' % keys)\n finally:\n _releaseLock()\n\n#---------------------------------------------------------------------------\n# Utility functions at module level.\n# Basically delegate everything to the root logger.\n#---------------------------------------------------------------------------\n\ndef getLogger(name=None):\n """\n Return a logger with the specified name, creating it if necessary.\n\n If no name is specified, return the root logger.\n """\n if name:\n return Logger.manager.getLogger(name)\n else:\n return root\n\ndef critical(msg, *args, **kwargs):\n """\n Log a message with severity 'CRITICAL' on the root logger. If the logger\n has no handlers, call basicConfig() to add a console handler with a\n pre-defined format.\n """\n if len(root.handlers) == 0:\n basicConfig()\n root.critical(msg, *args, **kwargs)\n\nfatal = critical\n\ndef error(msg, *args, **kwargs):\n """\n Log a message with severity 'ERROR' on the root logger. If the logger has\n no handlers, call basicConfig() to add a console handler with a pre-defined\n format.\n """\n if len(root.handlers) == 0:\n basicConfig()\n root.error(msg, *args, **kwargs)\n\ndef exception(msg, *args, exc_info=True, **kwargs):\n """\n Log a message with severity 'ERROR' on the root logger, with exception\n information. If the logger has no handlers, basicConfig() is called to add\n a console handler with a pre-defined format.\n """\n error(msg, *args, exc_info=exc_info, **kwargs)\n\ndef warning(msg, *args, **kwargs):\n """\n Log a message with severity 'WARNING' on the root logger. If the logger has\n no handlers, call basicConfig() to add a console handler with a pre-defined\n format.\n """\n if len(root.handlers) == 0:\n basicConfig()\n root.warning(msg, *args, **kwargs)\n\ndef warn(msg, *args, **kwargs):\n warnings.warn("The 'warn' function is deprecated, "\n "use 'warning' instead", DeprecationWarning, 2)\n warning(msg, *args, **kwargs)\n\ndef info(msg, *args, **kwargs):\n """\n Log a message with severity 'INFO' on the root logger. If the logger has\n no handlers, call basicConfig() to add a console handler with a pre-defined\n format.\n """\n if len(root.handlers) == 0:\n basicConfig()\n root.info(msg, *args, **kwargs)\n\ndef debug(msg, *args, **kwargs):\n """\n Log a message with severity 'DEBUG' on the root logger. If the logger has\n no handlers, call basicConfig() to add a console handler with a pre-defined\n format.\n """\n if len(root.handlers) == 0:\n basicConfig()\n root.debug(msg, *args, **kwargs)\n\ndef log(level, msg, *args, **kwargs):\n """\n Log 'msg % args' with the integer severity 'level' on the root logger. If\n the logger has no handlers, call basicConfig() to add a console handler\n with a pre-defined format.\n """\n if len(root.handlers) == 0:\n basicConfig()\n root.log(level, msg, *args, **kwargs)\n\ndef disable(level):\n """\n Disable all logging calls of severity 'level' and below.\n """\n root.manager.disable = level\n\ndef shutdown(handlerList=_handlerList):\n """\n Perform any cleanup actions in the logging system (e.g. flushing\n buffers).\n\n Should be called at application exit.\n """\n for wr in reversed(handlerList[:]):\n #errors might occur, for example, if files are locked\n #we just ignore them if raiseExceptions is not set\n try:\n h = wr()\n if h:\n try:\n h.acquire()\n h.flush()\n h.close()\n except (OSError, ValueError):\n # Ignore errors which might be caused\n # because handlers have been closed but\n # references to them are still around at\n # application exit.\n pass\n finally:\n h.release()\n except: # ignore everything, as we're shutting down\n if raiseExceptions:\n raise\n #else, swallow\n\n#Let's try and shutdown automatically on application exit...\nimport atexit\natexit.register(shutdown)\n\n# Null handler\n\nclass NullHandler(Handler):\n """\n This handler does nothing. It's intended to be used to avoid the\n "No handlers could be found for logger XXX" one-off warning. This is\n important for library code, which may contain code to log events. If a user\n of the library does not configure logging, the one-off warning might be\n produced; to avoid this, the library developer simply needs to instantiate\n a NullHandler and add it to the top-level logger of the library module or\n package.\n """\n def handle(self, record):\n """Stub."""\n\n def emit(self, record):\n """Stub."""\n\n def createLock(self):\n self.lock = None\n\n# Warnings integration\n\n_warnings_showwarning = None\n\ndef _showwarning(message, category, filename, lineno, file=None, line=None):\n """\n Implementation of showwarnings which redirects to logging, which will first\n check to see if the file parameter is None. If a file is specified, it will\n delegate to the original warnings implementation of showwarning. Otherwise,\n it will call warnings.formatwarning and will log the resulting string to a\n warnings logger named "py.warnings" with level logging.WARNING.\n """\n if file is not None:\n if _warnings_showwarning is not None:\n _warnings_showwarning(message, category, filename, lineno, file, line)\n else:\n s = warnings.formatwarning(message, category, filename, lineno, line)\n logger = getLogger("py.warnings")\n if not logger.handlers:\n logger.addHandler(NullHandler())\n logger.warning("%s", s)\n\ndef captureWarnings(capture):\n """\n If capture is true, redirect all warnings to the logging package.\n If capture is False, ensure that warnings are not redirected to logging\n but to their original destinations.\n """\n global _warnings_showwarning\n if capture:\n if _warnings_showwarning is None:\n _warnings_showwarning = warnings.showwarning\n warnings.showwarning = _showwarning\n else:\n if _warnings_showwarning is not None:\n warnings.showwarning = _warnings_showwarning\n _warnings_showwarning = None\n\nimport fnmatch\nimport functools\nimport io\nimport ntpath\nimport os\nimport posixpath\nimport re\nimport sys\nfrom collections import Sequence\nfrom contextlib import contextmanager\nfrom errno import EINVAL, ENOENT, ENOTDIR\nfrom operator import attrgetter\nfrom stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO\nfrom urllib.parse import quote_from_bytes as urlquote_from_bytes\n\n\nsupports_symlinks = True\nif os.name == 'nt':\n import nt\n if sys.getwindowsversion()[:2] >= (6, 0):\n from nt import _getfinalpathname\n else:\n supports_symlinks = False\n _getfinalpathname = None\nelse:\n nt = None\n\n\n__all__ = [\n "PurePath", "PurePosixPath", "PureWindowsPath",\n "Path", "PosixPath", "WindowsPath",\n ]\n\n#\n# Internals\n#\n\ndef _is_wildcard_pattern(pat):\n # Whether this pattern needs actual matching using fnmatch, or can\n # be looked up directly as a file.\n return "*" in pat or "?" in pat or "[" in pat\n\n\nclass _Flavour(object):\n """A flavour implements a particular (platform-specific) set of path\n semantics."""\n\n def __init__(self):\n self.join = self.sep.join\n\n def parse_parts(self, parts):\n parsed = []\n sep = self.sep\n altsep = self.altsep\n drv = root = ''\n it = reversed(parts)\n for part in it:\n if not part:\n continue\n if altsep:\n part = part.replace(altsep, sep)\n drv, root, rel = self.splitroot(part)\n if sep in rel:\n for x in reversed(rel.split(sep)):\n if x and x != '.':\n parsed.append(sys.intern(x))\n else:\n if rel and rel != '.':\n parsed.append(sys.intern(rel))\n if drv or root:\n if not drv:\n # If no drive is present, try to find one in the previous\n # parts. This makes the result of parsing e.g.\n # ("C:", "/", "a") reasonably intuitive.\n for part in it:\n if not part:\n continue\n if altsep:\n part = part.replace(altsep, sep)\n drv = self.splitroot(part)[0]\n if drv:\n break\n break\n if drv or root:\n parsed.append(drv + root)\n parsed.reverse()\n return drv, root, parsed\n\n def join_parsed_parts(self, drv, root, parts, drv2, root2, parts2):\n """\n Join the two paths represented by the respective\n (drive, root, parts) tuples. Return a new (drive, root, parts) tuple.\n """\n if root2:\n if not drv2 and drv:\n return drv, root2, [drv + root2] + parts2[1:]\n elif drv2:\n if drv2 == drv or self.casefold(drv2) == self.casefold(drv):\n # Same drive => second path is relative to the first\n return drv, root, parts + parts2[1:]\n else:\n # Second path is non-anchored (common case)\n return drv, root, parts + parts2\n return drv2, root2, parts2\n\n\nclass _WindowsFlavour(_Flavour):\n # Reference for Windows paths can be found at\n # http://msdn.microsoft.com/en-us/library/aa365247%28v=vs.85%29.aspx\n\n sep = '\\\\'\n altsep = '/'\n has_drv = True\n pathmod = ntpath\n\n is_supported = (os.name == 'nt')\n\n drive_letters = (\n set(chr(x) for x in range(ord('a'), ord('z') + 1)) |\n set(chr(x) for x in range(ord('A'), ord('Z') + 1))\n )\n ext_namespace_prefix = '\\\\\\\\?\\\\'\n\n reserved_names = (\n {'CON', 'PRN', 'AUX', 'NUL'} |\n {'COM%d' % i for i in range(1, 10)} |\n {'LPT%d' % i for i in range(1, 10)}\n )\n\n # Interesting findings about extended paths:\n # - '\\\\?\\c:\\a', '//?/c:\\a' and '//?/c:/a' are all supported\n # but '\\\\?\\c:/a' is not\n # - extended paths are always absolute; "relative" extended paths will\n # fail.\n\n def splitroot(self, part, sep=sep):\n first = part[0:1]\n second = part[1:2]\n if (second == sep and first == sep):\n # XXX extended paths should also disable the collapsing of "."\n # components (according to MSDN docs).\n prefix, part = self._split_extended_path(part)\n first = part[0:1]\n second = part[1:2]\n else:\n prefix = ''\n third = part[2:3]\n if (second == sep and first == sep and third != sep):\n # is a UNC path:\n # vvvvvvvvvvvvvvvvvvvvv root\n # \\\\machine\\mountpoint\\directory\\etc\\...\n # directory ^^^^^^^^^^^^^^\n index = part.find(sep, 2)\n if index != -1:\n index2 = part.find(sep, index + 1)\n # a UNC path can't have two slashes in a row\n # (after the initial two)\n if index2 != index + 1:\n if index2 == -1:\n index2 = len(part)\n if prefix:\n return prefix + part[1:index2], sep, part[index2+1:]\n else:\n return part[:index2], sep, part[index2+1:]\n drv = root = ''\n if second == ':' and first in self.drive_letters:\n drv = part[:2]\n part = part[2:]\n first = third\n if first == sep:\n root = first\n part = part.lstrip(sep)\n return prefix + drv, root, part\n\n def casefold(self, s):\n return s.lower()\n\n def casefold_parts(self, parts):\n return [p.lower() for p in parts]\n\n def resolve(self, path, strict=False):\n s = str(path)\n if not s:\n return os.getcwd()\n previous_s = None\n if _getfinalpathname is not None:\n if strict:\n return self._ext_to_normal(_getfinalpathname(s))\n else:\n tail_parts = [] # End of the path after the first one not found\n while True:\n try:\n s = self._ext_to_normal(_getfinalpathname(s))\n except FileNotFoundError:\n previous_s = s\n s, tail = os.path.split(s)\n tail_parts.append(tail)\n if previous_s == s:\n return path\n else:\n return os.path.join(s, *reversed(tail_parts))\n # Means fallback on absolute\n return None\n\n def _split_extended_path(self, s, ext_prefix=ext_namespace_prefix):\n prefix = ''\n if s.startswith(ext_prefix):\n prefix = s[:4]\n s = s[4:]\n if s.startswith('UNC\\\\'):\n prefix += s[:3]\n s = '\\\\' + s[3:]\n return prefix, s\n\n def _ext_to_normal(self, s):\n # Turn back an extended path into a normal DOS-like path\n return self._split_extended_path(s)[1]\n\n def is_reserved(self, parts):\n # NOTE: the rules for reserved names seem somewhat complicated\n # (e.g. r"..\\NUL" is reserved but not r"foo\\NUL").\n # We err on the side of caution and return True for paths which are\n # not considered reserved by Windows.\n if not parts:\n return False\n if parts[0].startswith('\\\\\\\\'):\n # UNC paths are never reserved\n return False\n return parts[-1].partition('.')[0].upper() in self.reserved_names\n\n def make_uri(self, path):\n # Under Windows, file URIs use the UTF-8 encoding.\n drive = path.drive\n if len(drive) == 2 and drive[1] == ':':\n # It's a path on a local drive => 'file:///c:/a/b'\n rest = path.as_posix()[2:].lstrip('/')\n return 'file:///%s/%s' % (\n drive, urlquote_from_bytes(rest.encode('utf-8')))\n else:\n # It's a path on a network drive => 'file://host/share/a/b'\n return 'file:' + urlquote_from_bytes(path.as_posix().encode('utf-8'))\n\n def gethomedir(self, username):\n if 'HOME' in os.environ:\n userhome = os.environ['HOME']\n elif 'USERPROFILE' in os.environ:\n userhome = os.environ['USERPROFILE']\n elif 'HOMEPATH' in os.environ:\n try:\n drv = os.environ['HOMEDRIVE']\n except KeyError:\n drv = ''\n userhome = drv + os.environ['HOMEPATH']\n else:\n raise RuntimeError("Can't determine home directory")\n\n if username:\n # Try to guess user home directory. By default all users\n # directories are located in the same place and are named by\n # corresponding usernames. If current user home directory points\n # to nonstandard place, this guess is likely wrong.\n if os.environ['USERNAME'] != username:\n drv, root, parts = self.parse_parts((userhome,))\n if parts[-1] != os.environ['USERNAME']:\n raise RuntimeError("Can't determine home directory "\n "for %r" % username)\n parts[-1] = username\n if drv or root:\n userhome = drv + root + self.join(parts[1:])\n else:\n userhome = self.join(parts)\n return userhome\n\nclass _PosixFlavour(_Flavour):\n sep = '/'\n altsep = ''\n has_drv = False\n pathmod = posixpath\n\n is_supported = (os.name != 'nt')\n\n def splitroot(self, part, sep=sep):\n if part and part[0] == sep:\n stripped_part = part.lstrip(sep)\n # According to POSIX path resolution:\n # http://pubs.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap04.html#tag_04_11\n # "A pathname that begins with two successive slashes may be\n # interpreted in an implementation-defined manner, although more\n # than two leading slashes shall be treated as a single slash".\n if len(part) - len(stripped_part) == 2:\n return '', sep * 2, stripped_part\n else:\n return '', sep, stripped_part\n else:\n return '', '', part\n\n def casefold(self, s):\n return s\n\n def casefold_parts(self, parts):\n return parts\n\n def resolve(self, path, strict=False):\n sep = self.sep\n accessor = path._accessor\n seen = {}\n def _resolve(path, rest):\n if rest.startswith(sep):\n path = ''\n\n for name in rest.split(sep):\n if not name or name == '.':\n # current dir\n continue\n if name == '..':\n # parent dir\n path, _, _ = path.rpartition(sep)\n continue\n newpath = path + sep + name\n if newpath in seen:\n # Already seen this path\n path = seen[newpath]\n if path is not None:\n # use cached value\n continue\n # The symlink is not resolved, so we must have a symlink loop.\n raise RuntimeError("Symlink loop from %r" % newpath)\n # Resolve the symbolic link\n try:\n target = accessor.readlink(newpath)\n except OSError as e:\n if e.errno != EINVAL and strict:\n raise\n # Not a symlink, or non-strict mode. We just leave the path\n # untouched.\n path = newpath\n else:\n seen[newpath] = None # not resolved symlink\n path = _resolve(path, target)\n seen[newpath] = path # resolved symlink\n\n return path\n # NOTE: according to POSIX, getcwd() cannot contain path components\n # which are symlinks.\n base = '' if path.is_absolute() else os.getcwd()\n return _resolve(base, str(path)) or sep\n\n def is_reserved(self, parts):\n return False\n\n def make_uri(self, path):\n # We represent the path using the local filesystem encoding,\n # for portability to other applications.\n bpath = bytes(path)\n return 'file://' + urlquote_from_bytes(bpath)\n\n def gethomedir(self, username):\n if not username:\n try:\n return os.environ['HOME']\n except KeyError:\n import pwd\n return pwd.getpwuid(os.getuid()).pw_dir\n else:\n import pwd\n try:\n return pwd.getpwnam(username).pw_dir\n except KeyError:\n raise RuntimeError("Can't determine home directory "\n "for %r" % username)\n\n\n_windows_flavour = _WindowsFlavour()\n_posix_flavour = _PosixFlavour()\n\n\nclass _Accessor:\n """An accessor implements a particular (system-specific or not) way of\n accessing paths on the filesystem."""\n\n\nclass _NormalAccessor(_Accessor):\n\n def _wrap_strfunc(strfunc):\n @functools.wraps(strfunc)\n def wrapped(pathobj, *args):\n return strfunc(str(pathobj), *args)\n return staticmethod(wrapped)\n\n def _wrap_binary_strfunc(strfunc):\n @functools.wraps(strfunc)\n def wrapped(pathobjA, pathobjB, *args):\n return strfunc(str(pathobjA), str(pathobjB), *args)\n return staticmethod(wrapped)\n\n stat = _wrap_strfunc(os.stat)\n\n lstat = _wrap_strfunc(os.lstat)\n\n open = _wrap_strfunc(os.open)\n\n listdir = _wrap_strfunc(os.listdir)\n\n scandir = _wrap_strfunc(os.scandir)\n\n chmod = _wrap_strfunc(os.chmod)\n\n if hasattr(os, "lchmod"):\n lchmod = _wrap_strfunc(os.lchmod)\n else:\n def lchmod(self, pathobj, mode):\n raise NotImplementedError("lchmod() not available on this system")\n\n mkdir = _wrap_strfunc(os.mkdir)\n\n unlink = _wrap_strfunc(os.unlink)\n\n rmdir = _wrap_strfunc(os.rmdir)\n\n rename = _wrap_binary_strfunc(os.rename)\n\n replace = _wrap_binary_strfunc(os.replace)\n\n if nt:\n if supports_symlinks:\n symlink = _wrap_binary_strfunc(os.symlink)\n else:\n def symlink(a, b, target_is_directory):\n raise NotImplementedError("symlink() not available on this system")\n else:\n # Under POSIX, os.symlink() takes two args\n @staticmethod\n def symlink(a, b, target_is_directory):\n return os.symlink(str(a), str(b))\n\n utime = _wrap_strfunc(os.utime)\n\n # Helper for resolve()\n def readlink(self, path):\n return os.readlink(path)\n\n\n_normal_accessor = _NormalAccessor()\n\n\n#\n# Globbing helpers\n#\n\ndef _make_selector(pattern_parts):\n pat = pattern_parts[0]\n child_parts = pattern_parts[1:]\n if pat == '**':\n cls = _RecursiveWildcardSelector\n elif '**' in pat:\n raise ValueError("Invalid pattern: '**' can only be an entire path component")\n elif _is_wildcard_pattern(pat):\n cls = _WildcardSelector\n else:\n cls = _PreciseSelector\n return cls(pat, child_parts)\n\nif hasattr(functools, "lru_cache"):\n _make_selector = functools.lru_cache()(_make_selector)\n\n\nclass _Selector:\n """A selector matches a specific glob pattern part against the children\n of a given path."""\n\n def __init__(self, child_parts):\n self.child_parts = child_parts\n if child_parts:\n self.successor = _make_selector(child_parts)\n self.dironly = True\n else:\n self.successor = _TerminatingSelector()\n self.dironly = False\n\n def select_from(self, parent_path):\n """Iterate over all child paths of `parent_path` matched by this\n selector. This can contain parent_path itself."""\n path_cls = type(parent_path)\n is_dir = path_cls.is_dir\n exists = path_cls.exists\n scandir = parent_path._accessor.scandir\n if not is_dir(parent_path):\n return iter([])\n return self._select_from(parent_path, is_dir, exists, scandir)\n\n\nclass _TerminatingSelector:\n\n def _select_from(self, parent_path, is_dir, exists, scandir):\n yield parent_path\n\n\nclass _PreciseSelector(_Selector):\n\n def __init__(self, name, child_parts):\n self.name = name\n _Selector.__init__(self, child_parts)\n\n def _select_from(self, parent_path, is_dir, exists, scandir):\n try:\n path = parent_path._make_child_relpath(self.name)\n if (is_dir if self.dironly else exists)(path):\n for p in self.successor._select_from(path, is_dir, exists, scandir):\n yield p\n except PermissionError:\n return\n\n\nclass _WildcardSelector(_Selector):\n\n def __init__(self, pat, child_parts):\n self.pat = re.compile(fnmatch.translate(pat))\n _Selector.__init__(self, child_parts)\n\n def _select_from(self, parent_path, is_dir, exists, scandir):\n try:\n cf = parent_path._flavour.casefold\n entries = list(scandir(parent_path))\n for entry in entries:\n if not self.dironly or entry.is_dir():\n name = entry.name\n casefolded = cf(name)\n if self.pat.match(casefolded):\n path = parent_path._make_child_relpath(name)\n for p in self.successor._select_from(path, is_dir, exists, scandir):\n yield p\n except PermissionError:\n return\n\n\n\nclass _RecursiveWildcardSelector(_Selector):\n\n def __init__(self, pat, child_parts):\n _Selector.__init__(self, child_parts)\n\n def _iterate_directories(self, parent_path, is_dir, scandir):\n yield parent_path\n try:\n entries = list(scandir(parent_path))\n for entry in entries:\n if entry.is_dir() and not entry.is_symlink():\n path = parent_path._make_child_relpath(entry.name)\n for p in self._iterate_directories(path, is_dir, scandir):\n yield p\n except PermissionError:\n return\n\n def _select_from(self, parent_path, is_dir, exists, scandir):\n try:\n yielded = set()\n try:\n successor_select = self.successor._select_from\n for starting_point in self._iterate_directories(parent_path, is_dir, scandir):\n for p in successor_select(starting_point, is_dir, exists, scandir):\n if p not in yielded:\n yield p\n yielded.add(p)\n finally:\n yielded.clear()\n except PermissionError:\n return\n\n\n#\n# Public API\n#\n\nclass _PathParents(Sequence):\n """This object provides sequence-like access to the logical ancestors\n of a path. Don't try to construct it yourself."""\n __slots__ = ('_pathcls', '_drv', '_root', '_parts')\n\n def __init__(self, path):\n # We don't store the instance to avoid reference cycles\n self._pathcls = type(path)\n self._drv = path._drv\n self._root = path._root\n self._parts = path._parts\n\n def __len__(self):\n if self._drv or self._root:\n return len(self._parts) - 1\n else:\n return len(self._parts)\n\n def __getitem__(self, idx):\n if idx < 0 or idx >= len(self):\n raise IndexError(idx)\n return self._pathcls._from_parsed_parts(self._drv, self._root,\n self._parts[:-idx - 1])\n\n def __repr__(self):\n return "<{}.parents>".format(self._pathcls.__name__)\n\n\nclass PurePath(object):\n """Base class for manipulating paths without I/O.\n\n PurePath represents a filesystem path and offers operations which\n don't imply any actual filesystem I/O. Depending on your system,\n instantiating a PurePath will return either a PurePosixPath or a\n PureWindowsPath object. You can also instantiate either of these classes\n directly, regardless of your system.\n """\n __slots__ = (\n '_drv', '_root', '_parts',\n '_str', '_hash', '_pparts', '_cached_cparts',\n )\n\n def __new__(cls, *args):\n """Construct a PurePath from one or several strings and or existing\n PurePath objects. The strings and path objects are combined so as\n to yield a canonicalized path, which is incorporated into the\n new PurePath object.\n """\n if cls is PurePath:\n cls = PureWindowsPath if os.name == 'nt' else PurePosixPath\n return cls._from_parts(args)\n\n def __reduce__(self):\n # Using the parts tuple helps share interned path parts\n # when pickling related paths.\n return (self.__class__, tuple(self._parts))\n\n @classmethod\n def _parse_args(cls, args):\n # This is useful when you don't want to create an instance, just\n # canonicalize some constructor arguments.\n parts = []\n for a in args:\n if isinstance(a, PurePath):\n parts += a._parts\n else:\n a = os.fspath(a)\n if isinstance(a, str):\n # Force-cast str subclasses to str (issue #21127)\n parts.append(str(a))\n else:\n raise TypeError(\n "argument should be a str object or an os.PathLike "\n "object returning str, not %r"\n % type(a))\n return cls._flavour.parse_parts(parts)\n\n @classmethod\n def _from_parts(cls, args, init=True):\n # We need to call _parse_args on the instance, so as to get the\n # right flavour.\n self = object.__new__(cls)\n drv, root, parts = self._parse_args(args)\n self._drv = drv\n self._root = root\n self._parts = parts\n if init:\n self._init()\n return self\n\n @classmethod\n def _from_parsed_parts(cls, drv, root, parts, init=True):\n self = object.__new__(cls)\n self._drv = drv\n self._root = root\n self._parts = parts\n if init:\n self._init()\n return self\n\n @classmethod\n def _format_parsed_parts(cls, drv, root, parts):\n if drv or root:\n return drv + root + cls._flavour.join(parts[1:])\n else:\n return cls._flavour.join(parts)\n\n def _init(self):\n # Overridden in concrete Path\n pass\n\n def _make_child(self, args):\n drv, root, parts = self._parse_args(args)\n drv, root, parts = self._flavour.join_parsed_parts(\n self._drv, self._root, self._parts, drv, root, parts)\n return self._from_parsed_parts(drv, root, parts)\n\n def __str__(self):\n """Return the string representation of the path, suitable for\n passing to system calls."""\n try:\n return self._str\n except AttributeError:\n self._str = self._format_parsed_parts(self._drv, self._root,\n self._parts) or '.'\n return self._str\n\n def __fspath__(self):\n return str(self)\n\n def as_posix(self):\n """Return the string representation of the path with forward (/)\n slashes."""\n f = self._flavour\n return str(self).replace(f.sep, '/')\n\n def __bytes__(self):\n """Return the bytes representation of the path. This is only\n recommended to use under Unix."""\n return os.fsencode(str(self))\n\n def __repr__(self):\n return "{}({!r})".format(self.__class__.__name__, self.as_posix())\n\n def as_uri(self):\n """Return the path as a 'file' URI."""\n if not self.is_absolute():\n raise ValueError("relative path can't be expressed as a file URI")\n return self._flavour.make_uri(self)\n\n @property\n def _cparts(self):\n # Cached casefolded parts, for hashing and comparison\n try:\n return self._cached_cparts\n except AttributeError:\n self._cached_cparts = self._flavour.casefold_parts(self._parts)\n return self._cached_cparts\n\n def __eq__(self, other):\n if not isinstance(other, PurePath):\n return NotImplemented\n return self._cparts == other._cparts and self._flavour is other._flavour\n\n def __hash__(self):\n try:\n return self._hash\n except AttributeError:\n self._hash = hash(tuple(self._cparts))\n return self._hash\n\n def __lt__(self, other):\n if not isinstance(other, PurePath) or self._flavour is not other._flavour:\n return NotImplemented\n return self._cparts < other._cparts\n\n def __le__(self, other):\n if not isinstance(other, PurePath) or self._flavour is not other._flavour:\n return NotImplemented\n return self._cparts <= other._cparts\n\n def __gt__(self, other):\n if not isinstance(other, PurePath) or self._flavour is not other._flavour:\n return NotImplemented\n return self._cparts > other._cparts\n\n def __ge__(self, other):\n if not isinstance(other, PurePath) or self._flavour is not other._flavour:\n return NotImplemented\n return self._cparts >= other._cparts\n\n drive = property(attrgetter('_drv'),\n doc="""The drive prefix (letter or UNC path), if any.""")\n\n root = property(attrgetter('_root'),\n doc="""The root of the path, if any.""")\n\n @property\n def anchor(self):\n """The concatenation of the drive and root, or ''."""\n anchor = self._drv + self._root\n return anchor\n\n @property\n def name(self):\n """The final path component, if any."""\n parts = self._parts\n if len(parts) == (1 if (self._drv or self._root) else 0):\n return ''\n return parts[-1]\n\n @property\n def suffix(self):\n """The final component's last suffix, if any."""\n name = self.name\n i = name.rfind('.')\n if 0 < i < len(name) - 1:\n return name[i:]\n else:\n return ''\n\n @property\n def suffixes(self):\n """A list of the final component's suffixes, if any."""\n name = self.name\n if name.endswith('.'):\n return []\n name = name.lstrip('.')\n return ['.' + suffix for suffix in name.split('.')[1:]]\n\n @property\n def stem(self):\n """The final path component, minus its last suffix."""\n name = self.name\n i = name.rfind('.')\n if 0 < i < len(name) - 1:\n return name[:i]\n else:\n return name\n\n def with_name(self, name):\n """Return a new path with the file name changed."""\n if not self.name:\n raise ValueError("%r has an empty name" % (self,))\n drv, root, parts = self._flavour.parse_parts((name,))\n if (not name or name[-1] in [self._flavour.sep, self._flavour.altsep]\n or drv or root or len(parts) != 1):\n raise ValueError("Invalid name %r" % (name))\n return self._from_parsed_parts(self._drv, self._root,\n self._parts[:-1] + [name])\n\n def with_suffix(self, suffix):\n """Return a new path with the file suffix changed. If the path\n has no suffix, add given suffix. If the given suffix is an empty\n string, remove the suffix from the path.\n """\n f = self._flavour\n if f.sep in suffix or f.altsep and f.altsep in suffix:\n raise ValueError("Invalid suffix %r" % (suffix))\n if suffix and not suffix.startswith('.') or suffix == '.':\n raise ValueError("Invalid suffix %r" % (suffix))\n name = self.name\n if not name:\n raise ValueError("%r has an empty name" % (self,))\n old_suffix = self.suffix\n if not old_suffix:\n name = name + suffix\n else:\n name = name[:-len(old_suffix)] + suffix\n return self._from_parsed_parts(self._drv, self._root,\n self._parts[:-1] + [name])\n\n def relative_to(self, *other):\n """Return the relative path to another path identified by the passed\n arguments. If the operation is not possible (because this is not\n a subpath of the other path), raise ValueError.\n """\n # For the purpose of this method, drive and root are considered\n # separate parts, i.e.:\n # Path('c:/').relative_to('c:') gives Path('/')\n # Path('c:/').relative_to('/') raise ValueError\n if not other:\n raise TypeError("need at least one argument")\n parts = self._parts\n drv = self._drv\n root = self._root\n if root:\n abs_parts = [drv, root] + parts[1:]\n else:\n abs_parts = parts\n to_drv, to_root, to_parts = self._parse_args(other)\n if to_root:\n to_abs_parts = [to_drv, to_root] + to_parts[1:]\n else:\n to_abs_parts = to_parts\n n = len(to_abs_parts)\n cf = self._flavour.casefold_parts\n if (root or drv) if n == 0 else cf(abs_parts[:n]) != cf(to_abs_parts):\n formatted = self._format_parsed_parts(to_drv, to_root, to_parts)\n raise ValueError("{!r} does not start with {!r}"\n .format(str(self), str(formatted)))\n return self._from_parsed_parts('', root if n == 1 else '',\n abs_parts[n:])\n\n @property\n def parts(self):\n """An object providing sequence-like access to the\n components in the filesystem path."""\n # We cache the tuple to avoid building a new one each time .parts\n # is accessed. XXX is this necessary?\n try:\n return self._pparts\n except AttributeError:\n self._pparts = tuple(self._parts)\n return self._pparts\n\n def joinpath(self, *args):\n """Combine this path with one or several arguments, and return a\n new path representing either a subpath (if all arguments are relative\n paths) or a totally different path (if one of the arguments is\n anchored).\n """\n return self._make_child(args)\n\n def __truediv__(self, key):\n return self._make_child((key,))\n\n def __rtruediv__(self, key):\n return self._from_parts([key] + self._parts)\n\n @property\n def parent(self):\n """The logical parent of the path."""\n drv = self._drv\n root = self._root\n parts = self._parts\n if len(parts) == 1 and (drv or root):\n return self\n return self._from_parsed_parts(drv, root, parts[:-1])\n\n @property\n def parents(self):\n """A sequence of this path's logical parents."""\n return _PathParents(self)\n\n def is_absolute(self):\n """True if the path is absolute (has both a root and, if applicable,\n a drive)."""\n if not self._root:\n return False\n return not self._flavour.has_drv or bool(self._drv)\n\n def is_reserved(self):\n """Return True if the path contains one of the special names reserved\n by the system, if any."""\n return self._flavour.is_reserved(self._parts)\n\n def match(self, path_pattern):\n """\n Return True if this path matches the given pattern.\n """\n cf = self._flavour.casefold\n path_pattern = cf(path_pattern)\n drv, root, pat_parts = self._flavour.parse_parts((path_pattern,))\n if not pat_parts:\n raise ValueError("empty pattern")\n if drv and drv != cf(self._drv):\n return False\n if root and root != cf(self._root):\n return False\n parts = self._cparts\n if drv or root:\n if len(pat_parts) != len(parts):\n return False\n pat_parts = pat_parts[1:]\n elif len(pat_parts) > len(parts):\n return False\n for part, pat in zip(reversed(parts), reversed(pat_parts)):\n if not fnmatch.fnmatchcase(part, pat):\n return False\n return True\n\n# Can't subclass os.PathLike from PurePath and keep the constructor\n# optimizations in PurePath._parse_args().\nos.PathLike.register(PurePath)\n\n\nclass PurePosixPath(PurePath):\n """PurePath subclass for non-Windows systems.\n\n On a POSIX system, instantiating a PurePath should return this object.\n However, you can also instantiate it directly on any system.\n """\n _flavour = _posix_flavour\n __slots__ = ()\n\n\nclass PureWindowsPath(PurePath):\n """PurePath subclass for Windows systems.\n\n On a Windows system, instantiating a PurePath should return this object.\n However, you can also instantiate it directly on any system.\n """\n _flavour = _windows_flavour\n __slots__ = ()\n\n\n# Filesystem-accessing classes\n\n\nclass Path(PurePath):\n """PurePath subclass that can make system calls.\n\n Path represents a filesystem path but unlike PurePath, also offers\n methods to do system calls on path objects. Depending on your system,\n instantiating a Path will return either a PosixPath or a WindowsPath\n object. You can also instantiate a PosixPath or WindowsPath directly,\n but cannot instantiate a WindowsPath on a POSIX system or vice versa.\n """\n __slots__ = (\n '_accessor',\n '_closed',\n )\n\n def __new__(cls, *args, **kwargs):\n if cls is Path:\n cls = WindowsPath if os.name == 'nt' else PosixPath\n self = cls._from_parts(args, init=False)\n if not self._flavour.is_supported:\n raise NotImplementedError("cannot instantiate %r on your system"\n % (cls.__name__,))\n self._init()\n return self\n\n def _init(self,\n # Private non-constructor arguments\n template=None,\n ):\n self._closed = False\n if template is not None:\n self._accessor = template._accessor\n else:\n self._accessor = _normal_accessor\n\n def _make_child_relpath(self, part):\n # This is an optimization used for dir walking. `part` must be\n # a single part relative to this path.\n parts = self._parts + [part]\n return self._from_parsed_parts(self._drv, self._root, parts)\n\n def __enter__(self):\n if self._closed:\n self._raise_closed()\n return self\n\n def __exit__(self, t, v, tb):\n self._closed = True\n\n def _raise_closed(self):\n raise ValueError("I/O operation on closed path")\n\n def _opener(self, name, flags, mode=0o666):\n # A stub for the opener argument to built-in open()\n return self._accessor.open(self, flags, mode)\n\n def _raw_open(self, flags, mode=0o777):\n """\n Open the file pointed by this path and return a file descriptor,\n as os.open() does.\n """\n if self._closed:\n self._raise_closed()\n return self._accessor.open(self, flags, mode)\n\n # Public API\n\n @classmethod\n def cwd(cls):\n """Return a new path pointing to the current working directory\n (as returned by os.getcwd()).\n """\n return cls(os.getcwd())\n\n @classmethod\n def home(cls):\n """Return a new path pointing to the user's home directory (as\n returned by os.path.expanduser('~')).\n """\n return cls(cls()._flavour.gethomedir(None))\n\n def samefile(self, other_path):\n """Return whether other_path is the same or not as this file\n (as returned by os.path.samefile()).\n """\n st = self.stat()\n try:\n other_st = other_path.stat()\n except AttributeError:\n other_st = os.stat(other_path)\n return os.path.samestat(st, other_st)\n\n def iterdir(self):\n """Iterate over the files in this directory. Does not yield any\n result for the special paths '.' and '..'.\n """\n if self._closed:\n self._raise_closed()\n for name in self._accessor.listdir(self):\n if name in {'.', '..'}:\n # Yielding a path object for these makes little sense\n continue\n yield self._make_child_relpath(name)\n if self._closed:\n self._raise_closed()\n\n def glob(self, pattern):\n """Iterate over this subtree and yield all existing files (of any\n kind, including directories) matching the given pattern.\n """\n if not pattern:\n raise ValueError("Unacceptable pattern: {!r}".format(pattern))\n pattern = self._flavour.casefold(pattern)\n drv, root, pattern_parts = self._flavour.parse_parts((pattern,))\n if drv or root:\n raise NotImplementedError("Non-relative patterns are unsupported")\n selector = _make_selector(tuple(pattern_parts))\n for p in selector.select_from(self):\n yield p\n\n def rglob(self, pattern):\n """Recursively yield all existing files (of any kind, including\n directories) matching the given pattern, anywhere in this subtree.\n """\n pattern = self._flavour.casefold(pattern)\n drv, root, pattern_parts = self._flavour.parse_parts((pattern,))\n if drv or root:\n raise NotImplementedError("Non-relative patterns are unsupported")\n selector = _make_selector(("**",) + tuple(pattern_parts))\n for p in selector.select_from(self):\n yield p\n\n def absolute(self):\n """Return an absolute version of this path. This function works\n even if the path doesn't point to anything.\n\n No normalization is done, i.e. all '.' and '..' will be kept along.\n Use resolve() to get the canonical path to a file.\n """\n # XXX untested yet!\n if self._closed:\n self._raise_closed()\n if self.is_absolute():\n return self\n # FIXME this must defer to the specific flavour (and, under Windows,\n # use nt._getfullpathname())\n obj = self._from_parts([os.getcwd()] + self._parts, init=False)\n obj._init(template=self)\n return obj\n\n def resolve(self, strict=False):\n """\n Make the path absolute, resolving all symlinks on the way and also\n normalizing it (for example turning slashes into backslashes under\n Windows).\n """\n if self._closed:\n self._raise_closed()\n s = self._flavour.resolve(self, strict=strict)\n if s is None:\n # No symlink resolution => for consistency, raise an error if\n # the path doesn't exist or is forbidden\n self.stat()\n s = str(self.absolute())\n # Now we have no symlinks in the path, it's safe to normalize it.\n normed = self._flavour.pathmod.normpath(s)\n obj = self._from_parts((normed,), init=False)\n obj._init(template=self)\n return obj\n\n def stat(self):\n """\n Return the result of the stat() system call on this path, like\n os.stat() does.\n """\n return self._accessor.stat(self)\n\n def owner(self):\n """\n Return the login name of the file owner.\n """\n import pwd\n return pwd.getpwuid(self.stat().st_uid).pw_name\n\n def group(self):\n """\n Return the group name of the file gid.\n """\n import grp\n return grp.getgrgid(self.stat().st_gid).gr_name\n\n def open(self, mode='r', buffering=-1, encoding=None,\n errors=None, newline=None):\n """\n Open the file pointed by this path and return a file object, as\n the built-in open() function does.\n """\n if self._closed:\n self._raise_closed()\n return io.open(str(self), mode, buffering, encoding, errors, newline,\n opener=self._opener)\n\n def read_bytes(self):\n """\n Open the file in bytes mode, read it, and close the file.\n """\n with self.open(mode='rb') as f:\n return f.read()\n\n def read_text(self, encoding=None, errors=None):\n """\n Open the file in text mode, read it, and close the file.\n """\n with self.open(mode='r', encoding=encoding, errors=errors) as f:\n return f.read()\n\n def write_bytes(self, data):\n """\n Open the file in bytes mode, write to it, and close the file.\n """\n # type-check for the buffer interface before truncating the file\n view = memoryview(data)\n with self.open(mode='wb') as f:\n return f.write(view)\n\n def write_text(self, data, encoding=None, errors=None):\n """\n Open the file in text mode, write to it, and close the file.\n """\n if not isinstance(data, str):\n raise TypeError('data must be str, not %s' %\n data.__class__.__name__)\n with self.open(mode='w', encoding=encoding, errors=errors) as f:\n return f.write(data)\n\n def touch(self, mode=0o666, exist_ok=True):\n """\n Create this file with the given access mode, if it doesn't exist.\n """\n if self._closed:\n self._raise_closed()\n if exist_ok:\n # First try to bump modification time\n # Implementation note: GNU touch uses the UTIME_NOW option of\n # the utimensat() / futimens() functions.\n try:\n self._accessor.utime(self, None)\n except OSError:\n # Avoid exception chaining\n pass\n else:\n return\n flags = os.O_CREAT | os.O_WRONLY\n if not exist_ok:\n flags |= os.O_EXCL\n fd = self._raw_open(flags, mode)\n os.close(fd)\n\n def mkdir(self, mode=0o777, parents=False, exist_ok=False):\n """\n Create a new directory at this given path.\n """\n if self._closed:\n self._raise_closed()\n try:\n self._accessor.mkdir(self, mode)\n except FileNotFoundError:\n if not parents or self.parent == self:\n raise\n self.parent.mkdir(parents=True, exist_ok=True)\n self.mkdir(mode, parents=False, exist_ok=exist_ok)\n except OSError:\n # Cannot rely on checking for EEXIST, since the operating system\n # could give priority to other errors like EACCES or EROFS\n if not exist_ok or not self.is_dir():\n raise\n\n def chmod(self, mode):\n """\n Change the permissions of the path, like os.chmod().\n """\n if self._closed:\n self._raise_closed()\n self._accessor.chmod(self, mode)\n\n def lchmod(self, mode):\n """\n Like chmod(), except if the path points to a symlink, the symlink's\n permissions are changed, rather than its target's.\n """\n if self._closed:\n self._raise_closed()\n self._accessor.lchmod(self, mode)\n\n def unlink(self):\n """\n Remove this file or link.\n If the path is a directory, use rmdir() instead.\n """\n if self._closed:\n self._raise_closed()\n self._accessor.unlink(self)\n\n def rmdir(self):\n """\n Remove this directory. The directory must be empty.\n """\n if self._closed:\n self._raise_closed()\n self._accessor.rmdir(self)\n\n def lstat(self):\n """\n Like stat(), except if the path points to a symlink, the symlink's\n status information is returned, rather than its target's.\n """\n if self._closed:\n self._raise_closed()\n return self._accessor.lstat(self)\n\n def rename(self, target):\n """\n Rename this path to the given path.\n """\n if self._closed:\n self._raise_closed()\n self._accessor.rename(self, target)\n\n def replace(self, target):\n """\n Rename this path to the given path, clobbering the existing\n destination if it exists.\n """\n if self._closed:\n self._raise_closed()\n self._accessor.replace(self, target)\n\n def symlink_to(self, target, target_is_directory=False):\n """\n Make this path a symlink pointing to the given path.\n Note the order of arguments (self, target) is the reverse of os.symlink's.\n """\n if self._closed:\n self._raise_closed()\n self._accessor.symlink(target, self, target_is_directory)\n\n # Convenience functions for querying the stat results\n\n def exists(self):\n """\n Whether this path exists.\n """\n try:\n self.stat()\n except OSError as e:\n if e.errno not in (ENOENT, ENOTDIR):\n raise\n return False\n return True\n\n def is_dir(self):\n """\n Whether this path is a directory.\n """\n try:\n return S_ISDIR(self.stat().st_mode)\n except OSError as e:\n if e.errno not in (ENOENT, ENOTDIR):\n raise\n # Path doesn't exist or is a broken symlink\n # (see https://bitbucket.org/pitrou/pathlib/issue/12/)\n return False\n\n def is_file(self):\n """\n Whether this path is a regular file (also True for symlinks pointing\n to regular files).\n """\n try:\n return S_ISREG(self.stat().st_mode)\n except OSError as e:\n if e.errno not in (ENOENT, ENOTDIR):\n raise\n # Path doesn't exist or is a broken symlink\n # (see https://bitbucket.org/pitrou/pathlib/issue/12/)\n return False\n\n def is_symlink(self):\n """\n Whether this path is a symbolic link.\n """\n try:\n return S_ISLNK(self.lstat().st_mode)\n except OSError as e:\n if e.errno not in (ENOENT, ENOTDIR):\n raise\n # Path doesn't exist\n return False\n\n def is_block_device(self):\n """\n Whether this path is a block device.\n """\n try:\n return S_ISBLK(self.stat().st_mode)\n except OSError as e:\n if e.errno not in (ENOENT, ENOTDIR):\n raise\n # Path doesn't exist or is a broken symlink\n # (see https://bitbucket.org/pitrou/pathlib/issue/12/)\n return False\n\n def is_char_device(self):\n """\n Whether this path is a character device.\n """\n try:\n return S_ISCHR(self.stat().st_mode)\n except OSError as e:\n if e.errno not in (ENOENT, ENOTDIR):\n raise\n # Path doesn't exist or is a broken symlink\n # (see https://bitbucket.org/pitrou/pathlib/issue/12/)\n return False\n\n def is_fifo(self):\n """\n Whether this path is a FIFO.\n """\n try:\n return S_ISFIFO(self.stat().st_mode)\n except OSError as e:\n if e.errno not in (ENOENT, ENOTDIR):\n raise\n # Path doesn't exist or is a broken symlink\n # (see https://bitbucket.org/pitrou/pathlib/issue/12/)\n return False\n\n def is_socket(self):\n """\n Whether this path is a socket.\n """\n try:\n return S_ISSOCK(self.stat().st_mode)\n except OSError as e:\n if e.errno not in (ENOENT, ENOTDIR):\n raise\n # Path doesn't exist or is a broken symlink\n # (see https://bitbucket.org/pitrou/pathlib/issue/12/)\n return False\n\n def expanduser(self):\n """ Return a new path with expanded ~ and ~user constructs\n (as returned by os.path.expanduser)\n """\n if (not (self._drv or self._root) and\n self._parts and self._parts[0][:1] == '~'):\n homedir = self._flavour.gethomedir(self._parts[0][1:])\n return self._from_parts([homedir] + self._parts[1:])\n\n return self\n\n\nclass PosixPath(Path, PurePosixPath):\n """Path subclass for non-Windows systems.\n\n On a POSIX system, instantiating a Path should return this object.\n """\n __slots__ = ()\n\nclass WindowsPath(Path, PureWindowsPath):\n """Path subclass for Windows systems.\n\n On a Windows system, instantiating a Path should return this object.\n """\n __slots__ = ()\n\n def owner(self):\n raise NotImplementedError("Path.owner() is unsupported on this system")\n\n def group(self):\n raise NotImplementedError("Path.group() is unsupported on this system")\n' + _('Hide Search Matches') + '
')\n .appendTo($('#searchbox'));\n }\n },\n\n /**\n * init the domain index toggle buttons\n */\n initIndexTable : function() {\n var togglers = $('img.toggler').click(function() {\n var src = $(this).attr('src');\n var idnum = $(this).attr('id').substr(7);\n $('tr.cg-' + idnum).toggle();\n if (src.substr(-9) === 'minus.png')\n $(this).attr('src', src.substr(0, src.length-9) + 'plus.png');\n else\n $(this).attr('src', src.substr(0, src.length-8) + 'minus.png');\n }).css('display', '');\n if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) {\n togglers.click();\n }\n },\n\n /**\n * helper function to hide the search marks again\n */\n hideSearchWords : function() {\n $('#searchbox .highlight-link').fadeOut(300);\n $('span.highlighted').removeClass('highlighted');\n },\n\n /**\n * make the url absolute\n */\n makeURL : function(relativeURL) {\n return DOCUMENTATION_OPTIONS.URL_ROOT + '/' + relativeURL;\n },\n\n /**\n * get the current relative url\n */\n getCurrentURL : function() {\n var path = document.location.pathname;\n var parts = path.split(/\\//);\n $.each(DOCUMENTATION_OPTIONS.URL_ROOT.split(/\\//), function() {\n if (this === '..')\n parts.pop();\n });\n var url = parts.join('/');\n return path.substring(url.lastIndexOf('/') + 1, path.length - 1);\n },\n\n initOnKeyListeners: function() {\n $(document).keyup(function(event) {\n var activeElementType = document.activeElement.tagName;\n // don't navigate when in search box or textarea\n if (activeElementType !== 'TEXTAREA' && activeElementType !== 'INPUT' && activeElementType !== 'SELECT') {\n switch (event.keyCode) {\n case 37: // left\n var prevHref = $('link[rel=\"prev\"]').prop('href');\n if (prevHref) {\n window.location.href = prevHref;\n return false;\n }\n case 39: // right\n var nextHref = $('link[rel=\"next\"]').prop('href');\n if (nextHref) {\n window.location.href = nextHref;\n return false;\n }\n }\n }\n });\n }\n};\n\n// quick alias for translations\n_ = Documentation.gettext;\n\n$(document).ready(function() {\n Documentation.init();\n});\n" }, { "path": "docs/_build/html/_static/documentation_options.js", "content": "var DOCUMENTATION_OPTIONS = {\n URL_ROOT: document.getElementById(\"documentation_options\").getAttribute('data-url_root'),\n VERSION: '0.2.0',\n LANGUAGE: 'None',\n COLLAPSE_INDEX: false,\n FILE_SUFFIX: '.html',\n HAS_SOURCE: true,\n SOURCELINK_SUFFIX: '.txt',\n NAVIGATION_WITH_KEYS: false\n};" }, { "path": "docs/_build/html/_static/jquery-3.2.1.js", "content": "/*!\n * jQuery JavaScript Library v3.2.1\n * https://jquery.com/\n *\n * Includes Sizzle.js\n * https://sizzlejs.com/\n *\n * Copyright JS Foundation and other contributors\n * Released under the MIT license\n * https://jquery.org/license\n *\n * Date: 2017-03-20T18:59Z\n */\n( function( global, factory ) {\n\n\t\"use strict\";\n\n\tif ( typeof module === \"object\" && typeof module.exports === \"object\" ) {\n\n\t\t// For CommonJS and CommonJS-like environments where a proper `window`\n\t\t// is present, execute the factory and get jQuery.\n\t\t// For environments that do not have a `window` with a `document`\n\t\t// (such as Node.js), expose a factory as module.exports.\n\t\t// This accentuates the need for the creation of a real `window`.\n\t\t// e.g. var jQuery = require(\"jquery\")(window);\n\t\t// See ticket #14549 for more info.\n\t\tmodule.exports = global.document ?\n\t\t\tfactory( global, true ) :\n\t\t\tfunction( w ) {\n\t\t\t\tif ( !w.document ) {\n\t\t\t\t\tthrow new Error( \"jQuery requires a window with a document\" );\n\t\t\t\t}\n\t\t\t\treturn factory( w );\n\t\t\t};\n\t} else {\n\t\tfactory( global );\n\t}\n\n// Pass this if window is not defined yet\n} )( typeof window !== \"undefined\" ? window : this, function( window, noGlobal ) {\n\n// Edge <= 12 - 13+, Firefox <=18 - 45+, IE 10 - 11, Safari 5.1 - 9+, iOS 6 - 9.1\n// throw exceptions when non-strict code (e.g., ASP.NET 4.5) accesses strict mode\n// arguments.callee.caller (trac-13335). But as of jQuery 3.0 (2016), strict mode should be common\n// enough that all such attempts are guarded in a try block.\n\"use strict\";\n\nvar arr = [];\n\nvar document = window.document;\n\nvar getProto = Object.getPrototypeOf;\n\nvar slice = arr.slice;\n\nvar concat = arr.concat;\n\nvar push = arr.push;\n\nvar indexOf = arr.indexOf;\n\nvar class2type = {};\n\nvar toString = class2type.toString;\n\nvar hasOwn = class2type.hasOwnProperty;\n\nvar fnToString = hasOwn.toString;\n\nvar ObjectFunctionString = fnToString.call( Object );\n\nvar support = {};\n\n\n\n\tfunction DOMEval( code, doc ) {\n\t\tdoc = doc || document;\n\n\t\tvar script = doc.createElement( \"script\" );\n\n\t\tscript.text = code;\n\t\tdoc.head.appendChild( script ).parentNode.removeChild( script );\n\t}\n/* global Symbol */\n// Defining this global in .eslintrc.json would create a danger of using the global\n// unguarded in another place, it seems safer to define global only for this module\n\n\n\nvar\n\tversion = \"3.2.1\",\n\n\t// Define a local copy of jQuery\n\tjQuery = function( selector, context ) {\n\n\t\t// The jQuery object is actually just the init constructor 'enhanced'\n\t\t// Need init if jQuery is called (just allow error to be thrown if not included)\n\t\treturn new jQuery.fn.init( selector, context );\n\t},\n\n\t// Support: Android <=4.0 only\n\t// Make sure we trim BOM and NBSP\n\trtrim = /^[\\s\\uFEFF\\xA0]+|[\\s\\uFEFF\\xA0]+$/g,\n\n\t// Matches dashed string for camelizing\n\trmsPrefix = /^-ms-/,\n\trdashAlpha = /-([a-z])/g,\n\n\t// Used by jQuery.camelCase as callback to replace()\n\tfcamelCase = function( all, letter ) {\n\t\treturn letter.toUpperCase();\n\t};\n\njQuery.fn = jQuery.prototype = {\n\n\t// The current version of jQuery being used\n\tjquery: version,\n\n\tconstructor: jQuery,\n\n\t// The default length of a jQuery object is 0\n\tlength: 0,\n\n\ttoArray: function() {\n\t\treturn slice.call( this );\n\t},\n\n\t// Get the Nth element in the matched element set OR\n\t// Get the whole matched element set as a clean array\n\tget: function( num ) {\n\n\t\t// Return all the elements in a clean array\n\t\tif ( num == null ) {\n\t\t\treturn slice.call( this );\n\t\t}\n\n\t\t// Return just the one element from the set\n\t\treturn num < 0 ? this[ num + this.length ] : this[ num ];\n\t},\n\n\t// Take an array of elements and push it onto the stack\n\t// (returning the new matched element set)\n\tpushStack: function( elems ) {\n\n\t\t// Build a new jQuery matched element set\n\t\tvar ret = jQuery.merge( this.constructor(), elems );\n\n\t\t// Add the old object onto the stack (as a reference)\n\t\tret.prevObject = this;\n\n\t\t// Return the newly-formed element set\n\t\treturn ret;\n\t},\n\n\t// Execute a callback for every element in the matched set.\n\teach: function( callback ) {\n\t\treturn jQuery.each( this, callback );\n\t},\n\n\tmap: function( callback ) {\n\t\treturn this.pushStack( jQuery.map( this, function( elem, i ) {\n\t\t\treturn callback.call( elem, i, elem );\n\t\t} ) );\n\t},\n\n\tslice: function() {\n\t\treturn this.pushStack( slice.apply( this, arguments ) );\n\t},\n\n\tfirst: function() {\n\t\treturn this.eq( 0 );\n\t},\n\n\tlast: function() {\n\t\treturn this.eq( -1 );\n\t},\n\n\teq: function( i ) {\n\t\tvar len = this.length,\n\t\t\tj = +i + ( i < 0 ? len : 0 );\n\t\treturn this.pushStack( j >= 0 && j < len ? [ this[ j ] ] : [] );\n\t},\n\n\tend: function() {\n\t\treturn this.prevObject || this.constructor();\n\t},\n\n\t// For internal use only.\n\t// Behaves like an Array's method, not like a jQuery method.\n\tpush: push,\n\tsort: arr.sort,\n\tsplice: arr.splice\n};\n\njQuery.extend = jQuery.fn.extend = function() {\n\tvar options, name, src, copy, copyIsArray, clone,\n\t\ttarget = arguments[ 0 ] || {},\n\t\ti = 1,\n\t\tlength = arguments.length,\n\t\tdeep = false;\n\n\t// Handle a deep copy situation\n\tif ( typeof target === \"boolean\" ) {\n\t\tdeep = target;\n\n\t\t// Skip the boolean and the target\n\t\ttarget = arguments[ i ] || {};\n\t\ti++;\n\t}\n\n\t// Handle case when target is a string or something (possible in deep copy)\n\tif ( typeof target !== \"object\" && !jQuery.isFunction( target ) ) {\n\t\ttarget = {};\n\t}\n\n\t// Extend jQuery itself if only one argument is passed\n\tif ( i === length ) {\n\t\ttarget = this;\n\t\ti--;\n\t}\n\n\tfor ( ; i < length; i++ ) {\n\n\t\t// Only deal with non-null/undefined values\n\t\tif ( ( options = arguments[ i ] ) != null ) {\n\n\t\t\t// Extend the base object\n\t\t\tfor ( name in options ) {\n\t\t\t\tsrc = target[ name ];\n\t\t\t\tcopy = options[ name ];\n\n\t\t\t\t// Prevent never-ending loop\n\t\t\t\tif ( target === copy ) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\n\t\t\t\t// Recurse if we're merging plain objects or arrays\n\t\t\t\tif ( deep && copy && ( jQuery.isPlainObject( copy ) ||\n\t\t\t\t\t( copyIsArray = Array.isArray( copy ) ) ) ) {\n\n\t\t\t\t\tif ( copyIsArray ) {\n\t\t\t\t\t\tcopyIsArray = false;\n\t\t\t\t\t\tclone = src && Array.isArray( src ) ? src : [];\n\n\t\t\t\t\t} else {\n\t\t\t\t\t\tclone = src && jQuery.isPlainObject( src ) ? src : {};\n\t\t\t\t\t}\n\n\t\t\t\t\t// Never move original objects, clone them\n\t\t\t\t\ttarget[ name ] = jQuery.extend( deep, clone, copy );\n\n\t\t\t\t// Don't bring in undefined values\n\t\t\t\t} else if ( copy !== undefined ) {\n\t\t\t\t\ttarget[ name ] = copy;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Return the modified object\n\treturn target;\n};\n\njQuery.extend( {\n\n\t// Unique for each copy of jQuery on the page\n\texpando: \"jQuery\" + ( version + Math.random() ).replace( /\\D/g, \"\" ),\n\n\t// Assume jQuery is ready without the ready module\n\tisReady: true,\n\n\terror: function( msg ) {\n\t\tthrow new Error( msg );\n\t},\n\n\tnoop: function() {},\n\n\tisFunction: function( obj ) {\n\t\treturn jQuery.type( obj ) === \"function\";\n\t},\n\n\tisWindow: function( obj ) {\n\t\treturn obj != null && obj === obj.window;\n\t},\n\n\tisNumeric: function( obj ) {\n\n\t\t// As of jQuery 3.0, isNumeric is limited to\n\t\t// strings and numbers (primitives or objects)\n\t\t// that can be coerced to finite numbers (gh-2662)\n\t\tvar type = jQuery.type( obj );\n\t\treturn ( type === \"number\" || type === \"string\" ) &&\n\n\t\t\t// parseFloat NaNs numeric-cast false positives (\"\")\n\t\t\t// ...but misinterprets leading-number strings, particularly hex literals (\"0x...\")\n\t\t\t// subtraction forces infinities to NaN\n\t\t\t!isNaN( obj - parseFloat( obj ) );\n\t},\n\n\tisPlainObject: function( obj ) {\n\t\tvar proto, Ctor;\n\n\t\t// Detect obvious negatives\n\t\t// Use toString instead of jQuery.type to catch host objects\n\t\tif ( !obj || toString.call( obj ) !== \"[object Object]\" ) {\n\t\t\treturn false;\n\t\t}\n\n\t\tproto = getProto( obj );\n\n\t\t// Objects with no prototype (e.g., `Object.create( null )`) are plain\n\t\tif ( !proto ) {\n\t\t\treturn true;\n\t\t}\n\n\t\t// Objects with prototype are plain iff they were constructed by a global Object function\n\t\tCtor = hasOwn.call( proto, \"constructor\" ) && proto.constructor;\n\t\treturn typeof Ctor === \"function\" && fnToString.call( Ctor ) === ObjectFunctionString;\n\t},\n\n\tisEmptyObject: function( obj ) {\n\n\t\t/* eslint-disable no-unused-vars */\n\t\t// See https://github.com/eslint/eslint/issues/6125\n\t\tvar name;\n\n\t\tfor ( name in obj ) {\n\t\t\treturn false;\n\t\t}\n\t\treturn true;\n\t},\n\n\ttype: function( obj ) {\n\t\tif ( obj == null ) {\n\t\t\treturn obj + \"\";\n\t\t}\n\n\t\t// Support: Android <=2.3 only (functionish RegExp)\n\t\treturn typeof obj === \"object\" || typeof obj === \"function\" ?\n\t\t\tclass2type[ toString.call( obj ) ] || \"object\" :\n\t\t\ttypeof obj;\n\t},\n\n\t// Evaluates a script in a global context\n\tglobalEval: function( code ) {\n\t\tDOMEval( code );\n\t},\n\n\t// Convert dashed to camelCase; used by the css and data modules\n\t// Support: IE <=9 - 11, Edge 12 - 13\n\t// Microsoft forgot to hump their vendor prefix (#9572)\n\tcamelCase: function( string ) {\n\t\treturn string.replace( rmsPrefix, \"ms-\" ).replace( rdashAlpha, fcamelCase );\n\t},\n\n\teach: function( obj, callback ) {\n\t\tvar length, i = 0;\n\n\t\tif ( isArrayLike( obj ) ) {\n\t\t\tlength = obj.length;\n\t\t\tfor ( ; i < length; i++ ) {\n\t\t\t\tif ( callback.call( obj[ i ], i, obj[ i ] ) === false ) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tfor ( i in obj ) {\n\t\t\t\tif ( callback.call( obj[ i ], i, obj[ i ] ) === false ) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn obj;\n\t},\n\n\t// Support: Android <=4.0 only\n\ttrim: function( text ) {\n\t\treturn text == null ?\n\t\t\t\"\" :\n\t\t\t( text + \"\" ).replace( rtrim, \"\" );\n\t},\n\n\t// results is for internal usage only\n\tmakeArray: function( arr, results ) {\n\t\tvar ret = results || [];\n\n\t\tif ( arr != null ) {\n\t\t\tif ( isArrayLike( Object( arr ) ) ) {\n\t\t\t\tjQuery.merge( ret,\n\t\t\t\t\ttypeof arr === \"string\" ?\n\t\t\t\t\t[ arr ] : arr\n\t\t\t\t);\n\t\t\t} else {\n\t\t\t\tpush.call( ret, arr );\n\t\t\t}\n\t\t}\n\n\t\treturn ret;\n\t},\n\n\tinArray: function( elem, arr, i ) {\n\t\treturn arr == null ? -1 : indexOf.call( arr, elem, i );\n\t},\n\n\t// Support: Android <=4.0 only, PhantomJS 1 only\n\t// push.apply(_, arraylike) throws on ancient WebKit\n\tmerge: function( first, second ) {\n\t\tvar len = +second.length,\n\t\t\tj = 0,\n\t\t\ti = first.length;\n\n\t\tfor ( ; j < len; j++ ) {\n\t\t\tfirst[ i++ ] = second[ j ];\n\t\t}\n\n\t\tfirst.length = i;\n\n\t\treturn first;\n\t},\n\n\tgrep: function( elems, callback, invert ) {\n\t\tvar callbackInverse,\n\t\t\tmatches = [],\n\t\t\ti = 0,\n\t\t\tlength = elems.length,\n\t\t\tcallbackExpect = !invert;\n\n\t\t// Go through the array, only saving the items\n\t\t// that pass the validator function\n\t\tfor ( ; i < length; i++ ) {\n\t\t\tcallbackInverse = !callback( elems[ i ], i );\n\t\t\tif ( callbackInverse !== callbackExpect ) {\n\t\t\t\tmatches.push( elems[ i ] );\n\t\t\t}\n\t\t}\n\n\t\treturn matches;\n\t},\n\n\t// arg is for internal usage only\n\tmap: function( elems, callback, arg ) {\n\t\tvar length, value,\n\t\t\ti = 0,\n\t\t\tret = [];\n\n\t\t// Go through the array, translating each of the items to their new values\n\t\tif ( isArrayLike( elems ) ) {\n\t\t\tlength = elems.length;\n\t\t\tfor ( ; i < length; i++ ) {\n\t\t\t\tvalue = callback( elems[ i ], i, arg );\n\n\t\t\t\tif ( value != null ) {\n\t\t\t\t\tret.push( value );\n\t\t\t\t}\n\t\t\t}\n\n\t\t// Go through every key on the object,\n\t\t} else {\n\t\t\tfor ( i in elems ) {\n\t\t\t\tvalue = callback( elems[ i ], i, arg );\n\n\t\t\t\tif ( value != null ) {\n\t\t\t\t\tret.push( value );\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Flatten any nested arrays\n\t\treturn concat.apply( [], ret );\n\t},\n\n\t// A global GUID counter for objects\n\tguid: 1,\n\n\t// Bind a function to a context, optionally partially applying any\n\t// arguments.\n\tproxy: function( fn, context ) {\n\t\tvar tmp, args, proxy;\n\n\t\tif ( typeof context === \"string\" ) {\n\t\t\ttmp = fn[ context ];\n\t\t\tcontext = fn;\n\t\t\tfn = tmp;\n\t\t}\n\n\t\t// Quick check to determine if target is callable, in the spec\n\t\t// this throws a TypeError, but we will just return undefined.\n\t\tif ( !jQuery.isFunction( fn ) ) {\n\t\t\treturn undefined;\n\t\t}\n\n\t\t// Simulated bind\n\t\targs = slice.call( arguments, 2 );\n\t\tproxy = function() {\n\t\t\treturn fn.apply( context || this, args.concat( slice.call( arguments ) ) );\n\t\t};\n\n\t\t// Set the guid of unique handler to the same of original handler, so it can be removed\n\t\tproxy.guid = fn.guid = fn.guid || jQuery.guid++;\n\n\t\treturn proxy;\n\t},\n\n\tnow: Date.now,\n\n\t// jQuery.support is not used in Core but other projects attach their\n\t// properties to it so it needs to exist.\n\tsupport: support\n} );\n\nif ( typeof Symbol === \"function\" ) {\n\tjQuery.fn[ Symbol.iterator ] = arr[ Symbol.iterator ];\n}\n\n// Populate the class2type map\njQuery.each( \"Boolean Number String Function Array Date RegExp Object Error Symbol\".split( \" \" ),\nfunction( i, name ) {\n\tclass2type[ \"[object \" + name + \"]\" ] = name.toLowerCase();\n} );\n\nfunction isArrayLike( obj ) {\n\n\t// Support: real iOS 8.2 only (not reproducible in simulator)\n\t// `in` check used to prevent JIT error (gh-2145)\n\t// hasOwn isn't used here due to false negatives\n\t// regarding Nodelist length in IE\n\tvar length = !!obj && \"length\" in obj && obj.length,\n\t\ttype = jQuery.type( obj );\n\n\tif ( type === \"function\" || jQuery.isWindow( obj ) ) {\n\t\treturn false;\n\t}\n\n\treturn type === \"array\" || length === 0 ||\n\t\ttypeof length === \"number\" && length > 0 && ( length - 1 ) in obj;\n}\nvar Sizzle =\n/*!\n * Sizzle CSS Selector Engine v2.3.3\n * https://sizzlejs.com/\n *\n * Copyright jQuery Foundation and other contributors\n * Released under the MIT license\n * http://jquery.org/license\n *\n * Date: 2016-08-08\n */\n(function( window ) {\n\nvar i,\n\tsupport,\n\tExpr,\n\tgetText,\n\tisXML,\n\ttokenize,\n\tcompile,\n\tselect,\n\toutermostContext,\n\tsortInput,\n\thasDuplicate,\n\n\t// Local document vars\n\tsetDocument,\n\tdocument,\n\tdocElem,\n\tdocumentIsHTML,\n\trbuggyQSA,\n\trbuggyMatches,\n\tmatches,\n\tcontains,\n\n\t// Instance-specific data\n\texpando = \"sizzle\" + 1 * new Date(),\n\tpreferredDoc = window.document,\n\tdirruns = 0,\n\tdone = 0,\n\tclassCache = createCache(),\n\ttokenCache = createCache(),\n\tcompilerCache = createCache(),\n\tsortOrder = function( a, b ) {\n\t\tif ( a === b ) {\n\t\t\thasDuplicate = true;\n\t\t}\n\t\treturn 0;\n\t},\n\n\t// Instance methods\n\thasOwn = ({}).hasOwnProperty,\n\tarr = [],\n\tpop = arr.pop,\n\tpush_native = arr.push,\n\tpush = arr.push,\n\tslice = arr.slice,\n\t// Use a stripped-down indexOf as it's faster than native\n\t// https://jsperf.com/thor-indexof-vs-for/5\n\tindexOf = function( list, elem ) {\n\t\tvar i = 0,\n\t\t\tlen = list.length;\n\t\tfor ( ; i < len; i++ ) {\n\t\t\tif ( list[i] === elem ) {\n\t\t\t\treturn i;\n\t\t\t}\n\t\t}\n\t\treturn -1;\n\t},\n\n\tbooleans = \"checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped\",\n\n\t// Regular expressions\n\n\t// http://www.w3.org/TR/css3-selectors/#whitespace\n\twhitespace = \"[\\\\x20\\\\t\\\\r\\\\n\\\\f]\",\n\n\t// http://www.w3.org/TR/CSS21/syndata.html#value-def-identifier\n\tidentifier = \"(?:\\\\\\\\.|[\\\\w-]|[^\\0-\\\\xa0])+\",\n\n\t// Attribute selectors: http://www.w3.org/TR/selectors/#attribute-selectors\n\tattributes = \"\\\\[\" + whitespace + \"*(\" + identifier + \")(?:\" + whitespace +\n\t\t// Operator (capture 2)\n\t\t\"*([*^$|!~]?=)\" + whitespace +\n\t\t// \"Attribute values must be CSS identifiers [capture 5] or strings [capture 3 or capture 4]\"\n\t\t\"*(?:'((?:\\\\\\\\.|[^\\\\\\\\'])*)'|\\\"((?:\\\\\\\\.|[^\\\\\\\\\\\"])*)\\\"|(\" + identifier + \"))|)\" + whitespace +\n\t\t\"*\\\\]\",\n\n\tpseudos = \":(\" + identifier + \")(?:\\\\((\" +\n\t\t// To reduce the number of selectors needing tokenize in the preFilter, prefer arguments:\n\t\t// 1. quoted (capture 3; capture 4 or capture 5)\n\t\t\"('((?:\\\\\\\\.|[^\\\\\\\\'])*)'|\\\"((?:\\\\\\\\.|[^\\\\\\\\\\\"])*)\\\")|\" +\n\t\t// 2. simple (capture 6)\n\t\t\"((?:\\\\\\\\.|[^\\\\\\\\()[\\\\]]|\" + attributes + \")*)|\" +\n\t\t// 3. anything else (capture 2)\n\t\t\".*\" +\n\t\t\")\\\\)|)\",\n\n\t// Leading and non-escaped trailing whitespace, capturing some non-whitespace characters preceding the latter\n\trwhitespace = new RegExp( whitespace + \"+\", \"g\" ),\n\trtrim = new RegExp( \"^\" + whitespace + \"+|((?:^|[^\\\\\\\\])(?:\\\\\\\\.)*)\" + whitespace + \"+$\", \"g\" ),\n\n\trcomma = new RegExp( \"^\" + whitespace + \"*,\" + whitespace + \"*\" ),\n\trcombinators = new RegExp( \"^\" + whitespace + \"*([>+~]|\" + whitespace + \")\" + whitespace + \"*\" ),\n\n\trattributeQuotes = new RegExp( \"=\" + whitespace + \"*([^\\\\]'\\\"]*?)\" + whitespace + \"*\\\\]\", \"g\" ),\n\n\trpseudo = new RegExp( pseudos ),\n\tridentifier = new RegExp( \"^\" + identifier + \"$\" ),\n\n\tmatchExpr = {\n\t\t\"ID\": new RegExp( \"^#(\" + identifier + \")\" ),\n\t\t\"CLASS\": new RegExp( \"^\\\\.(\" + identifier + \")\" ),\n\t\t\"TAG\": new RegExp( \"^(\" + identifier + \"|[*])\" ),\n\t\t\"ATTR\": new RegExp( \"^\" + attributes ),\n\t\t\"PSEUDO\": new RegExp( \"^\" + pseudos ),\n\t\t\"CHILD\": new RegExp( \"^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\\\(\" + whitespace +\n\t\t\t\"*(even|odd|(([+-]|)(\\\\d*)n|)\" + whitespace + \"*(?:([+-]|)\" + whitespace +\n\t\t\t\"*(\\\\d+)|))\" + whitespace + \"*\\\\)|)\", \"i\" ),\n\t\t\"bool\": new RegExp( \"^(?:\" + booleans + \")$\", \"i\" ),\n\t\t// For use in libraries implementing .is()\n\t\t// We use this for POS matching in `select`\n\t\t\"needsContext\": new RegExp( \"^\" + whitespace + \"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\\\(\" +\n\t\t\twhitespace + \"*((?:-\\\\d)?\\\\d*)\" + whitespace + \"*\\\\)|)(?=[^-]|$)\", \"i\" )\n\t},\n\n\trinputs = /^(?:input|select|textarea|button)$/i,\n\trheader = /^h\\d$/i,\n\n\trnative = /^[^{]+\\{\\s*\\[native \\w/,\n\n\t// Easily-parseable/retrievable ID or TAG or CLASS selectors\n\trquickExpr = /^(?:#([\\w-]+)|(\\w+)|\\.([\\w-]+))$/,\n\n\trsibling = /[+~]/,\n\n\t// CSS escapes\n\t// http://www.w3.org/TR/CSS21/syndata.html#escaped-characters\n\trunescape = new RegExp( \"\\\\\\\\([\\\\da-f]{1,6}\" + whitespace + \"?|(\" + whitespace + \")|.)\", \"ig\" ),\n\tfunescape = function( _, escaped, escapedWhitespace ) {\n\t\tvar high = \"0x\" + escaped - 0x10000;\n\t\t// NaN means non-codepoint\n\t\t// Support: Firefox<24\n\t\t// Workaround erroneous numeric interpretation of +\"0x\"\n\t\treturn high !== high || escapedWhitespace ?\n\t\t\tescaped :\n\t\t\thigh < 0 ?\n\t\t\t\t// BMP codepoint\n\t\t\t\tString.fromCharCode( high + 0x10000 ) :\n\t\t\t\t// Supplemental Plane codepoint (surrogate pair)\n\t\t\t\tString.fromCharCode( high >> 10 | 0xD800, high & 0x3FF | 0xDC00 );\n\t},\n\n\t// CSS string/identifier serialization\n\t// https://drafts.csswg.org/cssom/#common-serializing-idioms\n\trcssescape = /([\\0-\\x1f\\x7f]|^-?\\d)|^-$|[^\\0-\\x1f\\x7f-\\uFFFF\\w-]/g,\n\tfcssescape = function( ch, asCodePoint ) {\n\t\tif ( asCodePoint ) {\n\n\t\t\t// U+0000 NULL becomes U+FFFD REPLACEMENT CHARACTER\n\t\t\tif ( ch === \"\\0\" ) {\n\t\t\t\treturn \"\\uFFFD\";\n\t\t\t}\n\n\t\t\t// Control characters and (dependent upon position) numbers get escaped as code points\n\t\t\treturn ch.slice( 0, -1 ) + \"\\\\\" + ch.charCodeAt( ch.length - 1 ).toString( 16 ) + \" \";\n\t\t}\n\n\t\t// Other potentially-special ASCII characters get backslash-escaped\n\t\treturn \"\\\\\" + ch;\n\t},\n\n\t// Used for iframes\n\t// See setDocument()\n\t// Removing the function wrapper causes a \"Permission Denied\"\n\t// error in IE\n\tunloadHandler = function() {\n\t\tsetDocument();\n\t},\n\n\tdisabledAncestor = addCombinator(\n\t\tfunction( elem ) {\n\t\t\treturn elem.disabled === true && (\"form\" in elem || \"label\" in elem);\n\t\t},\n\t\t{ dir: \"parentNode\", next: \"legend\" }\n\t);\n\n// Optimize for push.apply( _, NodeList )\ntry {\n\tpush.apply(\n\t\t(arr = slice.call( preferredDoc.childNodes )),\n\t\tpreferredDoc.childNodes\n\t);\n\t// Support: Android<4.0\n\t// Detect silently failing push.apply\n\tarr[ preferredDoc.childNodes.length ].nodeType;\n} catch ( e ) {\n\tpush = { apply: arr.length ?\n\n\t\t// Leverage slice if possible\n\t\tfunction( target, els ) {\n\t\t\tpush_native.apply( target, slice.call(els) );\n\t\t} :\n\n\t\t// Support: IE<9\n\t\t// Otherwise append directly\n\t\tfunction( target, els ) {\n\t\t\tvar j = target.length,\n\t\t\t\ti = 0;\n\t\t\t// Can't trust NodeList.length\n\t\t\twhile ( (target[j++] = els[i++]) ) {}\n\t\t\ttarget.length = j - 1;\n\t\t}\n\t};\n}\n\nfunction Sizzle( selector, context, results, seed ) {\n\tvar m, i, elem, nid, match, groups, newSelector,\n\t\tnewContext = context && context.ownerDocument,\n\n\t\t// nodeType defaults to 9, since context defaults to document\n\t\tnodeType = context ? context.nodeType : 9;\n\n\tresults = results || [];\n\n\t// Return early from calls with invalid selector or context\n\tif ( typeof selector !== \"string\" || !selector ||\n\t\tnodeType !== 1 && nodeType !== 9 && nodeType !== 11 ) {\n\n\t\treturn results;\n\t}\n\n\t// Try to shortcut find operations (as opposed to filters) in HTML documents\n\tif ( !seed ) {\n\n\t\tif ( ( context ? context.ownerDocument || context : preferredDoc ) !== document ) {\n\t\t\tsetDocument( context );\n\t\t}\n\t\tcontext = context || document;\n\n\t\tif ( documentIsHTML ) {\n\n\t\t\t// If the selector is sufficiently simple, try using a \"get*By*\" DOM method\n\t\t\t// (excepting DocumentFragment context, where the methods don't exist)\n\t\t\tif ( nodeType !== 11 && (match = rquickExpr.exec( selector )) ) {\n\n\t\t\t\t// ID selector\n\t\t\t\tif ( (m = match[1]) ) {\n\n\t\t\t\t\t// Document context\n\t\t\t\t\tif ( nodeType === 9 ) {\n\t\t\t\t\t\tif ( (elem = context.getElementById( m )) ) {\n\n\t\t\t\t\t\t\t// Support: IE, Opera, Webkit\n\t\t\t\t\t\t\t// TODO: identify versions\n\t\t\t\t\t\t\t// getElementById can match elements by name instead of ID\n\t\t\t\t\t\t\tif ( elem.id === m ) {\n\t\t\t\t\t\t\t\tresults.push( elem );\n\t\t\t\t\t\t\t\treturn results;\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\treturn results;\n\t\t\t\t\t\t}\n\n\t\t\t\t\t// Element context\n\t\t\t\t\t} else {\n\n\t\t\t\t\t\t// Support: IE, Opera, Webkit\n\t\t\t\t\t\t// TODO: identify versions\n\t\t\t\t\t\t// getElementById can match elements by name instead of ID\n\t\t\t\t\t\tif ( newContext && (elem = newContext.getElementById( m )) &&\n\t\t\t\t\t\t\tcontains( context, elem ) &&\n\t\t\t\t\t\t\telem.id === m ) {\n\n\t\t\t\t\t\t\tresults.push( elem );\n\t\t\t\t\t\t\treturn results;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t// Type selector\n\t\t\t\t} else if ( match[2] ) {\n\t\t\t\t\tpush.apply( results, context.getElementsByTagName( selector ) );\n\t\t\t\t\treturn results;\n\n\t\t\t\t// Class selector\n\t\t\t\t} else if ( (m = match[3]) && support.getElementsByClassName &&\n\t\t\t\t\tcontext.getElementsByClassName ) {\n\n\t\t\t\t\tpush.apply( results, context.getElementsByClassName( m ) );\n\t\t\t\t\treturn results;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Take advantage of querySelectorAll\n\t\t\tif ( support.qsa &&\n\t\t\t\t!compilerCache[ selector + \" \" ] &&\n\t\t\t\t(!rbuggyQSA || !rbuggyQSA.test( selector )) ) {\n\n\t\t\t\tif ( nodeType !== 1 ) {\n\t\t\t\t\tnewContext = context;\n\t\t\t\t\tnewSelector = selector;\n\n\t\t\t\t// qSA looks outside Element context, which is not what we want\n\t\t\t\t// Thanks to Andrew Dupont for this workaround technique\n\t\t\t\t// Support: IE <=8\n\t\t\t\t// Exclude object elements\n\t\t\t\t} else if ( context.nodeName.toLowerCase() !== \"object\" ) {\n\n\t\t\t\t\t// Capture the context ID, setting it first if necessary\n\t\t\t\t\tif ( (nid = context.getAttribute( \"id\" )) ) {\n\t\t\t\t\t\tnid = nid.replace( rcssescape, fcssescape );\n\t\t\t\t\t} else {\n\t\t\t\t\t\tcontext.setAttribute( \"id\", (nid = expando) );\n\t\t\t\t\t}\n\n\t\t\t\t\t// Prefix every selector in the list\n\t\t\t\t\tgroups = tokenize( selector );\n\t\t\t\t\ti = groups.length;\n\t\t\t\t\twhile ( i-- ) {\n\t\t\t\t\t\tgroups[i] = \"#\" + nid + \" \" + toSelector( groups[i] );\n\t\t\t\t\t}\n\t\t\t\t\tnewSelector = groups.join( \",\" );\n\n\t\t\t\t\t// Expand context for sibling selectors\n\t\t\t\t\tnewContext = rsibling.test( selector ) && testContext( context.parentNode ) ||\n\t\t\t\t\t\tcontext;\n\t\t\t\t}\n\n\t\t\t\tif ( newSelector ) {\n\t\t\t\t\ttry {\n\t\t\t\t\t\tpush.apply( results,\n\t\t\t\t\t\t\tnewContext.querySelectorAll( newSelector )\n\t\t\t\t\t\t);\n\t\t\t\t\t\treturn results;\n\t\t\t\t\t} catch ( qsaError ) {\n\t\t\t\t\t} finally {\n\t\t\t\t\t\tif ( nid === expando ) {\n\t\t\t\t\t\t\tcontext.removeAttribute( \"id\" );\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// All others\n\treturn select( selector.replace( rtrim, \"$1\" ), context, results, seed );\n}\n\n/**\n * Create key-value caches of limited size\n * @returns {function(string, object)} Returns the Object data after storing it on itself with\n *\tproperty name the (space-suffixed) string and (if the cache is larger than Expr.cacheLength)\n *\tdeleting the oldest entry\n */\nfunction createCache() {\n\tvar keys = [];\n\n\tfunction cache( key, value ) {\n\t\t// Use (key + \" \") to avoid collision with native prototype properties (see Issue #157)\n\t\tif ( keys.push( key + \" \" ) > Expr.cacheLength ) {\n\t\t\t// Only keep the most recent entries\n\t\t\tdelete cache[ keys.shift() ];\n\t\t}\n\t\treturn (cache[ key + \" \" ] = value);\n\t}\n\treturn cache;\n}\n\n/**\n * Mark a function for special use by Sizzle\n * @param {Function} fn The function to mark\n */\nfunction markFunction( fn ) {\n\tfn[ expando ] = true;\n\treturn fn;\n}\n\n/**\n * Support testing using an element\n * @param {Function} fn Passed the created element and returns a boolean result\n */\nfunction assert( fn ) {\n\tvar el = document.createElement(\"fieldset\");\n\n\ttry {\n\t\treturn !!fn( el );\n\t} catch (e) {\n\t\treturn false;\n\t} finally {\n\t\t// Remove from its parent by default\n\t\tif ( el.parentNode ) {\n\t\t\tel.parentNode.removeChild( el );\n\t\t}\n\t\t// release memory in IE\n\t\tel = null;\n\t}\n}\n\n/**\n * Adds the same handler for all of the specified attrs\n * @param {String} attrs Pipe-separated list of attributes\n * @param {Function} handler The method that will be applied\n */\nfunction addHandle( attrs, handler ) {\n\tvar arr = attrs.split(\"|\"),\n\t\ti = arr.length;\n\n\twhile ( i-- ) {\n\t\tExpr.attrHandle[ arr[i] ] = handler;\n\t}\n}\n\n/**\n * Checks document order of two siblings\n * @param {Element} a\n * @param {Element} b\n * @returns {Number} Returns less than 0 if a precedes b, greater than 0 if a follows b\n */\nfunction siblingCheck( a, b ) {\n\tvar cur = b && a,\n\t\tdiff = cur && a.nodeType === 1 && b.nodeType === 1 &&\n\t\t\ta.sourceIndex - b.sourceIndex;\n\n\t// Use IE sourceIndex if available on both nodes\n\tif ( diff ) {\n\t\treturn diff;\n\t}\n\n\t// Check if b follows a\n\tif ( cur ) {\n\t\twhile ( (cur = cur.nextSibling) ) {\n\t\t\tif ( cur === b ) {\n\t\t\t\treturn -1;\n\t\t\t}\n\t\t}\n\t}\n\n\treturn a ? 1 : -1;\n}\n\n/**\n * Returns a function to use in pseudos for input types\n * @param {String} type\n */\nfunction createInputPseudo( type ) {\n\treturn function( elem ) {\n\t\tvar name = elem.nodeName.toLowerCase();\n\t\treturn name === \"input\" && elem.type === type;\n\t};\n}\n\n/**\n * Returns a function to use in pseudos for buttons\n * @param {String} type\n */\nfunction createButtonPseudo( type ) {\n\treturn function( elem ) {\n\t\tvar name = elem.nodeName.toLowerCase();\n\t\treturn (name === \"input\" || name === \"button\") && elem.type === type;\n\t};\n}\n\n/**\n * Returns a function to use in pseudos for :enabled/:disabled\n * @param {Boolean} disabled true for :disabled; false for :enabled\n */\nfunction createDisabledPseudo( disabled ) {\n\n\t// Known :disabled false positives: fieldset[disabled] > legend:nth-of-type(n+2) :can-disable\n\treturn function( elem ) {\n\n\t\t// Only certain elements can match :enabled or :disabled\n\t\t// https://html.spec.whatwg.org/multipage/scripting.html#selector-enabled\n\t\t// https://html.spec.whatwg.org/multipage/scripting.html#selector-disabled\n\t\tif ( \"form\" in elem ) {\n\n\t\t\t// Check for inherited disabledness on relevant non-disabled elements:\n\t\t\t// * listed form-associated elements in a disabled fieldset\n\t\t\t// https://html.spec.whatwg.org/multipage/forms.html#category-listed\n\t\t\t// https://html.spec.whatwg.org/multipage/forms.html#concept-fe-disabled\n\t\t\t// * option elements in a disabled optgroup\n\t\t\t// https://html.spec.whatwg.org/multipage/forms.html#concept-option-disabled\n\t\t\t// All such elements have a \"form\" property.\n\t\t\tif ( elem.parentNode && elem.disabled === false ) {\n\n\t\t\t\t// Option elements defer to a parent optgroup if present\n\t\t\t\tif ( \"label\" in elem ) {\n\t\t\t\t\tif ( \"label\" in elem.parentNode ) {\n\t\t\t\t\t\treturn elem.parentNode.disabled === disabled;\n\t\t\t\t\t} else {\n\t\t\t\t\t\treturn elem.disabled === disabled;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// Support: IE 6 - 11\n\t\t\t\t// Use the isDisabled shortcut property to check for disabled fieldset ancestors\n\t\t\t\treturn elem.isDisabled === disabled ||\n\n\t\t\t\t\t// Where there is no isDisabled, check manually\n\t\t\t\t\t/* jshint -W018 */\n\t\t\t\t\telem.isDisabled !== !disabled &&\n\t\t\t\t\t\tdisabledAncestor( elem ) === disabled;\n\t\t\t}\n\n\t\t\treturn elem.disabled === disabled;\n\n\t\t// Try to winnow out elements that can't be disabled before trusting the disabled property.\n\t\t// Some victims get caught in our net (label, legend, menu, track), but it shouldn't\n\t\t// even exist on them, let alone have a boolean value.\n\t\t} else if ( \"label\" in elem ) {\n\t\t\treturn elem.disabled === disabled;\n\t\t}\n\n\t\t// Remaining elements are neither :enabled nor :disabled\n\t\treturn false;\n\t};\n}\n\n/**\n * Returns a function to use in pseudos for positionals\n * @param {Function} fn\n */\nfunction createPositionalPseudo( fn ) {\n\treturn markFunction(function( argument ) {\n\t\targument = +argument;\n\t\treturn markFunction(function( seed, matches ) {\n\t\t\tvar j,\n\t\t\t\tmatchIndexes = fn( [], seed.length, argument ),\n\t\t\t\ti = matchIndexes.length;\n\n\t\t\t// Match elements found at the specified indexes\n\t\t\twhile ( i-- ) {\n\t\t\t\tif ( seed[ (j = matchIndexes[i]) ] ) {\n\t\t\t\t\tseed[j] = !(matches[j] = seed[j]);\n\t\t\t\t}\n\t\t\t}\n\t\t});\n\t});\n}\n\n/**\n * Checks a node for validity as a Sizzle context\n * @param {Element|Object=} context\n * @returns {Element|Object|Boolean} The input node if acceptable, otherwise a falsy value\n */\nfunction testContext( context ) {\n\treturn context && typeof context.getElementsByTagName !== \"undefined\" && context;\n}\n\n// Expose support vars for convenience\nsupport = Sizzle.support = {};\n\n/**\n * Detects XML nodes\n * @param {Element|Object} elem An element or a document\n * @returns {Boolean} True iff elem is a non-HTML XML node\n */\nisXML = Sizzle.isXML = function( elem ) {\n\t// documentElement is verified for cases where it doesn't yet exist\n\t// (such as loading iframes in IE - #4833)\n\tvar documentElement = elem && (elem.ownerDocument || elem).documentElement;\n\treturn documentElement ? documentElement.nodeName !== \"HTML\" : false;\n};\n\n/**\n * Sets document-related variables once based on the current document\n * @param {Element|Object} [doc] An element or document object to use to set the document\n * @returns {Object} Returns the current document\n */\nsetDocument = Sizzle.setDocument = function( node ) {\n\tvar hasCompare, subWindow,\n\t\tdoc = node ? node.ownerDocument || node : preferredDoc;\n\n\t// Return early if doc is invalid or already selected\n\tif ( doc === document || doc.nodeType !== 9 || !doc.documentElement ) {\n\t\treturn document;\n\t}\n\n\t// Update global variables\n\tdocument = doc;\n\tdocElem = document.documentElement;\n\tdocumentIsHTML = !isXML( document );\n\n\t// Support: IE 9-11, Edge\n\t// Accessing iframe documents after unload throws \"permission denied\" errors (jQuery #13936)\n\tif ( preferredDoc !== document &&\n\t\t(subWindow = document.defaultView) && subWindow.top !== subWindow ) {\n\n\t\t// Support: IE 11, Edge\n\t\tif ( subWindow.addEventListener ) {\n\t\t\tsubWindow.addEventListener( \"unload\", unloadHandler, false );\n\n\t\t// Support: IE 9 - 10 only\n\t\t} else if ( subWindow.attachEvent ) {\n\t\t\tsubWindow.attachEvent( \"onunload\", unloadHandler );\n\t\t}\n\t}\n\n\t/* Attributes\n\t---------------------------------------------------------------------- */\n\n\t// Support: IE<8\n\t// Verify that getAttribute really returns attributes and not properties\n\t// (excepting IE8 booleans)\n\tsupport.attributes = assert(function( el ) {\n\t\tel.className = \"i\";\n\t\treturn !el.getAttribute(\"className\");\n\t});\n\n\t/* getElement(s)By*\n\t---------------------------------------------------------------------- */\n\n\t// Check if getElementsByTagName(\"*\") returns only elements\n\tsupport.getElementsByTagName = assert(function( el ) {\n\t\tel.appendChild( document.createComment(\"\") );\n\t\treturn !el.getElementsByTagName(\"*\").length;\n\t});\n\n\t// Support: IE<9\n\tsupport.getElementsByClassName = rnative.test( document.getElementsByClassName );\n\n\t// Support: IE<10\n\t// Check if getElementById returns elements by name\n\t// The broken getElementById methods don't pick up programmatically-set names,\n\t// so use a roundabout getElementsByName test\n\tsupport.getById = assert(function( el ) {\n\t\tdocElem.appendChild( el ).id = expando;\n\t\treturn !document.getElementsByName || !document.getElementsByName( expando ).length;\n\t});\n\n\t// ID filter and find\n\tif ( support.getById ) {\n\t\tExpr.filter[\"ID\"] = function( id ) {\n\t\t\tvar attrId = id.replace( runescape, funescape );\n\t\t\treturn function( elem ) {\n\t\t\t\treturn elem.getAttribute(\"id\") === attrId;\n\t\t\t};\n\t\t};\n\t\tExpr.find[\"ID\"] = function( id, context ) {\n\t\t\tif ( typeof context.getElementById !== \"undefined\" && documentIsHTML ) {\n\t\t\t\tvar elem = context.getElementById( id );\n\t\t\t\treturn elem ? [ elem ] : [];\n\t\t\t}\n\t\t};\n\t} else {\n\t\tExpr.filter[\"ID\"] = function( id ) {\n\t\t\tvar attrId = id.replace( runescape, funescape );\n\t\t\treturn function( elem ) {\n\t\t\t\tvar node = typeof elem.getAttributeNode !== \"undefined\" &&\n\t\t\t\t\telem.getAttributeNode(\"id\");\n\t\t\t\treturn node && node.value === attrId;\n\t\t\t};\n\t\t};\n\n\t\t// Support: IE 6 - 7 only\n\t\t// getElementById is not reliable as a find shortcut\n\t\tExpr.find[\"ID\"] = function( id, context ) {\n\t\t\tif ( typeof context.getElementById !== \"undefined\" && documentIsHTML ) {\n\t\t\t\tvar node, i, elems,\n\t\t\t\t\telem = context.getElementById( id );\n\n\t\t\t\tif ( elem ) {\n\n\t\t\t\t\t// Verify the id attribute\n\t\t\t\t\tnode = elem.getAttributeNode(\"id\");\n\t\t\t\t\tif ( node && node.value === id ) {\n\t\t\t\t\t\treturn [ elem ];\n\t\t\t\t\t}\n\n\t\t\t\t\t// Fall back on getElementsByName\n\t\t\t\t\telems = context.getElementsByName( id );\n\t\t\t\t\ti = 0;\n\t\t\t\t\twhile ( (elem = elems[i++]) ) {\n\t\t\t\t\t\tnode = elem.getAttributeNode(\"id\");\n\t\t\t\t\t\tif ( node && node.value === id ) {\n\t\t\t\t\t\t\treturn [ elem ];\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn [];\n\t\t\t}\n\t\t};\n\t}\n\n\t// Tag\n\tExpr.find[\"TAG\"] = support.getElementsByTagName ?\n\t\tfunction( tag, context ) {\n\t\t\tif ( typeof context.getElementsByTagName !== \"undefined\" ) {\n\t\t\t\treturn context.getElementsByTagName( tag );\n\n\t\t\t// DocumentFragment nodes don't have gEBTN\n\t\t\t} else if ( support.qsa ) {\n\t\t\t\treturn context.querySelectorAll( tag );\n\t\t\t}\n\t\t} :\n\n\t\tfunction( tag, context ) {\n\t\t\tvar elem,\n\t\t\t\ttmp = [],\n\t\t\t\ti = 0,\n\t\t\t\t// By happy coincidence, a (broken) gEBTN appears on DocumentFragment nodes too\n\t\t\t\tresults = context.getElementsByTagName( tag );\n\n\t\t\t// Filter out possible comments\n\t\t\tif ( tag === \"*\" ) {\n\t\t\t\twhile ( (elem = results[i++]) ) {\n\t\t\t\t\tif ( elem.nodeType === 1 ) {\n\t\t\t\t\t\ttmp.push( elem );\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn tmp;\n\t\t\t}\n\t\t\treturn results;\n\t\t};\n\n\t// Class\n\tExpr.find[\"CLASS\"] = support.getElementsByClassName && function( className, context ) {\n\t\tif ( typeof context.getElementsByClassName !== \"undefined\" && documentIsHTML ) {\n\t\t\treturn context.getElementsByClassName( className );\n\t\t}\n\t};\n\n\t/* QSA/matchesSelector\n\t---------------------------------------------------------------------- */\n\n\t// QSA and matchesSelector support\n\n\t// matchesSelector(:active) reports false when true (IE9/Opera 11.5)\n\trbuggyMatches = [];\n\n\t// qSa(:focus) reports false when true (Chrome 21)\n\t// We allow this because of a bug in IE8/9 that throws an error\n\t// whenever `document.activeElement` is accessed on an iframe\n\t// So, we allow :focus to pass through QSA all the time to avoid the IE error\n\t// See https://bugs.jquery.com/ticket/13378\n\trbuggyQSA = [];\n\n\tif ( (support.qsa = rnative.test( document.querySelectorAll )) ) {\n\t\t// Build QSA regex\n\t\t// Regex strategy adopted from Diego Perini\n\t\tassert(function( el ) {\n\t\t\t// Select is set to empty string on purpose\n\t\t\t// This is to test IE's treatment of not explicitly\n\t\t\t// setting a boolean content attribute,\n\t\t\t// since its presence should be enough\n\t\t\t// https://bugs.jquery.com/ticket/12359\n\t\t\tdocElem.appendChild( el ).innerHTML = \"\" +\n\t\t\t\t\"\";\n\n\t\t\t// Support: IE8, Opera 11-12.16\n\t\t\t// Nothing should be selected when empty strings follow ^= or $= or *=\n\t\t\t// The test attribute must be unknown in Opera but \"safe\" for WinRT\n\t\t\t// https://msdn.microsoft.com/en-us/library/ie/hh465388.aspx#attribute_section\n\t\t\tif ( el.querySelectorAll(\"[msallowcapture^='']\").length ) {\n\t\t\t\trbuggyQSA.push( \"[*^$]=\" + whitespace + \"*(?:''|\\\"\\\")\" );\n\t\t\t}\n\n\t\t\t// Support: IE8\n\t\t\t// Boolean attributes and \"value\" are not treated correctly\n\t\t\tif ( !el.querySelectorAll(\"[selected]\").length ) {\n\t\t\t\trbuggyQSA.push( \"\\\\[\" + whitespace + \"*(?:value|\" + booleans + \")\" );\n\t\t\t}\n\n\t\t\t// Support: Chrome<29, Android<4.4, Safari<7.0+, iOS<7.0+, PhantomJS<1.9.8+\n\t\t\tif ( !el.querySelectorAll( \"[id~=\" + expando + \"-]\" ).length ) {\n\t\t\t\trbuggyQSA.push(\"~=\");\n\t\t\t}\n\n\t\t\t// Webkit/Opera - :checked should return selected option elements\n\t\t\t// http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked\n\t\t\t// IE8 throws error here and will not see later tests\n\t\t\tif ( !el.querySelectorAll(\":checked\").length ) {\n\t\t\t\trbuggyQSA.push(\":checked\");\n\t\t\t}\n\n\t\t\t// Support: Safari 8+, iOS 8+\n\t\t\t// https://bugs.webkit.org/show_bug.cgi?id=136851\n\t\t\t// In-page `selector#id sibling-combinator selector` fails\n\t\t\tif ( !el.querySelectorAll( \"a#\" + expando + \"+*\" ).length ) {\n\t\t\t\trbuggyQSA.push(\".#.+[+~]\");\n\t\t\t}\n\t\t});\n\n\t\tassert(function( el ) {\n\t\t\tel.innerHTML = \"\" +\n\t\t\t\t\"\";\n\n\t\t\t// Support: Windows 8 Native Apps\n\t\t\t// The type and name attributes are restricted during .innerHTML assignment\n\t\t\tvar input = document.createElement(\"input\");\n\t\t\tinput.setAttribute( \"type\", \"hidden\" );\n\t\t\tel.appendChild( input ).setAttribute( \"name\", \"D\" );\n\n\t\t\t// Support: IE8\n\t\t\t// Enforce case-sensitivity of name attribute\n\t\t\tif ( el.querySelectorAll(\"[name=d]\").length ) {\n\t\t\t\trbuggyQSA.push( \"name\" + whitespace + \"*[*^$|!~]?=\" );\n\t\t\t}\n\n\t\t\t// FF 3.5 - :enabled/:disabled and hidden elements (hidden elements are still enabled)\n\t\t\t// IE8 throws error here and will not see later tests\n\t\t\tif ( el.querySelectorAll(\":enabled\").length !== 2 ) {\n\t\t\t\trbuggyQSA.push( \":enabled\", \":disabled\" );\n\t\t\t}\n\n\t\t\t// Support: IE9-11+\n\t\t\t// IE's :disabled selector does not pick up the children of disabled fieldsets\n\t\t\tdocElem.appendChild( el ).disabled = true;\n\t\t\tif ( el.querySelectorAll(\":disabled\").length !== 2 ) {\n\t\t\t\trbuggyQSA.push( \":enabled\", \":disabled\" );\n\t\t\t}\n\n\t\t\t// Opera 10-11 does not throw on post-comma invalid pseudos\n\t\t\tel.querySelectorAll(\"*,:x\");\n\t\t\trbuggyQSA.push(\",.*:\");\n\t\t});\n\t}\n\n\tif ( (support.matchesSelector = rnative.test( (matches = docElem.matches ||\n\t\tdocElem.webkitMatchesSelector ||\n\t\tdocElem.mozMatchesSelector ||\n\t\tdocElem.oMatchesSelector ||\n\t\tdocElem.msMatchesSelector) )) ) {\n\n\t\tassert(function( el ) {\n\t\t\t// Check to see if it's possible to do matchesSelector\n\t\t\t// on a disconnected node (IE 9)\n\t\t\tsupport.disconnectedMatch = matches.call( el, \"*\" );\n\n\t\t\t// This should fail with an exception\n\t\t\t// Gecko does not error, returns false instead\n\t\t\tmatches.call( el, \"[s!='']:x\" );\n\t\t\trbuggyMatches.push( \"!=\", pseudos );\n\t\t});\n\t}\n\n\trbuggyQSA = rbuggyQSA.length && new RegExp( rbuggyQSA.join(\"|\") );\n\trbuggyMatches = rbuggyMatches.length && new RegExp( rbuggyMatches.join(\"|\") );\n\n\t/* Contains\n\t---------------------------------------------------------------------- */\n\thasCompare = rnative.test( docElem.compareDocumentPosition );\n\n\t// Element contains another\n\t// Purposefully self-exclusive\n\t// As in, an element does not contain itself\n\tcontains = hasCompare || rnative.test( docElem.contains ) ?\n\t\tfunction( a, b ) {\n\t\t\tvar adown = a.nodeType === 9 ? a.documentElement : a,\n\t\t\t\tbup = b && b.parentNode;\n\t\t\treturn a === bup || !!( bup && bup.nodeType === 1 && (\n\t\t\t\tadown.contains ?\n\t\t\t\t\tadown.contains( bup ) :\n\t\t\t\t\ta.compareDocumentPosition && a.compareDocumentPosition( bup ) & 16\n\t\t\t));\n\t\t} :\n\t\tfunction( a, b ) {\n\t\t\tif ( b ) {\n\t\t\t\twhile ( (b = b.parentNode) ) {\n\t\t\t\t\tif ( b === a ) {\n\t\t\t\t\t\treturn true;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn false;\n\t\t};\n\n\t/* Sorting\n\t---------------------------------------------------------------------- */\n\n\t// Document order sorting\n\tsortOrder = hasCompare ?\n\tfunction( a, b ) {\n\n\t\t// Flag for duplicate removal\n\t\tif ( a === b ) {\n\t\t\thasDuplicate = true;\n\t\t\treturn 0;\n\t\t}\n\n\t\t// Sort on method existence if only one input has compareDocumentPosition\n\t\tvar compare = !a.compareDocumentPosition - !b.compareDocumentPosition;\n\t\tif ( compare ) {\n\t\t\treturn compare;\n\t\t}\n\n\t\t// Calculate position if both inputs belong to the same document\n\t\tcompare = ( a.ownerDocument || a ) === ( b.ownerDocument || b ) ?\n\t\t\ta.compareDocumentPosition( b ) :\n\n\t\t\t// Otherwise we know they are disconnected\n\t\t\t1;\n\n\t\t// Disconnected nodes\n\t\tif ( compare & 1 ||\n\t\t\t(!support.sortDetached && b.compareDocumentPosition( a ) === compare) ) {\n\n\t\t\t// Choose the first element that is related to our preferred document\n\t\t\tif ( a === document || a.ownerDocument === preferredDoc && contains(preferredDoc, a) ) {\n\t\t\t\treturn -1;\n\t\t\t}\n\t\t\tif ( b === document || b.ownerDocument === preferredDoc && contains(preferredDoc, b) ) {\n\t\t\t\treturn 1;\n\t\t\t}\n\n\t\t\t// Maintain original order\n\t\t\treturn sortInput ?\n\t\t\t\t( indexOf( sortInput, a ) - indexOf( sortInput, b ) ) :\n\t\t\t\t0;\n\t\t}\n\n\t\treturn compare & 4 ? -1 : 1;\n\t} :\n\tfunction( a, b ) {\n\t\t// Exit early if the nodes are identical\n\t\tif ( a === b ) {\n\t\t\thasDuplicate = true;\n\t\t\treturn 0;\n\t\t}\n\n\t\tvar cur,\n\t\t\ti = 0,\n\t\t\taup = a.parentNode,\n\t\t\tbup = b.parentNode,\n\t\t\tap = [ a ],\n\t\t\tbp = [ b ];\n\n\t\t// Parentless nodes are either documents or disconnected\n\t\tif ( !aup || !bup ) {\n\t\t\treturn a === document ? -1 :\n\t\t\t\tb === document ? 1 :\n\t\t\t\taup ? -1 :\n\t\t\t\tbup ? 1 :\n\t\t\t\tsortInput ?\n\t\t\t\t( indexOf( sortInput, a ) - indexOf( sortInput, b ) ) :\n\t\t\t\t0;\n\n\t\t// If the nodes are siblings, we can do a quick check\n\t\t} else if ( aup === bup ) {\n\t\t\treturn siblingCheck( a, b );\n\t\t}\n\n\t\t// Otherwise we need full lists of their ancestors for comparison\n\t\tcur = a;\n\t\twhile ( (cur = cur.parentNode) ) {\n\t\t\tap.unshift( cur );\n\t\t}\n\t\tcur = b;\n\t\twhile ( (cur = cur.parentNode) ) {\n\t\t\tbp.unshift( cur );\n\t\t}\n\n\t\t// Walk down the tree looking for a discrepancy\n\t\twhile ( ap[i] === bp[i] ) {\n\t\t\ti++;\n\t\t}\n\n\t\treturn i ?\n\t\t\t// Do a sibling check if the nodes have a common ancestor\n\t\t\tsiblingCheck( ap[i], bp[i] ) :\n\n\t\t\t// Otherwise nodes in our document sort first\n\t\t\tap[i] === preferredDoc ? -1 :\n\t\t\tbp[i] === preferredDoc ? 1 :\n\t\t\t0;\n\t};\n\n\treturn document;\n};\n\nSizzle.matches = function( expr, elements ) {\n\treturn Sizzle( expr, null, null, elements );\n};\n\nSizzle.matchesSelector = function( elem, expr ) {\n\t// Set document vars if needed\n\tif ( ( elem.ownerDocument || elem ) !== document ) {\n\t\tsetDocument( elem );\n\t}\n\n\t// Make sure that attribute selectors are quoted\n\texpr = expr.replace( rattributeQuotes, \"='$1']\" );\n\n\tif ( support.matchesSelector && documentIsHTML &&\n\t\t!compilerCache[ expr + \" \" ] &&\n\t\t( !rbuggyMatches || !rbuggyMatches.test( expr ) ) &&\n\t\t( !rbuggyQSA || !rbuggyQSA.test( expr ) ) ) {\n\n\t\ttry {\n\t\t\tvar ret = matches.call( elem, expr );\n\n\t\t\t// IE 9's matchesSelector returns false on disconnected nodes\n\t\t\tif ( ret || support.disconnectedMatch ||\n\t\t\t\t\t// As well, disconnected nodes are said to be in a document\n\t\t\t\t\t// fragment in IE 9\n\t\t\t\t\telem.document && elem.document.nodeType !== 11 ) {\n\t\t\t\treturn ret;\n\t\t\t}\n\t\t} catch (e) {}\n\t}\n\n\treturn Sizzle( expr, document, null, [ elem ] ).length > 0;\n};\n\nSizzle.contains = function( context, elem ) {\n\t// Set document vars if needed\n\tif ( ( context.ownerDocument || context ) !== document ) {\n\t\tsetDocument( context );\n\t}\n\treturn contains( context, elem );\n};\n\nSizzle.attr = function( elem, name ) {\n\t// Set document vars if needed\n\tif ( ( elem.ownerDocument || elem ) !== document ) {\n\t\tsetDocument( elem );\n\t}\n\n\tvar fn = Expr.attrHandle[ name.toLowerCase() ],\n\t\t// Don't get fooled by Object.prototype properties (jQuery #13807)\n\t\tval = fn && hasOwn.call( Expr.attrHandle, name.toLowerCase() ) ?\n\t\t\tfn( elem, name, !documentIsHTML ) :\n\t\t\tundefined;\n\n\treturn val !== undefined ?\n\t\tval :\n\t\tsupport.attributes || !documentIsHTML ?\n\t\t\telem.getAttribute( name ) :\n\t\t\t(val = elem.getAttributeNode(name)) && val.specified ?\n\t\t\t\tval.value :\n\t\t\t\tnull;\n};\n\nSizzle.escape = function( sel ) {\n\treturn (sel + \"\").replace( rcssescape, fcssescape );\n};\n\nSizzle.error = function( msg ) {\n\tthrow new Error( \"Syntax error, unrecognized expression: \" + msg );\n};\n\n/**\n * Document sorting and removing duplicates\n * @param {ArrayLike} results\n */\nSizzle.uniqueSort = function( results ) {\n\tvar elem,\n\t\tduplicates = [],\n\t\tj = 0,\n\t\ti = 0;\n\n\t// Unless we *know* we can detect duplicates, assume their presence\n\thasDuplicate = !support.detectDuplicates;\n\tsortInput = !support.sortStable && results.slice( 0 );\n\tresults.sort( sortOrder );\n\n\tif ( hasDuplicate ) {\n\t\twhile ( (elem = results[i++]) ) {\n\t\t\tif ( elem === results[ i ] ) {\n\t\t\t\tj = duplicates.push( i );\n\t\t\t}\n\t\t}\n\t\twhile ( j-- ) {\n\t\t\tresults.splice( duplicates[ j ], 1 );\n\t\t}\n\t}\n\n\t// Clear input after sorting to release objects\n\t// See https://github.com/jquery/sizzle/pull/225\n\tsortInput = null;\n\n\treturn results;\n};\n\n/**\n * Utility function for retrieving the text value of an array of DOM nodes\n * @param {Array|Element} elem\n */\ngetText = Sizzle.getText = function( elem ) {\n\tvar node,\n\t\tret = \"\",\n\t\ti = 0,\n\t\tnodeType = elem.nodeType;\n\n\tif ( !nodeType ) {\n\t\t// If no nodeType, this is expected to be an array\n\t\twhile ( (node = elem[i++]) ) {\n\t\t\t// Do not traverse comment nodes\n\t\t\tret += getText( node );\n\t\t}\n\t} else if ( nodeType === 1 || nodeType === 9 || nodeType === 11 ) {\n\t\t// Use textContent for elements\n\t\t// innerText usage removed for consistency of new lines (jQuery #11153)\n\t\tif ( typeof elem.textContent === \"string\" ) {\n\t\t\treturn elem.textContent;\n\t\t} else {\n\t\t\t// Traverse its children\n\t\t\tfor ( elem = elem.firstChild; elem; elem = elem.nextSibling ) {\n\t\t\t\tret += getText( elem );\n\t\t\t}\n\t\t}\n\t} else if ( nodeType === 3 || nodeType === 4 ) {\n\t\treturn elem.nodeValue;\n\t}\n\t// Do not include comment or processing instruction nodes\n\n\treturn ret;\n};\n\nExpr = Sizzle.selectors = {\n\n\t// Can be adjusted by the user\n\tcacheLength: 50,\n\n\tcreatePseudo: markFunction,\n\n\tmatch: matchExpr,\n\n\tattrHandle: {},\n\n\tfind: {},\n\n\trelative: {\n\t\t\">\": { dir: \"parentNode\", first: true },\n\t\t\" \": { dir: \"parentNode\" },\n\t\t\"+\": { dir: \"previousSibling\", first: true },\n\t\t\"~\": { dir: \"previousSibling\" }\n\t},\n\n\tpreFilter: {\n\t\t\"ATTR\": function( match ) {\n\t\t\tmatch[1] = match[1].replace( runescape, funescape );\n\n\t\t\t// Move the given value to match[3] whether quoted or unquoted\n\t\t\tmatch[3] = ( match[3] || match[4] || match[5] || \"\" ).replace( runescape, funescape );\n\n\t\t\tif ( match[2] === \"~=\" ) {\n\t\t\t\tmatch[3] = \" \" + match[3] + \" \";\n\t\t\t}\n\n\t\t\treturn match.slice( 0, 4 );\n\t\t},\n\n\t\t\"CHILD\": function( match ) {\n\t\t\t/* matches from matchExpr[\"CHILD\"]\n\t\t\t\t1 type (only|nth|...)\n\t\t\t\t2 what (child|of-type)\n\t\t\t\t3 argument (even|odd|\\d*|\\d*n([+-]\\d+)?|...)\n\t\t\t\t4 xn-component of xn+y argument ([+-]?\\d*n|)\n\t\t\t\t5 sign of xn-component\n\t\t\t\t6 x of xn-component\n\t\t\t\t7 sign of y-component\n\t\t\t\t8 y of y-component\n\t\t\t*/\n\t\t\tmatch[1] = match[1].toLowerCase();\n\n\t\t\tif ( match[1].slice( 0, 3 ) === \"nth\" ) {\n\t\t\t\t// nth-* requires argument\n\t\t\t\tif ( !match[3] ) {\n\t\t\t\t\tSizzle.error( match[0] );\n\t\t\t\t}\n\n\t\t\t\t// numeric x and y parameters for Expr.filter.CHILD\n\t\t\t\t// remember that false/true cast respectively to 0/1\n\t\t\t\tmatch[4] = +( match[4] ? match[5] + (match[6] || 1) : 2 * ( match[3] === \"even\" || match[3] === \"odd\" ) );\n\t\t\t\tmatch[5] = +( ( match[7] + match[8] ) || match[3] === \"odd\" );\n\n\t\t\t// other types prohibit arguments\n\t\t\t} else if ( match[3] ) {\n\t\t\t\tSizzle.error( match[0] );\n\t\t\t}\n\n\t\t\treturn match;\n\t\t},\n\n\t\t\"PSEUDO\": function( match ) {\n\t\t\tvar excess,\n\t\t\t\tunquoted = !match[6] && match[2];\n\n\t\t\tif ( matchExpr[\"CHILD\"].test( match[0] ) ) {\n\t\t\t\treturn null;\n\t\t\t}\n\n\t\t\t// Accept quoted arguments as-is\n\t\t\tif ( match[3] ) {\n\t\t\t\tmatch[2] = match[4] || match[5] || \"\";\n\n\t\t\t// Strip excess characters from unquoted arguments\n\t\t\t} else if ( unquoted && rpseudo.test( unquoted ) &&\n\t\t\t\t// Get excess from tokenize (recursively)\n\t\t\t\t(excess = tokenize( unquoted, true )) &&\n\t\t\t\t// advance to the next closing parenthesis\n\t\t\t\t(excess = unquoted.indexOf( \")\", unquoted.length - excess ) - unquoted.length) ) {\n\n\t\t\t\t// excess is a negative index\n\t\t\t\tmatch[0] = match[0].slice( 0, excess );\n\t\t\t\tmatch[2] = unquoted.slice( 0, excess );\n\t\t\t}\n\n\t\t\t// Return only captures needed by the pseudo filter method (type and argument)\n\t\t\treturn match.slice( 0, 3 );\n\t\t}\n\t},\n\n\tfilter: {\n\n\t\t\"TAG\": function( nodeNameSelector ) {\n\t\t\tvar nodeName = nodeNameSelector.replace( runescape, funescape ).toLowerCase();\n\t\t\treturn nodeNameSelector === \"*\" ?\n\t\t\t\tfunction() { return true; } :\n\t\t\t\tfunction( elem ) {\n\t\t\t\t\treturn elem.nodeName && elem.nodeName.toLowerCase() === nodeName;\n\t\t\t\t};\n\t\t},\n\n\t\t\"CLASS\": function( className ) {\n\t\t\tvar pattern = classCache[ className + \" \" ];\n\n\t\t\treturn pattern ||\n\t\t\t\t(pattern = new RegExp( \"(^|\" + whitespace + \")\" + className + \"(\" + whitespace + \"|$)\" )) &&\n\t\t\t\tclassCache( className, function( elem ) {\n\t\t\t\t\treturn pattern.test( typeof elem.className === \"string\" && elem.className || typeof elem.getAttribute !== \"undefined\" && elem.getAttribute(\"class\") || \"\" );\n\t\t\t\t});\n\t\t},\n\n\t\t\"ATTR\": function( name, operator, check ) {\n\t\t\treturn function( elem ) {\n\t\t\t\tvar result = Sizzle.attr( elem, name );\n\n\t\t\t\tif ( result == null ) {\n\t\t\t\t\treturn operator === \"!=\";\n\t\t\t\t}\n\t\t\t\tif ( !operator ) {\n\t\t\t\t\treturn true;\n\t\t\t\t}\n\n\t\t\t\tresult += \"\";\n\n\t\t\t\treturn operator === \"=\" ? result === check :\n\t\t\t\t\toperator === \"!=\" ? result !== check :\n\t\t\t\t\toperator === \"^=\" ? check && result.indexOf( check ) === 0 :\n\t\t\t\t\toperator === \"*=\" ? check && result.indexOf( check ) > -1 :\n\t\t\t\t\toperator === \"$=\" ? check && result.slice( -check.length ) === check :\n\t\t\t\t\toperator === \"~=\" ? ( \" \" + result.replace( rwhitespace, \" \" ) + \" \" ).indexOf( check ) > -1 :\n\t\t\t\t\toperator === \"|=\" ? result === check || result.slice( 0, check.length + 1 ) === check + \"-\" :\n\t\t\t\t\tfalse;\n\t\t\t};\n\t\t},\n\n\t\t\"CHILD\": function( type, what, argument, first, last ) {\n\t\t\tvar simple = type.slice( 0, 3 ) !== \"nth\",\n\t\t\t\tforward = type.slice( -4 ) !== \"last\",\n\t\t\t\tofType = what === \"of-type\";\n\n\t\t\treturn first === 1 && last === 0 ?\n\n\t\t\t\t// Shortcut for :nth-*(n)\n\t\t\t\tfunction( elem ) {\n\t\t\t\t\treturn !!elem.parentNode;\n\t\t\t\t} :\n\n\t\t\t\tfunction( elem, context, xml ) {\n\t\t\t\t\tvar cache, uniqueCache, outerCache, node, nodeIndex, start,\n\t\t\t\t\t\tdir = simple !== forward ? \"nextSibling\" : \"previousSibling\",\n\t\t\t\t\t\tparent = elem.parentNode,\n\t\t\t\t\t\tname = ofType && elem.nodeName.toLowerCase(),\n\t\t\t\t\t\tuseCache = !xml && !ofType,\n\t\t\t\t\t\tdiff = false;\n\n\t\t\t\t\tif ( parent ) {\n\n\t\t\t\t\t\t// :(first|last|only)-(child|of-type)\n\t\t\t\t\t\tif ( simple ) {\n\t\t\t\t\t\t\twhile ( dir ) {\n\t\t\t\t\t\t\t\tnode = elem;\n\t\t\t\t\t\t\t\twhile ( (node = node[ dir ]) ) {\n\t\t\t\t\t\t\t\t\tif ( ofType ?\n\t\t\t\t\t\t\t\t\t\tnode.nodeName.toLowerCase() === name :\n\t\t\t\t\t\t\t\t\t\tnode.nodeType === 1 ) {\n\n\t\t\t\t\t\t\t\t\t\treturn false;\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t// Reverse direction for :only-* (if we haven't yet done so)\n\t\t\t\t\t\t\t\tstart = dir = type === \"only\" && !start && \"nextSibling\";\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\treturn true;\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tstart = [ forward ? parent.firstChild : parent.lastChild ];\n\n\t\t\t\t\t\t// non-xml :nth-child(...) stores cache data on `parent`\n\t\t\t\t\t\tif ( forward && useCache ) {\n\n\t\t\t\t\t\t\t// Seek `elem` from a previously-cached index\n\n\t\t\t\t\t\t\t// ...in a gzip-friendly way\n\t\t\t\t\t\t\tnode = parent;\n\t\t\t\t\t\t\touterCache = node[ expando ] || (node[ expando ] = {});\n\n\t\t\t\t\t\t\t// Support: IE <9 only\n\t\t\t\t\t\t\t// Defend against cloned attroperties (jQuery gh-1709)\n\t\t\t\t\t\t\tuniqueCache = outerCache[ node.uniqueID ] ||\n\t\t\t\t\t\t\t\t(outerCache[ node.uniqueID ] = {});\n\n\t\t\t\t\t\t\tcache = uniqueCache[ type ] || [];\n\t\t\t\t\t\t\tnodeIndex = cache[ 0 ] === dirruns && cache[ 1 ];\n\t\t\t\t\t\t\tdiff = nodeIndex && cache[ 2 ];\n\t\t\t\t\t\t\tnode = nodeIndex && parent.childNodes[ nodeIndex ];\n\n\t\t\t\t\t\t\twhile ( (node = ++nodeIndex && node && node[ dir ] ||\n\n\t\t\t\t\t\t\t\t// Fallback to seeking `elem` from the start\n\t\t\t\t\t\t\t\t(diff = nodeIndex = 0) || start.pop()) ) {\n\n\t\t\t\t\t\t\t\t// When found, cache indexes on `parent` and break\n\t\t\t\t\t\t\t\tif ( node.nodeType === 1 && ++diff && node === elem ) {\n\t\t\t\t\t\t\t\t\tuniqueCache[ type ] = [ dirruns, nodeIndex, diff ];\n\t\t\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t// Use previously-cached element index if available\n\t\t\t\t\t\t\tif ( useCache ) {\n\t\t\t\t\t\t\t\t// ...in a gzip-friendly way\n\t\t\t\t\t\t\t\tnode = elem;\n\t\t\t\t\t\t\t\touterCache = node[ expando ] || (node[ expando ] = {});\n\n\t\t\t\t\t\t\t\t// Support: IE <9 only\n\t\t\t\t\t\t\t\t// Defend against cloned attroperties (jQuery gh-1709)\n\t\t\t\t\t\t\t\tuniqueCache = outerCache[ node.uniqueID ] ||\n\t\t\t\t\t\t\t\t\t(outerCache[ node.uniqueID ] = {});\n\n\t\t\t\t\t\t\t\tcache = uniqueCache[ type ] || [];\n\t\t\t\t\t\t\t\tnodeIndex = cache[ 0 ] === dirruns && cache[ 1 ];\n\t\t\t\t\t\t\t\tdiff = nodeIndex;\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// xml :nth-child(...)\n\t\t\t\t\t\t\t// or :nth-last-child(...) or :nth(-last)?-of-type(...)\n\t\t\t\t\t\t\tif ( diff === false ) {\n\t\t\t\t\t\t\t\t// Use the same loop as above to seek `elem` from the start\n\t\t\t\t\t\t\t\twhile ( (node = ++nodeIndex && node && node[ dir ] ||\n\t\t\t\t\t\t\t\t\t(diff = nodeIndex = 0) || start.pop()) ) {\n\n\t\t\t\t\t\t\t\t\tif ( ( ofType ?\n\t\t\t\t\t\t\t\t\t\tnode.nodeName.toLowerCase() === name :\n\t\t\t\t\t\t\t\t\t\tnode.nodeType === 1 ) &&\n\t\t\t\t\t\t\t\t\t\t++diff ) {\n\n\t\t\t\t\t\t\t\t\t\t// Cache the index of each encountered element\n\t\t\t\t\t\t\t\t\t\tif ( useCache ) {\n\t\t\t\t\t\t\t\t\t\t\touterCache = node[ expando ] || (node[ expando ] = {});\n\n\t\t\t\t\t\t\t\t\t\t\t// Support: IE <9 only\n\t\t\t\t\t\t\t\t\t\t\t// Defend against cloned attroperties (jQuery gh-1709)\n\t\t\t\t\t\t\t\t\t\t\tuniqueCache = outerCache[ node.uniqueID ] ||\n\t\t\t\t\t\t\t\t\t\t\t\t(outerCache[ node.uniqueID ] = {});\n\n\t\t\t\t\t\t\t\t\t\t\tuniqueCache[ type ] = [ dirruns, diff ];\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\tif ( node === elem ) {\n\t\t\t\t\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\t// Incorporate the offset, then check against cycle size\n\t\t\t\t\t\tdiff -= last;\n\t\t\t\t\t\treturn diff === first || ( diff % first === 0 && diff / first >= 0 );\n\t\t\t\t\t}\n\t\t\t\t};\n\t\t},\n\n\t\t\"PSEUDO\": function( pseudo, argument ) {\n\t\t\t// pseudo-class names are case-insensitive\n\t\t\t// http://www.w3.org/TR/selectors/#pseudo-classes\n\t\t\t// Prioritize by case sensitivity in case custom pseudos are added with uppercase letters\n\t\t\t// Remember that setFilters inherits from pseudos\n\t\t\tvar args,\n\t\t\t\tfn = Expr.pseudos[ pseudo ] || Expr.setFilters[ pseudo.toLowerCase() ] ||\n\t\t\t\t\tSizzle.error( \"unsupported pseudo: \" + pseudo );\n\n\t\t\t// The user may use createPseudo to indicate that\n\t\t\t// arguments are needed to create the filter function\n\t\t\t// just as Sizzle does\n\t\t\tif ( fn[ expando ] ) {\n\t\t\t\treturn fn( argument );\n\t\t\t}\n\n\t\t\t// But maintain support for old signatures\n\t\t\tif ( fn.length > 1 ) {\n\t\t\t\targs = [ pseudo, pseudo, \"\", argument ];\n\t\t\t\treturn Expr.setFilters.hasOwnProperty( pseudo.toLowerCase() ) ?\n\t\t\t\t\tmarkFunction(function( seed, matches ) {\n\t\t\t\t\t\tvar idx,\n\t\t\t\t\t\t\tmatched = fn( seed, argument ),\n\t\t\t\t\t\t\ti = matched.length;\n\t\t\t\t\t\twhile ( i-- ) {\n\t\t\t\t\t\t\tidx = indexOf( seed, matched[i] );\n\t\t\t\t\t\t\tseed[ idx ] = !( matches[ idx ] = matched[i] );\n\t\t\t\t\t\t}\n\t\t\t\t\t}) :\n\t\t\t\t\tfunction( elem ) {\n\t\t\t\t\t\treturn fn( elem, 0, args );\n\t\t\t\t\t};\n\t\t\t}\n\n\t\t\treturn fn;\n\t\t}\n\t},\n\n\tpseudos: {\n\t\t// Potentially complex pseudos\n\t\t\"not\": markFunction(function( selector ) {\n\t\t\t// Trim the selector passed to compile\n\t\t\t// to avoid treating leading and trailing\n\t\t\t// spaces as combinators\n\t\t\tvar input = [],\n\t\t\t\tresults = [],\n\t\t\t\tmatcher = compile( selector.replace( rtrim, \"$1\" ) );\n\n\t\t\treturn matcher[ expando ] ?\n\t\t\t\tmarkFunction(function( seed, matches, context, xml ) {\n\t\t\t\t\tvar elem,\n\t\t\t\t\t\tunmatched = matcher( seed, null, xml, [] ),\n\t\t\t\t\t\ti = seed.length;\n\n\t\t\t\t\t// Match elements unmatched by `matcher`\n\t\t\t\t\twhile ( i-- ) {\n\t\t\t\t\t\tif ( (elem = unmatched[i]) ) {\n\t\t\t\t\t\t\tseed[i] = !(matches[i] = elem);\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}) :\n\t\t\t\tfunction( elem, context, xml ) {\n\t\t\t\t\tinput[0] = elem;\n\t\t\t\t\tmatcher( input, null, xml, results );\n\t\t\t\t\t// Don't keep the element (issue #299)\n\t\t\t\t\tinput[0] = null;\n\t\t\t\t\treturn !results.pop();\n\t\t\t\t};\n\t\t}),\n\n\t\t\"has\": markFunction(function( selector ) {\n\t\t\treturn function( elem ) {\n\t\t\t\treturn Sizzle( selector, elem ).length > 0;\n\t\t\t};\n\t\t}),\n\n\t\t\"contains\": markFunction(function( text ) {\n\t\t\ttext = text.replace( runescape, funescape );\n\t\t\treturn function( elem ) {\n\t\t\t\treturn ( elem.textContent || elem.innerText || getText( elem ) ).indexOf( text ) > -1;\n\t\t\t};\n\t\t}),\n\n\t\t// \"Whether an element is represented by a :lang() selector\n\t\t// is based solely on the element's language value\n\t\t// being equal to the identifier C,\n\t\t// or beginning with the identifier C immediately followed by \"-\".\n\t\t// The matching of C against the element's language value is performed case-insensitively.\n\t\t// The identifier C does not have to be a valid language name.\"\n\t\t// http://www.w3.org/TR/selectors/#lang-pseudo\n\t\t\"lang\": markFunction( function( lang ) {\n\t\t\t// lang value must be a valid identifier\n\t\t\tif ( !ridentifier.test(lang || \"\") ) {\n\t\t\t\tSizzle.error( \"unsupported lang: \" + lang );\n\t\t\t}\n\t\t\tlang = lang.replace( runescape, funescape ).toLowerCase();\n\t\t\treturn function( elem ) {\n\t\t\t\tvar elemLang;\n\t\t\t\tdo {\n\t\t\t\t\tif ( (elemLang = documentIsHTML ?\n\t\t\t\t\t\telem.lang :\n\t\t\t\t\t\telem.getAttribute(\"xml:lang\") || elem.getAttribute(\"lang\")) ) {\n\n\t\t\t\t\t\telemLang = elemLang.toLowerCase();\n\t\t\t\t\t\treturn elemLang === lang || elemLang.indexOf( lang + \"-\" ) === 0;\n\t\t\t\t\t}\n\t\t\t\t} while ( (elem = elem.parentNode) && elem.nodeType === 1 );\n\t\t\t\treturn false;\n\t\t\t};\n\t\t}),\n\n\t\t// Miscellaneous\n\t\t\"target\": function( elem ) {\n\t\t\tvar hash = window.location && window.location.hash;\n\t\t\treturn hash && hash.slice( 1 ) === elem.id;\n\t\t},\n\n\t\t\"root\": function( elem ) {\n\t\t\treturn elem === docElem;\n\t\t},\n\n\t\t\"focus\": function( elem ) {\n\t\t\treturn elem === document.activeElement && (!document.hasFocus || document.hasFocus()) && !!(elem.type || elem.href || ~elem.tabIndex);\n\t\t},\n\n\t\t// Boolean properties\n\t\t\"enabled\": createDisabledPseudo( false ),\n\t\t\"disabled\": createDisabledPseudo( true ),\n\n\t\t\"checked\": function( elem ) {\n\t\t\t// In CSS3, :checked should return both checked and selected elements\n\t\t\t// http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked\n\t\t\tvar nodeName = elem.nodeName.toLowerCase();\n\t\t\treturn (nodeName === \"input\" && !!elem.checked) || (nodeName === \"option\" && !!elem.selected);\n\t\t},\n\n\t\t\"selected\": function( elem ) {\n\t\t\t// Accessing this property makes selected-by-default\n\t\t\t// options in Safari work properly\n\t\t\tif ( elem.parentNode ) {\n\t\t\t\telem.parentNode.selectedIndex;\n\t\t\t}\n\n\t\t\treturn elem.selected === true;\n\t\t},\n\n\t\t// Contents\n\t\t\"empty\": function( elem ) {\n\t\t\t// http://www.w3.org/TR/selectors/#empty-pseudo\n\t\t\t// :empty is negated by element (1) or content nodes (text: 3; cdata: 4; entity ref: 5),\n\t\t\t// but not by others (comment: 8; processing instruction: 7; etc.)\n\t\t\t// nodeType < 6 works because attributes (2) do not appear as children\n\t\t\tfor ( elem = elem.firstChild; elem; elem = elem.nextSibling ) {\n\t\t\t\tif ( elem.nodeType < 6 ) {\n\t\t\t\t\treturn false;\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true;\n\t\t},\n\n\t\t\"parent\": function( elem ) {\n\t\t\treturn !Expr.pseudos[\"empty\"]( elem );\n\t\t},\n\n\t\t// Element/input types\n\t\t\"header\": function( elem ) {\n\t\t\treturn rheader.test( elem.nodeName );\n\t\t},\n\n\t\t\"input\": function( elem ) {\n\t\t\treturn rinputs.test( elem.nodeName );\n\t\t},\n\n\t\t\"button\": function( elem ) {\n\t\t\tvar name = elem.nodeName.toLowerCase();\n\t\t\treturn name === \"input\" && elem.type === \"button\" || name === \"button\";\n\t\t},\n\n\t\t\"text\": function( elem ) {\n\t\t\tvar attr;\n\t\t\treturn elem.nodeName.toLowerCase() === \"input\" &&\n\t\t\t\telem.type === \"text\" &&\n\n\t\t\t\t// Support: IE<8\n\t\t\t\t// New HTML5 attribute values (e.g., \"search\") appear with elem.type === \"text\"\n\t\t\t\t( (attr = elem.getAttribute(\"type\")) == null || attr.toLowerCase() === \"text\" );\n\t\t},\n\n\t\t// Position-in-collection\n\t\t\"first\": createPositionalPseudo(function() {\n\t\t\treturn [ 0 ];\n\t\t}),\n\n\t\t\"last\": createPositionalPseudo(function( matchIndexes, length ) {\n\t\t\treturn [ length - 1 ];\n\t\t}),\n\n\t\t\"eq\": createPositionalPseudo(function( matchIndexes, length, argument ) {\n\t\t\treturn [ argument < 0 ? argument + length : argument ];\n\t\t}),\n\n\t\t\"even\": createPositionalPseudo(function( matchIndexes, length ) {\n\t\t\tvar i = 0;\n\t\t\tfor ( ; i < length; i += 2 ) {\n\t\t\t\tmatchIndexes.push( i );\n\t\t\t}\n\t\t\treturn matchIndexes;\n\t\t}),\n\n\t\t\"odd\": createPositionalPseudo(function( matchIndexes, length ) {\n\t\t\tvar i = 1;\n\t\t\tfor ( ; i < length; i += 2 ) {\n\t\t\t\tmatchIndexes.push( i );\n\t\t\t}\n\t\t\treturn matchIndexes;\n\t\t}),\n\n\t\t\"lt\": createPositionalPseudo(function( matchIndexes, length, argument ) {\n\t\t\tvar i = argument < 0 ? argument + length : argument;\n\t\t\tfor ( ; --i >= 0; ) {\n\t\t\t\tmatchIndexes.push( i );\n\t\t\t}\n\t\t\treturn matchIndexes;\n\t\t}),\n\n\t\t\"gt\": createPositionalPseudo(function( matchIndexes, length, argument ) {\n\t\t\tvar i = argument < 0 ? argument + length : argument;\n\t\t\tfor ( ; ++i < length; ) {\n\t\t\t\tmatchIndexes.push( i );\n\t\t\t}\n\t\t\treturn matchIndexes;\n\t\t})\n\t}\n};\n\nExpr.pseudos[\"nth\"] = Expr.pseudos[\"eq\"];\n\n// Add button/input type pseudos\nfor ( i in { radio: true, checkbox: true, file: true, password: true, image: true } ) {\n\tExpr.pseudos[ i ] = createInputPseudo( i );\n}\nfor ( i in { submit: true, reset: true } ) {\n\tExpr.pseudos[ i ] = createButtonPseudo( i );\n}\n\n// Easy API for creating new setFilters\nfunction setFilters() {}\nsetFilters.prototype = Expr.filters = Expr.pseudos;\nExpr.setFilters = new setFilters();\n\ntokenize = Sizzle.tokenize = function( selector, parseOnly ) {\n\tvar matched, match, tokens, type,\n\t\tsoFar, groups, preFilters,\n\t\tcached = tokenCache[ selector + \" \" ];\n\n\tif ( cached ) {\n\t\treturn parseOnly ? 0 : cached.slice( 0 );\n\t}\n\n\tsoFar = selector;\n\tgroups = [];\n\tpreFilters = Expr.preFilter;\n\n\twhile ( soFar ) {\n\n\t\t// Comma and first run\n\t\tif ( !matched || (match = rcomma.exec( soFar )) ) {\n\t\t\tif ( match ) {\n\t\t\t\t// Don't consume trailing commas as valid\n\t\t\t\tsoFar = soFar.slice( match[0].length ) || soFar;\n\t\t\t}\n\t\t\tgroups.push( (tokens = []) );\n\t\t}\n\n\t\tmatched = false;\n\n\t\t// Combinators\n\t\tif ( (match = rcombinators.exec( soFar )) ) {\n\t\t\tmatched = match.shift();\n\t\t\ttokens.push({\n\t\t\t\tvalue: matched,\n\t\t\t\t// Cast descendant combinators to space\n\t\t\t\ttype: match[0].replace( rtrim, \" \" )\n\t\t\t});\n\t\t\tsoFar = soFar.slice( matched.length );\n\t\t}\n\n\t\t// Filters\n\t\tfor ( type in Expr.filter ) {\n\t\t\tif ( (match = matchExpr[ type ].exec( soFar )) && (!preFilters[ type ] ||\n\t\t\t\t(match = preFilters[ type ]( match ))) ) {\n\t\t\t\tmatched = match.shift();\n\t\t\t\ttokens.push({\n\t\t\t\t\tvalue: matched,\n\t\t\t\t\ttype: type,\n\t\t\t\t\tmatches: match\n\t\t\t\t});\n\t\t\t\tsoFar = soFar.slice( matched.length );\n\t\t\t}\n\t\t}\n\n\t\tif ( !matched ) {\n\t\t\tbreak;\n\t\t}\n\t}\n\n\t// Return the length of the invalid excess\n\t// if we're just parsing\n\t// Otherwise, throw an error or return tokens\n\treturn parseOnly ?\n\t\tsoFar.length :\n\t\tsoFar ?\n\t\t\tSizzle.error( selector ) :\n\t\t\t// Cache the tokens\n\t\t\ttokenCache( selector, groups ).slice( 0 );\n};\n\nfunction toSelector( tokens ) {\n\tvar i = 0,\n\t\tlen = tokens.length,\n\t\tselector = \"\";\n\tfor ( ; i < len; i++ ) {\n\t\tselector += tokens[i].value;\n\t}\n\treturn selector;\n}\n\nfunction addCombinator( matcher, combinator, base ) {\n\tvar dir = combinator.dir,\n\t\tskip = combinator.next,\n\t\tkey = skip || dir,\n\t\tcheckNonElements = base && key === \"parentNode\",\n\t\tdoneName = done++;\n\n\treturn combinator.first ?\n\t\t// Check against closest ancestor/preceding element\n\t\tfunction( elem, context, xml ) {\n\t\t\twhile ( (elem = elem[ dir ]) ) {\n\t\t\t\tif ( elem.nodeType === 1 || checkNonElements ) {\n\t\t\t\t\treturn matcher( elem, context, xml );\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn false;\n\t\t} :\n\n\t\t// Check against all ancestor/preceding elements\n\t\tfunction( elem, context, xml ) {\n\t\t\tvar oldCache, uniqueCache, outerCache,\n\t\t\t\tnewCache = [ dirruns, doneName ];\n\n\t\t\t// We can't set arbitrary data on XML nodes, so they don't benefit from combinator caching\n\t\t\tif ( xml ) {\n\t\t\t\twhile ( (elem = elem[ dir ]) ) {\n\t\t\t\t\tif ( elem.nodeType === 1 || checkNonElements ) {\n\t\t\t\t\t\tif ( matcher( elem, context, xml ) ) {\n\t\t\t\t\t\t\treturn true;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\twhile ( (elem = elem[ dir ]) ) {\n\t\t\t\t\tif ( elem.nodeType === 1 || checkNonElements ) {\n\t\t\t\t\t\touterCache = elem[ expando ] || (elem[ expando ] = {});\n\n\t\t\t\t\t\t// Support: IE <9 only\n\t\t\t\t\t\t// Defend against cloned attroperties (jQuery gh-1709)\n\t\t\t\t\t\tuniqueCache = outerCache[ elem.uniqueID ] || (outerCache[ elem.uniqueID ] = {});\n\n\t\t\t\t\t\tif ( skip && skip === elem.nodeName.toLowerCase() ) {\n\t\t\t\t\t\t\telem = elem[ dir ] || elem;\n\t\t\t\t\t\t} else if ( (oldCache = uniqueCache[ key ]) &&\n\t\t\t\t\t\t\toldCache[ 0 ] === dirruns && oldCache[ 1 ] === doneName ) {\n\n\t\t\t\t\t\t\t// Assign to newCache so results back-propagate to previous elements\n\t\t\t\t\t\t\treturn (newCache[ 2 ] = oldCache[ 2 ]);\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t// Reuse newcache so results back-propagate to previous elements\n\t\t\t\t\t\t\tuniqueCache[ key ] = newCache;\n\n\t\t\t\t\t\t\t// A match means we're done; a fail means we have to keep checking\n\t\t\t\t\t\t\tif ( (newCache[ 2 ] = matcher( elem, context, xml )) ) {\n\t\t\t\t\t\t\t\treturn true;\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn false;\n\t\t};\n}\n\nfunction elementMatcher( matchers ) {\n\treturn matchers.length > 1 ?\n\t\tfunction( elem, context, xml ) {\n\t\t\tvar i = matchers.length;\n\t\t\twhile ( i-- ) {\n\t\t\t\tif ( !matchers[i]( elem, context, xml ) ) {\n\t\t\t\t\treturn false;\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true;\n\t\t} :\n\t\tmatchers[0];\n}\n\nfunction multipleContexts( selector, contexts, results ) {\n\tvar i = 0,\n\t\tlen = contexts.length;\n\tfor ( ; i < len; i++ ) {\n\t\tSizzle( selector, contexts[i], results );\n\t}\n\treturn results;\n}\n\nfunction condense( unmatched, map, filter, context, xml ) {\n\tvar elem,\n\t\tnewUnmatched = [],\n\t\ti = 0,\n\t\tlen = unmatched.length,\n\t\tmapped = map != null;\n\n\tfor ( ; i < len; i++ ) {\n\t\tif ( (elem = unmatched[i]) ) {\n\t\t\tif ( !filter || filter( elem, context, xml ) ) {\n\t\t\t\tnewUnmatched.push( elem );\n\t\t\t\tif ( mapped ) {\n\t\t\t\t\tmap.push( i );\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn newUnmatched;\n}\n\nfunction setMatcher( preFilter, selector, matcher, postFilter, postFinder, postSelector ) {\n\tif ( postFilter && !postFilter[ expando ] ) {\n\t\tpostFilter = setMatcher( postFilter );\n\t}\n\tif ( postFinder && !postFinder[ expando ] ) {\n\t\tpostFinder = setMatcher( postFinder, postSelector );\n\t}\n\treturn markFunction(function( seed, results, context, xml ) {\n\t\tvar temp, i, elem,\n\t\t\tpreMap = [],\n\t\t\tpostMap = [],\n\t\t\tpreexisting = results.length,\n\n\t\t\t// Get initial elements from seed or context\n\t\t\telems = seed || multipleContexts( selector || \"*\", context.nodeType ? [ context ] : context, [] ),\n\n\t\t\t// Prefilter to get matcher input, preserving a map for seed-results synchronization\n\t\t\tmatcherIn = preFilter && ( seed || !selector ) ?\n\t\t\t\tcondense( elems, preMap, preFilter, context, xml ) :\n\t\t\t\telems,\n\n\t\t\tmatcherOut = matcher ?\n\t\t\t\t// If we have a postFinder, or filtered seed, or non-seed postFilter or preexisting results,\n\t\t\t\tpostFinder || ( seed ? preFilter : preexisting || postFilter ) ?\n\n\t\t\t\t\t// ...intermediate processing is necessary\n\t\t\t\t\t[] :\n\n\t\t\t\t\t// ...otherwise use results directly\n\t\t\t\t\tresults :\n\t\t\t\tmatcherIn;\n\n\t\t// Find primary matches\n\t\tif ( matcher ) {\n\t\t\tmatcher( matcherIn, matcherOut, context, xml );\n\t\t}\n\n\t\t// Apply postFilter\n\t\tif ( postFilter ) {\n\t\t\ttemp = condense( matcherOut, postMap );\n\t\t\tpostFilter( temp, [], context, xml );\n\n\t\t\t// Un-match failing elements by moving them back to matcherIn\n\t\t\ti = temp.length;\n\t\t\twhile ( i-- ) {\n\t\t\t\tif ( (elem = temp[i]) ) {\n\t\t\t\t\tmatcherOut[ postMap[i] ] = !(matcherIn[ postMap[i] ] = elem);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif ( seed ) {\n\t\t\tif ( postFinder || preFilter ) {\n\t\t\t\tif ( postFinder ) {\n\t\t\t\t\t// Get the final matcherOut by condensing this intermediate into postFinder contexts\n\t\t\t\t\ttemp = [];\n\t\t\t\t\ti = matcherOut.length;\n\t\t\t\t\twhile ( i-- ) {\n\t\t\t\t\t\tif ( (elem = matcherOut[i]) ) {\n\t\t\t\t\t\t\t// Restore matcherIn since elem is not yet a final match\n\t\t\t\t\t\t\ttemp.push( (matcherIn[i] = elem) );\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tpostFinder( null, (matcherOut = []), temp, xml );\n\t\t\t\t}\n\n\t\t\t\t// Move matched elements from seed to results to keep them synchronized\n\t\t\t\ti = matcherOut.length;\n\t\t\t\twhile ( i-- ) {\n\t\t\t\t\tif ( (elem = matcherOut[i]) &&\n\t\t\t\t\t\t(temp = postFinder ? indexOf( seed, elem ) : preMap[i]) > -1 ) {\n\n\t\t\t\t\t\tseed[temp] = !(results[temp] = elem);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t// Add elements to results, through postFinder if defined\n\t\t} else {\n\t\t\tmatcherOut = condense(\n\t\t\t\tmatcherOut === results ?\n\t\t\t\t\tmatcherOut.splice( preexisting, matcherOut.length ) :\n\t\t\t\t\tmatcherOut\n\t\t\t);\n\t\t\tif ( postFinder ) {\n\t\t\t\tpostFinder( null, results, matcherOut, xml );\n\t\t\t} else {\n\t\t\t\tpush.apply( results, matcherOut );\n\t\t\t}\n\t\t}\n\t});\n}\n\nfunction matcherFromTokens( tokens ) {\n\tvar checkContext, matcher, j,\n\t\tlen = tokens.length,\n\t\tleadingRelative = Expr.relative[ tokens[0].type ],\n\t\timplicitRelative = leadingRelative || Expr.relative[\" \"],\n\t\ti = leadingRelative ? 1 : 0,\n\n\t\t// The foundational matcher ensures that elements are reachable from top-level context(s)\n\t\tmatchContext = addCombinator( function( elem ) {\n\t\t\treturn elem === checkContext;\n\t\t}, implicitRelative, true ),\n\t\tmatchAnyContext = addCombinator( function( elem ) {\n\t\t\treturn indexOf( checkContext, elem ) > -1;\n\t\t}, implicitRelative, true ),\n\t\tmatchers = [ function( elem, context, xml ) {\n\t\t\tvar ret = ( !leadingRelative && ( xml || context !== outermostContext ) ) || (\n\t\t\t\t(checkContext = context).nodeType ?\n\t\t\t\t\tmatchContext( elem, context, xml ) :\n\t\t\t\t\tmatchAnyContext( elem, context, xml ) );\n\t\t\t// Avoid hanging onto element (issue #299)\n\t\t\tcheckContext = null;\n\t\t\treturn ret;\n\t\t} ];\n\n\tfor ( ; i < len; i++ ) {\n\t\tif ( (matcher = Expr.relative[ tokens[i].type ]) ) {\n\t\t\tmatchers = [ addCombinator(elementMatcher( matchers ), matcher) ];\n\t\t} else {\n\t\t\tmatcher = Expr.filter[ tokens[i].type ].apply( null, tokens[i].matches );\n\n\t\t\t// Return special upon seeing a positional matcher\n\t\t\tif ( matcher[ expando ] ) {\n\t\t\t\t// Find the next relative operator (if any) for proper handling\n\t\t\t\tj = ++i;\n\t\t\t\tfor ( ; j < len; j++ ) {\n\t\t\t\t\tif ( Expr.relative[ tokens[j].type ] ) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn setMatcher(\n\t\t\t\t\ti > 1 && elementMatcher( matchers ),\n\t\t\t\t\ti > 1 && toSelector(\n\t\t\t\t\t\t// If the preceding token was a descendant combinator, insert an implicit any-element `*`\n\t\t\t\t\t\ttokens.slice( 0, i - 1 ).concat({ value: tokens[ i - 2 ].type === \" \" ? \"*\" : \"\" })\n\t\t\t\t\t).replace( rtrim, \"$1\" ),\n\t\t\t\t\tmatcher,\n\t\t\t\t\ti < j && matcherFromTokens( tokens.slice( i, j ) ),\n\t\t\t\t\tj < len && matcherFromTokens( (tokens = tokens.slice( j )) ),\n\t\t\t\t\tj < len && toSelector( tokens )\n\t\t\t\t);\n\t\t\t}\n\t\t\tmatchers.push( matcher );\n\t\t}\n\t}\n\n\treturn elementMatcher( matchers );\n}\n\nfunction matcherFromGroupMatchers( elementMatchers, setMatchers ) {\n\tvar bySet = setMatchers.length > 0,\n\t\tbyElement = elementMatchers.length > 0,\n\t\tsuperMatcher = function( seed, context, xml, results, outermost ) {\n\t\t\tvar elem, j, matcher,\n\t\t\t\tmatchedCount = 0,\n\t\t\t\ti = \"0\",\n\t\t\t\tunmatched = seed && [],\n\t\t\t\tsetMatched = [],\n\t\t\t\tcontextBackup = outermostContext,\n\t\t\t\t// We must always have either seed elements or outermost context\n\t\t\t\telems = seed || byElement && Expr.find[\"TAG\"]( \"*\", outermost ),\n\t\t\t\t// Use integer dirruns iff this is the outermost matcher\n\t\t\t\tdirrunsUnique = (dirruns += contextBackup == null ? 1 : Math.random() || 0.1),\n\t\t\t\tlen = elems.length;\n\n\t\t\tif ( outermost ) {\n\t\t\t\toutermostContext = context === document || context || outermost;\n\t\t\t}\n\n\t\t\t// Add elements passing elementMatchers directly to results\n\t\t\t// Support: IE<9, Safari\n\t\t\t// Tolerate NodeList properties (IE: \"length\"; Safari: