Showing preview only (8,286K chars total). Download the full file or copy to clipboard to get everything.
Repository: naver/claf
Branch: master
Commit: 6f45b1ecca0a
Files: 660
Total size: 7.8 MB
Directory structure:
gitextract_t48_1t0e/
├── .coveragerc
├── .gitignore
├── .nojekyll
├── .readthedocs.yml
├── .travis.yml
├── CONTRIBUTING.md
├── Dockerfile
├── LICENSE
├── NOTICE
├── README.md
├── base_config/
│ ├── cola/
│ │ ├── bert_base_uncased.json
│ │ ├── bert_large_uncased.json
│ │ └── structured_self_attention.json
│ ├── conll2003/
│ │ └── bert_large_cased.json
│ ├── glue/
│ │ ├── cola_bert.json
│ │ ├── cola_roberta.json
│ │ ├── mnlim_bert.json
│ │ ├── mnlim_roberta.json
│ │ ├── mnlimm_bert.json
│ │ ├── mnlimm_roberta.json
│ │ ├── mrpc_bert.json
│ │ ├── mrpc_roberta.json
│ │ ├── qnli_bert.json
│ │ ├── qnli_roberta.json
│ │ ├── qqp_bert.json
│ │ ├── qqp_roberta.json
│ │ ├── rte_bert.json
│ │ ├── rte_roberta.json
│ │ ├── sst_bert.json
│ │ ├── sst_roberta.json
│ │ ├── stsb_bert.json
│ │ ├── stsb_roberta.json
│ │ ├── wnli_bert.json
│ │ └── wnli_roberta.json
│ ├── korquad/
│ │ ├── bert_base_multilingual_cased.yaml
│ │ ├── bert_base_multilingual_uncased.json
│ │ ├── bidaf.json
│ │ └── docqa.json
│ ├── multi_task/
│ │ ├── bert_base_glue+squad.json
│ │ ├── bert_base_glue.json
│ │ ├── bert_large_glue+squad.json
│ │ └── bert_large_glue.json
│ ├── squad/
│ │ ├── bert_base_uncased.json
│ │ ├── bert_large_uncased.json
│ │ ├── bidaf+elmo.json
│ │ ├── bidaf.json
│ │ ├── bidaf_no_answer.json
│ │ ├── docqa+elmo.json
│ │ ├── docqa.json
│ │ ├── docqa_no_answer.json
│ │ ├── drqa.json
│ │ ├── drqa_paper.json
│ │ ├── qanet.json
│ │ ├── qanet_paper.json
│ │ ├── roberta_base.json
│ │ └── roberta_large.json
│ ├── test/
│ │ ├── bert_for_multi_task.json
│ │ ├── bert_for_qa.yaml
│ │ ├── bert_for_seq_cls.json
│ │ ├── bert_for_tok_cls.json
│ │ ├── bidaf+bert.json
│ │ ├── bidaf+cove.json
│ │ ├── bidaf+elmo.json
│ │ ├── bidaf.yaml
│ │ ├── bidaf_no_answer.json
│ │ ├── cola_bert.json
│ │ ├── cola_roberta.json
│ │ ├── docqa.json
│ │ ├── docqa_no_answer.json
│ │ ├── drqa.json
│ │ ├── drqa_sparse_to_embedding.json
│ │ ├── mnlim_bert.json
│ │ ├── mrpc_bert.json
│ │ ├── mt_bert.json
│ │ ├── open_qa.json
│ │ ├── qanet.json
│ │ ├── qnli_bert.json
│ │ ├── qqp_bert.json
│ │ ├── roberta_for_qa.json
│ │ ├── rte_bert.json
│ │ ├── rte_roberta.json
│ │ ├── sqlnet.json
│ │ ├── ssa.json
│ │ ├── sst_bert.json
│ │ ├── stsb_bert.json
│ │ ├── stsb_roberta.json
│ │ └── wnli_bert.json
│ └── wikisql/
│ └── sqlnet.json
├── claf/
│ ├── __init__.py
│ ├── __version__.py
│ ├── config/
│ │ ├── __init__.py
│ │ ├── args.py
│ │ ├── namespace.py
│ │ ├── pattern.py
│ │ ├── registry.py
│ │ └── utils.py
│ ├── data/
│ │ ├── __init__.py
│ │ ├── collate.py
│ │ ├── data_handler.py
│ │ ├── dataset/
│ │ │ ├── __init__.py
│ │ │ ├── base.py
│ │ │ ├── bert/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── multi_task.py
│ │ │ │ ├── regression.py
│ │ │ │ ├── seq_cls.py
│ │ │ │ ├── squad.py
│ │ │ │ └── tok_cls.py
│ │ │ ├── seq_cls.py
│ │ │ ├── squad.py
│ │ │ └── wikisql.py
│ │ ├── dto/
│ │ │ ├── __init__.py
│ │ │ ├── batch.py
│ │ │ ├── bert_feature.py
│ │ │ └── helper.py
│ │ ├── reader/
│ │ │ ├── __init__.py
│ │ │ ├── base.py
│ │ │ ├── bert/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── conll2003.py
│ │ │ │ ├── glue/
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── cola.py
│ │ │ │ │ ├── mnli.py
│ │ │ │ │ ├── mrpc.py
│ │ │ │ │ ├── qnli.py
│ │ │ │ │ ├── qqp.py
│ │ │ │ │ ├── rte.py
│ │ │ │ │ ├── sst.py
│ │ │ │ │ ├── stsb.py
│ │ │ │ │ └── wnli.py
│ │ │ │ ├── multi_task.py
│ │ │ │ ├── regression.py
│ │ │ │ ├── seq_cls.py
│ │ │ │ ├── squad.py
│ │ │ │ └── tok_cls.py
│ │ │ ├── cola.py
│ │ │ ├── seq_cls.py
│ │ │ ├── squad.py
│ │ │ └── wikisql.py
│ │ └── utils.py
│ ├── decorator/
│ │ ├── __init__.py
│ │ ├── arguments.py
│ │ └── register.py
│ ├── factory/
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── data_loader.py
│ │ ├── data_reader.py
│ │ ├── model.py
│ │ ├── optimizer.py
│ │ └── tokens.py
│ ├── learn/
│ │ ├── __init__.py
│ │ ├── experiment.py
│ │ ├── mode.py
│ │ ├── optimization/
│ │ │ ├── __init__.py
│ │ │ ├── exponential_moving_avarage.py
│ │ │ ├── learning_rate_scheduler.py
│ │ │ └── optimizer.py
│ │ ├── tensorboard.py
│ │ ├── trainer.py
│ │ └── utils.py
│ ├── machine/
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── components/
│ │ │ ├── __init__.py
│ │ │ └── retrieval/
│ │ │ ├── __init__.py
│ │ │ └── tfidf.py
│ │ ├── ensemble_topk.py
│ │ ├── knowlege_base/
│ │ │ ├── __init__.py
│ │ │ └── docs.py
│ │ ├── module.py
│ │ ├── nlu.py
│ │ └── open_qa.py
│ ├── metric/
│ │ ├── __init__.py
│ │ ├── classification.py
│ │ ├── glue.py
│ │ ├── korquad_v1_official.py
│ │ ├── regression.py
│ │ ├── squad_v1_official.py
│ │ ├── squad_v2_official.py
│ │ ├── wikisql_lib/
│ │ │ ├── __init__.py
│ │ │ ├── dbengine.py
│ │ │ └── query.py
│ │ └── wikisql_official.py
│ ├── model/
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── cls_utils.py
│ │ ├── multi_task/
│ │ │ ├── __init__.py
│ │ │ ├── bert.py
│ │ │ ├── category.py
│ │ │ └── mixin.py
│ │ ├── reading_comprehension/
│ │ │ ├── __init__.py
│ │ │ ├── bert.py
│ │ │ ├── bidaf.py
│ │ │ ├── bidaf_no_answer.py
│ │ │ ├── docqa.py
│ │ │ ├── docqa_no_answer.py
│ │ │ ├── drqa.py
│ │ │ ├── mixin.py
│ │ │ ├── qanet.py
│ │ │ └── roberta.py
│ │ ├── regression/
│ │ │ ├── __init__.py
│ │ │ ├── bert.py
│ │ │ ├── mixin.py
│ │ │ └── roberta.py
│ │ ├── semantic_parsing/
│ │ │ ├── __init__.py
│ │ │ ├── mixin.py
│ │ │ ├── sqlnet.py
│ │ │ └── utils.py
│ │ ├── sequence_classification/
│ │ │ ├── __init__.py
│ │ │ ├── bert.py
│ │ │ ├── mixin.py
│ │ │ ├── roberta.py
│ │ │ └── structured_self_attention.py
│ │ └── token_classification/
│ │ ├── __init__.py
│ │ ├── bert.py
│ │ └── mixin.py
│ ├── modules/
│ │ ├── __init__.py
│ │ ├── activation.py
│ │ ├── attention/
│ │ │ ├── __init__.py
│ │ │ ├── bi_attention.py
│ │ │ ├── co_attention.py
│ │ │ ├── docqa_attention.py
│ │ │ ├── multi_head_attention.py
│ │ │ └── seq_attention.py
│ │ ├── conv/
│ │ │ ├── __init__.py
│ │ │ ├── depthwise_separable_conv.py
│ │ │ └── pointwise_conv.py
│ │ ├── encoder/
│ │ │ ├── __init__.py
│ │ │ ├── lstm_cell_with_projection.py
│ │ │ └── positional.py
│ │ ├── functional.py
│ │ ├── initializer.py
│ │ └── layer/
│ │ ├── __init__.py
│ │ ├── highway.py
│ │ ├── normalization.py
│ │ ├── positionwise.py
│ │ ├── residual.py
│ │ └── scalar_mix.py
│ ├── nsml.py
│ ├── tokens/
│ │ ├── __init__.py
│ │ ├── cove.py
│ │ ├── elmo.py
│ │ ├── embedding/
│ │ │ ├── __init__.py
│ │ │ ├── base.py
│ │ │ ├── bert_embedding.py
│ │ │ ├── char_embedding.py
│ │ │ ├── cove_embedding.py
│ │ │ ├── elmo_embedding.py
│ │ │ ├── frequent_word_embedding.py
│ │ │ ├── sparse_feature.py
│ │ │ └── word_embedding.py
│ │ ├── hangul.py
│ │ ├── indexer/
│ │ │ ├── __init__.py
│ │ │ ├── base.py
│ │ │ ├── bert_indexer.py
│ │ │ ├── char_indexer.py
│ │ │ ├── elmo_indexer.py
│ │ │ ├── exact_match_indexer.py
│ │ │ ├── linguistic_indexer.py
│ │ │ └── word_indexer.py
│ │ ├── linguistic.py
│ │ ├── text_handler.py
│ │ ├── token_embedder/
│ │ │ ├── __init__.py
│ │ │ ├── base.py
│ │ │ ├── basic_embedder.py
│ │ │ └── reading_comprehension_embedder.py
│ │ ├── token_maker.py
│ │ ├── tokenizer/
│ │ │ ├── __init__.py
│ │ │ ├── base.py
│ │ │ ├── bpe.py
│ │ │ ├── char.py
│ │ │ ├── pass_text.py
│ │ │ ├── sent.py
│ │ │ ├── subword.py
│ │ │ ├── utils.py
│ │ │ └── word.py
│ │ └── vocabulary.py
│ └── utils.py
├── docs/
│ ├── Makefile
│ ├── _build/
│ │ ├── doctrees/
│ │ │ ├── claf.config.doctree
│ │ │ ├── claf.config.factory.doctree
│ │ │ ├── claf.data.dataset.doctree
│ │ │ ├── claf.data.doctree
│ │ │ ├── claf.data.reader.bert.doctree
│ │ │ ├── claf.data.reader.doctree
│ │ │ ├── claf.decorator.doctree
│ │ │ ├── claf.doctree
│ │ │ ├── claf.learn.doctree
│ │ │ ├── claf.machine.components.doctree
│ │ │ ├── claf.machine.components.retrieval.doctree
│ │ │ ├── claf.machine.doctree
│ │ │ ├── claf.metric.doctree
│ │ │ ├── claf.model.doctree
│ │ │ ├── claf.model.reading_comprehension.doctree
│ │ │ ├── claf.model.semantic_parsing.doctree
│ │ │ ├── claf.model.sequence_classification.doctree
│ │ │ ├── claf.model.token_classification.doctree
│ │ │ ├── claf.modules.attention.doctree
│ │ │ ├── claf.modules.conv.doctree
│ │ │ ├── claf.modules.doctree
│ │ │ ├── claf.modules.encoder.doctree
│ │ │ ├── claf.modules.layer.doctree
│ │ │ ├── claf.tokens.doctree
│ │ │ ├── claf.tokens.embedding.doctree
│ │ │ ├── claf.tokens.indexer.doctree
│ │ │ ├── claf.tokens.token_embedder.doctree
│ │ │ ├── claf.tokens.tokenizer.doctree
│ │ │ ├── contents/
│ │ │ │ ├── dataset_and_model.doctree
│ │ │ │ ├── pretrained_vector.doctree
│ │ │ │ └── tokens.doctree
│ │ │ ├── environment.pickle
│ │ │ ├── index.doctree
│ │ │ ├── modules.doctree
│ │ │ ├── references.doctree
│ │ │ ├── reports/
│ │ │ │ ├── glue.doctree
│ │ │ │ ├── historyqa.doctree
│ │ │ │ ├── korquad.doctree
│ │ │ │ ├── squad.doctree
│ │ │ │ └── wikisql.doctree
│ │ │ └── summary/
│ │ │ └── reading_comprehension.doctree
│ │ └── html/
│ │ ├── .buildinfo
│ │ ├── .nojekyll
│ │ ├── _modules/
│ │ │ ├── claf/
│ │ │ │ ├── config/
│ │ │ │ │ ├── args.html
│ │ │ │ │ ├── factory/
│ │ │ │ │ │ ├── base.html
│ │ │ │ │ │ ├── data_loader.html
│ │ │ │ │ │ ├── data_reader.html
│ │ │ │ │ │ ├── model.html
│ │ │ │ │ │ ├── optimizer.html
│ │ │ │ │ │ └── tokens.html
│ │ │ │ │ ├── namespace.html
│ │ │ │ │ ├── pattern.html
│ │ │ │ │ ├── registry.html
│ │ │ │ │ └── utils.html
│ │ │ │ ├── data/
│ │ │ │ │ ├── batch.html
│ │ │ │ │ ├── collate.html
│ │ │ │ │ ├── data_handler.html
│ │ │ │ │ ├── dataset/
│ │ │ │ │ │ ├── base.html
│ │ │ │ │ │ ├── bert/
│ │ │ │ │ │ │ ├── multi_task.html
│ │ │ │ │ │ │ ├── regression.html
│ │ │ │ │ │ │ ├── seq_cls.html
│ │ │ │ │ │ │ ├── squad.html
│ │ │ │ │ │ │ └── tok_cls.html
│ │ │ │ │ │ ├── seq_cls.html
│ │ │ │ │ │ ├── seq_cls_bert.html
│ │ │ │ │ │ ├── squad.html
│ │ │ │ │ │ ├── squad_bert.html
│ │ │ │ │ │ ├── tok_cls_bert.html
│ │ │ │ │ │ └── wikisql.html
│ │ │ │ │ ├── reader/
│ │ │ │ │ │ ├── base.html
│ │ │ │ │ │ ├── bert/
│ │ │ │ │ │ │ ├── cola.html
│ │ │ │ │ │ │ ├── conll2003.html
│ │ │ │ │ │ │ ├── glue/
│ │ │ │ │ │ │ │ ├── cola.html
│ │ │ │ │ │ │ │ ├── mnli.html
│ │ │ │ │ │ │ │ ├── mrpc.html
│ │ │ │ │ │ │ │ ├── qnli.html
│ │ │ │ │ │ │ │ ├── qqp.html
│ │ │ │ │ │ │ │ ├── rte.html
│ │ │ │ │ │ │ │ ├── sst.html
│ │ │ │ │ │ │ │ ├── stsb.html
│ │ │ │ │ │ │ │ └── wnli.html
│ │ │ │ │ │ │ ├── mnli.html
│ │ │ │ │ │ │ ├── mrpc.html
│ │ │ │ │ │ │ ├── multi_task.html
│ │ │ │ │ │ │ ├── qnli.html
│ │ │ │ │ │ │ ├── qqp.html
│ │ │ │ │ │ │ ├── regression.html
│ │ │ │ │ │ │ ├── rte.html
│ │ │ │ │ │ │ ├── seq_cls.html
│ │ │ │ │ │ │ ├── squad.html
│ │ │ │ │ │ │ ├── sst.html
│ │ │ │ │ │ │ ├── stsb.html
│ │ │ │ │ │ │ ├── tok_cls.html
│ │ │ │ │ │ │ └── wnli.html
│ │ │ │ │ │ ├── cola.html
│ │ │ │ │ │ ├── seq_cls.html
│ │ │ │ │ │ ├── squad.html
│ │ │ │ │ │ └── wikisql.html
│ │ │ │ │ └── utils.html
│ │ │ │ ├── decorator/
│ │ │ │ │ ├── arguments.html
│ │ │ │ │ └── register.html
│ │ │ │ ├── learn/
│ │ │ │ │ ├── experiment.html
│ │ │ │ │ ├── mode.html
│ │ │ │ │ ├── tensorboard.html
│ │ │ │ │ ├── trainer.html
│ │ │ │ │ └── utils.html
│ │ │ │ ├── machine/
│ │ │ │ │ ├── base.html
│ │ │ │ │ ├── components/
│ │ │ │ │ │ └── retrieval/
│ │ │ │ │ │ └── tfidf.html
│ │ │ │ │ ├── module.html
│ │ │ │ │ ├── nlu.html
│ │ │ │ │ └── open_qa.html
│ │ │ │ ├── metric/
│ │ │ │ │ ├── classification.html
│ │ │ │ │ ├── squad_v1_official.html
│ │ │ │ │ ├── squad_v2_official.html
│ │ │ │ │ └── wikisql_official.html
│ │ │ │ ├── model/
│ │ │ │ │ ├── base.html
│ │ │ │ │ ├── cls_utils.html
│ │ │ │ │ ├── reading_comprehension/
│ │ │ │ │ │ ├── bert.html
│ │ │ │ │ │ ├── bert_for_qa.html
│ │ │ │ │ │ ├── bidaf.html
│ │ │ │ │ │ ├── bidaf_no_answer.html
│ │ │ │ │ │ ├── docqa.html
│ │ │ │ │ │ ├── docqa_no_answer.html
│ │ │ │ │ │ ├── drqa.html
│ │ │ │ │ │ ├── mixin.html
│ │ │ │ │ │ ├── qanet.html
│ │ │ │ │ │ └── roberta.html
│ │ │ │ │ ├── semantic_parsing/
│ │ │ │ │ │ ├── mixin.html
│ │ │ │ │ │ ├── sqlnet.html
│ │ │ │ │ │ └── utils.html
│ │ │ │ │ ├── sequence_classification/
│ │ │ │ │ │ ├── bert.html
│ │ │ │ │ │ ├── bert_for_seq_cls.html
│ │ │ │ │ │ ├── mixin.html
│ │ │ │ │ │ ├── roberta.html
│ │ │ │ │ │ └── structured_self_attention.html
│ │ │ │ │ └── token_classification/
│ │ │ │ │ ├── bert.html
│ │ │ │ │ ├── bert_for_tok_cls.html
│ │ │ │ │ └── mixin.html
│ │ │ │ ├── modules/
│ │ │ │ │ ├── activation.html
│ │ │ │ │ ├── attention/
│ │ │ │ │ │ ├── bi_attention.html
│ │ │ │ │ │ ├── co_attention.html
│ │ │ │ │ │ ├── docqa_attention.html
│ │ │ │ │ │ ├── multi_head_attention.html
│ │ │ │ │ │ └── seq_attention.html
│ │ │ │ │ ├── conv/
│ │ │ │ │ │ ├── depthwise_separable_conv.html
│ │ │ │ │ │ └── pointwise_conv.html
│ │ │ │ │ ├── encoder/
│ │ │ │ │ │ ├── lstm_cell_with_projection.html
│ │ │ │ │ │ └── positional.html
│ │ │ │ │ ├── functional.html
│ │ │ │ │ ├── initializer.html
│ │ │ │ │ └── layer/
│ │ │ │ │ ├── highway.html
│ │ │ │ │ ├── normalization.html
│ │ │ │ │ ├── positionwise.html
│ │ │ │ │ ├── residual.html
│ │ │ │ │ └── scalar_mix.html
│ │ │ │ ├── tokens/
│ │ │ │ │ ├── cove.html
│ │ │ │ │ ├── elmo.html
│ │ │ │ │ ├── embedding/
│ │ │ │ │ │ ├── base.html
│ │ │ │ │ │ ├── bert_embedding.html
│ │ │ │ │ │ ├── char_embedding.html
│ │ │ │ │ │ ├── cove_embedding.html
│ │ │ │ │ │ ├── elmo_embedding.html
│ │ │ │ │ │ ├── frequent_word_embedding.html
│ │ │ │ │ │ ├── sparse_feature.html
│ │ │ │ │ │ └── word_embedding.html
│ │ │ │ │ ├── hangul.html
│ │ │ │ │ ├── indexer/
│ │ │ │ │ │ ├── base.html
│ │ │ │ │ │ ├── bert_indexer.html
│ │ │ │ │ │ ├── char_indexer.html
│ │ │ │ │ │ ├── elmo_indexer.html
│ │ │ │ │ │ ├── exact_match_indexer.html
│ │ │ │ │ │ ├── linguistic_indexer.html
│ │ │ │ │ │ └── word_indexer.html
│ │ │ │ │ ├── linguistic.html
│ │ │ │ │ ├── text_handler.html
│ │ │ │ │ ├── token_embedder/
│ │ │ │ │ │ ├── base.html
│ │ │ │ │ │ ├── basic_embedder.html
│ │ │ │ │ │ └── reading_comprehension_embedder.html
│ │ │ │ │ ├── token_maker.html
│ │ │ │ │ ├── tokenizer/
│ │ │ │ │ │ ├── base.html
│ │ │ │ │ │ ├── bpe.html
│ │ │ │ │ │ ├── char.html
│ │ │ │ │ │ ├── pass_text.html
│ │ │ │ │ │ ├── sent.html
│ │ │ │ │ │ ├── subword.html
│ │ │ │ │ │ ├── utils.html
│ │ │ │ │ │ └── word.html
│ │ │ │ │ └── vocabulary.html
│ │ │ │ ├── tokens.html
│ │ │ │ └── utils.html
│ │ │ ├── index.html
│ │ │ ├── logging.html
│ │ │ └── pathlib.html
│ │ ├── _sources/
│ │ │ ├── claf.config.factory.rst.txt
│ │ │ ├── claf.config.rst.txt
│ │ │ ├── claf.data.dataset.rst.txt
│ │ │ ├── claf.data.reader.bert.rst.txt
│ │ │ ├── claf.data.reader.rst.txt
│ │ │ ├── claf.data.rst.txt
│ │ │ ├── claf.decorator.rst.txt
│ │ │ ├── claf.learn.rst.txt
│ │ │ ├── claf.machine.components.retrieval.rst.txt
│ │ │ ├── claf.machine.components.rst.txt
│ │ │ ├── claf.machine.rst.txt
│ │ │ ├── claf.metric.rst.txt
│ │ │ ├── claf.model.reading_comprehension.rst.txt
│ │ │ ├── claf.model.rst.txt
│ │ │ ├── claf.model.semantic_parsing.rst.txt
│ │ │ ├── claf.model.sequence_classification.rst.txt
│ │ │ ├── claf.model.token_classification.rst.txt
│ │ │ ├── claf.modules.attention.rst.txt
│ │ │ ├── claf.modules.conv.rst.txt
│ │ │ ├── claf.modules.encoder.rst.txt
│ │ │ ├── claf.modules.layer.rst.txt
│ │ │ ├── claf.modules.rst.txt
│ │ │ ├── claf.rst.txt
│ │ │ ├── claf.tokens.embedding.rst.txt
│ │ │ ├── claf.tokens.indexer.rst.txt
│ │ │ ├── claf.tokens.rst.txt
│ │ │ ├── claf.tokens.token_embedder.rst.txt
│ │ │ ├── claf.tokens.tokenizer.rst.txt
│ │ │ ├── contents/
│ │ │ │ ├── dataset_and_model.md.txt
│ │ │ │ ├── pretrained_vector.md.txt
│ │ │ │ └── tokens.md.txt
│ │ │ ├── index.rst.txt
│ │ │ ├── modules.rst.txt
│ │ │ ├── references.md.txt
│ │ │ ├── reports/
│ │ │ │ ├── glue.md.txt
│ │ │ │ ├── historyqa.md.txt
│ │ │ │ ├── korquad.md.txt
│ │ │ │ ├── squad.md.txt
│ │ │ │ └── wikisql.md.txt
│ │ │ └── summary/
│ │ │ └── reading_comprehension.md.txt
│ │ ├── _static/
│ │ │ ├── basic.css
│ │ │ ├── css/
│ │ │ │ ├── badge_only.css
│ │ │ │ └── theme.css
│ │ │ ├── doctools.js
│ │ │ ├── documentation_options.js
│ │ │ ├── jquery-3.2.1.js
│ │ │ ├── jquery-3.4.1.js
│ │ │ ├── jquery.js
│ │ │ ├── js/
│ │ │ │ └── theme.js
│ │ │ ├── language_data.js
│ │ │ ├── pygments.css
│ │ │ ├── searchtools.js
│ │ │ ├── theme_overrides.css
│ │ │ ├── underscore-1.3.1.js
│ │ │ ├── underscore.js
│ │ │ └── websupport.js
│ │ ├── claf.config.factory.html
│ │ ├── claf.config.html
│ │ ├── claf.data.dataset.html
│ │ ├── claf.data.html
│ │ ├── claf.data.reader.bert.html
│ │ ├── claf.data.reader.html
│ │ ├── claf.decorator.html
│ │ ├── claf.html
│ │ ├── claf.learn.html
│ │ ├── claf.machine.components.html
│ │ ├── claf.machine.components.retrieval.html
│ │ ├── claf.machine.html
│ │ ├── claf.metric.html
│ │ ├── claf.model.html
│ │ ├── claf.model.reading_comprehension.html
│ │ ├── claf.model.semantic_parsing.html
│ │ ├── claf.model.sequence_classification.html
│ │ ├── claf.model.token_classification.html
│ │ ├── claf.modules.attention.html
│ │ ├── claf.modules.conv.html
│ │ ├── claf.modules.encoder.html
│ │ ├── claf.modules.html
│ │ ├── claf.modules.layer.html
│ │ ├── claf.tokens.embedding.html
│ │ ├── claf.tokens.html
│ │ ├── claf.tokens.indexer.html
│ │ ├── claf.tokens.token_embedder.html
│ │ ├── claf.tokens.tokenizer.html
│ │ ├── contents/
│ │ │ ├── dataset_and_model.html
│ │ │ ├── pretrained_vector.html
│ │ │ └── tokens.html
│ │ ├── genindex.html
│ │ ├── index.html
│ │ ├── modules.html
│ │ ├── objects.inv
│ │ ├── py-modindex.html
│ │ ├── references.html
│ │ ├── reports/
│ │ │ ├── glue.html
│ │ │ ├── historyqa.html
│ │ │ ├── korquad.html
│ │ │ ├── squad.html
│ │ │ └── wikisql.html
│ │ ├── search.html
│ │ ├── searchindex.js
│ │ └── summary/
│ │ └── reading_comprehension.html
│ ├── _static/
│ │ └── theme_overrides.css
│ ├── _templates/
│ │ ├── modules.rst
│ │ └── package.rst
│ ├── claf.config.factory.rst
│ ├── claf.config.rst
│ ├── claf.data.dataset.rst
│ ├── claf.data.reader.bert.rst
│ ├── claf.data.reader.rst
│ ├── claf.data.rst
│ ├── claf.decorator.rst
│ ├── claf.learn.rst
│ ├── claf.machine.components.retrieval.rst
│ ├── claf.machine.components.rst
│ ├── claf.machine.rst
│ ├── claf.metric.rst
│ ├── claf.model.reading_comprehension.rst
│ ├── claf.model.rst
│ ├── claf.model.semantic_parsing.rst
│ ├── claf.model.sequence_classification.rst
│ ├── claf.model.token_classification.rst
│ ├── claf.modules.attention.rst
│ ├── claf.modules.conv.rst
│ ├── claf.modules.encoder.rst
│ ├── claf.modules.layer.rst
│ ├── claf.modules.rst
│ ├── claf.rst
│ ├── claf.tokens.embedding.rst
│ ├── claf.tokens.indexer.rst
│ ├── claf.tokens.rst
│ ├── claf.tokens.token_embedder.rst
│ ├── claf.tokens.tokenizer.rst
│ ├── conf.py
│ ├── contents/
│ │ ├── dataset_and_model.md
│ │ ├── pretrained_vector.md
│ │ └── tokens.md
│ ├── index.rst
│ ├── make.bat
│ ├── modules.rst
│ ├── references.md
│ ├── reports/
│ │ ├── glue.md
│ │ ├── historyqa.md
│ │ ├── korquad.md
│ │ ├── squad.md
│ │ └── wikisql.md
│ ├── requirements.txt
│ └── summary/
│ └── reading_comprehension.md
├── eval.py
├── index.html
├── machine.py
├── machine_config/
│ ├── ko_wiki.json
│ └── nlu.json
├── predict.py
├── pyproject.toml
├── reports/
│ ├── inference_result/
│ │ ├── bert_for_qa-cpu.json
│ │ ├── bidaf+elmo-cpu.json
│ │ ├── bidaf-cpu.json
│ │ ├── docqa+elmo-cpu.json
│ │ ├── docqa-cpu.json
│ │ ├── drqa-cpu.json
│ │ └── qanet-cpu.json
│ └── summary/
│ ├── bert_for_qa.json
│ ├── bidaf+elmo.json
│ ├── bidaf.json
│ ├── docqa+elmo.json
│ ├── docqa.json
│ ├── drqa.json
│ └── qanet.json
├── requirements.txt
├── script/
│ ├── convert_checkpoint_to_bert_model.py
│ ├── convert_embedding_to_vocab_txt.py
│ ├── download_wikisql.sh
│ ├── install_mecab.sh
│ ├── make_squad_synthetic_data.py
│ └── plot.py
├── setup.py
├── tests/
│ ├── __init__.py
│ ├── claf/
│ │ ├── data/
│ │ │ └── test_batch.py
│ │ ├── machine/
│ │ │ └── knowlege_base/
│ │ │ └── test_docs.py
│ │ ├── modules/
│ │ │ └── test_functional.py
│ │ └── tokens/
│ │ └── test_vocabulary.py
│ └── integration/
│ ├── test_config.py
│ ├── test_machine.py
│ ├── test_multi_task.py
│ ├── test_reading_comprehension.py
│ ├── test_semantic_parsing.py
│ ├── test_sequence_classification.py
│ ├── test_token_classification.py
│ ├── test_tokenizers.py
│ └── utils.py
└── train.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .coveragerc
================================================
# .coveragerc to control coverage.py
[run]
branch = True
[report]
# Regexes for lines to exclude from consideration
exclude_lines =
# Have to re-enable the standard pragma
pragma: no cover
# Don't complain about missing debug-only code:
def __repr__
if self\.debug
# Don't complain if tests don't hit defensive assertion code:
raise AssertionError
raise NotImplementedError
raise ValueError
# Don't complain
pass
logger.info
# Dont't complain NSML and GPU env
if IS_ON_NSML
if torch.cuda.is_available()
if self.use_multi_gpu
# Don't complain if non-runnable code isn't run:
if 0:
if __name__ == .__main__.:
ignore_errors = True
[html]
directory = coverage_html_report
================================================
FILE: .gitignore
================================================
# Created by https://www.gitignore.io/api/macos,python,jupyternotebook
### CLaF: Clova Language Framework ###
/data/
logs/
/inference_result/hide
model_config/cache
model_config/short
/mecab
/src
### pytest
cov_html/
### JupyterNotebook ###
.ipynb_checkpoints
*/.ipynb_checkpoints/*
# Remove previous ipynb_checkpoints
# git rm -r .ipynb_checkpoints/
#
### macOS ###
*.DS_Store
.AppleDouble
.LSOverride
# Icon must end with two \r
Icon
# Thumbnails
._*
# Files that might appear in the root of a volume
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
.com.apple.timemachine.donotpresent
# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
.pytest_cache/
nosetests.xml
coverage.xml
*.cover
.hypothesis/
# Translations
*.mo
*.pot
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
# docs/_build/
# PyBuilder
target/
# Jupyter Notebook
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule.*
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
# End of https://www.gitignore.io/api/macos,python,jupyternotebook
================================================
FILE: .nojekyll
================================================
================================================
FILE: .readthedocs.yml
================================================
# .readthedocs.yml
# Read the Docs configuration file
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
# Required
version: 2
# Build documentation in the docs/ directory with Sphinx
sphinx:
configuration: docs/conf.py
# Build documentation with MkDocs
#mkdocs:
# configuration: mkdocs.yml
# Optionally build your docs in additional formats such as PDF and ePub
formats: all
# Optionally set the version of Python and requirements required to build your docs
python:
version: 3.6
install:
- requirements: docs/requirements.txt
================================================
FILE: .travis.yml
================================================
language: python
python:
- "3.6"
# command to install dependencies
install:
- pip install --upgrade pip
- pip install --progress-bar off -r requirements.txt
- pip install --progress-bar off codecov
- python -m nltk.downloader punkt
- python -m nltk.downloader wordnet
- export BOTO_CONFIG=/dev/nul
# command to run tests
script:
python -m pytest tests --cov-report term --cov claf
after_success:
codecov -t 2a4a166c-cd15-4121-b4a7-c2eed4c7390f
================================================
FILE: CONTRIBUTING.md
================================================
# Contributing to CLAF
First of all, thank you for considering contributing to CLAF. It's people like you that make CLaF such a great framework.
Following these guidelines helps to communicate that you respect the time of the developers managing and developing this open source project. In return, they should reciprocate that respect in addressing your issue, assessing changes, and helping you finalize your pull requests.
If you are not familiar with creating a Pull Request, here are some guides:
- [Create a Pull Request](https://help.github.com/articles/creating-a-pull-request/)
## Bug & Simple features
1. Search on [Issues](https://github.com/naver/claf/issues)
2. If there are similar issues, add Comments to those issues, otherwise, create new ones
3. Check `pytest`, `black (lint)` before Pull Request
- [pytest](https://github.com/pytest-dev/pytest)
- Add unittest to the `tests/claf` folder and integration test code if necessary
- Run test with coverage ```pytest --cov-config .coveragerc --cov-report html:cov_html --cov=rqa tests```
- [Black](https://github.com/ambv/black) (lint)
- ```black claf -l 120 ``` (reformat your code)
4. Clean up your work to create a Pull Request
(* When adding a new function (model, optimizer and so on), add it to `claf.config.ars` with a description.)
e.g. Exponential Learning Rate Scheduler
```
# ExponentialLR:
--exponential.gamma OPTIMIZER.EXPONENTIAL.GAMMA
Multiplicative factor of learning rate decay.
Default: 0.1.
--exponential.last_epoch OPTIMIZER.EXPONENTIAL.LAST_EPOCH
The index of last epoch.
Default: -1.
```
## The structure of the framework
1. Post it on the issue and discuss it with maintainers.
2. After discuss, organize according to priority and start working on.
================================================
FILE: Dockerfile
================================================
FROM jmin/pytorch:apex
RUN git clone https://github.com/naver/claf && cd claf && pip install -r requirements.txt && python setup.py install
RUN apt-get install g++ default-jdk
RUN bash <(curl -s https://raw.githubusercontent.com/konlpy/konlpy/master/scripts/mecab.sh)
RUN python -m nltk.downloader punkt --dir /usr/share/nltk_data
RUN python -m nltk.downloader wordnet --dir /usr/share/nltk_data
================================================
FILE: LICENSE
================================================
Copyright (c) 2019-present NAVER Corp.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: NOTICE
================================================
CLaF
Copyright 2019-present NAVER Corp.
This project contains subcomponents with separate copyright notices and license terms.
Your use of the source code for these subcomponents is subject to the terms and conditions of the following licenses.
=======================================================================
rhobot/Hangulpy from https://github.com/rhobot/Hangulpy
=======================================================================
Copyright (C) 2012 Ryan Rho, Hyunwoo Cho
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
=======================================================================
allenai/allennlp from https://github.com/allenai/allennlp
=======================================================================
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
=======================================================================
facebookresearch/DrQA from https://github.com/facebookresearch/DrQA
=======================================================================
BSD License
For DrQA software
Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name Facebook nor the names of its contributors may be used to
endorse or promote products derived from this software without specific
prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
=====
================================================
FILE: README.md
================================================
<p align="center">
<img src="images/logo.png" style="inline" width=300>
</p>
<h4 align="center">Clova Language Framework</h4>
<p align="center">
<a href="https://naver.github.io/claf">
<img src="https://img.shields.io/badge/docs-passing-brightgreen.svg" alt="Documentation Status">
</a>
<a href="https://travis-ci.org/naver/claf">
<img src='https://travis-ci.org/naver/claf.svg?branch=master'/>
</a>
<a href="https://github.com/ambv/black">
<img src="https://img.shields.io/badge/code%20style-black-000000.svg" alt="Code style: black">
<a href="https://codecov.io/gh/naver/claf">
<img src="https://codecov.io/gh/naver/claf/branch/master/graph/badge.svg" />
</a>
</p>
---
# CLaF: Clova Language Framework
- [Full Documentation](https://naver.github.io/claf/)
- [Dataset And Model](https://naver.github.io/claf/docs/_build/html/contents/dataset_and_model.html)
- [Pretrained Vector](https://naver.github.io/claf/docs/_build/html/contents/pretrained_vector.html)
- [Tokens](https://naver.github.io/claf/docs/_build/html/contents/tokens.html): `Tokenizers` and `TokenMakers`
- List of [BaseConfig](#baseconfig)
| Task | Language | Dataset | Model |
| ---- | -------- | ------- | ----- |
| Multi-Task Learning | English | [GLUE Benchmark](https://gluebenchmark.com/), [SQuAD v1.1](https://rajpurkar.github.io/SQuAD-explorer/) | `MT-DNN (BERT)` |
| Natural Language Understanding | English | [GLUE Benchmark](https://gluebenchmark.com/) | `BERT`, `RoBERTa` |
| Named Entity Recognition | English | CoNLL 2003 | `BERT` |
| Question Answering | Korean | [KorQuAD v1.0](https://korquad.github.io/category/1.0_KOR.html) | `BiDAF`, `DocQA`, `BERT` |
| Question Answering | Engilsh | [SQuAD v1.1 and v2.0](https://rajpurkar.github.io/SQuAD-explorer/) | - v1.1: `BiDAF`, `DrQA`, `DocQA`, `DocQA+ELMo`, `QANet`, `BERT`, `RoBERTa` <br/> - v2.0: `BiDAF + No Answer`, `DocQA + No Answer` |
| Semantic Parsing | English | [WikiSQL](https://github.com/salesforce/WikiSQL) | `SQLNet` |
- Reports
- [GLUE](https://naver.github.io/claf/docs/_build/html/reports/glue.html)
- [KorQuAD](https://naver.github.io/claf/docs/_build/html/reports/korquad.html)
- [SQuAD](https://naver.github.io/claf/docs/_build/html/reports/squad.html)
- [WikiSQL](https://naver.github.io/claf/docs/_build/html/reports/wikisql.html)
- Summary (1-example Inference Latency)
- [Reading Comprehension](https://naver.github.io/claf/docs/_build/html/summary/reading_comprehension.html)
- List of [MachineConfig](#machine)
| Name | Language | Pipeline | Note |
| ---- | -------- | ------- | ----- |
| KoWiki | Korean | `Wiki Dumps` -> `Document Retrieval` -> `Reading Comprehension` | - |
| NLU | All | `Query` -> `Intent Classification` & `Token Classification (Slot)` -> `Template Matching` | - |
---
## Table of Contents
- [Overview](#overview)
- [Features](#features)
- [Installation](#installation)
- [Requirements](#requirements)
- [Install via pip](#install-via-pip)
- [Experiment](#experiment)
- [Usage](#usage)
- [Training](#training)
- [Evaluate](#evaluate)
- [Predict](#predict)
- [Docker Images](#docker-images)
- [Machine](#machine)
- [Contributing](#contributing)
- [Maintainers](#maintainers)
- [Citing](#citing)
- [License](#license)
---
## Overview
**CLaF** is a Language Framework built on PyTorch that provides following two high-level features:
- `Experiment` enables the control of training flow in general NLP by offering various `TokenMaker` methods.
- CLaF is inspired by the design principle of [AllenNLP](https://github.com/allenai/allennlp) such as the higher level concepts and reusable code, but mostly based on PyTorch’s common module, so that user can easily modify the code on their demands.
- `Machine` helps to combine various modules to build a NLP Machine in one place.
- There are knowledge-based, components and trained experiments which infer 1-example in modules.
### Features
- **Multilingual** modeling support (currently, English and Korean are supported).
- Light weighted **Systemization** and Modularization.
- Easy extension and implementation of models.
- A wide variation of **Experiments** with reproducible and comprehensive logging
- The metrics for services such as "1\-example inference latency" are provided.
- Easy to build of a NLP **Machine** by combining modules.
## Installation
### Requirements
- Python 3.6
- PyTorch >= 1.3.1
- [MeCab](https://bitbucket.org/eunjeon/mecab-ko) for Korean Tokenizer
- ```sh script/install_mecab.sh```
It is recommended to use the virtual environment.
[Conda](https://conda.io/docs/download.html) is the easiest way to set up a virtual environment.
```
conda create -n claf python=3.6
conda activate claf
(claf) ✗ pip install -r requirements.txt
```
### Install via pip
Commands to install via pip
```
pip install claf
```
## Experiment
- Training Flow
### Usage
#### Training
1. only Arguments
```
python train.py --train_file_path {file_path} --valid_file_path {file_path} --model_name {name} ...
```
2. only BaseConfig (skip `/base_config` path)
```
python train.py --base_config {base_config}
```
3. BaseConfig + Arguments
```
python train.py --base_config {base_config} --learning_rate 0.002
```
- Load BaseConfig then overwrite `learning_rate` to 0.002
#### BaseConfig
Declarative experiment config (.json, .ymal)
- Simply matching with object's parameters
- Exists samples in `/base_config` directory
##### Defined BaseConfig
```
Base Config:
--base_config BASE_CONFIG
Use pre-defined base_config:
[]
* CoNLL 2003:
['conll2003/bert_large_cased']
* GLUE:
['glue/qqp_roberta_base', 'glue/qnli_bert_base', 'glue/rte_bert_base', 'glue/wnli_roberta_base', 'glue/mnlim_roberta_base', 'glue/wnli_bert_base', 'glue/mnlimm_roberta_base', 'glue/cola_bert_base', 'glue/mrpc_bert_base', 'glue/mnlimm_bert_base', 'glue/stsb_bert_base', 'glue/mnlim_bert_base', 'glue/qqp_bert_base', 'glue/rte_roberta_base', 'glue/qnli_roberta_base', 'glue/sst_bert_base', 'glue/mrpc_roberta_base', 'glue/cola_roberta_base', 'glue/stsb_roberta_base', 'glue/sst_roberta_base']
* KorQuAD:
['korquad/bert_base_multilingual_cased', 'korquad/bidaf', 'korquad/bert_base_multilingual_uncased', 'korquad/docqa']
* SQuAD:
['squad/bert_large_uncased', 'squad/bidaf', 'squad/drqa_paper', 'squad/drqa', 'squad/bert_base_uncased', 'squad/qanet', 'squad/docqa+elmo', 'squad/bidaf_no_answer', 'squad/docqa_no_answer', 'squad/qanet_paper', 'squad/bidaf+elmo', 'squad/docqa']
* WikiSQL:
['wikisql/sqlnet']
```
#### Evaluate
```
python eval.py <data_path> <model_checkpoint_path>
```
- Example
```
✗ python eval.py data/squad/dev-v1.1.json logs/squad/bidaf/checkpoint/model_19.pkl
...
[INFO] - {
"valid/loss": 2.59111491665019,
"valid/epoch_time": 60.7434446811676,
"valid/start_acc": 63.17880794701987,
"valid/end_acc": 67.19016083254493,
"valid/span_acc": 54.45600756859035,
"valid/em": 68.10785241248817,
"valid/f1": 77.77963381714842
}
# write predictions files (<log_dir>/predictions/predictions-valid-19.json)
```
- 1-example Inference Latency ([Summary](docs/_build/html/reports/summary.html))
```
✗ python eval.py data/squad/dev-v1.1.json logs/squad/bidaf/checkpoint/model_19.pkl
...
# Evaluate Inference Latency Mode.
...
[INFO] - saved inference_latency results. bidaf-cpu.json # file_format: {model_name}-{env}.json
```
#### Predict
```
python predict.py <model_checkpoint_path> --<arguments>
```
- Example
```
✗ python predict.py logs/squad/bidaf/checkpoint/model_19.pkl \
--question "When was the last Super Bowl in California?" \
--context "On May 21, 2013, NFL owners at their spring meetings in Boston voted and awarded the game to Levi's Stadium. The $1.2 billion stadium opened in 2014. It is the first Super Bowl held in the San Francisco Bay Area since Super Bowl XIX in 1985, and the first in California since Super Bowl XXXVII took place in San Diego in 2003."
>>> Predict: {'text': '2003', 'score': 4.1640071868896484}
```
#### Docker Images
- [Docker Hub](https://hub.docker.com/u/claf)
- Run with Docker Image
- Pull docker image
```✗ docker pull claf/claf:latest```
- Run
``` docker run --rm -i -t claf/claf:latest /bin/bash ```
---
### Machine
- Machine Architecture
#### Usage
- Define the config file (.json) like [BaseConfig](#baseconfig) in `machine_config/` directory
- Run CLaF Machine (skip `/machine_config` path)
```
✗ python machine.py --machine_config {machine_config}
```
* The list of pre-defined `Machine`:
```
Machine Config:
--machine_config MACHINE_CONFIG
Use pre-defined machine_config (.json (.json))
['ko_wiki', 'nlu']
```
#### Open QA (DrQA Style)
DrQA is a system for reading comprehension applied to open-domain question answering. The system has to combine the challenges of document retrieval (finding the relevant documents) with that of machine comprehension of text (identifying the answers from those documents).
- ko_wiki: Korean Wiki Version
```
✗ python machine.py --machine_config ko_wiki
...
Completed!
Question > 동학의 2대 교주 이름은?
--------------------------------------------------
Doc Scores:
- 교주 : 0.5347289443016052
- 이교주 : 0.4967213571071625
- 교주도 : 0.49036136269569397
- 동학 : 0.4800325632095337
- 동학중학교 : 0.4352934956550598
--------------------------------------------------
Answer: [
{
"text": "최시형",
"score": 11.073444366455078
},
{
"text": "충주목",
"score": 9.443866729736328
},
{
"text": "반월동",
"score": 9.37778091430664
},
{
"text": "환조 이자춘",
"score": 4.64817476272583
},
{
"text": "합포군",
"score": 3.3186707496643066
}
]
```
#### NLU (Dialog)
The reason why NLU machine does not return the full response is that response generation may require various task-specific post-processing techniques or additional logic(e.g. API calls, template-decision rules, template filling rules, nn-based response generation model) Therefore, for flexible usage, NLU machine returns only the NLU result.
```
✗ python machine.py --machine_config nlu
...
Utterance > "looking for a flight from Boston to Seoul or Incheon"
NLU Result: {
"intent": "flight",
"slots": {
"city.depart": ["Boston"],
"city.dest": ["Seoul", "Incheon"]
}
}
```
## Contributing
Thanks for your interest in contributing! There are many ways to contribute to this project.
Get started [here](./CONTRIBUTING.md).
## Maintainers
CLaF is currently maintained by
- [Dongjun Lee](https://github.com/DongjunLee) (Author)
- [Sohee Yang](https://github.com/soheeyang)
- [Minjeong Kim](https://github.com/Mjkim88)
## Citing
If you use CLaF for your work, please cite:
```bibtex
@misc{claf,
author = {Lee, Dongjun and Yang, Sohee and Kim, Minjeong},
title = {CLaF: Open-Source Clova Language Framework},
year = {2019},
publisher = {GitHub},
journal = {GitHub repository},
howpublished = {\url{https://github.com/naver/claf}}
}
```
We will update this bibtex with our paper.
## Acknowledgements
`docs/` directory which includes documentation created by [Sphinx](http://www.sphinx-doc.org/).
## License
MIT license
```
Copyright (c) 2019-present NAVER Corp.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
```
================================================
FILE: base_config/cola/bert_base_uncased.json
================================================
{
"data_reader": {
"dataset": "cola_bert",
"train_file_path": "<CoLA train.tsv path>",
"valid_file_path": "<CoLA dev.tsv path>",
"cola_bert": {
"sequence_max_length": 128
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.0
}
},
"trainer": {
"log_dir": "logs/cola_bert",
"num_epochs": 3,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 10000
}
},
"seed_num": 42
}
================================================
FILE: base_config/cola/bert_large_uncased.json
================================================
{
"data_reader": {
"dataset": "cola_bert",
"train_file_path": "<CoLA train.tsv path>",
"valid_file_path": "<CoLA dev.tsv path>",
"cola_bert": {
"sequence_max_length": 128
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-large-uncased",
"dropout": 0.0
}
},
"trainer": {
"log_dir": "logs/cola_bert",
"num_epochs": 3,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 10000
}
},
"seed_num": 42
}
================================================
FILE: base_config/cola/structured_self_attention.json
================================================
{
"data_reader": {
"dataset": "cola",
"train_file_path": "<CoLA train.tsv path>",
"valid_file_path": "<CoLA dev.tsv path>",
"cola": {
"sequence_max_length": 128
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["char", "glove"],
"types": ["char", "word"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"start_token": "<s>",
"end_token": "</s>",
"max_vocab_size": 260
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 16,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"glove": {
"vocab": {
"pretrained_path": "http://dev-reasoning-qa-data-ncl.nfra.io:7778/data/glove.6B.vocab.txt",
"pretrained_token": "intersect"
},
"indexer": {
"lowercase": true
},
"embedding": {
"embed_dim": 100,
"pretrained_path": "http://dev-reasoning-qa-data-ncl.nfra.io:7778/data/glove.6B.100d.txt",
"trainable": false,
"dropout": 0.2
}
}
},
"model": {
"name": "structured_self_attention",
"structured_self_attention": {
"encoding_rnn_hidden_dim": 300,
"encoding_rnn_num_layer": 2,
"encoding_rnn_dropout": 0,
"attention_dim": 350,
"num_attention_heads": 30,
"sequence_embed_dim": 2000,
"dropout": 0.5,
"penalization_coefficient": 1
}
},
"trainer": {
"log_dir": "logs/cola",
"num_epochs": 50,
"early_stopping_threshold": 10,
"grad_max_norm": 5.0,
"metric_key": "accuracy",
"verbose_step_count": 100,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adam",
"learning_rate": 0.001,
"exponential_moving_average": 0.999
},
"seed_num": 42
}
================================================
FILE: base_config/conll2003/bert_large_cased.json
================================================
{
"data_reader": {
"dataset": "conll2003_bert",
"train_file_path": "https://raw.githubusercontent.com/Franck-Dernoncourt/NeuroNER/master/data/conll2003/en/train.txt",
"valid_file_path": "https://raw.githubusercontent.com/Franck-Dernoncourt/NeuroNER/master/data/conll2003/en/valid.txt",
"conll2003_bert": {
"sequence_max_length": 128,
"ignore_tag_idx": -1
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": false
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_tok_cls",
"bert_for_tok_cls": {
"pretrained_model_name": "bert-large-cased",
"dropout": 0.0
}
},
"trainer": {
"log_dir": "logs/conll2003_bert",
"num_epochs": 3,
"early_stopping_threshold": 10,
"metric_key": "conlleval_f1",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 10000
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/cola_bert.json
================================================
{
"data_reader": {
"dataset": "cola_bert",
"train_file_path": "CoLA/train.tsv",
"valid_file_path": "CoLA/dev.tsv",
"cola_bert": {
"sequence_max_length": 128
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/cola_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "matthews_corr",
"eval_and_save_step_count": 100
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/cola_roberta.json
================================================
{
"data_reader": {
"dataset": "cola_bert",
"train_file_path": "CoLA/train.tsv",
"valid_file_path": "CoLA/dev.tsv",
"cola_bert": {
"sequence_max_length": 128,
"cls_token": "<s>",
"sep_token": "</s>",
"input_type": "roberta"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>",
"cls_token": "<s>",
"sep_token": "</s>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_seq_cls",
"roberta_for_seq_cls": {
"pretrained_model_name": "roberta-base",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/cola_roberta",
"num_epochs": 10,
"early_stopping_threshold": 20,
"metric_key": "matthews_corr",
"eval_and_save_step_count": 100
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.98],
"eps": 1e-6,
"weight_decay": 0.1
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.06
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/mnlim_bert.json
================================================
{
"data_reader": {
"dataset": "mnli_bert",
"train_file_path": "MNLI/train.tsv",
"valid_file_path": "MNLI/dev_matched.tsv",
"mnli_bert": {
"sequence_max_length": 128
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/mnlim_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/mnlim_roberta.json
================================================
{
"data_reader": {
"dataset": "mnli_bert",
"train_file_path": "MNLI/train.tsv",
"valid_file_path": "MNLI/dev_matched.tsv",
"mnli_bert": {
"sequence_max_length": 128,
"cls_token": "<s>",
"sep_token": "</s>",
"input_type": "roberta"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_seq_cls",
"roberta_for_seq_cls": {
"pretrained_model_name": "roberta-base",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/mnlim_roberta",
"num_epochs": 10,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.98],
"eps": 1e-6,
"weight_decay": 0.1
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.06
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/mnlimm_bert.json
================================================
{
"data_reader": {
"dataset": "mnli_bert",
"train_file_path": "MNLI/train.tsv",
"valid_file_path": "MNLI/dev_mismatched.tsv",
"mnli_bert": {
"sequence_max_length": 128
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/mnlimm_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/mnlimm_roberta.json
================================================
{
"data_reader": {
"dataset": "mnli_bert",
"train_file_path": "MNLI/train.tsv",
"valid_file_path": "MNLI/dev_mismatched.tsv",
"mnli_bert": {
"sequence_max_length": 128,
"cls_token": "<s>",
"sep_token": "</s>",
"input_type": "roberta"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_seq_cls",
"roberta_for_seq_cls": {
"pretrained_model_name": "roberta-base",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/mnlimm_roberta",
"num_epochs": 10,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.98],
"eps": 1e-6,
"weight_decay": 0.1
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.06
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/mrpc_bert.json
================================================
{
"data_reader": {
"dataset": "mrpc_bert",
"train_file_path": "MRPC/train.tsv",
"valid_file_path": "MRPC/dev.tsv",
"mrpc_bert": {
"sequence_max_length": 128
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/mrpc_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "f1",
"eval_and_save_step_count": 50
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/mrpc_roberta.json
================================================
{
"data_reader": {
"dataset": "mrpc_bert",
"train_file_path": "MRPC/train.tsv",
"valid_file_path": "MRPC/dev.tsv",
"mrpc_bert": {
"sequence_max_length": 128,
"cls_token": "<s>",
"sep_token": "</s>",
"input_type": "roberta"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_seq_cls",
"roberta_for_seq_cls": {
"pretrained_model_name": "roberta-base",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/mrpc_roberta",
"num_epochs": 10,
"early_stopping_threshold": 10,
"metric_key": "f1",
"eval_and_save_step_count": 100
},
"optimizer": {
"learning_rate": 1e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.98],
"eps": 1e-6,
"weight_decay": 0.1
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.06
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/qnli_bert.json
================================================
{
"data_reader": {
"dataset": "qnli_bert",
"train_file_path": "QNLI/train.tsv",
"valid_file_path": "QNLI/dev.tsv",
"qnli_bert": {
"sequence_max_length": 128
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/qnli_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/qnli_roberta.json
================================================
{
"data_reader": {
"dataset": "qnli_bert",
"train_file_path": "QNLI/train.tsv",
"valid_file_path": "QNLI/dev.tsv",
"qnli_bert": {
"sequence_max_length": 128,
"cls_token": "<s>",
"sep_token": "</s>",
"input_type": "roberta"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_seq_cls",
"roberta_for_seq_cls": {
"pretrained_model_name": "roberta-base",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/qnli_roberta",
"num_epochs": 10,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.98],
"eps": 1e-6,
"weight_decay": 0.1
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.06
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/qqp_bert.json
================================================
{
"data_reader": {
"dataset": "qqp_bert",
"train_file_path": "QQP/train.tsv",
"valid_file_path": "QQP/dev.tsv",
"qqp_bert": {
"sequence_max_length": 128
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/qqp_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "f1",
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/qqp_roberta.json
================================================
{
"data_reader": {
"dataset": "qqp_bert",
"train_file_path": "QQP/train.tsv",
"valid_file_path": "QQP/dev.tsv",
"qqp_bert": {
"sequence_max_length": 128,
"cls_token": "<s>",
"sep_token": "</s>",
"input_type": "roberta"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_seq_cls",
"roberta_for_seq_cls": {
"pretrained_model_name": "roberta-base",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/qqp_roberta",
"num_epochs": 10,
"early_stopping_threshold": 10,
"metric_key": "f1",
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.98],
"eps": 1e-6,
"weight_decay": 0.1
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.06
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/rte_bert.json
================================================
{
"data_reader": {
"dataset": "rte_bert",
"train_file_path": "RTE/train.tsv",
"valid_file_path": "RTE/dev.tsv",
"rte_bert": {
"sequence_max_length": 128
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/rte_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": 30
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/rte_roberta.json
================================================
{
"data_reader": {
"dataset": "rte_bert",
"train_file_path": "RTE/train.tsv",
"valid_file_path": "RTE/dev.tsv",
"rte_bert": {
"sequence_max_length": 128,
"cls_token": "<s>",
"sep_token": "</s>",
"input_type": "roberta"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_seq_cls",
"roberta_for_seq_cls": {
"pretrained_model_name": "roberta-base",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/rte_roberta",
"num_epochs": 20,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": 50
},
"optimizer": {
"learning_rate": 1e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.98],
"eps": 1e-6,
"weight_decay": 0.1
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.06
}
},
"seed_num": 21
}
================================================
FILE: base_config/glue/sst_bert.json
================================================
{
"data_reader": {
"dataset": "sst_bert",
"train_file_path": "SST-2/train.tsv",
"valid_file_path": "SST-2/dev.tsv",
"sst_bert": {
"sequence_max_length": 128
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/sst_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/sst_roberta.json
================================================
{
"data_reader": {
"dataset": "sst_bert",
"train_file_path": "SST-2/train.tsv",
"valid_file_path": "SST-2/dev.tsv",
"sst_bert": {
"sequence_max_length": 128,
"cls_token": "<s>",
"sep_token": "</s>",
"input_type": "roberta"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_seq_cls",
"roberta_for_seq_cls": {
"pretrained_model_name": "roberta-base",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/sst_roberta",
"num_epochs": 10,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.98],
"eps": 1e-6,
"weight_decay": 0.1
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.06
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/stsb_bert.json
================================================
{
"data_reader": {
"dataset": "stsb_bert",
"train_file_path": "STS-B/train.tsv",
"valid_file_path": "STS-B/dev.tsv",
"stsb_bert": {
"sequence_max_length": 128
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_reg",
"bert_for_reg": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/stsb_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "pearson_spearman_corr",
"eval_and_save_step_count": 100
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 22
}
================================================
FILE: base_config/glue/stsb_roberta.json
================================================
{
"data_reader": {
"dataset": "stsb_bert",
"train_file_path": "STS-B/train.tsv",
"valid_file_path": "STS-B/dev.tsv",
"stsb_bert": {
"sequence_max_length": 128,
"cls_token": "<s>",
"sep_token": "</s>",
"input_type": "roberta"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_reg",
"roberta_for_reg": {
"pretrained_model_name": "roberta-base",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/stsb_roberta",
"num_epochs": 10,
"early_stopping_threshold": 10,
"metric_key": "pearson_spearman_corr",
"eval_and_save_step_count": 100
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.98],
"eps": 1e-6,
"weight_decay": 0.1
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.06
}
},
"seed_num": 21
}
================================================
FILE: base_config/glue/wnli_bert.json
================================================
{
"data_reader": {
"dataset": "wnli_bert",
"train_file_path": "WNLI/train.tsv",
"valid_file_path": "WNLI/dev.tsv",
"wnli_bert": {
"sequence_max_length": 128
}
},
"iterator": {
"batch_size": 16
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/wnli_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": 10
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/glue/wnli_roberta.json
================================================
{
"data_reader": {
"dataset": "wnli_bert",
"train_file_path": "WNLI/train.tsv",
"valid_file_path": "WNLI/dev.tsv",
"wnli_bert": {
"sequence_max_length": 128,
"cls_token": "<s>",
"sep_token": "</s>",
"input_type": "roberta"
}
},
"iterator": {
"batch_size": 16
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_seq_cls",
"roberta_for_seq_cls": {
"pretrained_model_name": "roberta-base",
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/glue/wnli_roberta",
"num_epochs": 10,
"early_stopping_threshold": 30,
"metric_key": "accuracy",
"eval_and_save_step_count": 20
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.98],
"eps": 1e-6,
"weight_decay": 0.1
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.06
}
},
"seed_num": 21
}
================================================
FILE: base_config/korquad/bert_base_multilingual_cased.yaml
================================================
data_reader:
dataset: "squad_bert"
train_file_path: "https://korquad.github.io/dataset/KorQuAD_v1.0_train.json"
valid_file_path: "https://korquad.github.io/dataset/KorQuAD_v1.0_dev.json"
squad_bert:
lang_code: "ko"
max_seq_length: 512
context_stride: 64
max_question_length: 64
iterator:
batch_size: 12
token:
names:
- "feature"
types:
- "feature"
tokenizer:
subword:
name: "wordpiece"
wordpiece:
vocab_path: "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt"
word:
name: "bert_basic"
bert_basic:
do_lower_case: true
feature:
vocab:
pretrained_path: "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt"
pretrained_token: "all"
indexer":
do_tokenize: false
model:
name: "bert_for_qa"
bert_for_qa:
pretrained_model_name: "bert-base-multilingual-cased"
answer_maxlen: 30
trainer:
log_dir: "logs/test/bert_for_qa/"
num_epochs: 5
early_stopping_threshold: 10
metric_key: "f1"
verbose_step_count: 100
eval_and_save_step_count: 1000
optimizer:
learning_rate: 0.00003
op_type: "adamw"
adamw:
weight_decay: 0.01
lr_scheduler_type: "warmup_linear"
warmup_linear:
warmup_proportion: 0.1
seed_num: 42
================================================
FILE: base_config/korquad/bert_base_multilingual_uncased.json
================================================
{
"data_reader": {
"dataset": "squad_bert",
"train_file_path": "https://korquad.github.io/dataset/KorQuAD_v1.0_train.json",
"valid_file_path": "https://korquad.github.io/dataset/KorQuAD_v1.0_dev.json",
"squad_bert": {
"lang_code": "ko",
"max_seq_length": 512,
"context_stride": 64,
"max_question_length": 64
}
},
"iterator": {
"batch_size": 12
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_qa",
"bert_for_qa": {
"pretrained_model_name": "bert-base-multilingual-uncased",
"answer_maxlen": 30
}
},
"trainer": {
"log_dir": "logs/squad_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 0.00003,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/korquad/bidaf.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://korquad.github.io/dataset/KorQuAD_v1.0_train.json",
"valid_file_path": "https://korquad.github.io/dataset/KorQuAD_v1.0_dev.json",
"squad": {
"lang_code": "ko",
"context_max_length": 1000
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["char", "fasttext"],
"types": ["char", "word"],
"tokenizer": {
"char": {
"name": "jamo_ko"
},
"word": {
"name": "mecab_ko",
"split_with_regex": true
}
},
"char": {
"vocab": {
"start_token": "<s>",
"end_token": "</s>",
"max_vocab_size": 70
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 16,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"fasttext": {
"embedding": {
"embed_dim": 300,
"pretrained_path": "<fasttext.wiki.ko.300d.txt path>",
"trainable": false,
"dropout": 0.2
}
}
},
"model": {
"name": "bidaf",
"bidaf": {
"model_dim": 100,
"contextual_rnn_num_layer": 1,
"modeling_rnn_num_layer": 2,
"predict_rnn_num_layer": 1,
"dropout": 0.2
}
},
"trainer": {
"log_dir": "logs/korquad_bidaf",
"num_epochs": 50,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adadelta",
"learning_rate": 0.5,
"exponential_moving_average": 0.999
},
"seed_num": 2
}
================================================
FILE: base_config/korquad/docqa.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://korquad.github.io/dataset/KorQuAD_v1.0_train.json",
"valid_file_path": "https://korquad.github.io/dataset/KorQuAD_v1.0_dev.json",
"squad": {
"lang_code": "ko",
"context_max_length": 1000
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["char", "fasttext"],
"types": ["char", "word"],
"tokenizer": {
"char": {
"name": "jamo_ko"
},
"word": {
"name": "mecab_ko",
"split_with_regex": true
}
},
"char": {
"vocab": {
"start_token": "<s>",
"end_token": "</s>",
"max_vocab_size": 70
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 20,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"fasttext": {
"embedding": {
"embed_dim": 300,
"pretrained_path": "<fasttext.wiki.ko.300d.txt path>",
"trainable": false,
"dropout": 0.2
}
}
},
"model": {
"name": "docqa",
"docqa": {
"answer_maxlen": 17,
"rnn_dim": 100,
"linear_dim": 200,
"preprocess_rnn_num_layer": 1,
"modeling_rnn_num_layer": 1,
"predict_rnn_num_layer": 1,
"dropout": 0.2,
"weight_init": true
}
},
"trainer": {
"log_dir": "logs/korquad_docqa",
"num_epochs": 50,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adamax",
"learning_rate": 0.001,
"lr_scheduler_type": "reduce_on_plateau",
"reduce_on_plateau": {
"factor": 0.5,
"mode": "max",
"patience": 2
}
},
"seed_num": 2
}
================================================
FILE: base_config/multi_task/bert_base_glue+squad.json
================================================
{
"data_reader": {
"dataset": "multitask_bert",
"train_file_path": "train",
"valid_file_path": "valid",
"multitask_bert": {
"batch_sizes": [32, 32, 32, 32, 32, 32, 32, 32, 16, 12],
"readers": [{
"dataset": "cola_bert",
"train_file_path": "CoLA/train.tsv",
"valid_file_path": "CoLA/dev.tsv",
"cola_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "mnli_bert",
"train_file_path": "MNLI/train.tsv",
"valid_file_path": "MNLI/dev_matched.tsv",
"mnli_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "mrpc_bert",
"train_file_path": "MRPC/train.tsv",
"valid_file_path": "MRPC/dev.tsv",
"mrpc_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "qnli_bert",
"train_file_path": "QNLI/train.tsv",
"valid_file_path": "QNLI/dev.tsv",
"qnli_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "qqp_bert",
"train_file_path": "QQP/train.tsv",
"valid_file_path": "QQP/dev.tsv",
"qqp_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "rte_bert",
"train_file_path": "RTE/train.tsv",
"valid_file_path": "RTE/dev.tsv",
"rte_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "sst_bert",
"train_file_path": "SST-2/train.tsv",
"valid_file_path": "SST-2/dev.tsv",
"sst_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "stsb_bert",
"train_file_path": "STS-B/train.tsv",
"valid_file_path": "STS-B/dev.tsv",
"stsb_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "wnli_bert",
"train_file_path": "WNLI/train.tsv",
"valid_file_path": "WNLI/dev.tsv",
"wnli_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "squad_bert",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad_bert": {
"lang_code": "en",
"max_seq_length": 384,
"context_stride": 128,
"max_question_length": 64
}
}]
}
},
"iterator": {
"batch_size": 1
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_multi",
"bert_for_multi": {
"pretrained_model_name": "bert-base-uncased",
"dropouts": [0.05, 0.3, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0]
}
},
"trainer": {
"log_dir": "logs/multi_task/bert_glue+squad",
"num_epochs": 5,
"early_stopping_threshold": 10,
"grad_max_norm": 1,
"metric_key": "average",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.999],
"eps": 1e-6,
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/multi_task/bert_base_glue.json
================================================
{
"data_reader": {
"dataset": "multitask_bert",
"train_file_path": "train",
"valid_file_path": "valid",
"multitask_bert": {
"batch_sizes": [32, 32, 32, 32, 32, 32, 32, 32, 16],
"readers": [{
"dataset": "cola_bert",
"train_file_path": "CoLA/train.tsv",
"valid_file_path": "CoLA/dev.tsv",
"cola_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "mnli_bert",
"train_file_path": "MNLI/train.tsv",
"valid_file_path": "MNLI/dev_matched.tsv",
"mnli_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "mrpc_bert",
"train_file_path": "MRPC/train.tsv",
"valid_file_path": "MRPC/dev.tsv",
"mrpc_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "qnli_bert",
"train_file_path": "QNLI/train.tsv",
"valid_file_path": "QNLI/dev.tsv",
"qnli_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "qqp_bert",
"train_file_path": "QQP/train.tsv",
"valid_file_path": "QQP/dev.tsv",
"qqp_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "rte_bert",
"train_file_path": "RTE/train.tsv",
"valid_file_path": "RTE/dev.tsv",
"rte_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "sst_bert",
"train_file_path": "SST-2/train.tsv",
"valid_file_path": "SST-2/dev.tsv",
"sst_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "stsb_bert",
"train_file_path": "STS-B/train.tsv",
"valid_file_path": "STS-B/dev.tsv",
"stsb_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "wnli_bert",
"train_file_path": "WNLI/train.tsv",
"valid_file_path": "WNLI/dev.tsv",
"wnli_bert": {
"sequence_max_length": 128
}
}]
}
},
"iterator": {
"batch_size": 1
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_multi",
"bert_for_multi": {
"pretrained_model_name": "bert-base-uncased",
"dropouts": [0.05, 0.3, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1]
}
},
"trainer": {
"log_dir": "logs/multi_task/bert_glue",
"num_epochs": 5,
"early_stopping_threshold": 10,
"grad_max_norm": 1,
"metric_key": "average",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.999],
"eps": 1e-6,
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/multi_task/bert_large_glue+squad.json
================================================
{
"data_reader": {
"dataset": "multitask_bert",
"train_file_path": "train",
"valid_file_path": "valid",
"multitask_bert": {
"batch_sizes": [32, 32, 32, 32, 32, 32, 32, 32, 16, 6],
"readers": [{
"dataset": "cola_bert",
"train_file_path": "CoLA/train.tsv",
"valid_file_path": "CoLA/dev.tsv",
"cola_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "mnli_bert",
"train_file_path": "MNLI/train.tsv",
"valid_file_path": "MNLI/dev_matched.tsv",
"mnli_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "mrpc_bert",
"train_file_path": "MRPC/train.tsv",
"valid_file_path": "MRPC/dev.tsv",
"mrpc_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "qnli_bert",
"train_file_path": "QNLI/train.tsv",
"valid_file_path": "QNLI/dev.tsv",
"qnli_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "qqp_bert",
"train_file_path": "QQP/train.tsv",
"valid_file_path": "QQP/dev.tsv",
"qqp_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "rte_bert",
"train_file_path": "RTE/train.tsv",
"valid_file_path": "RTE/dev.tsv",
"rte_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "sst_bert",
"train_file_path": "SST-2/train.tsv",
"valid_file_path": "SST-2/dev.tsv",
"sst_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "stsb_bert",
"train_file_path": "STS-B/train.tsv",
"valid_file_path": "STS-B/dev.tsv",
"stsb_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "wnli_bert",
"train_file_path": "WNLI/train.tsv",
"valid_file_path": "WNLI/dev.tsv",
"wnli_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "squad_bert",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad_bert": {
"lang_code": "en",
"max_seq_length": 384,
"context_stride": 128,
"max_question_length": 64
}
}]
}
},
"iterator": {
"batch_size": 1
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_multi",
"bert_for_multi": {
"pretrained_model_name": "bert-large-uncased",
"dropouts": [0.05, 0.3, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0]
}
},
"trainer": {
"log_dir": "logs/multi_task/bert_glue+squad",
"num_epochs": 5,
"early_stopping_threshold": 10,
"grad_max_norm": 1,
"metric_key": "average",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.999],
"eps": 1e-6,
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/multi_task/bert_large_glue.json
================================================
{
"data_reader": {
"dataset": "multitask_bert",
"train_file_path": "train",
"valid_file_path": "valid",
"multitask_bert": {
"batch_sizes": [32, 32, 32, 32, 32, 32, 32, 32, 16],
"readers": [{
"dataset": "cola_bert",
"train_file_path": "CoLA/train.tsv",
"valid_file_path": "CoLA/dev.tsv",
"cola_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "mnli_bert",
"train_file_path": "MNLI/train.tsv",
"valid_file_path": "MNLI/dev_matched.tsv",
"mnli_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "mrpc_bert",
"train_file_path": "MRPC/train.tsv",
"valid_file_path": "MRPC/dev.tsv",
"mrpc_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "qnli_bert",
"train_file_path": "QNLI/train.tsv",
"valid_file_path": "QNLI/dev.tsv",
"qnli_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "qqp_bert",
"train_file_path": "QQP/train.tsv",
"valid_file_path": "QQP/dev.tsv",
"qqp_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "rte_bert",
"train_file_path": "RTE/train.tsv",
"valid_file_path": "RTE/dev.tsv",
"rte_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "sst_bert",
"train_file_path": "SST-2/train.tsv",
"valid_file_path": "SST-2/dev.tsv",
"sst_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "stsb_bert",
"train_file_path": "STS-B/train.tsv",
"valid_file_path": "STS-B/dev.tsv",
"stsb_bert": {
"sequence_max_length": 128
}
}, {
"dataset": "wnli_bert",
"train_file_path": "WNLI/train.tsv",
"valid_file_path": "WNLI/dev.tsv",
"wnli_bert": {
"sequence_max_length": 128
}
}]
}
},
"iterator": {
"batch_size": 1
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_multi",
"bert_for_multi": {
"pretrained_model_name": "bert-large-uncased",
"dropouts": [0.05, 0.3, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1]
}
},
"trainer": {
"log_dir": "logs/multi_task/bert_glue",
"num_epochs": 5,
"early_stopping_threshold": 10,
"grad_max_norm": 1,
"metric_key": "average",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.999],
"eps": 1e-6,
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/squad/bert_base_uncased.json
================================================
{
"data_reader": {
"dataset": "squad_bert",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad_bert": {
"lang_code": "en",
"max_seq_length": 384,
"context_stride": 128,
"max_question_length": 64
}
},
"iterator": {
"batch_size": 12
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_qa",
"bert_for_qa": {
"pretrained_model_name": "bert-base-uncased",
"answer_maxlen": 30
}
},
"trainer": {
"log_dir": "logs/squad_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 5e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.999],
"eps": 1e-6,
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/squad/bert_large_uncased.json
================================================
{
"data_reader": {
"dataset": "squad_bert",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad_bert": {
"lang_code": "en",
"max_seq_length": 384,
"context_stride": 128,
"max_question_length": 64
}
},
"iterator": {
"batch_size": 8
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_qa",
"bert_for_qa": {
"pretrained_model_name": "bert-large-uncased",
"answer_maxlen": 30
}
},
"trainer": {
"log_dir": "logs/squad_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 5e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.999],
"eps": 1e-6,
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/squad/bidaf+elmo.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["char", "glove", "elmo"],
"types": ["char", "word", "elmo"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"start_token": "<s>",
"end_token": "</s>",
"max_vocab_size": 260
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 16,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"glove": {
"indexer": {
"lowercase": true
},
"embedding": {
"embed_dim": 100,
"pretrained_path": "<glove.6B.100d.txt path>",
"trainable": false,
"dropout": 0.2
}
},
"elmo": {
"embedding": {
"options_file": "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json",
"weight_file": "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5",
"trainable": false,
"dropout": 0.5
}
}
},
"model": {
"name": "bidaf",
"bidaf": {
"model_dim": 200,
"dropout": 0.3
}
},
"trainer": {
"log_dir": "logs/squad_bidaf+elmo",
"num_epochs": 50,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adamax",
"learning_rate": 0.001,
"adam": {
"betas": [0.9, 0.9]
},
"lr_scheduler_type": "reduce_on_plateau",
"reduce_on_plateau": {
"factor": 0.5,
"mode": "max",
"patience": 2
}
},
"seed_num": 31
}
================================================
FILE: base_config/squad/bidaf.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["char", "glove"],
"types": ["char", "word"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"start_token": "<s>",
"end_token": "</s>",
"max_vocab_size": 260
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 16,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"glove": {
"vocab": {
"pretrained_path": "<glove.6B.vocab.txt path>",
"pretrained_token": "intersect"
},
"indexer": {
"lowercase": true
},
"embedding": {
"embed_dim": 100,
"pretrained_path": "<glove.6B.100d.txt path>",
"trainable": false,
"dropout": 0.2
}
}
},
"model": {
"name": "bidaf",
"bidaf": {
"model_dim": 100,
"contextual_rnn_num_layer": 1,
"modeling_rnn_num_layer": 2,
"predict_rnn_num_layer": 1,
"dropout": 0.2
}
},
"trainer": {
"log_dir": "logs/squad_bidaf",
"num_epochs": 50,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adadelta",
"learning_rate": 0.5,
"exponential_moving_average": 0.999
},
"seed_num": 2
}
================================================
FILE: base_config/squad/bidaf_no_answer.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json"
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["char", "glove"],
"types": ["char", "word"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"start_token": "<s>",
"end_token": "</s>",
"max_vocab_size": 260
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 16,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"glove": {
"indexer": {
"lowercase": true
},
"embedding": {
"embed_dim": 100,
"pretrained_path": "<glove.6B.100d.txt path>",
"trainable": false,
"dropout": 0.2
}
}
},
"model": {
"name": "bidaf_no_answer",
"bidaf": {
"model_dim": 100,
"dropout": 0.2
}
},
"trainer": {
"log_dir": "logs/squad_bidaf_no_answer",
"num_epochs": 50,
"early_stopping_threshold": 10,
"metric_key": "best_f1",
"verbose_step_count": 100,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adadelta",
"learning_rate": 0.5,
"exponential_moving_average": 0.999
},
"seed_num": 2
}
================================================
FILE: base_config/squad/docqa+elmo.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["char", "glove", "elmo"],
"types": ["char", "word", "elmo"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"max_vocab_size": 260,
"start_token": "<s>",
"end_token": "</s>"
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 20,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"glove": {
"indexer": {
"lowercase": false
},
"embedding": {
"embed_dim": 300,
"pretrained_path": "<glove.840B.300d.txt path>",
"trainable": false
}
},
"elmo": {
"embedding": {
"options_file": "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json",
"weight_file": "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5",
"trainable": false,
"dropout": 0.5
}
}
},
"model": {
"name": "docqa",
"docqa": {
"rnn_dim": 200,
"linear_dim": 400,
"dropout": 0.25,
"weight_init": true
}
},
"trainer": {
"log_dir": "logs/squad_docqa+elmo",
"num_epochs": 30,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adamax",
"learning_rate": 0.001,
"lr_scheduler_type": "reduce_on_plateau",
"reduce_on_plateau": {
"factor": 0.5,
"mode": "max",
"patience": 2
}
},
"seed_num": 2
}
================================================
FILE: base_config/squad/docqa.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["char", "glove"],
"types": ["char", "word"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"max_vocab_size": 260,
"start_token": "<s>",
"end_token": "</s>"
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 20,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"glove": {
"indexer": {
"lowercase": false
},
"embedding": {
"embed_dim": 300,
"pretrained_path": "<glove.840B.300d.txt path>",
"trainable": false,
"dropout": 0.2
}
}
},
"model": {
"name": "docqa",
"docqa": {
"answer_maxlen": 17,
"rnn_dim": 100,
"linear_dim": 200,
"preprocess_rnn_num_layer": 1,
"modeling_rnn_num_layer": 1,
"predict_rnn_num_layer": 1,
"dropout": 0.2,
"weight_init": true
}
},
"trainer": {
"log_dir": "logs/squad_docqa",
"num_epochs": 50,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adamax",
"learning_rate": 0.001,
"lr_scheduler_type": "reduce_on_plateau",
"reduce_on_plateau": {
"factor": 0.5,
"mode": "max",
"patience": 2
}
},
"seed_num": 2
}
================================================
FILE: base_config/squad/docqa_no_answer.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["char", "glove"],
"types": ["char", "word"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"max_vocab_size": 260,
"start_token": "<s>",
"end_token": "</s>"
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 20,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"glove": {
"indexer": {
"lowercase": false
},
"embedding": {
"embed_dim": 300,
"pretrained_path": "<glove.840B.300d.txt path>",
"trainable": false,
"dropout": 0.2
}
}
},
"model": {
"name": "docqa_no_answer",
"docqa": {
"answer_maxlen": 17,
"rnn_dim": 100,
"linear_dim": 200,
"dropout": 0.2,
"weight_init": true
}
},
"trainer": {
"log_dir": "logs/squad_docqa_no_answer",
"num_epochs": 50,
"early_stopping_threshold": 10,
"metric_key": "best_f1",
"verbose_step_count": 100,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adamax",
"learning_rate": 0.001,
"lr_scheduler_type": "reduce_on_plateau",
"reduce_on_plateau": {
"factor": 0.5,
"mode": "max",
"patience": 2
}
},
"seed_num": 2
}
================================================
FILE: base_config/squad/drqa.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["frequent_glove", "exact_match"],
"types": ["frequent_word", "exact_match"],
"tokenizer": {
"word": {
"name": "spacy_en",
"split_with_regex": true
}
},
"frequent_glove": {
"vocab": {
"frequent_count": 1000
},
"indexer": {
"lowercase": false
},
"embedding": {
"embed_dim": 300,
"pretrained_path": "<glove.840B.300d.txt paht>",
"dropout": 0.3
}
},
"exact_match": {
"indexer": {
"lower": true,
"lemma": true
},
"embedding": {
"type": "sparse"
}
}
},
"model": {
"name": "drqa",
"drqa": {
"aligned_query_embedding": true,
"answer_maxlen": 15,
"model_dim": 128,
"dropout": 0.3
}
},
"trainer": {
"log_dir": "logs/squad_drqa",
"num_epochs": 50,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adamax",
"learning_rate": 0.002,
"adamax": {
"betas": [0.9, 0.999],
"eps": 1e-08,
"weight_decay": 0
},
"lr_scheduler_type": "reduce_on_plateau",
"reduce_on_plateau": {
"factor": 0.5,
"mode": "max",
"patience": 2
}
},
"seed_num": 21
}
================================================
FILE: base_config/squad/drqa_paper.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["frequent_glove", "exact_match", "linguistic"],
"types": ["frequent_word", "exact_match", "linguistic"],
"tokenizer": {
"word": {
"name": "spacy_en",
"split_with_regex": true
}
},
"frequent_glove": {
"vocab": {
"frequent_count": 1000
},
"indexer": {
"lowercase": false
},
"embedding": {
"embed_dim": 300,
"pretrained_path": "<glove.840B.300d.txt path>",
"dropout": 0.3
}
},
"exact_match": {
"indexer": {
"lower": true,
"lemma": true
},
"embedding": {
"type": "sparse"
}
},
"linguistic": {
"indexer": {
"pos_tag": true,
"ner": true
},
"embedding": {
"type": "sparse"
}
}
},
"model": {
"name": "drqa",
"drqa": {
"aligned_query_embedding": true,
"answer_maxlen": 15,
"model_dim": 128,
"dropout": 0.3
}
},
"trainer": {
"log_dir": "logs/squad_drqa_paper",
"num_epochs": 50,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adamax",
"learning_rate": 0.003,
"adamax": {
"betas": [0.9, 0.999],
"eps": 1e-08,
"weight_decay": 0
},
"lr_scheduler_type": "reduce_on_plateau",
"reduce_on_plateau": {
"factor": 0.5,
"mode": "max",
"patience": 2
}
},
"seed_num": 21
}
================================================
FILE: base_config/squad/qanet.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en",
"context_max_length": 400
}
},
"iterator": {
"batch_size": 6
},
"token": {
"names": ["char", "glove"],
"types": ["char", "word"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"max_vocab_size": 260,
"start_token": "<s>",
"end_token": "</s>"
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 64,
"kernel_sizes": [5],
"num_filter": 200,
"activation": "relu",
"dropout": 0.05
}
},
"glove": {
"vocab": {
"pretrained_path": "<glove.840B.vocab.txt path>",
"pretrained_token": "intersect"
},
"indexer": {
"lowercase": false
},
"embedding": {
"embed_dim": 300,
"pretrained_path": "<glove.840B.300d.txt path>",
"trainable": false,
"dropout": 0.1
}
}
},
"model": {
"name": "qanet",
"qanet": {
"answer_maxlen": 30,
"model_dim": 128,
"kernel_size_in_embedding": 7,
"num_head_in_embedding": 8,
"num_conv_block_in_embedding": 4,
"num_embedding_encoder_block": 1,
"kernel_size_in_modeling": 5,
"num_head_in_modeling": 8,
"num_conv_block_in_modeling": 2,
"num_modeling_encoder_block": 7,
"layer_dropout": 0.9,
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/squad_qanet",
"num_epochs": 100,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adamax",
"learning_rate": 0.002,
"lr_scheduler_type": "reduce_on_plateau",
"reduce_on_plateau": {
"factor": 0.5,
"mode": "max",
"patience": 2
}
},
"seed_num": 2
}
================================================
FILE: base_config/squad/qanet_paper.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en",
"context_max_length": 400
}
},
"iterator": {
"batch_size": 6
},
"token": {
"names": ["char", "glove"],
"types": ["char", "word"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"max_vocab_size": 260,
"start_token": "<s>",
"end_token": "</s>"
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 64,
"kernel_sizes": [5],
"num_filter": 200,
"activation": "relu",
"dropout": 0.05
}
},
"glove": {
"vocab": {
"pretrained_path": "<glove.840B.vocab.txt path>",
"pretrained_token": "intersect"
},
"indexer": {
"lowercase": false
},
"embedding": {
"embed_dim": 300,
"pretrained_path": "<glove.840B.300d.txt path>",
"trainable": false,
"dropout": 0.1
}
}
},
"model": {
"name": "qanet",
"qanet": {
"answer_maxlen": 30,
"model_dim": 128,
"kernel_size_in_embedding": 7,
"num_head_in_embedding": 8,
"num_conv_block_in_embedding": 4,
"num_embedding_encoder_block": 1,
"kernel_size_in_modeling": 5,
"num_head_in_modeling": 8,
"num_conv_block_in_modeling": 2,
"num_modeling_encoder_block": 7,
"layer_dropout": 0.9,
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/squad_qanet_paper",
"num_epochs": 100,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adam",
"learning_rate": 0.001,
"adam": {
"betas": [0.8, 0.999],
"eps": 1e-7,
"weight_decay": 3e-7
},
"exponential_moving_average": 0.9999,
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 1000
}
},
"seed_num": 2
}
================================================
FILE: base_config/squad/roberta_base.json
================================================
{
"data_reader": {
"dataset": "squad_bert",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad_bert": {
"lang_code": "en",
"max_seq_length": 384,
"context_stride": 128,
"max_question_length": 64,
"cls_token": "<s>",
"sep_token": "</s>"
}
},
"iterator": {
"batch_size": 12
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>",
"cls_token": "<s>",
"sep_token": "</s>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_qa",
"roberta_for_qa": {
"pretrained_model_name": "roberta-base",
"answer_maxlen": 30
}
},
"trainer": {
"log_dir": "logs/squad_roberta_base",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 1.5e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.06
}
},
"seed_num": 42
}
================================================
FILE: base_config/squad/roberta_large.json
================================================
{
"data_reader": {
"dataset": "squad_bert",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad_bert": {
"lang_code": "en",
"max_seq_length": 384,
"context_stride": 128,
"max_question_length": 64,
"cls_token": "<s>",
"sep_token": "</s>"
}
},
"iterator": {
"batch_size": 8
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>",
"cls_token": "<s>",
"sep_token": "</s>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_qa",
"roberta_for_qa": {
"pretrained_model_name": "roberta-large",
"answer_maxlen": 30
}
},
"trainer": {
"log_dir": "logs/squad_roberta_large",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "f1",
"verbose_step_count": 100,
"eval_and_save_step_count": 1000
},
"optimizer": {
"learning_rate": 1.5e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.06
}
},
"seed_num": 42
}
================================================
FILE: base_config/test/bert_for_multi_task.json
================================================
{
"data_reader": {
"dataset": "multitask_bert",
"train_file_path": "train",
"valid_file_path": "valid",
"multitask_bert": {
"batch_sizes": [15, 13, 11],
"readers": [{
"dataset": "seq_cls_bert",
"train_file_path": "train.tsv",
"valid_file_path": "dev.tsv",
"seq_cls_bert": {
"sequence_max_length": 128,
"is_test": true
}
}, {
"dataset": "regression_bert",
"train_file_path": "train.tsv",
"valid_file_path": "dev.tsv",
"regression_bert": {
"sequence_max_length": 128,
"is_test": true
}
}, {
"dataset": "squad_bert",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad_bert": {
"lang_code": "en",
"max_seq_length": 384,
"context_stride": 128,
"max_question_length": 64
}
}]
}
},
"iterator": {
"batch_size": 1
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_multi",
"bert_for_multi": {
"pretrained_model_name": "bert-base-uncased",
"dropouts": [0.05, 0.3, 0.1]
}
},
"trainer": {
"log_dir": "logs/test/bert_for_multi_task",
"num_epochs": 1,
"early_stopping_threshold": 5,
"grad_max_norm": 1,
"metric_key": "average",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"betas": [0.9, 0.999],
"eps": 1e-6,
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_proportion": 0.1
}
},
"seed_num": 42
}
================================================
FILE: base_config/test/bert_for_qa.yaml
================================================
data_reader:
dataset: "squad_bert"
train_file_path: "data/bert/squad_synthetic_data.json"
valid_file_path: "data/bert/squad_synthetic_data.json"
squad_bert:
lang_code: "en"
max_seq_length: 384
context_stride: 128
max_question_length: 64
iterator:
batch_size: 10
token:
names:
- "feature"
types:
- "feature"
tokenizer:
subword:
name: "wordpiece"
wordpiece:
vocab_path: "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
word:
name: "bert_basic"
bert_basic:
do_lower_case: true
feature:
vocab:
pretrained_path: "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
pretrained_token: "all"
indexer:
do_tokenize: false
model:
name: "bert_for_qa"
bert_for_qa:
pretrained_model_name: "bert-base-uncased"
trainer:
log_dir: "logs/test/bert_for_qa/"
num_epochs: 2
early_stopping_threshold: 2
metric_key: "em"
verbose_step_count: 1
eval_and_save_step_count: 1
optimizer:
learning_rate: 0.00005
op_type: "adamw"
adamw:
weight_decay: 0.01
lr_scheduler_type: "warmup_linear"
warmup_linear:
warmup_steps: 10000
gradient_accumulation_steps: 2
seed_num: 25
================================================
FILE: base_config/test/bert_for_seq_cls.json
================================================
{
"data_reader": {
"dataset": "seq_cls_bert",
"train_file_path": "logs/test/seq_cls/synthetic_data.json",
"valid_file_path": "logs/test/seq_cls/synthetic_data.json",
"seq_cls_bert": {
"sequence_max_length": 128,
"class_key": "label"
}
},
"iterator": {
"batch_size": 64
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": false
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.0
}
},
"trainer": {
"log_dir": "logs/test/seq_cls/bert",
"num_epochs": 2,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 0.00001,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 10000
}
},
"seed_num": 42
}
================================================
FILE: base_config/test/bert_for_tok_cls.json
================================================
{
"data_reader": {
"dataset": "tok_cls_bert",
"train_file_path": "logs/test/tok_cls/synthetic_data.json",
"valid_file_path": "logs/test/tok_cls/synthetic_data.json",
"tok_cls_bert": {
"sequence_max_length": 128,
"tag_key": "label",
"ignore_tag_idx": -1
}
},
"iterator": {
"batch_size": 64
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": false
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_tok_cls",
"bert_for_tok_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.2
}
},
"trainer": {
"log_dir": "logs/test/tok_cls/bert",
"num_epochs": 2,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 0.00001,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 10000
}
},
"seed_num": 42
}
================================================
FILE: base_config/test/bidaf+bert.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 10
},
"token": {
"names": ["char", "glove", "bert"],
"types": ["char", "word", "bert"],
"tokenizer": {
"char": {
"name": "character"
},
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "treebank_en"
}
},
"char": {
"vocab": {
"start_token": "<s>",
"end_token": "</s>",
"max_vocab_size": 260
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 16,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"glove": {
"vocab": {},
"indexer": {
"lowercase": false
},
"embedding": {
"embed_dim": 50,
"trainable": false,
"dropout": 0.2
}
},
"bert": {
"vocab": {
"pad_token": "[PAD]",
"oov_token": "[UNK]",
"cls_token": "[CLS]",
"sep_token": "[SEP]",
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-vocab.txt",
"pretrained_token": "all"
},
"embedding": {
"pretrained_model_name": "bert-base-cased",
"trainable": false
}
}
},
"model": {
"name": "bidaf",
"bidaf": {
"model_dim": 50,
"dropout": 0.2
}
},
"trainer": {
"log_dir": "logs/test/bidaf+bert/",
"num_epochs": 1,
"early_stopping_threshold": 2,
"metric_key": "f1",
"verbose_step_count": 1,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adadelta",
"learning_rate": 1,
"exponential_moving_average": 0.999
},
"seed_num": 25
}
================================================
FILE: base_config/test/bidaf+cove.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 10
},
"token": {
"names": ["char", "glove", "cove"],
"types": ["char", "word", "cove"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"start_token": "<s>",
"end_token": "</s>",
"max_vocab_size": 260
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 16,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"glove": {
"indexer": {
"lowercase": true
},
"embedding": {
"embed_dim": 50,
"trainable": false,
"dropout": 0.2
}
},
"cove": {
"embedding": {
"glove_pretrained_path": "<glove.840B.300d.txt path>",
"model_pretrained_path": "https://s3.amazonaws.com/research.metamind.io/cove/wmtlstm-8f474287.pth",
"dropout": 0.2,
"trainable": false
}
}
},
"model": {
"name": "bidaf",
"bidaf": {
"model_dim": 50,
"contextual_rnn_num_layer": 1,
"modeling_rnn_num_layer": 2,
"predict_rnn_num_layer": 1,
"dropout": 0.2
}
},
"trainer": {
"log_dir": "logs/test/bidaf+cove/",
"num_epochs": 2,
"early_stopping_threshold": 2,
"metric_key": "em",
"verbose_step_count": 1,
"eval_and_save_step_count": 5
},
"optimizer": {
"op_type": "adadelta",
"learning_rate": 1,
"exponential_moving_average": 0.999
},
"seed_num": 25
}
================================================
FILE: base_config/test/bidaf+elmo.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 10
},
"token": {
"names": ["char", "glove", "elmo"],
"types": ["char", "word", "elmo"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"start_token": "<s>",
"end_token": "</s>",
"max_vocab_size": 260
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 16,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"glove": {
"vocab": {},
"indexer": {
"lowercase": false
},
"embedding": {
"embed_dim": 50,
"trainable": false,
"dropout": 0.2
}
},
"elmo": {
"indexer": {},
"embedding": {
"options_file": "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json",
"weight_file": "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5",
"trainable": false,
"dropout": 0.5
}
}
},
"model": {
"name": "bidaf",
"bidaf": {
"model_dim": 50,
"dropout": 0.2
}
},
"trainer": {
"log_dir": "logs/test/bidaf+elmo/",
"num_epochs": 1,
"early_stopping_threshold": 2,
"metric_key": "em",
"verbose_step_count": 1,
"eval_and_save_step_count": 5
},
"optimizer": {
"op_type": "adadelta",
"learning_rate": 1,
"exponential_moving_average": 0.999
},
"seed_num": 25
}
================================================
FILE: base_config/test/bidaf.yaml
================================================
data_reader:
dataset: "squad"
train_file_path: "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json"
valid_file_path: "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json"
squad:
lang_code: "ko"
iterator:
batch_size: 10
token:
names:
- "char"
- "glove"
types:
- "char"
- "word"
tokenizer:
char:
name: "character"
word:
name: "treebank_en"
split_with_regex: true
char:
vocab:
start_token: "<s>"
end_token: "</s>"
max_vocab_size: 260
indexer:
insert_char_start: true
insert_char_end: true
embedding:
embed_dim: 16
kernel_sizes:
- 5
num_filter: 100
activation: "relu"
dropout: 0.2
glove:
indexer:
lowercase: true
embedding:
embed_dim: 50
trainable: false
dropout: 0.2
model:
name: "bidaf"
bidaf:
model_dim: 50
contextual_rnn_num_layer: 1
modeling_rnn_num_layer: 2
predict_rnn_num_layer: 1
dropout: 0.2
trainer:
log_dir: "logs/test/bidaf/"
num_epochs: 2
early_stopping_threshold: 2
grad_max_norm: 0.5
metric_key: "em"
verbose_step_count: 1
eval_and_save_step_count: "epoch"
optimizer:
learning_rate: 1
op_type: "adadelta"
exponential_moving_average: 0.999
seed_num: 25
================================================
FILE: base_config/test/bidaf_no_answer.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 10
},
"token": {
"names": ["char", "glove"],
"types": ["char", "word"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"start_token": "<s>",
"end_token": "</s>",
"max_vocab_size": 260
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 16,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"glove": {
"vocab": {},
"indexer": {
"lowercase": true
},
"embedding": {
"embed_dim": 50,
"trainable": false,
"dropout": 0.2
}
}
},
"model": {
"name": "bidaf_no_answer",
"bidaf": {
"model_dim": 50,
"dropout": 0.2
}
},
"trainer": {
"log_dir": "logs/test/bidaf_no_answer/",
"num_epochs": 2,
"early_stopping_threshold": 2,
"metric_key": "em",
"verbose_step_count": 1,
"eval_and_save_step_count": 3
},
"optimizer": {
"op_type": "adadelta",
"learning_rate": 1,
"exponential_moving_average": 0.999
},
"seed_num": 25
}
================================================
FILE: base_config/test/cola_bert.json
================================================
{
"data_reader": {
"dataset": "cola_bert",
"train_file_path": "data/glue/CoLA/train.tsv",
"valid_file_path": "data/glue/CoLA/dev.tsv",
"cola_bert": {
"sequence_max_length": 128,
"is_test": true
}
},
"iterator": {
"batch_size": 2
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.0
}
},
"trainer": {
"log_dir": "logs/test/cola_bert",
"num_epochs": 1,
"early_stopping_threshold": 10,
"metric_key": "matthews_corr",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 10
}
},
"seed_num": 42
}
================================================
FILE: base_config/test/cola_roberta.json
================================================
{
"data_reader": {
"dataset": "cola_bert",
"train_file_path": "data/glue/CoLA/train.tsv",
"valid_file_path": "data/glue/CoLA/dev.tsv",
"cola_bert": {
"sequence_max_length": 128,
"cls_token": "<s>",
"sep_token": "</s>",
"input_type": "roberta",
"is_test": true
}
},
"iterator": {
"batch_size": 2
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>",
"cls_token": "<s>",
"sep_token": "</s>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_seq_cls",
"roberta_for_seq_cls": {
"pretrained_model_name": "roberta-base",
"dropout": 0.0
}
},
"trainer": {
"log_dir": "logs/test/cola_roberta",
"num_epochs": 1,
"early_stopping_threshold": 10,
"metric_key": "matthews_corr",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 10
}
},
"seed_num": 42
}
================================================
FILE: base_config/test/docqa.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 10
},
"token": {
"names": ["char", "glove"],
"types": ["char", "word"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"max_vocab_size": 260,
"start_token": "<s>",
"end_token": "</s>"
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 20,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"glove": {
"vocab": {},
"indexer": {
"lowercase": false
},
"embedding": {
"embed_dim": 50,
"trainable": false,
"dropout": 0.2
}
}
},
"model": {
"name": "docqa",
"docqa": {
"preprocess_rnn_num_layer": 1,
"modeling_rnn_num_layer": 2,
"predict_rnn_num_layer": 1
}
},
"trainer": {
"log_dir": "logs/test/docqa/",
"num_epochs": 1,
"early_stopping_threshold": 1,
"metric_key": "em",
"verbose_step_count": 1,
"eval_and_save_step_count": 1
},
"optimizer": {
"op_type": "adamax",
"learning_rate": 0.002
},
"seed_num": 25
}
================================================
FILE: base_config/test/docqa_no_answer.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 10
},
"token": {
"names": ["char", "glove"],
"types": ["char", "word"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"max_vocab_size": 260,
"start_token": "<s>",
"end_token": "</s>"
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 20,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"glove": {
"vocab": {},
"indexer": {
"lowercase": false
},
"embedding": {
"embed_dim": 50,
"trainable": false,
"dropout": 0.2
}
}
},
"model": {
"name": "docqa_no_answer"
},
"trainer": {
"log_dir": "logs/test/docqa_no_answer/",
"num_epochs": 1,
"early_stopping_threshold": 1,
"metric_key": "em",
"verbose_step_count": 1,
"eval_and_save_step_count": 1
},
"optimizer": {
"op_type": "adamax",
"learning_rate": 0.002
},
"seed_num": 25
}
================================================
FILE: base_config/test/drqa.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 10
},
"token": {
"names": ["frequent_word", "exact_match", "linguistic"],
"types": ["frequent_word", "exact_match", "linguistic"],
"tokenizer": {
"word": {
"name": "spacy_en",
"split_with_regex": true
}
},
"frequent_word": {
"vocab": {
"frequent_count": 1000
},
"indexer": {
"lowercase": false
},
"embedding": {
"embed_dim": 50,
"dropout": 0.3
}
},
"exact_match": {
"indexer": {
"lower": true,
"lemma": true
},
"embedding": {
"type": "sparse"
}
},
"linguistic": {
"indexer": {
"pos_tag": true,
"ner": true
},
"embedding": {
"type": "sparse"
}
}
},
"model": {
"name": "drqa",
"drqa": {
"aligned_query_embedding": false,
"answer_maxlen": 15,
"model_dim": 128,
"dropout": 0.3
}
},
"trainer": {
"log_dir": "logs/test/drqa/",
"num_epochs": 1,
"early_stopping_threshold": 1,
"metric_key": "em",
"verbose_step_count": 1,
"eval_and_save_step_count": 1
},
"optimizer": {
"op_type": "adamax",
"learning_rate": 0.003,
"adamax": {
"betas": [0.9, 0.999],
"eps": 1e-08,
"weight_decay": 0
}
},
"seed_num": 21
}
================================================
FILE: base_config/test/drqa_sparse_to_embedding.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 10
},
"token": {
"names": ["frequent_word", "exact_match", "linguistic"],
"types": ["frequent_word", "exact_match", "linguistic"],
"tokenizer": {
"word": {
"name": "spacy_en",
"split_with_regex": true
}
},
"frequent_word": {
"vocab": {
"frequent_count": 1000
},
"indexer": {
"lowercase": false
},
"embedding": {
"embed_dim": 50,
"dropout": 0.3
}
},
"exact_match": {
"indexer": {
"lower": true,
"lemma": true
},
"embedding": {
"type": "sparse"
}
},
"linguistic": {
"indexer": {
"pos_tag": true,
"ner": true
},
"embedding": {
"type": "embedding",
"embed_dim": 10
}
}
},
"model": {
"name": "drqa",
"drqa": {
"aligned_query_embedding": true,
"answer_maxlen": 15,
"model_dim": 128,
"dropout": 0.3
}
},
"trainer": {
"log_dir": "logs/test/drqa_with_sparse_to_embedding/",
"num_epochs": 1,
"early_stopping_threshold": 1,
"metric_key": "em",
"verbose_step_count": 1,
"eval_and_save_step_count": 1
},
"optimizer": {
"op_type": "adamax",
"learning_rate": 0.001,
"lr_scheduler_type": "reduce_on_plateau",
"reduce_on_plateau": {
"factor": 0.5,
"mode": "max",
"patience": 2
}
},
"seed_num": 21
}
================================================
FILE: base_config/test/mnlim_bert.json
================================================
{
"data_reader": {
"dataset": "mnli_bert",
"train_file_path": "data/glue/MNLI/train.tsv",
"valid_file_path": "data/glue/MNLI/dev_matched.tsv",
"mnli_bert": {
"sequence_max_length": 128,
"is_test": true
}
},
"iterator": {
"batch_size": 2
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.0
}
},
"trainer": {
"log_dir": "logs/test/mnlim_bert",
"num_epochs": 1,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 100
}
},
"seed_num": 42
}
================================================
FILE: base_config/test/mrpc_bert.json
================================================
{
"data_reader": {
"dataset": "mrpc_bert",
"train_file_path": "data/glue/MRPC/train.tsv",
"valid_file_path": "data/glue/MRPC/dev.tsv",
"mrpc_bert": {
"sequence_max_length": 128,
"is_test": true
}
},
"iterator": {
"batch_size": 2
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.0
}
},
"trainer": {
"log_dir": "logs/test/mrpc_bert",
"num_epochs": 1,
"early_stopping_threshold": 10,
"metric_key": "f1",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 10
}
},
"seed_num": 42
}
================================================
FILE: base_config/test/mt_bert.json
================================================
{
"data_reader": {
"dataset": "multitask_bert",
"train_file_path": "train",
"valid_file_path": "valid",
"multitask_bert": {
"batch_sizes": [2, 3, 4],
"readers": [{
"dataset": "cola_bert",
"train_file_path": "data/glue/CoLA/train.tsv",
"valid_file_path": "data/glue/CoLA/dev.tsv",
"cola_bert": {
"sequence_max_length": 128,
"is_test": true
}
}, {
"dataset": "stsb_bert",
"train_file_path": "data/glue/STS-B/train.tsv",
"valid_file_path": "data/glue/STS-B/dev.tsv",
"stsb_bert": {
"sequence_max_length": 128,
"is_test": true
}
}, {
"dataset": "squad_bert",
"train_file_path": "data/squad/dev-v1.1.json",
"valid_file_path": "data/squad/dev-v1.1.json",
"squad_bert": {
"lang_code": "en",
"max_seq_length": 384,
"context_stride": 128,
"max_question_length": 64
}
}]
}
},
"iterator": {
"batch_size": 1
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_multi",
"bert_for_multi": {
"pretrained_model_name": "bert-base-uncased",
"dropouts": [0.1, 0.2, 0]
}
},
"trainer": {
"log_dir": "logs/test/mt_bert",
"num_epochs": 2,
"early_stopping_threshold": 10,
"metric_key": "average",
"verbose_step_count": 1,
"eval_and_save_step_count": 5
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 10
}
},
"seed_num": 42
}
================================================
FILE: base_config/test/open_qa.json
================================================
{
"name": "open_qa",
"open_qa": {
"tokenizers": {
"sent": {
"name": "punkt"
},
"word": {
"name": "space_all",
"split_with_regex": true
}
},
"knowledge_base": {
"wiki": "<WikiExtractor output_path with --json>"
},
"reasoning": {
"document_retrieval": {
"type": "component",
"name": "tfidf",
"tfidf": {
"k": 3
}
},
"reading_comprehension": {
"type": "experiment",
"checkpoint_path": "<model_checkpoint_path>"
}
}
}
}
================================================
FILE: base_config/test/qanet.json
================================================
{
"data_reader": {
"dataset": "squad",
"train_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"valid_file_path": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json",
"squad": {
"lang_code": "en"
}
},
"iterator": {
"batch_size": 10
},
"token": {
"names": ["char", "glove"],
"types": ["char", "word"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"vocab": {
"max_vocab_size": 260,
"start_token": "<s>",
"end_token": "</s>"
},
"indexer": {
"insert_char_start": true,
"insert_char_end": true
},
"embedding": {
"embed_dim": 20,
"kernel_sizes": [5],
"num_filter": 50,
"activation": "relu",
"dropout": 0.05
}
},
"glove": {
"vocab": {},
"indexer": {
"lowercase": false
},
"embedding": {
"embed_dim": 50,
"trainable": true,
"dropout": 0.1
}
}
},
"model": {
"name": "qanet",
"qanet": {
"answer_maxlen": 30,
"model_dim": 128,
"kernel_size_in_embedding": 7,
"num_head_in_embedding": 8,
"num_conv_block_in_embedding": 4,
"num_embedding_encoder_block": 1,
"kernel_size_in_modeling": 5,
"num_head_in_modeling": 8,
"num_conv_block_in_modeling": 2,
"num_modeling_encoder_block": 7,
"layer_dropout": 0.9,
"dropout": 0.1
}
},
"trainer": {
"log_dir": "logs/test/qanet/",
"num_epochs": 1,
"early_stopping_threshold": 1,
"metric_key": "em",
"verbose_step_count": 1,
"eval_and_save_step_count": 1
},
"optimizer": {
"op_type": "adam",
"learning_rate": 0.001,
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 1000
}
},
"seed_num": 25
}
================================================
FILE: base_config/test/qnli_bert.json
================================================
{
"data_reader": {
"dataset": "qnli_bert",
"train_file_path": "data/glue/QNLI/train.tsv",
"valid_file_path": "data/glue/QNLI/dev.tsv",
"qnli_bert": {
"sequence_max_length": 128,
"is_test": true
}
},
"iterator": {
"batch_size": 2
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.0
}
},
"trainer": {
"log_dir": "logs/test/qnli_bert",
"num_epochs": 1,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 100
}
},
"seed_num": 42
}
================================================
FILE: base_config/test/qqp_bert.json
================================================
{
"data_reader": {
"dataset": "qqp_bert",
"train_file_path": "data/glue/QQP/train.tsv",
"valid_file_path": "data/glue/QQP/dev.tsv",
"qqp_bert": {
"sequence_max_length": 128,
"is_test": true
}
},
"iterator": {
"batch_size": 2
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.0
}
},
"trainer": {
"log_dir": "logs/test/qqp_bert_base",
"num_epochs": 1,
"early_stopping_threshold": 10,
"metric_key": "f1",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 100
}
},
"seed_num": 42
}
================================================
FILE: base_config/test/roberta_for_qa.json
================================================
{
"data_reader": {
"dataset": "squad_bert",
"train_file_path": "data/squad/dev-v1.1.json",
"valid_file_path": "data/squad/dev-v1.1.json",
"squad_bert": {
"lang_code": "en",
"max_seq_length": 384,
"context_stride": 128,
"max_question_length": 64
}
},
"iterator": {
"batch_size": 10
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_qa",
"roberta_for_qa": {
"pretrained_model_name": "roberta-base"
}
},
"trainer": {
"log_dir": "logs/test/bert_for_qa/",
"num_epochs": 2,
"early_stopping_threshold": 2,
"metric_key": "em",
"verbose_step_count": 1,
"eval_and_save_step_count": 1
},
"optimizer": {
"learning_rate": 0.00005,
"op_type": "adamw",
"adamw": {
"weight_decay": 0.01
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 10000
},
"gradient_accumulation_steps": 2
},
"seed_num": 25
}
================================================
FILE: base_config/test/rte_bert.json
================================================
{
"data_reader": {
"dataset": "rte_bert",
"train_file_path": "data/glue/RTE/train.tsv",
"valid_file_path": "data/glue/RTE/dev.tsv",
"rte_bert": {
"sequence_max_length": 128,
"is_test": true
}
},
"iterator": {
"batch_size": 2
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.0
}
},
"trainer": {
"log_dir": "logs/test/rte_bert",
"num_epochs": 1,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 5
}
},
"seed_num": 42
}
================================================
FILE: base_config/test/rte_roberta.json
================================================
{
"data_reader": {
"dataset": "rte_bert",
"train_file_path": "data/glue/RTE/train.tsv",
"valid_file_path": "data/glue/RTE/dev.tsv",
"rte_bert": {
"sequence_max_length": 128,
"cls_token": "<s>",
"sep_token": "</s>",
"input_type": "roberta",
"is_test": true
}
},
"iterator": {
"batch_size": 2
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"bpe": {
"name": "roberta",
"roberta": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"merges_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json",
"pretrained_token": "all",
"pad_token": "<pad>",
"oov_token": "<unk>"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "roberta_for_seq_cls",
"roberta_for_seq_cls": {
"pretrained_model_name": "roberta-base",
"dropout": 0.0
}
},
"trainer": {
"log_dir": "logs/test/rte_roberta",
"num_epochs": 1,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 5
}
},
"seed_num": 42
}
================================================
FILE: base_config/test/sqlnet.json
================================================
{
"data_reader": {
"dataset": "wikisql",
"train_file_path": "data/wikisql/dev.jsonl",
"valid_file_path": "data/wikisql/dev.jsonl",
"wikisql": {
"is_test": true
}
},
"iterator": {
"batch_size": 10
},
"token": {
"names": ["glove"],
"types": ["word"],
"tokenizer": {
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"glove": {
"vocab": {
"start_token": "<s>",
"end_token": "</s>"
},
"indexer": {
"lowercase": true,
"do_tokenize": true
},
"embedding": {
"embed_dim": 50,
"trainable": false,
"dropout": 0.2
}
}
},
"model": {
"name": "sqlnet",
"sqlnet": {
"column_attention": true,
"model_dim": 100,
"rnn_num_layer": 2,
"dropout": 0.3,
"column_maxlen": 4,
"token_maxlen": 200,
"conds_column_loss_alpha": 3
}
},
"trainer": {
"log_dir": "logs/test/sqlnet/",
"num_epochs": 2,
"early_stopping_threshold": 2,
"metric_key": "ex_accuracy",
"verbose_step_count": 1,
"eval_and_save_step_count": 1
},
"optimizer": {
"op_type": "adam",
"learning_rate": 0.001
},
"seed_num": 25
}
================================================
FILE: base_config/test/ssa.json
================================================
{
"data_reader": {
"dataset": "seq_cls",
"train_file_path": "logs/test/seq_cls/synthetic_data.json",
"valid_file_path": "logs/test/seq_cls/synthetic_data.json",
"seq_cls": {
"class_key": "label"
}
},
"iterator": {
"batch_size": 32
},
"token": {
"names": ["char", "fasttext"],
"types": ["char", "word"],
"tokenizer": {
"char": {
"name": "character"
},
"word": {
"name": "treebank_en",
"split_with_regex": true
}
},
"char": {
"indexer": {
"insert_char_start": false,
"insert_char_end": false
},
"embedding": {
"embed_dim": 16,
"kernel_sizes": [5],
"num_filter": 100,
"activation": "relu",
"dropout": 0.2
}
},
"fasttext": {
"embedding": {
"embed_dim": 300,
"trainable": false,
"dropout": 0.2
}
}
},
"model": {
"name": "structured_self_attention",
"structured_self_attention": {
"encoding_rnn_hidden_dim": 300,
"encoding_rnn_num_layer": 2,
"encoding_rnn_dropout": 0,
"attention_dim": 350,
"num_attention_heads": 30,
"sequence_embed_dim": 2000,
"dropout": 0.5,
"penalization_coefficient": 1
}
},
"trainer": {
"log_dir": "logs/test/seq_cls/ssa",
"num_epochs": 1,
"early_stopping_threshold": 10,
"grad_max_norm": 5.0,
"metric_key": "accuracy",
"verbose_step_count": 100,
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"op_type": "adam",
"learning_rate": 0.001,
"exponential_moving_average": 0.999
},
"seed_num": 42
}
================================================
FILE: base_config/test/sst_bert.json
================================================
{
"data_reader": {
"dataset": "sst_bert",
"train_file_path": "data/glue/SST-2/train.tsv",
"valid_file_path": "data/glue/SST-2/dev.tsv",
"sst_bert": {
"sequence_max_length": 128,
"is_test": true
}
},
"iterator": {
"batch_size": 2
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_seq_cls",
"bert_for_seq_cls": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.0
}
},
"trainer": {
"log_dir": "logs/test/sst_bert",
"num_epochs": 1,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 100
}
},
"seed_num": 42
}
================================================
FILE: base_config/test/stsb_bert.json
================================================
{
"data_reader": {
"dataset": "stsb_bert",
"train_file_path": "data/glue/STS-B/train.tsv",
"valid_file_path": "data/glue/STS-B/dev.tsv",
"stsb_bert": {
"sequence_max_length": 128,
"is_test": true
}
},
"iterator": {
"batch_size": 2
},
"token": {
"names": ["feature"],
"types": ["feature"],
"tokenizer": {
"subword": {
"name": "wordpiece",
"wordpiece": {
"vocab_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
}
},
"word": {
"name": "bert_basic",
"bert_basic": {
"do_lower_case": true
}
}
},
"feature": {
"vocab": {
"pretrained_path": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
"pretrained_token": "all"
},
"indexer": {
"do_tokenize": false
}
}
},
"model": {
"name": "bert_for_reg",
"bert_for_reg": {
"pretrained_model_name": "bert-base-uncased",
"dropout": 0.0
}
},
"trainer": {
"log_dir": "logs/test/stsb_bert",
"num_epochs": 1,
"early_stopping_threshold": 10,
"metric_key": "pearson_spearman_corr",
"eval_and_save_step_count": "epoch"
},
"optimizer": {
"learning_rate": 2e-5,
"op_type": "adamw",
"adamw": {
"weight_decay": 0
},
"lr_scheduler_type": "warmup_linear",
"warmup_linear": {
"warmup_steps": 100
}
},
"seed_num": 42
}
================================================
FILE:
gitextract_t48_1t0e/ ├── .coveragerc ├── .gitignore ├── .nojekyll ├── .readthedocs.yml ├── .travis.yml ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── NOTICE ├── README.md ├── base_config/ │ ├── cola/ │ │ ├── bert_base_uncased.json │ │ ├── bert_large_uncased.json │ │ └── structured_self_attention.json │ ├── conll2003/ │ │ └── bert_large_cased.json │ ├── glue/ │ │ ├── cola_bert.json │ │ ├── cola_roberta.json │ │ ├── mnlim_bert.json │ │ ├── mnlim_roberta.json │ │ ├── mnlimm_bert.json │ │ ├── mnlimm_roberta.json │ │ ├── mrpc_bert.json │ │ ├── mrpc_roberta.json │ │ ├── qnli_bert.json │ │ ├── qnli_roberta.json │ │ ├── qqp_bert.json │ │ ├── qqp_roberta.json │ │ ├── rte_bert.json │ │ ├── rte_roberta.json │ │ ├── sst_bert.json │ │ ├── sst_roberta.json │ │ ├── stsb_bert.json │ │ ├── stsb_roberta.json │ │ ├── wnli_bert.json │ │ └── wnli_roberta.json │ ├── korquad/ │ │ ├── bert_base_multilingual_cased.yaml │ │ ├── bert_base_multilingual_uncased.json │ │ ├── bidaf.json │ │ └── docqa.json │ ├── multi_task/ │ │ ├── bert_base_glue+squad.json │ │ ├── bert_base_glue.json │ │ ├── bert_large_glue+squad.json │ │ └── bert_large_glue.json │ ├── squad/ │ │ ├── bert_base_uncased.json │ │ ├── bert_large_uncased.json │ │ ├── bidaf+elmo.json │ │ ├── bidaf.json │ │ ├── bidaf_no_answer.json │ │ ├── docqa+elmo.json │ │ ├── docqa.json │ │ ├── docqa_no_answer.json │ │ ├── drqa.json │ │ ├── drqa_paper.json │ │ ├── qanet.json │ │ ├── qanet_paper.json │ │ ├── roberta_base.json │ │ └── roberta_large.json │ ├── test/ │ │ ├── bert_for_multi_task.json │ │ ├── bert_for_qa.yaml │ │ ├── bert_for_seq_cls.json │ │ ├── bert_for_tok_cls.json │ │ ├── bidaf+bert.json │ │ ├── bidaf+cove.json │ │ ├── bidaf+elmo.json │ │ ├── bidaf.yaml │ │ ├── bidaf_no_answer.json │ │ ├── cola_bert.json │ │ ├── cola_roberta.json │ │ ├── docqa.json │ │ ├── docqa_no_answer.json │ │ ├── drqa.json │ │ ├── drqa_sparse_to_embedding.json │ │ ├── mnlim_bert.json │ │ ├── mrpc_bert.json │ │ ├── mt_bert.json │ │ ├── open_qa.json │ │ ├── qanet.json │ │ ├── qnli_bert.json │ │ ├── qqp_bert.json │ │ ├── roberta_for_qa.json │ │ ├── rte_bert.json │ │ ├── rte_roberta.json │ │ ├── sqlnet.json │ │ ├── ssa.json │ │ ├── sst_bert.json │ │ ├── stsb_bert.json │ │ ├── stsb_roberta.json │ │ └── wnli_bert.json │ └── wikisql/ │ └── sqlnet.json ├── claf/ │ ├── __init__.py │ ├── __version__.py │ ├── config/ │ │ ├── __init__.py │ │ ├── args.py │ │ ├── namespace.py │ │ ├── pattern.py │ │ ├── registry.py │ │ └── utils.py │ ├── data/ │ │ ├── __init__.py │ │ ├── collate.py │ │ ├── data_handler.py │ │ ├── dataset/ │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── bert/ │ │ │ │ ├── __init__.py │ │ │ │ ├── multi_task.py │ │ │ │ ├── regression.py │ │ │ │ ├── seq_cls.py │ │ │ │ ├── squad.py │ │ │ │ └── tok_cls.py │ │ │ ├── seq_cls.py │ │ │ ├── squad.py │ │ │ └── wikisql.py │ │ ├── dto/ │ │ │ ├── __init__.py │ │ │ ├── batch.py │ │ │ ├── bert_feature.py │ │ │ └── helper.py │ │ ├── reader/ │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── bert/ │ │ │ │ ├── __init__.py │ │ │ │ ├── conll2003.py │ │ │ │ ├── glue/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── cola.py │ │ │ │ │ ├── mnli.py │ │ │ │ │ ├── mrpc.py │ │ │ │ │ ├── qnli.py │ │ │ │ │ ├── qqp.py │ │ │ │ │ ├── rte.py │ │ │ │ │ ├── sst.py │ │ │ │ │ ├── stsb.py │ │ │ │ │ └── wnli.py │ │ │ │ ├── multi_task.py │ │ │ │ ├── regression.py │ │ │ │ ├── seq_cls.py │ │ │ │ ├── squad.py │ │ │ │ └── tok_cls.py │ │ │ ├── cola.py │ │ │ ├── seq_cls.py │ │ │ ├── squad.py │ │ │ └── wikisql.py │ │ └── utils.py │ ├── decorator/ │ │ ├── __init__.py │ │ ├── arguments.py │ │ └── register.py │ ├── factory/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── data_loader.py │ │ ├── data_reader.py │ │ ├── model.py │ │ ├── optimizer.py │ │ └── tokens.py │ ├── learn/ │ │ ├── __init__.py │ │ ├── experiment.py │ │ ├── mode.py │ │ ├── optimization/ │ │ │ ├── __init__.py │ │ │ ├── exponential_moving_avarage.py │ │ │ ├── learning_rate_scheduler.py │ │ │ └── optimizer.py │ │ ├── tensorboard.py │ │ ├── trainer.py │ │ └── utils.py │ ├── machine/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── components/ │ │ │ ├── __init__.py │ │ │ └── retrieval/ │ │ │ ├── __init__.py │ │ │ └── tfidf.py │ │ ├── ensemble_topk.py │ │ ├── knowlege_base/ │ │ │ ├── __init__.py │ │ │ └── docs.py │ │ ├── module.py │ │ ├── nlu.py │ │ └── open_qa.py │ ├── metric/ │ │ ├── __init__.py │ │ ├── classification.py │ │ ├── glue.py │ │ ├── korquad_v1_official.py │ │ ├── regression.py │ │ ├── squad_v1_official.py │ │ ├── squad_v2_official.py │ │ ├── wikisql_lib/ │ │ │ ├── __init__.py │ │ │ ├── dbengine.py │ │ │ └── query.py │ │ └── wikisql_official.py │ ├── model/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── cls_utils.py │ │ ├── multi_task/ │ │ │ ├── __init__.py │ │ │ ├── bert.py │ │ │ ├── category.py │ │ │ └── mixin.py │ │ ├── reading_comprehension/ │ │ │ ├── __init__.py │ │ │ ├── bert.py │ │ │ ├── bidaf.py │ │ │ ├── bidaf_no_answer.py │ │ │ ├── docqa.py │ │ │ ├── docqa_no_answer.py │ │ │ ├── drqa.py │ │ │ ├── mixin.py │ │ │ ├── qanet.py │ │ │ └── roberta.py │ │ ├── regression/ │ │ │ ├── __init__.py │ │ │ ├── bert.py │ │ │ ├── mixin.py │ │ │ └── roberta.py │ │ ├── semantic_parsing/ │ │ │ ├── __init__.py │ │ │ ├── mixin.py │ │ │ ├── sqlnet.py │ │ │ └── utils.py │ │ ├── sequence_classification/ │ │ │ ├── __init__.py │ │ │ ├── bert.py │ │ │ ├── mixin.py │ │ │ ├── roberta.py │ │ │ └── structured_self_attention.py │ │ └── token_classification/ │ │ ├── __init__.py │ │ ├── bert.py │ │ └── mixin.py │ ├── modules/ │ │ ├── __init__.py │ │ ├── activation.py │ │ ├── attention/ │ │ │ ├── __init__.py │ │ │ ├── bi_attention.py │ │ │ ├── co_attention.py │ │ │ ├── docqa_attention.py │ │ │ ├── multi_head_attention.py │ │ │ └── seq_attention.py │ │ ├── conv/ │ │ │ ├── __init__.py │ │ │ ├── depthwise_separable_conv.py │ │ │ └── pointwise_conv.py │ │ ├── encoder/ │ │ │ ├── __init__.py │ │ │ ├── lstm_cell_with_projection.py │ │ │ └── positional.py │ │ ├── functional.py │ │ ├── initializer.py │ │ └── layer/ │ │ ├── __init__.py │ │ ├── highway.py │ │ ├── normalization.py │ │ ├── positionwise.py │ │ ├── residual.py │ │ └── scalar_mix.py │ ├── nsml.py │ ├── tokens/ │ │ ├── __init__.py │ │ ├── cove.py │ │ ├── elmo.py │ │ ├── embedding/ │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── bert_embedding.py │ │ │ ├── char_embedding.py │ │ │ ├── cove_embedding.py │ │ │ ├── elmo_embedding.py │ │ │ ├── frequent_word_embedding.py │ │ │ ├── sparse_feature.py │ │ │ └── word_embedding.py │ │ ├── hangul.py │ │ ├── indexer/ │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── bert_indexer.py │ │ │ ├── char_indexer.py │ │ │ ├── elmo_indexer.py │ │ │ ├── exact_match_indexer.py │ │ │ ├── linguistic_indexer.py │ │ │ └── word_indexer.py │ │ ├── linguistic.py │ │ ├── text_handler.py │ │ ├── token_embedder/ │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── basic_embedder.py │ │ │ └── reading_comprehension_embedder.py │ │ ├── token_maker.py │ │ ├── tokenizer/ │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── bpe.py │ │ │ ├── char.py │ │ │ ├── pass_text.py │ │ │ ├── sent.py │ │ │ ├── subword.py │ │ │ ├── utils.py │ │ │ └── word.py │ │ └── vocabulary.py │ └── utils.py ├── docs/ │ ├── Makefile │ ├── _build/ │ │ ├── doctrees/ │ │ │ ├── claf.config.doctree │ │ │ ├── claf.config.factory.doctree │ │ │ ├── claf.data.dataset.doctree │ │ │ ├── claf.data.doctree │ │ │ ├── claf.data.reader.bert.doctree │ │ │ ├── claf.data.reader.doctree │ │ │ ├── claf.decorator.doctree │ │ │ ├── claf.doctree │ │ │ ├── claf.learn.doctree │ │ │ ├── claf.machine.components.doctree │ │ │ ├── claf.machine.components.retrieval.doctree │ │ │ ├── claf.machine.doctree │ │ │ ├── claf.metric.doctree │ │ │ ├── claf.model.doctree │ │ │ ├── claf.model.reading_comprehension.doctree │ │ │ ├── claf.model.semantic_parsing.doctree │ │ │ ├── claf.model.sequence_classification.doctree │ │ │ ├── claf.model.token_classification.doctree │ │ │ ├── claf.modules.attention.doctree │ │ │ ├── claf.modules.conv.doctree │ │ │ ├── claf.modules.doctree │ │ │ ├── claf.modules.encoder.doctree │ │ │ ├── claf.modules.layer.doctree │ │ │ ├── claf.tokens.doctree │ │ │ ├── claf.tokens.embedding.doctree │ │ │ ├── claf.tokens.indexer.doctree │ │ │ ├── claf.tokens.token_embedder.doctree │ │ │ ├── claf.tokens.tokenizer.doctree │ │ │ ├── contents/ │ │ │ │ ├── dataset_and_model.doctree │ │ │ │ ├── pretrained_vector.doctree │ │ │ │ └── tokens.doctree │ │ │ ├── environment.pickle │ │ │ ├── index.doctree │ │ │ ├── modules.doctree │ │ │ ├── references.doctree │ │ │ ├── reports/ │ │ │ │ ├── glue.doctree │ │ │ │ ├── historyqa.doctree │ │ │ │ ├── korquad.doctree │ │ │ │ ├── squad.doctree │ │ │ │ └── wikisql.doctree │ │ │ └── summary/ │ │ │ └── reading_comprehension.doctree │ │ └── html/ │ │ ├── .buildinfo │ │ ├── .nojekyll │ │ ├── _modules/ │ │ │ ├── claf/ │ │ │ │ ├── config/ │ │ │ │ │ ├── args.html │ │ │ │ │ ├── factory/ │ │ │ │ │ │ ├── base.html │ │ │ │ │ │ ├── data_loader.html │ │ │ │ │ │ ├── data_reader.html │ │ │ │ │ │ ├── model.html │ │ │ │ │ │ ├── optimizer.html │ │ │ │ │ │ └── tokens.html │ │ │ │ │ ├── namespace.html │ │ │ │ │ ├── pattern.html │ │ │ │ │ ├── registry.html │ │ │ │ │ └── utils.html │ │ │ │ ├── data/ │ │ │ │ │ ├── batch.html │ │ │ │ │ ├── collate.html │ │ │ │ │ ├── data_handler.html │ │ │ │ │ ├── dataset/ │ │ │ │ │ │ ├── base.html │ │ │ │ │ │ ├── bert/ │ │ │ │ │ │ │ ├── multi_task.html │ │ │ │ │ │ │ ├── regression.html │ │ │ │ │ │ │ ├── seq_cls.html │ │ │ │ │ │ │ ├── squad.html │ │ │ │ │ │ │ └── tok_cls.html │ │ │ │ │ │ ├── seq_cls.html │ │ │ │ │ │ ├── seq_cls_bert.html │ │ │ │ │ │ ├── squad.html │ │ │ │ │ │ ├── squad_bert.html │ │ │ │ │ │ ├── tok_cls_bert.html │ │ │ │ │ │ └── wikisql.html │ │ │ │ │ ├── reader/ │ │ │ │ │ │ ├── base.html │ │ │ │ │ │ ├── bert/ │ │ │ │ │ │ │ ├── cola.html │ │ │ │ │ │ │ ├── conll2003.html │ │ │ │ │ │ │ ├── glue/ │ │ │ │ │ │ │ │ ├── cola.html │ │ │ │ │ │ │ │ ├── mnli.html │ │ │ │ │ │ │ │ ├── mrpc.html │ │ │ │ │ │ │ │ ├── qnli.html │ │ │ │ │ │ │ │ ├── qqp.html │ │ │ │ │ │ │ │ ├── rte.html │ │ │ │ │ │ │ │ ├── sst.html │ │ │ │ │ │ │ │ ├── stsb.html │ │ │ │ │ │ │ │ └── wnli.html │ │ │ │ │ │ │ ├── mnli.html │ │ │ │ │ │ │ ├── mrpc.html │ │ │ │ │ │ │ ├── multi_task.html │ │ │ │ │ │ │ ├── qnli.html │ │ │ │ │ │ │ ├── qqp.html │ │ │ │ │ │ │ ├── regression.html │ │ │ │ │ │ │ ├── rte.html │ │ │ │ │ │ │ ├── seq_cls.html │ │ │ │ │ │ │ ├── squad.html │ │ │ │ │ │ │ ├── sst.html │ │ │ │ │ │ │ ├── stsb.html │ │ │ │ │ │ │ ├── tok_cls.html │ │ │ │ │ │ │ └── wnli.html │ │ │ │ │ │ ├── cola.html │ │ │ │ │ │ ├── seq_cls.html │ │ │ │ │ │ ├── squad.html │ │ │ │ │ │ └── wikisql.html │ │ │ │ │ └── utils.html │ │ │ │ ├── decorator/ │ │ │ │ │ ├── arguments.html │ │ │ │ │ └── register.html │ │ │ │ ├── learn/ │ │ │ │ │ ├── experiment.html │ │ │ │ │ ├── mode.html │ │ │ │ │ ├── tensorboard.html │ │ │ │ │ ├── trainer.html │ │ │ │ │ └── utils.html │ │ │ │ ├── machine/ │ │ │ │ │ ├── base.html │ │ │ │ │ ├── components/ │ │ │ │ │ │ └── retrieval/ │ │ │ │ │ │ └── tfidf.html │ │ │ │ │ ├── module.html │ │ │ │ │ ├── nlu.html │ │ │ │ │ └── open_qa.html │ │ │ │ ├── metric/ │ │ │ │ │ ├── classification.html │ │ │ │ │ ├── squad_v1_official.html │ │ │ │ │ ├── squad_v2_official.html │ │ │ │ │ └── wikisql_official.html │ │ │ │ ├── model/ │ │ │ │ │ ├── base.html │ │ │ │ │ ├── cls_utils.html │ │ │ │ │ ├── reading_comprehension/ │ │ │ │ │ │ ├── bert.html │ │ │ │ │ │ ├── bert_for_qa.html │ │ │ │ │ │ ├── bidaf.html │ │ │ │ │ │ ├── bidaf_no_answer.html │ │ │ │ │ │ ├── docqa.html │ │ │ │ │ │ ├── docqa_no_answer.html │ │ │ │ │ │ ├── drqa.html │ │ │ │ │ │ ├── mixin.html │ │ │ │ │ │ ├── qanet.html │ │ │ │ │ │ └── roberta.html │ │ │ │ │ ├── semantic_parsing/ │ │ │ │ │ │ ├── mixin.html │ │ │ │ │ │ ├── sqlnet.html │ │ │ │ │ │ └── utils.html │ │ │ │ │ ├── sequence_classification/ │ │ │ │ │ │ ├── bert.html │ │ │ │ │ │ ├── bert_for_seq_cls.html │ │ │ │ │ │ ├── mixin.html │ │ │ │ │ │ ├── roberta.html │ │ │ │ │ │ └── structured_self_attention.html │ │ │ │ │ └── token_classification/ │ │ │ │ │ ├── bert.html │ │ │ │ │ ├── bert_for_tok_cls.html │ │ │ │ │ └── mixin.html │ │ │ │ ├── modules/ │ │ │ │ │ ├── activation.html │ │ │ │ │ ├── attention/ │ │ │ │ │ │ ├── bi_attention.html │ │ │ │ │ │ ├── co_attention.html │ │ │ │ │ │ ├── docqa_attention.html │ │ │ │ │ │ ├── multi_head_attention.html │ │ │ │ │ │ └── seq_attention.html │ │ │ │ │ ├── conv/ │ │ │ │ │ │ ├── depthwise_separable_conv.html │ │ │ │ │ │ └── pointwise_conv.html │ │ │ │ │ ├── encoder/ │ │ │ │ │ │ ├── lstm_cell_with_projection.html │ │ │ │ │ │ └── positional.html │ │ │ │ │ ├── functional.html │ │ │ │ │ ├── initializer.html │ │ │ │ │ └── layer/ │ │ │ │ │ ├── highway.html │ │ │ │ │ ├── normalization.html │ │ │ │ │ ├── positionwise.html │ │ │ │ │ ├── residual.html │ │ │ │ │ └── scalar_mix.html │ │ │ │ ├── tokens/ │ │ │ │ │ ├── cove.html │ │ │ │ │ ├── elmo.html │ │ │ │ │ ├── embedding/ │ │ │ │ │ │ ├── base.html │ │ │ │ │ │ ├── bert_embedding.html │ │ │ │ │ │ ├── char_embedding.html │ │ │ │ │ │ ├── cove_embedding.html │ │ │ │ │ │ ├── elmo_embedding.html │ │ │ │ │ │ ├── frequent_word_embedding.html │ │ │ │ │ │ ├── sparse_feature.html │ │ │ │ │ │ └── word_embedding.html │ │ │ │ │ ├── hangul.html │ │ │ │ │ ├── indexer/ │ │ │ │ │ │ ├── base.html │ │ │ │ │ │ ├── bert_indexer.html │ │ │ │ │ │ ├── char_indexer.html │ │ │ │ │ │ ├── elmo_indexer.html │ │ │ │ │ │ ├── exact_match_indexer.html │ │ │ │ │ │ ├── linguistic_indexer.html │ │ │ │ │ │ └── word_indexer.html │ │ │ │ │ ├── linguistic.html │ │ │ │ │ ├── text_handler.html │ │ │ │ │ ├── token_embedder/ │ │ │ │ │ │ ├── base.html │ │ │ │ │ │ ├── basic_embedder.html │ │ │ │ │ │ └── reading_comprehension_embedder.html │ │ │ │ │ ├── token_maker.html │ │ │ │ │ ├── tokenizer/ │ │ │ │ │ │ ├── base.html │ │ │ │ │ │ ├── bpe.html │ │ │ │ │ │ ├── char.html │ │ │ │ │ │ ├── pass_text.html │ │ │ │ │ │ ├── sent.html │ │ │ │ │ │ ├── subword.html │ │ │ │ │ │ ├── utils.html │ │ │ │ │ │ └── word.html │ │ │ │ │ └── vocabulary.html │ │ │ │ ├── tokens.html │ │ │ │ └── utils.html │ │ │ ├── index.html │ │ │ ├── logging.html │ │ │ └── pathlib.html │ │ ├── _sources/ │ │ │ ├── claf.config.factory.rst.txt │ │ │ ├── claf.config.rst.txt │ │ │ ├── claf.data.dataset.rst.txt │ │ │ ├── claf.data.reader.bert.rst.txt │ │ │ ├── claf.data.reader.rst.txt │ │ │ ├── claf.data.rst.txt │ │ │ ├── claf.decorator.rst.txt │ │ │ ├── claf.learn.rst.txt │ │ │ ├── claf.machine.components.retrieval.rst.txt │ │ │ ├── claf.machine.components.rst.txt │ │ │ ├── claf.machine.rst.txt │ │ │ ├── claf.metric.rst.txt │ │ │ ├── claf.model.reading_comprehension.rst.txt │ │ │ ├── claf.model.rst.txt │ │ │ ├── claf.model.semantic_parsing.rst.txt │ │ │ ├── claf.model.sequence_classification.rst.txt │ │ │ ├── claf.model.token_classification.rst.txt │ │ │ ├── claf.modules.attention.rst.txt │ │ │ ├── claf.modules.conv.rst.txt │ │ │ ├── claf.modules.encoder.rst.txt │ │ │ ├── claf.modules.layer.rst.txt │ │ │ ├── claf.modules.rst.txt │ │ │ ├── claf.rst.txt │ │ │ ├── claf.tokens.embedding.rst.txt │ │ │ ├── claf.tokens.indexer.rst.txt │ │ │ ├── claf.tokens.rst.txt │ │ │ ├── claf.tokens.token_embedder.rst.txt │ │ │ ├── claf.tokens.tokenizer.rst.txt │ │ │ ├── contents/ │ │ │ │ ├── dataset_and_model.md.txt │ │ │ │ ├── pretrained_vector.md.txt │ │ │ │ └── tokens.md.txt │ │ │ ├── index.rst.txt │ │ │ ├── modules.rst.txt │ │ │ ├── references.md.txt │ │ │ ├── reports/ │ │ │ │ ├── glue.md.txt │ │ │ │ ├── historyqa.md.txt │ │ │ │ ├── korquad.md.txt │ │ │ │ ├── squad.md.txt │ │ │ │ └── wikisql.md.txt │ │ │ └── summary/ │ │ │ └── reading_comprehension.md.txt │ │ ├── _static/ │ │ │ ├── basic.css │ │ │ ├── css/ │ │ │ │ ├── badge_only.css │ │ │ │ └── theme.css │ │ │ ├── doctools.js │ │ │ ├── documentation_options.js │ │ │ ├── jquery-3.2.1.js │ │ │ ├── jquery-3.4.1.js │ │ │ ├── jquery.js │ │ │ ├── js/ │ │ │ │ └── theme.js │ │ │ ├── language_data.js │ │ │ ├── pygments.css │ │ │ ├── searchtools.js │ │ │ ├── theme_overrides.css │ │ │ ├── underscore-1.3.1.js │ │ │ ├── underscore.js │ │ │ └── websupport.js │ │ ├── claf.config.factory.html │ │ ├── claf.config.html │ │ ├── claf.data.dataset.html │ │ ├── claf.data.html │ │ ├── claf.data.reader.bert.html │ │ ├── claf.data.reader.html │ │ ├── claf.decorator.html │ │ ├── claf.html │ │ ├── claf.learn.html │ │ ├── claf.machine.components.html │ │ ├── claf.machine.components.retrieval.html │ │ ├── claf.machine.html │ │ ├── claf.metric.html │ │ ├── claf.model.html │ │ ├── claf.model.reading_comprehension.html │ │ ├── claf.model.semantic_parsing.html │ │ ├── claf.model.sequence_classification.html │ │ ├── claf.model.token_classification.html │ │ ├── claf.modules.attention.html │ │ ├── claf.modules.conv.html │ │ ├── claf.modules.encoder.html │ │ ├── claf.modules.html │ │ ├── claf.modules.layer.html │ │ ├── claf.tokens.embedding.html │ │ ├── claf.tokens.html │ │ ├── claf.tokens.indexer.html │ │ ├── claf.tokens.token_embedder.html │ │ ├── claf.tokens.tokenizer.html │ │ ├── contents/ │ │ │ ├── dataset_and_model.html │ │ │ ├── pretrained_vector.html │ │ │ └── tokens.html │ │ ├── genindex.html │ │ ├── index.html │ │ ├── modules.html │ │ ├── objects.inv │ │ ├── py-modindex.html │ │ ├── references.html │ │ ├── reports/ │ │ │ ├── glue.html │ │ │ ├── historyqa.html │ │ │ ├── korquad.html │ │ │ ├── squad.html │ │ │ └── wikisql.html │ │ ├── search.html │ │ ├── searchindex.js │ │ └── summary/ │ │ └── reading_comprehension.html │ ├── _static/ │ │ └── theme_overrides.css │ ├── _templates/ │ │ ├── modules.rst │ │ └── package.rst │ ├── claf.config.factory.rst │ ├── claf.config.rst │ ├── claf.data.dataset.rst │ ├── claf.data.reader.bert.rst │ ├── claf.data.reader.rst │ ├── claf.data.rst │ ├── claf.decorator.rst │ ├── claf.learn.rst │ ├── claf.machine.components.retrieval.rst │ ├── claf.machine.components.rst │ ├── claf.machine.rst │ ├── claf.metric.rst │ ├── claf.model.reading_comprehension.rst │ ├── claf.model.rst │ ├── claf.model.semantic_parsing.rst │ ├── claf.model.sequence_classification.rst │ ├── claf.model.token_classification.rst │ ├── claf.modules.attention.rst │ ├── claf.modules.conv.rst │ ├── claf.modules.encoder.rst │ ├── claf.modules.layer.rst │ ├── claf.modules.rst │ ├── claf.rst │ ├── claf.tokens.embedding.rst │ ├── claf.tokens.indexer.rst │ ├── claf.tokens.rst │ ├── claf.tokens.token_embedder.rst │ ├── claf.tokens.tokenizer.rst │ ├── conf.py │ ├── contents/ │ │ ├── dataset_and_model.md │ │ ├── pretrained_vector.md │ │ └── tokens.md │ ├── index.rst │ ├── make.bat │ ├── modules.rst │ ├── references.md │ ├── reports/ │ │ ├── glue.md │ │ ├── historyqa.md │ │ ├── korquad.md │ │ ├── squad.md │ │ └── wikisql.md │ ├── requirements.txt │ └── summary/ │ └── reading_comprehension.md ├── eval.py ├── index.html ├── machine.py ├── machine_config/ │ ├── ko_wiki.json │ └── nlu.json ├── predict.py ├── pyproject.toml ├── reports/ │ ├── inference_result/ │ │ ├── bert_for_qa-cpu.json │ │ ├── bidaf+elmo-cpu.json │ │ ├── bidaf-cpu.json │ │ ├── docqa+elmo-cpu.json │ │ ├── docqa-cpu.json │ │ ├── drqa-cpu.json │ │ └── qanet-cpu.json │ └── summary/ │ ├── bert_for_qa.json │ ├── bidaf+elmo.json │ ├── bidaf.json │ ├── docqa+elmo.json │ ├── docqa.json │ ├── drqa.json │ └── qanet.json ├── requirements.txt ├── script/ │ ├── convert_checkpoint_to_bert_model.py │ ├── convert_embedding_to_vocab_txt.py │ ├── download_wikisql.sh │ ├── install_mecab.sh │ ├── make_squad_synthetic_data.py │ └── plot.py ├── setup.py ├── tests/ │ ├── __init__.py │ ├── claf/ │ │ ├── data/ │ │ │ └── test_batch.py │ │ ├── machine/ │ │ │ └── knowlege_base/ │ │ │ └── test_docs.py │ │ ├── modules/ │ │ │ └── test_functional.py │ │ └── tokens/ │ │ └── test_vocabulary.py │ └── integration/ │ ├── test_config.py │ ├── test_machine.py │ ├── test_multi_task.py │ ├── test_reading_comprehension.py │ ├── test_semantic_parsing.py │ ├── test_sequence_classification.py │ ├── test_token_classification.py │ ├── test_tokenizers.py │ └── utils.py └── train.py
SYMBOL INDEX (1378 symbols across 178 files)
FILE: claf/config/args.py
function config (line 15) | def config(argv=None, mode=None):
function train_config (line 47) | def train_config(parser, input_argv=None):
function get_input_arguments (line 87) | def get_input_arguments(parser, input_arguments):
function optimize_config (line 103) | def optimize_config(config, is_test=False):
function set_gpu_env (line 124) | def set_gpu_env(config):
function set_batch_size (line 141) | def set_batch_size(config):
function arg_str2bool (line 151) | def arg_str2bool(v):
function general (line 161) | def general(parser):
function data (line 181) | def data(parser):
function token (line 267) | def token(parser):
function model (line 662) | def model(parser):
function nsml_for_internal (line 1164) | def nsml_for_internal(parser):
function trainer (line 1179) | def trainer(parser):
function base_config (line 1790) | def base_config(parser):
function _get_define_config (line 1819) | def _get_define_config(category=None, config_dir="base_config"):
function evaluate (line 1834) | def evaluate(parser):
function predict (line 1855) | def predict(parser):
function machine (line 1913) | def machine(parser):
FILE: claf/config/namespace.py
class NestedNamespace (line 5) | class NestedNamespace(argparse.Namespace):
method __setattr__ (line 12) | def __setattr__(self, name, value):
method delete_unselected (line 21) | def delete_unselected(self, namespace, excepts=[]):
method overwrite (line 30) | def overwrite(self, config):
method load_from_json (line 48) | def load_from_json(self, dict_data):
FILE: claf/config/pattern.py
class Singleton (line 1) | class Singleton(type):
method __call__ (line 12) | def __call__(cls, *args, **kwargs):
FILE: claf/config/registry.py
class Registry (line 9) | class Registry(metaclass=Singleton):
method __init__ (line 14) | def __init__(self):
method add (line 23) | def add(self, name, obj):
method get (line 32) | def get(self, name):
method _split_component_type_and_name (line 41) | def _split_component_type_and_name(self, name):
FILE: claf/config/utils.py
function add_config_extension (line 17) | def add_config_extension(file_path):
function read_config (line 29) | def read_config(file_path):
function pretty_json_dumps (line 41) | def pretty_json_dumps(inputs):
function remove_none (line 49) | def remove_none(obj):
function convert_config2dict (line 62) | def convert_config2dict(config):
function set_global_seed (line 73) | def set_global_seed(seed=21):
FILE: claf/data/collate.py
class PadCollator (line 10) | class PadCollator:
method __init__ (line 21) | def __init__(self, cuda_device_id=None, pad_value=0, skip_keys=["text"]):
method __call__ (line 26) | def __call__(self, features, labels):
method collate (line 32) | def collate(self, datas, apply_pad=True, pad_value=0):
method _collate (line 41) | def _collate(self, value, apply_pad=True, token_name=None, pad_value=0):
method _apply_pad (line 46) | def _apply_pad(self, value, token_name=None, pad_value=0):
method _make_tensor (line 49) | def _make_tensor(self, value):
class FeatLabelPadCollator (line 63) | class FeatLabelPadCollator(PadCollator):
method __call__ (line 77) | def __call__(self, features, labels, apply_pad_labels=(), apply_pad_va...
method collate (line 85) | def collate(self, datas, apply_pad=True, apply_pad_labels=(), apply_pa...
FILE: claf/data/data_handler.py
class CachePath (line 18) | class CachePath:
class DataHandler (line 30) | class DataHandler:
method __init__ (line 39) | def __init__(self, cache_path=CachePath.ROOT):
method convert_cache_path (line 46) | def convert_cache_path(self, path):
method read_embedding (line 50) | def read_embedding(self, file_path):
method read (line 53) | def read(self, file_path, encoding="utf-8", return_path=False):
method _read_from_http (line 79) | def _read_from_http(self, file_path, encoding, return_path=False):
method _download_from_http (line 95) | def _download_from_http(self, temp_file, url):
method cache_token_counter (line 105) | def cache_token_counter(self, data_reader_config, tokenizer_name, obj=...
method load (line 119) | def load(self, file_path, encoding="utf-8"):
method _load_msgpack (line 133) | def _load_msgpack(self, path, encoding):
method _load_pickle (line 137) | def _load_pickle(self, path, encoding):
method dump (line 141) | def dump(self, file_path, obj, encoding="utf-8"):
FILE: claf/data/dataset/base.py
class DatasetBase (line 7) | class DatasetBase(Dataset):
method __init__ (line 13) | def __init__(self):
method __getitem__ (line 18) | def __getitem__(self, index):
method _get_feature_maxlen (line 21) | def _get_feature_maxlen(self, features):
method collate_fn (line 33) | def collate_fn(self, cuda_device_id):
method get_ground_truths (line 36) | def get_ground_truths(self, data_idxs):
method get_ground_truth (line 48) | def get_ground_truth(self):
method get_predict (line 51) | def get_predict(self):
method lazy_evaluation (line 54) | def lazy_evaluation(self, index):
FILE: claf/data/dataset/bert/multi_task.py
class MultiTaskBertDataset (line 11) | class MultiTaskBertDataset(DatasetBase):
method __init__ (line 22) | def __init__(self, batches, vocab, helper=None):
method init_iterators (line 49) | def init_iterators(self):
method collate_fn (line 62) | def collate_fn(self, cuda_device_id=None):
method __getitem__ (line 76) | def __getitem__(self, index):
method __len__ (line 89) | def __len__(self):
method __repr__ (line 92) | def __repr__(self):
FILE: claf/data/dataset/bert/regression.py
class RegressionBertDataset (line 10) | class RegressionBertDataset(DatasetBase):
method __init__ (line 21) | def __init__(self, batch, vocab, helper=None):
method collate_fn (line 49) | def collate_fn(self, cuda_device_id=None):
method __getitem__ (line 69) | def __getitem__(self, index):
method __len__ (line 79) | def __len__(self):
method __repr__ (line 82) | def __repr__(self):
method sequence_maxlen (line 91) | def sequence_maxlen(self):
method get_id (line 94) | def get_id(self, data_index):
method get_ground_truth (line 98) | def get_ground_truth(self, data_id):
FILE: claf/data/dataset/bert/seq_cls.py
class SeqClsBertDataset (line 10) | class SeqClsBertDataset(DatasetBase):
method __init__ (line 21) | def __init__(self, batch, vocab, helper=None):
method collate_fn (line 53) | def collate_fn(self, cuda_device_id=None):
method __getitem__ (line 73) | def __getitem__(self, index):
method __len__ (line 83) | def __len__(self):
method __repr__ (line 86) | def __repr__(self):
method num_classes (line 97) | def num_classes(self):
method sequence_maxlen (line 101) | def sequence_maxlen(self):
method get_id (line 104) | def get_id(self, data_index):
method get_ground_truth (line 108) | def get_ground_truth(self, data_id):
method get_class_text_with_idx (line 111) | def get_class_text_with_idx(self, class_index):
FILE: claf/data/dataset/bert/squad.py
class SQuADBertDataset (line 10) | class SQuADBertDataset(DatasetBase):
method __init__ (line 22) | def __init__(self, batch, vocab, helper=None):
method collate_fn (line 53) | def collate_fn(self, cuda_device_id=None):
method __getitem__ (line 77) | def __getitem__(self, index):
method __len__ (line 89) | def __len__(self):
method __repr__ (line 92) | def __repr__(self):
method bert_input_maxlen (line 103) | def bert_input_maxlen(self):
method get_qid (line 106) | def get_qid(self, data_index):
method get_id (line 112) | def get_id(self, data_index):
method get_qid_index (line 115) | def get_qid_index(self, data_index):
method get_context (line 121) | def get_context(self, data_index):
method get_ground_truths (line 126) | def get_ground_truths(self, data_index):
method get_predict (line 133) | def get_predict(self, data_index, start, end):
method get_text_with_index (line 136) | def get_text_with_index(self, data_index, start, end):
method get_bert_tokens (line 158) | def get_bert_tokens(self, data_index):
FILE: claf/data/dataset/bert/tok_cls.py
class TokClsBertDataset (line 10) | class TokClsBertDataset(DatasetBase):
method __init__ (line 21) | def __init__(self, batch, vocab, helper=None):
method collate_fn (line 57) | def collate_fn(self, cuda_device_id=None):
method __getitem__ (line 84) | def __getitem__(self, index):
method __len__ (line 96) | def __len__(self):
method __repr__ (line 99) | def __repr__(self):
method num_tags (line 110) | def num_tags(self):
method sequence_maxlen (line 114) | def sequence_maxlen(self):
method get_id (line 117) | def get_id(self, data_index):
method get_ground_truth (line 121) | def get_ground_truth(self, data_id):
method get_tag_texts_with_idxs (line 124) | def get_tag_texts_with_idxs(self, tag_idxs):
method get_tag_text_with_idx (line 127) | def get_tag_text_with_idx(self, tag_index):
FILE: claf/data/dataset/seq_cls.py
class SeqClsDataset (line 11) | class SeqClsDataset(DatasetBase):
method __init__ (line 22) | def __init__(self, batch, vocab, helper=None):
method collate_fn (line 54) | def collate_fn(self, cuda_device_id=None):
method __getitem__ (line 73) | def __getitem__(self, index):
method __len__ (line 82) | def __len__(self):
method __repr__ (line 85) | def __repr__(self):
method num_classes (line 96) | def num_classes(self):
method sequence_maxlen (line 100) | def sequence_maxlen(self):
method get_id (line 103) | def get_id(self, data_index):
method get_ground_truth (line 107) | def get_ground_truth(self, data_id):
method get_class_text_with_idx (line 110) | def get_class_text_with_idx(self, class_index):
FILE: claf/data/dataset/squad.py
class SQuADDataset (line 10) | class SQuADDataset(DatasetBase):
method __init__ (line 22) | def __init__(self, batch, vocab, helper=None):
method collate_fn (line 52) | def collate_fn(self, cuda_device_id=None):
method __getitem__ (line 75) | def __getitem__(self, index):
method __len__ (line 87) | def __len__(self):
method __repr__ (line 90) | def __repr__(self):
method context_maxlen (line 102) | def context_maxlen(self):
method question_maxlen (line 106) | def question_maxlen(self):
method get_qid (line 109) | def get_qid(self, data_index):
method get_context (line 112) | def get_context(self, data_index):
method get_text_span (line 116) | def get_text_span(self, data_index):
method get_ground_truths (line 121) | def get_ground_truths(self, data_index):
method get_predict (line 128) | def get_predict(self, data_index, start, end):
method get_text_with_index (line 131) | def get_text_with_index(self, data_index, start, end):
FILE: claf/data/dataset/wikisql.py
class WikiSQLDataset (line 13) | class WikiSQLDataset(DatasetBase):
method __init__ (line 24) | def __init__(self, batch, vocab, helper=None):
method collate_fn (line 61) | def collate_fn(self, cuda_device_id=None):
method __getitem__ (line 80) | def __getitem__(self, index):
method __len__ (line 89) | def __len__(self):
method __repr__ (line 92) | def __repr__(self):
method question_maxlen (line 101) | def question_maxlen(self):
method get_id (line 104) | def get_id(self, data_index):
method get_table_id (line 109) | def get_table_id(self, data_index):
method get_tokenized_question (line 114) | def get_tokenized_question(self, data_index):
method get_ground_truth (line 119) | def get_ground_truth(self, data_index):
FILE: claf/data/dto/batch.py
class Batch (line 7) | class Batch:
method __init__ (line 16) | def __init__(self, **kwargs):
method __repr__ (line 21) | def __repr__(self):
method __len__ (line 24) | def __len__(self):
method sort_by_key (line 27) | def sort_by_key(self, sort_key):
method to_dict (line 51) | def to_dict(self, flatten=False, recursive=True):
FILE: claf/data/dto/bert_feature.py
class BertFeature (line 5) | class BertFeature:
method __init__ (line 17) | def __init__(self, **kwargs):
method set_input (line 20) | def set_input(self, bert_input):
method set_input_with_speical_token (line 24) | def set_input_with_speical_token(self, *args, **kwargs):
method set_feature (line 28) | def set_feature(self, key, value):
method to_dict (line 31) | def to_dict(self):
FILE: claf/data/dto/helper.py
class Helper (line 4) | class Helper:
method __init__ (line 19) | def __init__(self, **kwargs):
method set_example (line 27) | def set_example(self, uid, example, update=False):
method set_model_parameter (line 33) | def set_model_parameter(self, parameters):
method set_predict_helper (line 36) | def set_predict_helper(self, predict_helper):
method to_dict (line 39) | def to_dict(self):
FILE: claf/data/reader/base.py
class DataReader (line 10) | class DataReader:
method __init__ (line 19) | def __init__(self, file_paths, dataset_obj):
method filter_texts (line 26) | def filter_texts(self, dataset):
method read (line 43) | def read(self):
method _read (line 62) | def _read(self, file_path, desc=None):
method read_one_example (line 65) | def read_one_example(self, inputs):
method convert_to_dataset (line 69) | def convert_to_dataset(self, datas, vocab, helpers=None):
FILE: claf/data/reader/bert/conll2003.py
class CoNLL2003BertReader (line 14) | class CoNLL2003BertReader(TokClsBertReader):
method __init__ (line 25) | def __init__(
method _get_data (line 46) | def _get_data(self, file_path):
method _get_tag_dicts (line 64) | def _get_tag_dicts(self, **kwargs):
FILE: claf/data/reader/bert/glue/cola.py
class CoLABertReader (line 13) | class CoLABertReader(SeqClsBertReader):
method __init__ (line 25) | def __init__(
method _get_data (line 48) | def _get_data(self, file_path, **kwargs):
FILE: claf/data/reader/bert/glue/mnli.py
class MNLIBertReader (line 13) | class MNLIBertReader(SeqClsBertReader):
method __init__ (line 25) | def __init__(
method _get_data (line 48) | def _get_data(self, file_path, **kwargs):
FILE: claf/data/reader/bert/glue/mrpc.py
class MRPCBertReader (line 13) | class MRPCBertReader(SeqClsBertReader):
method __init__ (line 25) | def __init__(
method _get_data (line 48) | def _get_data(self, file_path, **kwargs):
FILE: claf/data/reader/bert/glue/qnli.py
class QNLIBertReader (line 13) | class QNLIBertReader(SeqClsBertReader):
method __init__ (line 25) | def __init__(
method _get_data (line 48) | def _get_data(self, file_path, **kwargs):
FILE: claf/data/reader/bert/glue/qqp.py
class QQPBertReader (line 13) | class QQPBertReader(SeqClsBertReader):
method __init__ (line 25) | def __init__(
method _get_data (line 48) | def _get_data(self, file_path, **kwargs):
FILE: claf/data/reader/bert/glue/rte.py
class RTEBertReader (line 13) | class RTEBertReader(SeqClsBertReader):
method __init__ (line 25) | def __init__(
method _get_data (line 48) | def _get_data(self, file_path, **kwargs):
FILE: claf/data/reader/bert/glue/sst.py
class SSTBertReader (line 13) | class SSTBertReader(SeqClsBertReader):
method __init__ (line 25) | def __init__(
method _get_data (line 48) | def _get_data(self, file_path, **kwargs):
FILE: claf/data/reader/bert/glue/stsb.py
class STSBBertReader (line 13) | class STSBBertReader(RegressionBertReader):
method __init__ (line 24) | def __init__(
method _get_data (line 47) | def _get_data(self, file_path, **kwargs):
FILE: claf/data/reader/bert/glue/wnli.py
class WNLIBertReader (line 13) | class WNLIBertReader(SeqClsBertReader):
method __init__ (line 25) | def __init__(
method _get_data (line 48) | def _get_data(self, file_path, **kwargs):
FILE: claf/data/reader/bert/multi_task.py
class MultiTaskBertReader (line 24) | class MultiTaskBertReader(DataReader):
method __init__ (line 38) | def __init__(
method make_data_reader (line 72) | def make_data_reader(self, config_dict):
method make_task_by_reader (line 79) | def make_task_by_reader(self, name, data_reader, helper):
method _read (line 103) | def _read(self, file_path, data_type=None):
method read_one_example (line 123) | def read_one_example(self, inputs):
FILE: claf/data/reader/bert/regression.py
class RegressionBertReader (line 19) | class RegressionBertReader(DataReader):
method __init__ (line 30) | def __init__(
method _get_data (line 66) | def _get_data(self, file_path, **kwargs):
method _read (line 73) | def _read(self, file_path, data_type=None):
method read_one_example (line 155) | def read_one_example(self, inputs):
FILE: claf/data/reader/bert/seq_cls.py
class SeqClsBertReader (line 19) | class SeqClsBertReader(DataReader):
method __init__ (line 34) | def __init__(
method _get_data (line 70) | def _get_data(self, file_path, **kwargs):
method _get_class_dicts (line 76) | def _get_class_dicts(self, **kwargs):
method _read (line 93) | def _read(self, file_path, data_type=None):
method read_one_example (line 192) | def read_one_example(self, inputs):
FILE: claf/data/reader/bert/squad.py
class Token (line 21) | class Token:
method __init__ (line 22) | def __init__(self, text, text_span=None):
class SQuADBertReader (line 28) | class SQuADBertReader(DataReader):
method __init__ (line 39) | def __init__(
method _read (line 77) | def _read(self, file_path, data_type=None):
method read_one_example (line 224) | def read_one_example(self, inputs):
method _find_one_most_common (line 265) | def _find_one_most_common(self, answers):
method _convert_to_spans (line 270) | def _convert_to_spans(self, raw_text, tokenized_text):
method _is_rebuild (line 312) | def _is_rebuild(self, char_answer_text, word_answer_text):
method _make_features_and_labels (line 321) | def _make_features_and_labels(
method _get_closest_answer_spans (line 360) | def _get_closest_answer_spans(self, tokens, char_start, char_end):
FILE: claf/data/reader/bert/tok_cls.py
class TokClsBertReader (line 21) | class TokClsBertReader(DataReader):
method __init__ (line 35) | def __init__(
method _get_data (line 68) | def _get_data(self, file_path):
method _get_tag_dicts (line 74) | def _get_tag_dicts(self, **kwargs):
method _read (line 90) | def _read(self, file_path, data_type=None):
method read_one_example (line 201) | def read_one_example(self, inputs):
FILE: claf/data/reader/cola.py
class CoLAReader (line 13) | class CoLAReader(SeqClsReader):
method __init__ (line 24) | def __init__(
method _get_data (line 38) | def _get_data(self, file_path, **kwargs):
FILE: claf/data/reader/seq_cls.py
class SeqClsReader (line 19) | class SeqClsReader(DataReader):
method __init__ (line 33) | def __init__(self, file_paths, tokenizers, sequence_max_length=None, c...
method _get_data (line 45) | def _get_data(self, file_path, **kwargs):
method _get_class_dicts (line 51) | def _get_class_dicts(self, **kwargs):
method _read (line 68) | def _read(self, file_path, data_type=None):
method read_one_example (line 144) | def read_one_example(self, inputs):
FILE: claf/data/reader/squad.py
class SQuADReader (line 21) | class SQuADReader(DataReader):
method __init__ (line 30) | def __init__(self, file_paths, lang_code, tokenizers, context_max_leng...
method _read (line 42) | def _read(self, file_path, data_type=None):
method read_one_example (line 142) | def read_one_example(self, inputs):
method _clean_text (line 159) | def _clean_text(self, text):
method _find_one_most_common (line 170) | def _find_one_most_common(self, answers):
method _convert_to_spans (line 175) | def _convert_to_spans(self, raw_text, tokenized_text):
method _get_word_span_idxs (line 198) | def _get_word_span_idxs(self, spans, start, end):
method _is_rebuild (line 208) | def _is_rebuild(self, char_answer_text, word_answer_text):
FILE: claf/data/reader/wikisql.py
class WikiSQLReader (line 22) | class WikiSQLReader(DataReader):
method __init__ (line 32) | def __init__(self, file_paths, tokenizers, context_max_length=None, is...
method _read (line 43) | def _read(self, file_path, data_type=None):
method read_one_example (line 108) | def read_one_example(self, inputs):
method load_data (line 114) | def load_data(self, sql_path, table_path, data_type=None):
method get_coditions_value_position (line 134) | def get_coditions_value_position(self, question, values):
FILE: claf/data/utils.py
function make_batch (line 10) | def make_batch(features, labels):
function make_bert_input (line 14) | def make_bert_input(
function make_bert_token_types (line 42) | def make_bert_token_types(bert_inputs, SEP_token="[SEP]"):
function make_bert_token_type (line 70) | def make_bert_token_type(bert_input_text, SEP_token="[SEP]"):
function padding_tokens (line 80) | def padding_tokens(tokens, max_len=None, token_name=None, pad_value=0):
function get_sequence_a (line 137) | def get_sequence_a(example):
function get_token_dim (line 146) | def get_token_dim(tokens, dim=0):
function get_token_type (line 164) | def get_token_type(tokens):
function is_lazy (line 171) | def is_lazy(tokens):
function transpose (line 181) | def transpose(list_of_dict, skip_keys=[]):
function sanity_check_iob (line 194) | def sanity_check_iob(naive_tokens, tag_texts):
function get_is_head_of_word (line 248) | def get_is_head_of_word(naive_tokens, sequence_tokens):
FILE: claf/decorator/arguments.py
class arguments_required (line 1) | class arguments_required:
method __init__ (line 8) | def __init__(self, required_fields):
method __call__ (line 11) | def __call__(self, fn):
FILE: claf/decorator/register.py
class register (line 5) | class register:
method __init__ (line 12) | def __init__(self, name):
method __call__ (line 15) | def __call__(self, obj):
FILE: claf/factory/base.py
class Factory (line 1) | class Factory:
method __init__ (line 15) | def __init__(self):
method create (line 18) | def create(self):
FILE: claf/factory/data_loader.py
function make_data_loader (line 8) | def make_data_loader(dataset, batch_size=32, shuffle=True, cuda_device_i...
class DataLoaderFactory (line 21) | class DataLoaderFactory(Factory):
method __init__ (line 29) | def __init__(self):
method create (line 33) | def create(self, config, datasets):
FILE: claf/factory/data_reader.py
class DataReaderFactory (line 9) | class DataReaderFactory(Factory):
method __init__ (line 20) | def __init__(self):
method create (line 24) | def create(self, config):
FILE: claf/factory/model.py
class ModelFactory (line 12) | class ModelFactory(Factory):
method __init__ (line 23) | def __init__(self):
method create (line 27) | def create(self, config, token_makers, **params):
method create_token_embedder (line 46) | def create_token_embedder(self, model, token_makers):
FILE: claf/factory/optimizer.py
class OptimizerFactory (line 17) | class OptimizerFactory(Factory):
method __init__ (line 27) | def __init__(self):
method create (line 31) | def create(self, config, model):
method get_model_parameters (line 61) | def get_model_parameters(self, model, optimizer_params):
method _group_parameters_for_transformers (line 69) | def _group_parameters_for_transformers(self, model, weight_decay=0):
method make_lr_scheduler (line 91) | def make_lr_scheduler(self, config, optimizer):
method set_warmup_steps (line 114) | def set_warmup_steps(self, lr_scheduler_config):
FILE: claf/factory/tokens.py
function make_tokenizer (line 11) | def make_tokenizer(tokenizer_cls, tokenizer_config, parent_tokenizers={}):
function make_all_tokenizers (line 26) | def make_all_tokenizers(all_tokenizer_config):
class TokenMakersFactory (line 61) | class TokenMakersFactory(Factory):
method __init__ (line 71) | def __init__(self):
method create (line 75) | def create(self, config):
FILE: claf/learn/experiment.py
class Experiment (line 28) | class Experiment:
method __init__ (line 37) | def __init__(self, mode, config):
method common_setting (line 62) | def common_setting(self, mode, config):
method _get_cuda_devices (line 70) | def _get_cuda_devices(self):
method load_setting (line 82) | def load_setting(self):
method _read_checkpoint (line 93) | def _read_checkpoint(self, cuda_devices, checkpoint_path, prev_cuda_de...
method _set_saved_config (line 108) | def _set_saved_config(self, cuda_devices):
method __call__ (line 124) | def __call__(self):
method set_train_mode (line 172) | def set_train_mode(self):
method _create_data_and_token_makers (line 234) | def _create_data_and_token_makers(self):
method _create_by_factory (line 243) | def _create_by_factory(self, factory_cls, item_config, param={}):
method _get_num_train_steps (line 247) | def _get_num_train_steps(self, train_loader):
method _load_exist_checkpoints (line 259) | def _load_exist_checkpoints(self, checkpoint_dir): # pragma: no cover
method _create_model (line 284) | def _create_model(self, token_makers, checkpoint=None, helpers=None):
method _set_gpu_env (line 310) | def _set_gpu_env(self, model):
method set_trainer (line 330) | def set_trainer(self, model, op_dict={}, save_params={}):
method _summary_experiments (line 346) | def _summary_experiments(self):
method set_eval_mode (line 372) | def set_eval_mode(self):
method set_eval_inference_latency_mode (line 414) | def set_eval_inference_latency_mode(self):
method predict (line 446) | def predict(self, raw_features):
method set_predict_mode (line 465) | def set_predict_mode(self, preload=False):
FILE: claf/learn/mode.py
class Mode (line 1) | class Mode:
FILE: claf/learn/optimization/exponential_moving_avarage.py
class EMA (line 1) | class EMA:
method __init__ (line 13) | def __init__(self, model, mu):
method register (line 21) | def register(self, name, val):
method __call__ (line 24) | def __call__(self, name, x):
FILE: claf/learn/optimization/learning_rate_scheduler.py
function get_lr_schedulers (line 16) | def get_lr_schedulers():
class LearningRateScheduler (line 31) | class LearningRateScheduler:
method __init__ (line 32) | def __init__(self, lr_scheduler):
method step (line 35) | def step(self, metric, epoch=None):
method step_batch (line 38) | def step_batch(self, batch_num_total):
class LearningRateWithoutMetricsWrapper (line 45) | class LearningRateWithoutMetricsWrapper(LearningRateScheduler):
method __init__ (line 50) | def __init__(
method step (line 57) | def step(self, metric, epoch=None):
class LearningRateWithMetricsWrapper (line 61) | class LearningRateWithMetricsWrapper(LearningRateScheduler):
method __init__ (line 67) | def __init__(self, lr_scheduler: torch.optim.lr_scheduler.ReduceLROnPl...
method step (line 72) | def step(self, metric, epoch=None):
class NoamLR (line 82) | class NoamLR(torch.optim.lr_scheduler._LRScheduler): # pylint: disable=...
method __init__ (line 98) | def __init__(
method step (line 111) | def step(self, epoch=None):
method step_batch (line 114) | def step_batch(self, epoch=None):
method get_lr (line 121) | def get_lr(self):
FILE: claf/learn/optimization/optimizer.py
function get_optimizer_by_name (line 6) | def get_optimizer_by_name(name):
FILE: claf/learn/tensorboard.py
class TensorBoard (line 9) | class TensorBoard:
method __init__ (line 12) | def __init__(self, log_dir):
method scalar_summaries (line 17) | def scalar_summaries(self, step, summary):
method scalar_summary (line 29) | def scalar_summary(self, step, tag, value):
method image_summary (line 36) | def image_summary(self, tag, images, step):
method embedding_summary (line 40) | def embedding_summary(self, features, metadata=None, label_img=None):
method histogram_summary (line 43) | def histogram_summary(self, tag, values, step, bins=1000):
method graph_summary (line 47) | def graph_summary(self, model, input_to_model=None):
FILE: claf/learn/trainer.py
class Trainer (line 22) | class Trainer:
method __init__ (line 52) | def __init__(
method set_model_base_properties (line 113) | def set_model_base_properties(self, config, log_dir):
method train_and_evaluate (line 123) | def train_and_evaluate(self, train_loader, valid_loader, optimizer):
method train (line 155) | def train(self, data_loader, optimizer):
method evaluate (line 175) | def evaluate(self, data_loader):
method evaluate_inference_latency (line 182) | def evaluate_inference_latency(self, raw_examples, raw_to_tensor_fn, t...
method _is_early_stopping (line 258) | def _is_early_stopping(self, metrics):
method _report_metrics (line 272) | def _report_metrics(self, tensorboard=True, train_metrics=None, valid_...
method _update_metric_logs (line 300) | def _update_metric_logs(self, total_metrics):
method _estimate_remainig_time (line 319) | def _estimate_remainig_time(self, start_time):
method _report_trainings (line 327) | def _report_trainings(self, start_time, train_loader=None, valid_loade...
method _run_epoch (line 336) | def _run_epoch(
method _set_dataset_to_model (line 466) | def _set_dataset_to_model(self, dataset):
method _get_model_parameters (line 472) | def _get_model_parameters(self):
method _check_valid_results (line 478) | def _check_valid_results(self, metrics, report=False):
method _make_metrics (line 492) | def _make_metrics(self, predictions):
method _update_predictions (line 500) | def _update_predictions(self, predictions, output_dict):
method _print_examples (line 506) | def _print_examples(self, index, inputs, predictions):
method predict (line 515) | def predict(self, raw_feature, raw_to_tensor_fn, arguments, interactiv...
method save (line 537) | def save(self, optimizer):
FILE: claf/learn/utils.py
class TrainCounter (line 23) | class TrainCounter:
method __init__ (line 28) | def __init__(self, display_unit="epoch"):
method get_display (line 33) | def get_display(self):
function load_model_checkpoint (line 43) | def load_model_checkpoint(model, checkpoint):
function load_optimizer_checkpoint (line 56) | def load_optimizer_checkpoint(optimizer, checkpoint):
function load_vocabs (line 63) | def load_vocabs(model_checkpoint):
function save_checkpoint (line 75) | def save_checkpoint(path, model, optimizer, max_to_keep=10):
function get_sorted_path (line 125) | def get_sorted_path(checkpoint_dir, both_exist=False):
function bind_nsml (line 159) | def bind_nsml(model, **kwargs): # pragma: no cover
function get_session_name (line 212) | def get_session_name():
function send_message_to_slack (line 219) | def send_message_to_slack(webhook_url, title=None, message=None): # pra...
FILE: claf/machine/base.py
class Machine (line 12) | class Machine:
method __init__ (line 20) | def __init__(self, config):
method load (line 24) | def load(self):
method load_from_config (line 28) | def load_from_config(cls, config_path):
method __call__ (line 37) | def __call__(self, text):
method make_module (line 40) | def make_module(self, config):
FILE: claf/machine/components/retrieval/tfidf.py
class TFIDF (line 14) | class TFIDF:
method __init__ (line 30) | def __init__(self, texts, word_tokenizer, k=1):
method init (line 37) | def init(self):
method init_model (line 45) | def init_model(self):
method get_closest (line 53) | def get_closest(self, query):
method parse (line 64) | def parse(self, query, ngram=1):
method text_to_tfidf (line 68) | def text_to_tfidf(self, query):
method save (line 78) | def save(self, dir_path):
method load (line 90) | def load(self, dir_path):
FILE: claf/machine/ensemble_topk.py
class MRCEnsemble (line 20) | class MRCEnsemble(Machine):
method __init__ (line 28) | def __init__(self, config):
method load (line 35) | def load(self):
method evaluate (line 56) | def evaluate(self, file_path, output_path):
method get_predict (line 236) | def get_predict(self, context, question):
FILE: claf/machine/knowlege_base/docs.py
function read_wiki_articles (line 12) | def read_wiki_articles(dir_path):
function get_subdir_paths (line 37) | def get_subdir_paths(dir_path):
function get_file_paths (line 46) | def get_file_paths(dir_path):
function read_wiki_article (line 55) | def read_wiki_article(file_path):
class WikiArticle (line 70) | class WikiArticle: # pragma: no cover
method __init__ (line 71) | def __init__(self, id=None, url=None, title=None, text=None):
method id (line 78) | def id(self):
method id (line 82) | def id(self, id):
method url (line 86) | def url(self):
method url (line 90) | def url(self, url):
method title (line 94) | def title(self):
method title (line 98) | def title(self, title):
method text (line 102) | def text(self):
method text (line 106) | def text(self, text):
FILE: claf/machine/module.py
class Module (line 1) | class Module:
FILE: claf/machine/nlu.py
class NLU (line 16) | class NLU(Machine):
method __init__ (line 24) | def __init__(self, config):
method load (line 31) | def load(self):
method __call__ (line 43) | def __call__(self, utterance):
method intent_classification (line 55) | def intent_classification(self, utterance):
method slot_filling (line 59) | def slot_filling(self, utterance):
FILE: claf/machine/open_qa.py
class OpenQA (line 20) | class OpenQA(Machine):
method __init__ (line 30) | def __init__(self, config):
method load (line 37) | def load(self):
method _load_knowledge_base (line 58) | def _load_knowledge_base(self, config):
method _load_document_retrieval (line 63) | def _load_document_retrieval(self, config, word_tokenizer, basename="d...
method __call__ (line 84) | def __call__(self, question):
method search_documents (line 106) | def search_documents(self, question):
method machine_reading (line 109) | def machine_reading(self, context, question):
FILE: claf/metric/classification.py
function recall (line 2) | def recall(pycm_obj):
function precision (line 6) | def precision(pycm_obj):
function f1 (line 10) | def f1(pycm_obj):
function macro_recall (line 14) | def macro_recall(pycm_obj):
function macro_precision (line 18) | def macro_precision(pycm_obj):
function macro_f1 (line 22) | def macro_f1(pycm_obj):
FILE: claf/metric/glue.py
function simple_accuracy (line 7) | def simple_accuracy(preds, labels):
function f1 (line 13) | def f1(preds, labels):
function matthews_corr (line 19) | def matthews_corr(preds, labels):
function pearson_and_spearman (line 25) | def pearson_and_spearman(preds, labels):
FILE: claf/metric/korquad_v1_official.py
function normalize_answer (line 12) | def normalize_answer(s):
function f1_score (line 42) | def f1_score(prediction, ground_truth):
function exact_match_score (line 69) | def exact_match_score(prediction, ground_truth):
function metric_max_over_ground_truths (line 73) | def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
function evaluate (line 81) | def evaluate(dataset, predictions):
FILE: claf/metric/regression.py
function mse (line 5) | def mse(outputs, labels):
FILE: claf/metric/squad_v1_official.py
function normalize_answer (line 11) | def normalize_answer(s): # pragma: no cover
function f1_score (line 30) | def f1_score(prediction, ground_truth): # pragma: no cover
function exact_match_score (line 43) | def exact_match_score(prediction, ground_truth): # pragma: no cover
function metric_max_over_ground_truths (line 47) | def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):...
function evaluate (line 55) | def evaluate(dataset, predictions):
FILE: claf/metric/squad_v2_official.py
function parse_args (line 20) | def parse_args(): # pragma: no cover
function make_qid_to_has_ans (line 57) | def make_qid_to_has_ans(dataset): # pragma: no cover
function normalize_answer (line 66) | def normalize_answer(s): # pragma: no cover
function get_tokens (line 86) | def get_tokens(s): # pragma: no cover
function compute_exact (line 92) | def compute_exact(a_gold, a_pred): # pragma: no cover
function compute_f1 (line 96) | def compute_f1(a_gold, a_pred): # pragma: no cover
function get_raw_scores (line 112) | def get_raw_scores(dataset, preds): # pragma: no cover
function apply_no_ans_threshold (line 134) | def apply_no_ans_threshold(scores, na_probs, qid_to_has_ans, na_prob_thr...
function make_eval_dict (line 145) | def make_eval_dict(exact_scores, f1_scores, qid_list=None): # pragma: n...
function merge_eval (line 166) | def merge_eval(main_eval, new_eval, prefix): # pragma: no cover
function plot_pr_curve (line 171) | def plot_pr_curve(precisions, recalls, out_image, title): # pragma: no ...
function make_precision_recall_eval (line 183) | def make_precision_recall_eval(
function run_precision_recall_analysis (line 208) | def run_precision_recall_analysis(
function histogram_na_prob (line 246) | def histogram_na_prob(na_probs, qid_list, image_dir, name): # pragma: n...
function find_best_thresh (line 259) | def find_best_thresh(preds, scores, na_probs, qid_to_has_ans): # pragma...
function find_all_best_thresh (line 282) | def find_all_best_thresh(
function evaluate (line 293) | def evaluate(dataset, na_probs, preds, na_prob_thresh=1.0):
function main (line 316) | def main(): # pragma: no cover
FILE: claf/metric/wikisql_lib/dbengine.py
class DBEngine (line 12) | class DBEngine: # pragma: no cover
method __init__ (line 14) | def __init__(self, fdb):
method execute_query (line 18) | def execute_query(self, table_id, query, *args, **kwargs):
method execute (line 21) | def execute(self, table_id, select_index, aggregation_index, condition...
FILE: claf/metric/wikisql_lib/query.py
function detokenize (line 9) | def detokenize(tokens): # pragma: no cover
class Query (line 16) | class Query: # pragma: no cover
method __init__ (line 37) | def __init__(self, sel_index, agg_index, conditions=tuple(), ordered=F...
method __eq__ (line 43) | def __eq__(self, other):
method __ne__ (line 58) | def __ne__(self, other):
method __hash__ (line 63) | def __hash__(self):
method __repr__ (line 66) | def __repr__(self):
method to_dict (line 79) | def to_dict(self):
method lower (line 82) | def lower(self):
method from_dict (line 89) | def from_dict(cls, d, ordered=False):
method from_tokenized_dict (line 93) | def from_tokenized_dict(cls, d):
method from_generated_dict (line 100) | def from_generated_dict(cls, d):
method from_sequence (line 108) | def from_sequence(cls, sequence, table, lowercase=True):
method from_partial_sequence (line 205) | def from_partial_sequence(cls, agg_col, agg_op, sequence, table, lower...
FILE: claf/metric/wikisql_official.py
function count_lines (line 10) | def count_lines(fname): # pragma: no cover
function evaluate (line 15) | def evaluate(labels, predictions, db_path, ordered=True): # pragma: no ...
FILE: claf/model/base.py
class ModelBase (line 8) | class ModelBase(nn.Module):
method __init__ (line 16) | def __init__(self):
method forward (line 19) | def forward(self, inputs):
method make_metrics (line 22) | def make_metrics(self, predictions):
method make_predictions (line 26) | def make_predictions(self, features):
method predict (line 33) | def predict(self, features):
method print_examples (line 40) | def print_examples(self, params):
method write_predictions (line 47) | def write_predictions(self, predictions, file_path=None, is_dict=True):
method is_ready (line 63) | def is_ready(self):
method config (line 76) | def config(self):
method config (line 80) | def config(self, config):
method log_dir (line 84) | def log_dir(self):
method log_dir (line 88) | def log_dir(self, log_dir):
method dataset (line 92) | def dataset(self):
method dataset (line 96) | def dataset(self, dataset):
method metrics (line 100) | def metrics(self):
method metrics (line 104) | def metrics(self, metrics):
method train_counter (line 108) | def train_counter(self):
method train_counter (line 112) | def train_counter(self, train_counter):
method vocabs (line 116) | def vocabs(self):
method vocabs (line 120) | def vocabs(self, vocabs):
class ModelWithTokenEmbedder (line 124) | class ModelWithTokenEmbedder(ModelBase):
method __init__ (line 125) | def __init__(self, token_embedder):
class ModelWithoutTokenEmbedder (line 133) | class ModelWithoutTokenEmbedder(ModelBase):
method __init__ (line 134) | def __init__(self, token_makers):
FILE: claf/model/cls_utils.py
function write_confusion_matrix_to_csv (line 8) | def write_confusion_matrix_to_csv(file_path, pycm_obj):
function get_tag_dict (line 37) | def get_tag_dict(sequence, tag_texts):
FILE: claf/model/multi_task/bert.py
class BertForMultiTask (line 15) | class BertForMultiTask(MultiTask, ModelWithoutTokenEmbedder):
method __init__ (line 30) | def __init__(self, token_makers, tasks, pretrained_model_name=None, dr...
method _init_criterions (line 47) | def _init_criterions(self, tasks):
method _init_task_layers (line 65) | def _init_task_layers(self, tasks, dropouts):
method forward (line 89) | def forward(self, features, labels=None):
method _task_forward (line 145) | def _task_forward(self, task_index, shared_outputs):
method _task_calculate_loss (line 181) | def _task_calculate_loss(self, task_index, output_dict, labels):
method print_examples (line 240) | def print_examples(self, index, inputs, predictions):
FILE: claf/model/multi_task/category.py
class TaskCategory (line 2) | class TaskCategory:
FILE: claf/model/multi_task/mixin.py
class MultiTask (line 13) | class MultiTask:
method make_predictions (line 16) | def make_predictions(self, output_dict):
method predict (line 25) | def predict(self, output_dict, arguments, helper):
method make_metrics (line 30) | def make_metrics(self, predictions):
method _split_predictions_by_task_index (line 55) | def _split_predictions_by_task_index(self, predictions):
method _make_task_mixin_obj (line 63) | def _make_task_mixin_obj(self, task_index):
method _set_model_properties (line 80) | def _set_model_properties(self, mixin_obj, task_index=None):
FILE: claf/model/reading_comprehension/bert.py
class BertForQA (line 14) | class BertForQA(SQuADv1ForBert, ModelWithoutTokenEmbedder):
method __init__ (line 31) | def __init__(self, token_makers, lang_code="en", pretrained_model_name...
method forward (line 44) | def forward(self, features, labels=None):
FILE: claf/model/reading_comprehension/bidaf.py
class BiDAF (line 15) | class BiDAF(SQuADv1, ModelWithTokenEmbedder):
method __init__ (line 45) | def __init__(
method forward (line 127) | def forward(self, features, labels=None):
FILE: claf/model/reading_comprehension/bidaf_no_answer.py
class BiDAF_No_Answer (line 15) | class BiDAF_No_Answer(SQuADv2, ModelWithTokenEmbedder):
method __init__ (line 40) | def __init__(
method forward (line 124) | def forward(self, features, labels=None):
FILE: claf/model/reading_comprehension/docqa.py
class DocQA (line 14) | class DocQA(SQuADv1, ModelWithTokenEmbedder):
method __init__ (line 45) | def __init__(
method forward (line 145) | def forward(self, features, labels=None):
class SelfAttention (line 254) | class SelfAttention(nn.Module):
method __init__ (line 259) | def __init__(self, rnn_dim, linear_dim, dropout=0.2, weight_init=True):
method forward (line 272) | def forward(self, context, context_mask):
FILE: claf/model/reading_comprehension/docqa_no_answer.py
class DocQA_No_Answer (line 14) | class DocQA_No_Answer(SQuADv2, ModelWithTokenEmbedder):
method __init__ (line 45) | def __init__(
method forward (line 147) | def forward(self, features, labels=None):
class SelfAttention (line 270) | class SelfAttention(nn.Module):
method __init__ (line 275) | def __init__(self, rnn_dim, linear_dim, dropout=0.2, weight_init=True):
method forward (line 288) | def forward(self, context, context_mask):
class NoAnswer (line 295) | class NoAnswer(nn.Module):
method __init__ (line 304) | def __init__(self, embed_dim, bias_hidden_dim):
method forward (line 312) | def forward(self, context_embed, span_start_logits, span_end_logits):
FILE: claf/model/reading_comprehension/drqa.py
class DrQA (line 13) | class DrQA(SQuADv1, ModelWithTokenEmbedder):
method __init__ (line 38) | def __init__(
method forward (line 85) | def forward(self, features, labels=None):
FILE: claf/model/reading_comprehension/mixin.py
class ReadingComprehension (line 13) | class ReadingComprehension:
method get_best_span (line 22) | def get_best_span(self, span_start_logits, span_end_logits, answer_max...
method _make_span_metrics (line 61) | def _make_span_metrics(self, predictions):
method make_predictions (line 82) | def make_predictions(self, output_dict):
method predict (line 127) | def predict(self, output_dict, arguments, helper):
method print_examples (line 160) | def print_examples(self, index, inputs, predictions):
method write_predictions (line 195) | def write_predictions(self, predictions, file_path=None, is_dict=True):
class SQuADv1 (line 211) | class SQuADv1(ReadingComprehension):
method make_metrics (line 221) | def make_metrics(self, predictions):
method _make_metrics_with_official (line 254) | def _make_metrics_with_official(self, preds):
class SQuADv1ForBert (line 265) | class SQuADv1ForBert(SQuADv1):
method make_metrics (line 275) | def make_metrics(self, predictions):
method predict (line 299) | def predict(self, output_dict, arguments, helper):
class SQuADv2 (line 352) | class SQuADv2(ReadingComprehension):
method make_metrics (line 362) | def make_metrics(self, predictions):
method _make_metrics_with_official (line 422) | def _make_metrics_with_official(self, preds, na_probs, na_prob_thresh=...
FILE: claf/model/reading_comprehension/qanet.py
class QANet (line 17) | class QANet(SQuADv1, ModelWithTokenEmbedder):
method __init__ (line 57) | def __init__(
method forward (line 143) | def forward(self, features, labels=None):
class EncoderBlock (line 251) | class EncoderBlock(nn.Module):
method __init__ (line 272) | def __init__(
method forward (line 314) | def forward(self, x, mask=None):
FILE: claf/model/reading_comprehension/roberta.py
class RoBertaForQA (line 13) | class RoBertaForQA(SQuADv1ForBert, ModelWithoutTokenEmbedder):
method __init__ (line 30) | def __init__(self, token_makers, lang_code="en", pretrained_model_name...
method forward (line 44) | def forward(self, features, labels=None):
FILE: claf/model/regression/bert.py
class BertForRegression (line 13) | class BertForRegression(Regression, ModelWithoutTokenEmbedder):
method __init__ (line 27) | def __init__(self, token_makers, pretrained_model_name=None, dropout=0...
method forward (line 40) | def forward(self, features, labels=None):
method print_examples (line 102) | def print_examples(self, index, inputs, predictions):
FILE: claf/model/regression/mixin.py
class Regression (line 11) | class Regression:
method make_predictions (line 14) | def make_predictions(self, output_dict):
method predict (line 44) | def predict(self, output_dict, arguments, helper):
method make_metrics (line 66) | def make_metrics(self, predictions):
method write_predictions (line 103) | def write_predictions(self, predictions, ):
method print_examples (line 115) | def print_examples(self, index, inputs, predictions):
FILE: claf/model/regression/roberta.py
class RobertaForRegression (line 13) | class RobertaForRegression(Regression, ModelWithoutTokenEmbedder):
method __init__ (line 27) | def __init__(self, token_makers, pretrained_model_name=None, dropout=0...
method forward (line 40) | def forward(self, features, labels=None):
method print_examples (line 94) | def print_examples(self, index, inputs, predictions):
FILE: claf/model/semantic_parsing/mixin.py
class WikiSQL (line 10) | class WikiSQL:
method make_metrics (line 22) | def make_metrics(self, predictions):
method _make_metrics_with_official (line 70) | def _make_metrics_with_official(self, preds):
method make_predictions (line 87) | def make_predictions(self, output_dict):
method generate_queries (line 101) | def generate_queries(self, output_dict):
method decode_pointer (line 159) | def decode_pointer(self, tokenized_question, cond_value_logits):
method merge_tokens (line 174) | def merge_tokens(self, tok_list, raw_tok_str):
method predict (line 215) | def predict(self, output_dict, arguments, helper):
method print_examples (line 244) | def print_examples(self, index, inputs, predictions):
FILE: claf/model/semantic_parsing/sqlnet.py
class SQLNet (line 19) | class SQLNet(WikiSQL, ModelWithTokenEmbedder):
method __init__ (line 42) | def __init__(
method forward (line 86) | def forward(self, features, labels=None):
class AggPredictor (line 227) | class AggPredictor(nn.Module):
method __init__ (line 228) | def __init__(self, embed_dim, model_dim, rnn_num_layer, dropout, agg_c...
method forward (line 244) | def forward(self, question_embed, question_mask):
class SelPredictor (line 252) | class SelPredictor(nn.Module):
method __init__ (line 253) | def __init__(self, embed_dim, model_dim, rnn_num_layer, dropout, colum...
method forward (line 284) | def forward(self, question_embed, question_mask, column_embed, column_...
class CondsPredictor (line 310) | class CondsPredictor(nn.Module):
method __init__ (line 311) | def __init__(
method forward (line 343) | def forward(
class CondsNumPredictor (line 377) | class CondsNumPredictor(nn.Module):
method __init__ (line 378) | def __init__(self, embed_dim, model_dim, rnn_num_layer, dropout, colum...
method forward (line 410) | def forward(self, question_embed, question_mask, column_embed, column_...
class CondsColPredictor (line 438) | class CondsColPredictor(nn.Module):
method __init__ (line 439) | def __init__(self, embed_dim, model_dim, rnn_num_layer, dropout, colum...
method forward (line 470) | def forward(self, question_embed, question_mask, column_embed, column_...
class CondsOpPredictor (line 496) | class CondsOpPredictor(nn.Module):
method __init__ (line 497) | def __init__(
method forward (line 540) | def forward(self, question_embed, question_mask, column_embed, column_...
class CondsValuePointer (line 567) | class CondsValuePointer(nn.Module):
method __init__ (line 568) | def __init__(self, embed_dim, model_dim, rnn_num_layer, dropout, colum...
method forward (line 607) | def forward(
method concat_start_and_end_zero_padding (line 666) | def concat_start_and_end_zero_padding(self, question_embed, mask):
method decode_then_output (line 688) | def decode_then_output(
FILE: claf/model/semantic_parsing/utils.py
function encode_column (line 6) | def encode_column(column_embed, column_name_mask, rnn_module):
function get_column_lengths (line 29) | def get_column_lengths(column_embed, column_name_mask):
function filter_used_column (line 36) | def filter_used_column(encoded_columns, col_idx, padding_count=4):
function convert_position_to_decoder_input (line 52) | def convert_position_to_decoder_input(conds_val_pos, token_maxlen=200):
FILE: claf/model/sequence_classification/bert.py
class BertForSeqCls (line 13) | class BertForSeqCls(SequenceClassification, ModelWithoutTokenEmbedder):
method __init__ (line 28) | def __init__(self, token_makers, num_classes, pretrained_model_name=No...
method forward (line 41) | def forward(self, features, labels=None):
method print_examples (line 103) | def print_examples(self, index, inputs, predictions):
FILE: claf/model/sequence_classification/mixin.py
class SequenceClassification (line 17) | class SequenceClassification:
method make_predictions (line 20) | def make_predictions(self, output_dict):
method predict (line 51) | def predict(self, output_dict, arguments, helper):
method make_metrics (line 78) | def make_metrics(self, predictions):
method write_predictions (line 125) | def write_predictions(self, predictions, file_path=None, is_dict=True,...
method print_examples (line 155) | def print_examples(self, index, inputs, predictions):
FILE: claf/model/sequence_classification/roberta.py
class RobertaForSeqCls (line 13) | class RobertaForSeqCls(SequenceClassification, ModelWithoutTokenEmbedder):
method __init__ (line 28) | def __init__(self, token_makers, num_classes, pretrained_model_name=No...
method forward (line 41) | def forward(self, features, labels=None):
method print_examples (line 96) | def print_examples(self, index, inputs, predictions):
FILE: claf/model/sequence_classification/structured_self_attention.py
class StructuredSelfAttention (line 16) | class StructuredSelfAttention(SequenceClassification, ModelWithTokenEmbe...
method __init__ (line 37) | def __init__(
method forward (line 86) | def forward(self, features, labels=None):
method penalty (line 148) | def penalty(self, attention):
FILE: claf/model/token_classification/bert.py
class BertForTokCls (line 15) | class BertForTokCls(TokenClassification, ModelWithoutTokenEmbedder):
method __init__ (line 31) | def __init__(
method forward (line 46) | def forward(self, features, labels=None):
method print_examples (line 122) | def print_examples(self, index, inputs, predictions):
FILE: claf/model/token_classification/mixin.py
class TokenClassification (line 20) | class TokenClassification:
method make_predictions (line 23) | def make_predictions(self, output_dict):
method predict (line 57) | def predict(self, output_dict, arguments, helper):
method make_metrics (line 88) | def make_metrics(self, predictions):
method write_predictions (line 171) | def write_predictions(self, predictions, file_path=None, is_dict=True,...
method print_examples (line 193) | def print_examples(self, index, inputs, predictions):
FILE: claf/modules/activation.py
function get_activation_fn (line 5) | def get_activation_fn(name):
FILE: claf/modules/attention/bi_attention.py
class BiAttention (line 8) | class BiAttention(nn.Module):
method __init__ (line 21) | def __init__(self, model_dim):
method forward (line 26) | def forward(self, context, context_mask, query, query_mask):
method _make_similiarity_matrix (line 39) | def _make_similiarity_matrix(self, c, q):
method _context2query (line 53) | def _context2query(self, S, q, q_mask):
method _query2context (line 59) | def _query2context(self, S, c, c_mask):
FILE: claf/modules/attention/co_attention.py
class CoAttention (line 9) | class CoAttention(nn.Module):
method __init__ (line 20) | def __init__(self, embed_dim):
method forward (line 25) | def forward(self, context_embed, question_embed, context_mask=None, qu...
FILE: claf/modules/attention/docqa_attention.py
class DocQAAttention (line 9) | class DocQAAttention(nn.Module):
method __init__ (line 24) | def __init__(self, rnn_dim, linear_dim, self_attn=False, weight_init=T...
method forward (line 41) | def forward(self, x, x_mask, key, key_mask):
method _compute_attention_mask (line 61) | def _compute_attention_mask(self, x_mask, key_mask):
method _trilinear (line 67) | def _trilinear(self, x, key):
method _x2key (line 79) | def _x2key(self, S, key, key_mask):
method _key2x (line 90) | def _key2x(self, S, x, x_mask):
FILE: claf/modules/attention/multi_head_attention.py
class MultiHeadAttention (line 10) | class MultiHeadAttention(nn.Module):
method __init__ (line 22) | def __init__(
method forward (line 50) | def forward(self, q, k, v, mask=None):
method _linear_projection (line 57) | def _linear_projection(self, query, key, value):
method _split_heads (line 63) | def _split_heads(self, query, key, value):
method _scaled_dot_product (line 71) | def _scaled_dot_product(self, query, key, value, mask=None):
method _concat_heads (line 84) | def _concat_heads(self, outputs):
FILE: claf/modules/attention/seq_attention.py
class SeqAttnMatch (line 16) | class SeqAttnMatch(nn.Module):
method __init__ (line 23) | def __init__(self, embed_dim, identity=False):
method forward (line 30) | def forward(self, x, y, y_mask):
class LinearSeqAttn (line 52) | class LinearSeqAttn(nn.Module):
method __init__ (line 58) | def __init__(self, input_size):
method forward (line 62) | def forward(self, x, x_mask):
class BilinearSeqAttn (line 70) | class BilinearSeqAttn(nn.Module):
method __init__ (line 77) | def __init__(self, x_size, y_size, identity=False, normalize=True):
method forward (line 86) | def forward(self, x, y, x_mask):
FILE: claf/modules/conv/depthwise_separable_conv.py
class DepSepConv (line 8) | class DepSepConv(nn.Module):
method __init__ (line 21) | def __init__(self, input_size=None, num_filters=None, kernel_size=None):
method forward (line 35) | def forward(self, x):
FILE: claf/modules/conv/pointwise_conv.py
class PointwiseConv (line 6) | class PointwiseConv(nn.Module):
method __init__ (line 20) | def __init__(self, input_size, num_filters):
method forward (line 31) | def forward(self, x):
FILE: claf/modules/encoder/lstm_cell_with_projection.py
class LstmCellWithProjection (line 13) | class LstmCellWithProjection(torch.nn.Module): # pragma: no cover
method __init__ (line 54) | def __init__(
method reset_parameters (line 83) | def reset_parameters(self):
method forward (line 93) | def forward(
function get_dropout_mask (line 251) | def get_dropout_mask(
function block_orthogonal (line 278) | def block_orthogonal(
function sort_batch_by_length (line 322) | def sort_batch_by_length(tensor: torch.Tensor, sequence_lengths: torch.T...
class _EncoderBase (line 368) | class _EncoderBase(torch.nn.Module): # pragma: no cover
method __init__ (line 380) | def __init__(self, stateful: bool = False) -> None:
method sort_and_run_forward (line 385) | def sort_and_run_forward(
method _get_initial_states (line 481) | def _get_initial_states(
method _update_states (line 562) | def _update_states(
method reset_states (line 638) | def reset_states(self):
FILE: claf/modules/encoder/positional.py
class PositionalEncoding (line 8) | class PositionalEncoding(nn.Module):
method __init__ (line 26) | def __init__(self, embed_dim, max_length=2000):
method _get_timing_signal (line 32) | def _get_timing_signal(self, length, channels, min_timescale=1.0, max_...
method forward (line 49) | def forward(self, x):
FILE: claf/modules/functional.py
function add_masked_value (line 16) | def add_masked_value(tensor, mask, value=-1e7):
function get_mask_from_tokens (line 23) | def get_mask_from_tokens(tokens):
function last_dim_masked_softmax (line 38) | def last_dim_masked_softmax(x, mask):
function masked_softmax (line 51) | def masked_softmax(x, mask):
function weighted_sum (line 61) | def weighted_sum(attention, matrix): # pragma: no cover
function masked_zero (line 72) | def masked_zero(tensor, mask):
function masked_log_softmax (line 87) | def masked_log_softmax(vector, mask): # pragma: no cover
function get_sorted_seq_config (line 93) | def get_sorted_seq_config(features, pad_index=0):
function forward_rnn_with_pack (line 110) | def forward_rnn_with_pack(rnn_module, tensor, seq_config):
FILE: claf/modules/initializer.py
function weight (line 10) | def weight(module):
FILE: claf/modules/layer/highway.py
class Highway (line 8) | class Highway(nn.Module):
method __init__ (line 19) | def __init__(self, input_size, num_layers=2, activation="relu"):
method forward (line 33) | def forward(self, x):
FILE: claf/modules/layer/normalization.py
class LayerNorm (line 6) | class LayerNorm(nn.Module):
method __init__ (line 12) | def __init__(self, normalized_shape, eps=1e-5):
method forward (line 18) | def forward(self, x):
FILE: claf/modules/layer/positionwise.py
class PositionwiseFeedForward (line 8) | class PositionwiseFeedForward(nn.Module):
method __init__ (line 20) | def __init__(self, input_size, hidden_size, dropout=0.1):
method forward (line 27) | def forward(self, x):
FILE: claf/modules/layer/residual.py
class ResidualConnection (line 7) | class ResidualConnection(nn.Module):
method __init__ (line 22) | def __init__(self, dim, layer_dropout=None, layernorm=False):
method forward (line 33) | def forward(self, x, sub_layer_fn):
FILE: claf/modules/layer/scalar_mix.py
class ScalarMix (line 12) | class ScalarMix(torch.nn.Module): # pragma: no cover
method __init__ (line 20) | def __init__(
method forward (line 49) | def forward(
FILE: claf/tokens/__init__.py
function basic_embedding_fn (line 9) | def basic_embedding_fn(embedding_config, module):
class FeatureTokenMaker (line 18) | class FeatureTokenMaker(TokenMaker):
method __init__ (line 35) | def __init__(self, tokenizers, indexer_config, embedding_config, vocab...
class BertTokenMaker (line 56) | class BertTokenMaker(TokenMaker):
method __init__ (line 71) | def __init__(self, tokenizers, indexer_config, embedding_config, vocab...
class CharTokenMaker (line 83) | class CharTokenMaker(TokenMaker):
method __init__ (line 100) | def __init__(self, tokenizers, indexer_config, embedding_config, vocab...
class CoveTokenMaker (line 111) | class CoveTokenMaker(TokenMaker):
method __init__ (line 128) | def __init__(self, tokenizers, indexer_config, embedding_config, vocab...
class ElmoTokenMaker (line 139) | class ElmoTokenMaker(TokenMaker):
method __init__ (line 157) | def __init__(self, tokenizers, indexer_config, embedding_config, vocab...
class ExactMatchTokenMaker (line 168) | class ExactMatchTokenMaker(TokenMaker):
method __init__ (line 185) | def __init__(self, tokenizers, indexer_config, embedding_config, vocab...
method _embedding_fn (line 194) | def _embedding_fn(self, embedding_config, indexer_config):
class WordTokenMaker (line 220) | class WordTokenMaker(TokenMaker):
method __init__ (line 233) | def __init__(self, tokenizers, indexer_config, embedding_config, vocab...
class FrequentWordTokenMaker (line 244) | class FrequentWordTokenMaker(TokenMaker):
method __init__ (line 261) | def __init__(self, tokenizers, indexer_config, embedding_config, vocab...
class LinguisticTokenMaker (line 274) | class LinguisticTokenMaker(TokenMaker):
method __init__ (line 291) | def __init__(self, tokenizers, indexer_config, embedding_config, vocab...
method _embedding_fn (line 300) | def _embedding_fn(self, embedding_config, indexer_config):
FILE: claf/tokens/cove.py
class MTLSTM (line 12) | class MTLSTM(nn.Module):
method __init__ (line 13) | def __init__(
method forward (line 39) | def forward(self, inputs):
FILE: claf/tokens/elmo.py
class Elmo (line 31) | class Elmo(torch.nn.Module): # pragma: no cover
method __init__ (line 72) | def __init__(
method get_output_dim (line 105) | def get_output_dim(self):
method forward (line 108) | def forward(
method from_params (line 186) | def from_params(cls, params) -> "Elmo":
function remove_sentence_boundaries (line 209) | def remove_sentence_boundaries(
class _ElmoBiLm (line 249) | class _ElmoBiLm(torch.nn.Module): # pragma: no cover
method __init__ (line 272) | def __init__(
method get_output_dim (line 315) | def get_output_dim(self):
method forward (line 318) | def forward(
function add_sentence_boundary_token_ids (line 379) | def add_sentence_boundary_token_ids(
function _make_bos_eos (line 432) | def _make_bos_eos(
class _ElmoCharacterEncoder (line 446) | class _ElmoCharacterEncoder(torch.nn.Module): # pragma: no cover
method __init__ (line 478) | def __init__(self, options_file: str, weight_file: str, requires_grad:...
method get_output_dim (line 523) | def get_output_dim(self):
method forward (line 527) | def forward(
method _load_weights (line 596) | def _load_weights(self):
method _load_char_embedding (line 602) | def _load_char_embedding(self):
method _load_cnn_weights (line 615) | def _load_cnn_weights(self):
method _load_highway (line 644) | def _load_highway(self):
method _load_projection (line 673) | def _load_projection(self):
class ElmoLstm (line 689) | class ElmoLstm(_EncoderBase): # pragma: no cover
method __init__ (line 725) | def __init__(
method forward (line 778) | def forward(
method _lstm_forward (line 835) | def _lstm_forward(
method load_weights (line 927) | def load_weights(self, weight_file: str) -> None:
FILE: claf/tokens/embedding/base.py
class TokenEmbedding (line 5) | class TokenEmbedding(torch.nn.Module):
method __init__ (line 15) | def __init__(self, vocab):
method forward (line 20) | def forward(self, tokens):
method get_output_dim (line 24) | def get_output_dim(self):
method get_vocab_size (line 28) | def get_vocab_size(self):
FILE: claf/tokens/embedding/bert_embedding.py
class BertEmbedding (line 11) | class BertEmbedding(TokenEmbedding):
method __init__ (line 27) | def __init__(self, vocab, pretrained_model_name=None, trainable=False,...
method forward (line 40) | def forward(self, inputs):
method get_output_dim (line 62) | def get_output_dim(self):
method remove_cls_sep_token (line 65) | def remove_cls_sep_token(self, inputs, outputs):
FILE: claf/tokens/embedding/char_embedding.py
class CharEmbedding (line 12) | class CharEmbedding(TokenEmbedding):
method __init__ (line 28) | def __init__(
method _init_weight (line 57) | def _init_weight(self, trainable=False):
method forward (line 64) | def forward(self, chars):
method get_output_dim (line 95) | def get_output_dim(self):
FILE: claf/tokens/embedding/cove_embedding.py
class CoveEmbedding (line 13) | class CoveEmbedding(TokenEmbedding):
method __init__ (line 30) | def __init__(
method forward (line 60) | def forward(self, words):
method get_output_dim (line 65) | def get_output_dim(self):
FILE: claf/tokens/embedding/elmo_embedding.py
class ELMoEmbedding (line 17) | class ELMoEmbedding(TokenEmbedding):
method __init__ (line 38) | def __init__(
method forward (line 61) | def forward(self, chars):
method get_output_dim (line 70) | def get_output_dim(self):
FILE: claf/tokens/embedding/frequent_word_embedding.py
class FrequentTuningWordEmbedding (line 12) | class FrequentTuningWordEmbedding(TokenEmbedding):
method __init__ (line 36) | def __init__(
method forward (line 82) | def forward(self, words, frequent_tuning=False):
method get_output_dim (line 109) | def get_output_dim(self):
FILE: claf/tokens/embedding/sparse_feature.py
class SparseFeature (line 13) | class SparseFeature(TokenEmbedding):
method __init__ (line 29) | def __init__(self, vocab, embed_type, feature_count, params={}):
method forward (line 48) | def forward(self, inputs):
method get_output_dim (line 59) | def get_output_dim(self):
class SparseToEmbedding (line 63) | class SparseToEmbedding(nn.Module):
method __init__ (line 86) | def __init__(
method forward (line 124) | def forward(self, inputs):
method get_output_dim (line 127) | def get_output_dim(self):
class OneHotEncoding (line 131) | class OneHotEncoding(nn.Module):
method __init__ (line 140) | def __init__(self, index, token_name, classes):
method forward (line 155) | def forward(self, inputs):
method get_output_dim (line 162) | def get_output_dim(self):
FILE: claf/tokens/embedding/word_embedding.py
class WordEmbedding (line 16) | class WordEmbedding(TokenEmbedding):
method __init__ (line 40) | def __init__(
method _init_weight (line 78) | def _init_weight(self, trainable=True):
method forward (line 85) | def forward(self, words):
method _read_pretrained_file (line 105) | def _read_pretrained_file(self, file_path):
method get_output_dim (line 156) | def get_output_dim(self):
FILE: claf/tokens/hangul.py
function is_hangul (line 119) | def is_hangul(phrase): # pragma: no cover
function is_all_hangul (line 136) | def is_all_hangul(phrase): # pragma: no cover
function has_jongsung (line 149) | def has_jongsung(letter): # pragma: no cover
function has_batchim (line 160) | def has_batchim(letter): # pragma: no cover
function has_approximant (line 165) | def has_approximant(letter): # pragma: no cover
function compose (line 186) | def compose(chosung, joongsung, jongsung=""): # pragma: no cover
function decompose (line 212) | def decompose(hangul_letter): # pragma: no cover
function josa_en (line 235) | def josa_en(word): # pragma: no cover
function josa_eg (line 246) | def josa_eg(word): # pragma: no cover
function josa_el (line 257) | def josa_el(word): # pragma: no cover
function josa_ro (line 268) | def josa_ro(word): # pragma: no cover
function josa_gwa (line 285) | def josa_gwa(word): # pragma: no cover
function josa_ida (line 296) | def josa_ida(word): # pragma: no cover
function add_ryul (line 313) | def add_ryul(word): # pragma: no cover
function ili (line 336) | def ili(word): # pragma: no cover
class NotHangulException (line 355) | class NotHangulException(Exception): # pragma: no cover
class NotLetterException (line 359) | class NotLetterException(Exception): # pragma: no cover
class NotWordException (line 363) | class NotWordException(Exception): # pragma: no cover
FILE: claf/tokens/indexer/base.py
class TokenIndexer (line 1) | class TokenIndexer:
method __init__ (line 8) | def __init__(self, tokenizer):
method index (line 12) | def index(self, token):
method set_vocab (line 16) | def set_vocab(self, vocab):
FILE: claf/tokens/indexer/bert_indexer.py
class BertIndexer (line 7) | class BertIndexer(TokenIndexer):
method __init__ (line 23) | def __init__(self, tokenizer, do_tokenize=True):
method index (line 28) | def index(self, text):
method _index_text (line 38) | def _index_text(self, text):
FILE: claf/tokens/indexer/char_indexer.py
class CharIndexer (line 7) | class CharIndexer(TokenIndexer):
method __init__ (line 24) | def __init__(self, tokenizer, insert_char_start=None, insert_char_end=...
method index (line 31) | def index(self, text):
method index_token (line 35) | def index_token(self, chars):
FILE: claf/tokens/indexer/elmo_indexer.py
function _make_bos_eos (line 11) | def _make_bos_eos(
class ELMoIndexer (line 25) | class ELMoIndexer(TokenIndexer):
method __init__ (line 61) | def __init__(self, tokenizer):
method index (line 65) | def index(self, text):
method index_token (line 69) | def index_token(self, word):
FILE: claf/tokens/indexer/exact_match_indexer.py
class ExactMatchIndexer (line 9) | class ExactMatchIndexer(TokenIndexer):
method __init__ (line 24) | def __init__(self, tokenizer, lower=True, lemma=True):
method index (line 34) | def index(self, text, query_text):
method index_token (line 49) | def index_token(self, token, query_tokens):
FILE: claf/tokens/indexer/linguistic_indexer.py
class LinguisticIndexer (line 10) | class LinguisticIndexer(TokenIndexer):
method __init__ (line 26) | def __init__(self, tokenizer, pos_tag=None, ner=None, dep=None):
method index (line 43) | def index(self, text):
method _mecab_ko (line 49) | def _mecab_ko(self, text):
method _nltk_en (line 52) | def _nltk_en(self, text):
method _spacy_en (line 55) | def _spacy_en(self, text):
FILE: claf/tokens/indexer/word_indexer.py
class WordIndexer (line 7) | class WordIndexer(TokenIndexer):
method __init__ (line 23) | def __init__(
method index (line 35) | def index(self, text):
method _index_text (line 52) | def _index_text(self, text):
method _index_list_of_text (line 58) | def _index_list_of_text(self, list_of_text):
method _index_token (line 68) | def _index_token(self, token):
FILE: claf/tokens/linguistic.py
class POSTag (line 1) | class POSTag:
class NER (line 30) | class NER:
FILE: claf/tokens/text_handler.py
class TextHandler (line 17) | class TextHandler:
method __init__ (line 34) | def __init__(self, token_makers, lazy_indexing=True):
method build_vocabs (line 40) | def build_vocabs(self, token_counters):
method _build_vocab_with_config (line 67) | def _build_vocab_with_config(self, token_name, token_maker, token_coun...
method is_all_vocab_use_pretrained (line 77) | def is_all_vocab_use_pretrained(self):
method make_token_counters (line 85) | def make_token_counters(self, texts, config=None):
method _make_token_counter (line 103) | def _make_token_counter(self, texts, tokenizer, config=None, desc=None):
method index (line 128) | def index(self, datas, text_columns):
method _index_features (line 147) | def _index_features(self, features, text_columns, desc=None, suppress_...
method _index_token (line 166) | def _index_token(self, token_maker, text, data):
method raw_to_tensor_fn (line 182) | def raw_to_tensor_fn(self, data_reader, cuda_device=None, helper={}):
FILE: claf/tokens/token_embedder/base.py
class TokenEmbedder (line 6) | class TokenEmbedder(torch.nn.Module):
method __init__ (line 16) | def __init__(self, token_makers):
method add_embedding_modules (line 26) | def add_embedding_modules(self, token_makers):
method get_embed_dim (line 38) | def get_embed_dim(self):
method forward (line 41) | def forward(self, inputs, params={}):
FILE: claf/tokens/token_embedder/basic_embedder.py
class BasicTokenEmbedder (line 9) | class BasicTokenEmbedder(TokenEmbedder):
method __init__ (line 20) | def __init__(self, token_makers):
method get_embed_dim (line 24) | def get_embed_dim(self, except_keys=[]):
method forward (line 28) | def forward(self, inputs, except_keys=[], params={}):
FILE: claf/tokens/token_embedder/reading_comprehension_embedder.py
class RCTokenEmbedder (line 11) | class RCTokenEmbedder(TokenEmbedder):
method __init__ (line 26) | def __init__(self, token_makers):
method get_embed_dim (line 35) | def get_embed_dim(self):
method forward (line 39) | def forward(self, context, query, context_params={}, query_params={}, ...
method _filter (line 101) | def _filter(self, token_data, exclusive=False):
FILE: claf/tokens/token_maker.py
class TokenMaker (line 1) | class TokenMaker:
method __init__ (line 26) | def __init__(
method tokenizer (line 36) | def tokenizer(self):
method tokenizer (line 40) | def tokenizer(self, tokenizer):
method indexer (line 44) | def indexer(self):
method indexer (line 48) | def indexer(self, indexer):
method embedding_fn (line 52) | def embedding_fn(self):
method embedding_fn (line 56) | def embedding_fn(self, embedding_fn):
method vocab_config (line 60) | def vocab_config(self):
method vocab_config (line 64) | def vocab_config(self, vocab_config):
method vocab (line 68) | def vocab(self):
method vocab (line 72) | def vocab(self, vocab):
method set_vocab (line 75) | def set_vocab(self, vocab):
FILE: claf/tokens/tokenizer/base.py
class Tokenizer (line 1) | class Tokenizer:
method __init__ (line 8) | def __init__(self, name, cache_name):
method tokenize (line 13) | def tokenize(self, text, unit="text"):
method _tokenize (line 28) | def _tokenize(self, text, unit="text"):
FILE: claf/tokens/tokenizer/bpe.py
class BPETokenizer (line 9) | class BPETokenizer(Tokenizer):
method __init__ (line 17) | def __init__(self, name, config={}):
method _roberta (line 26) | def _roberta(self, text, unit="text"):
FILE: claf/tokens/tokenizer/char.py
class CharTokenizer (line 7) | class CharTokenizer(Tokenizer):
method __init__ (line 18) | def __init__(self, name, word_tokenizer, config={}):
method _character (line 25) | def _character(self, text, unit="text"):
method _jamo_ko (line 34) | def _jamo_ko(self, text, unit="text"):
FILE: claf/tokens/tokenizer/pass_text.py
class PassText (line 1) | class PassText:
method __init__ (line 6) | def __init__(self):
method tokenize (line 10) | def tokenize(self, text):
FILE: claf/tokens/tokenizer/sent.py
class SentTokenizer (line 7) | class SentTokenizer(Tokenizer):
method __init__ (line 17) | def __init__(self, name, config={}):
method _punkt (line 23) | def _punkt(self, text, unit="text"):
FILE: claf/tokens/tokenizer/subword.py
class SubwordTokenizer (line 11) | class SubwordTokenizer(Tokenizer):
method __init__ (line 21) | def __init__(self, name, word_tokenizer, config={}):
method _wordpiece (line 30) | def _wordpiece(self, text, unit="text"):
FILE: claf/tokens/tokenizer/utils.py
function create_tokenizer_with_regex (line 5) | def create_tokenizer_with_regex(nlp, split_regex):
function load_spacy_model_for_tokenizer (line 20) | def load_spacy_model_for_tokenizer(split_regex):
FILE: claf/tokens/tokenizer/word.py
class WordTokenizer (line 12) | class WordTokenizer(Tokenizer):
method __init__ (line 24) | def __init__(self, name, sent_tokenizer, config={}, split_with_regex=T...
method make_split_regex_expression (line 34) | def make_split_regex_expression(self):
method _tokenize (line 72) | def _tokenize(self, text, unit="text"):
method _split_with_regex (line 88) | def _split_with_regex(self, sentences):
method _post_split_tokens (line 93) | def _post_split_tokens(self, tokens):
method _space_all (line 98) | def _space_all(self, text):
method _treebank_en (line 117) | def _treebank_en(self, text):
method _spacy_en (line 128) | def _spacy_en(self, text):
method _bert_basic (line 139) | def _bert_basic(self, text):
method _mecab_ko (line 147) | def _mecab_ko(self, text):
FILE: claf/tokens/vocabulary.py
class VocabDict (line 8) | class VocabDict(defaultdict):
method __init__ (line 16) | def __init__(self, oov_value):
method __missing__ (line 19) | def __missing__(self, key):
class Vocab (line 23) | class Vocab:
method __init__ (line 56) | def __init__(
method init (line 101) | def init(self):
method build (line 114) | def build(self, token_counter, predefine_vocab=None):
method build_with_pretrained_file (line 158) | def build_with_pretrained_file(self, token_counter):
method __len__ (line 173) | def __len__(self):
method add (line 176) | def add(self, token, predefine_vocab=None):
method get_index (line 188) | def get_index(self, token):
method get_token (line 191) | def get_token(self, index):
method get_all_tokens (line 194) | def get_all_tokens(self):
method dump (line 197) | def dump(self, path):
method load (line 201) | def load(self, path):
method to_text (line 207) | def to_text(self):
method from_texts (line 210) | def from_texts(self, texts):
FILE: claf/utils.py
function get_user_input (line 12) | def get_user_input(category):
function flatten (line 23) | def flatten(l):
function set_logging_config (line 35) | def set_logging_config(mode, config):
FILE: docs/_build/html/_static/doctools.js
function highlight (line 69) | function highlight(node, addItems) {
FILE: docs/_build/html/_static/jquery-3.2.1.js
function DOMEval (line 76) | function DOMEval( code, doc ) {
function isArrayLike (line 522) | function isArrayLike( obj ) {
function Sizzle (line 754) | function Sizzle( selector, context, results, seed ) {
function createCache (line 893) | function createCache() {
function markFunction (line 911) | function markFunction( fn ) {
function assert (line 920) | function assert( fn ) {
function addHandle (line 942) | function addHandle( attrs, handler ) {
function siblingCheck (line 957) | function siblingCheck( a, b ) {
function createInputPseudo (line 983) | function createInputPseudo( type ) {
function createButtonPseudo (line 994) | function createButtonPseudo( type ) {
function createDisabledPseudo (line 1005) | function createDisabledPseudo( disabled ) {
function createPositionalPseudo (line 1061) | function createPositionalPseudo( fn ) {
function testContext (line 1084) | function testContext( context ) {
function setFilters (line 2166) | function setFilters() {}
function toSelector (line 2237) | function toSelector( tokens ) {
function addCombinator (line 2247) | function addCombinator( matcher, combinator, base ) {
function elementMatcher (line 2311) | function elementMatcher( matchers ) {
function multipleContexts (line 2325) | function multipleContexts( selector, contexts, results ) {
function condense (line 2334) | function condense( unmatched, map, filter, context, xml ) {
function setMatcher (line 2355) | function setMatcher( preFilter, selector, matcher, postFilter, postFinde...
function matcherFromTokens (line 2448) | function matcherFromTokens( tokens ) {
function matcherFromGroupMatchers (line 2506) | function matcherFromGroupMatchers( elementMatchers, setMatchers ) {
function nodeName (line 2842) | function nodeName( elem, name ) {
function winnow (line 2854) | function winnow( elements, qualifier, not ) {
function sibling (line 3157) | function sibling( cur, dir ) {
function createOptions (line 3244) | function createOptions( options ) {
function Identity (line 3469) | function Identity( v ) {
function Thrower (line 3472) | function Thrower( ex ) {
function adoptValue (line 3476) | function adoptValue( value, resolve, reject, noValue ) {
function resolve (line 3569) | function resolve( depth, deferred, handler, special ) {
function completed (line 3927) | function completed() {
function Data (line 4029) | function Data() {
function getData (line 4198) | function getData( data ) {
function dataAttr (line 4223) | function dataAttr( elem, key, data ) {
function adjustCSS (line 4536) | function adjustCSS( elem, prop, valueParts, tween ) {
function getDefaultDisplay (line 4601) | function getDefaultDisplay( elem ) {
function showHide (line 4624) | function showHide( elements, show ) {
function getAll (line 4725) | function getAll( context, tag ) {
function setGlobalEval (line 4750) | function setGlobalEval( elems, refElements ) {
function buildFragment (line 4766) | function buildFragment( elems, context, scripts, selection, ignored ) {
function returnTrue (line 4889) | function returnTrue() {
function returnFalse (line 4893) | function returnFalse() {
function safeActiveElement (line 4899) | function safeActiveElement() {
function on (line 4905) | function on( elem, types, selector, data, fn, one ) {
function manipulationTarget (line 5634) | function manipulationTarget( elem, content ) {
function disableScript (line 5645) | function disableScript( elem ) {
function restoreScript (line 5649) | function restoreScript( elem ) {
function cloneCopyEvent (line 5661) | function cloneCopyEvent( src, dest ) {
function fixInput (line 5696) | function fixInput( src, dest ) {
function domManip (line 5709) | function domManip( collection, args, callback, ignored ) {
function remove (line 5799) | function remove( elem, selector, keepData ) {
function computeStyleTests (line 6092) | function computeStyleTests() {
function curCSS (line 6166) | function curCSS( elem, name, computed ) {
function addGetHookIf (line 6219) | function addGetHookIf( conditionFn, hookFn ) {
function vendorPropName (line 6256) | function vendorPropName( name ) {
function finalPropName (line 6277) | function finalPropName( name ) {
function setPositiveNumber (line 6285) | function setPositiveNumber( elem, value, subtract ) {
function augmentWidthOrHeight (line 6297) | function augmentWidthOrHeight( elem, name, extra, isBorderBox, styles ) {
function getWidthOrHeight (line 6343) | function getWidthOrHeight( elem, name, extra ) {
function Tween (line 6652) | function Tween( elem, options, prop, end, easing ) {
function schedule (line 6775) | function schedule() {
function createFxNow (line 6788) | function createFxNow() {
function genFx (line 6796) | function genFx( type, includeWidth ) {
function createTween (line 6816) | function createTween( value, prop, animation ) {
function defaultPrefilter (line 6830) | function defaultPrefilter( elem, props, opts ) {
function propFilter (line 7001) | function propFilter( props, specialEasing ) {
function Animation (line 7038) | function Animation( elem, properties, options ) {
function stripAndCollapse (line 7753) | function stripAndCollapse( value ) {
function getClass (line 7759) | function getClass( elem ) {
function buildParams (line 8383) | function buildParams( prefix, obj, traditional, add ) {
function addToPrefiltersOrTransports (line 8533) | function addToPrefiltersOrTransports( structure ) {
function inspectPrefiltersOrTransports (line 8567) | function inspectPrefiltersOrTransports( structure, options, originalOpti...
function ajaxExtend (line 8596) | function ajaxExtend( target, src ) {
function ajaxHandleResponses (line 8616) | function ajaxHandleResponses( s, jqXHR, responses ) {
function ajaxConvert (line 8674) | function ajaxConvert( s, response, jqXHR, isSuccess ) {
function done (line 9187) | function done( status, nativeStatusText, responses, headers ) {
FILE: docs/_build/html/_static/jquery-3.4.1.js
function DOMEval (line 98) | function DOMEval( code, node, doc ) {
function toType (line 128) | function toType( obj ) {
function isArrayLike (line 496) | function isArrayLike( obj ) {
function Sizzle (line 729) | function Sizzle( selector, context, results, seed ) {
function createCache (line 871) | function createCache() {
function markFunction (line 889) | function markFunction( fn ) {
function assert (line 898) | function assert( fn ) {
function addHandle (line 920) | function addHandle( attrs, handler ) {
function siblingCheck (line 935) | function siblingCheck( a, b ) {
function createInputPseudo (line 961) | function createInputPseudo( type ) {
function createButtonPseudo (line 972) | function createButtonPseudo( type ) {
function createDisabledPseudo (line 983) | function createDisabledPseudo( disabled ) {
function createPositionalPseudo (line 1039) | function createPositionalPseudo( fn ) {
function testContext (line 1062) | function testContext( context ) {
function setFilters (line 2150) | function setFilters() {}
function toSelector (line 2221) | function toSelector( tokens ) {
function addCombinator (line 2231) | function addCombinator( matcher, combinator, base ) {
function elementMatcher (line 2295) | function elementMatcher( matchers ) {
function multipleContexts (line 2309) | function multipleContexts( selector, contexts, results ) {
function condense (line 2318) | function condense( unmatched, map, filter, context, xml ) {
function setMatcher (line 2339) | function setMatcher( preFilter, selector, matcher, postFilter, postFinde...
function matcherFromTokens (line 2432) | function matcherFromTokens( tokens ) {
function matcherFromGroupMatchers (line 2490) | function matcherFromGroupMatchers( elementMatchers, setMatchers ) {
function nodeName (line 2826) | function nodeName( elem, name ) {
function winnow (line 2836) | function winnow( elements, qualifier, not ) {
function sibling (line 3131) | function sibling( cur, dir ) {
function createOptions (line 3218) | function createOptions( options ) {
function Identity (line 3443) | function Identity( v ) {
function Thrower (line 3446) | function Thrower( ex ) {
function adoptValue (line 3450) | function adoptValue( value, resolve, reject, noValue ) {
function resolve (line 3543) | function resolve( depth, deferred, handler, special ) {
function completed (line 3908) | function completed() {
function fcamelCase (line 4003) | function fcamelCase( all, letter ) {
function camelCase (line 4010) | function camelCase( string ) {
function Data (line 4027) | function Data() {
function getData (line 4196) | function getData( data ) {
function dataAttr (line 4221) | function dataAttr( elem, key, data ) {
function adjustCSS (line 4554) | function adjustCSS( elem, prop, valueParts, tween ) {
function getDefaultDisplay (line 4622) | function getDefaultDisplay( elem ) {
function showHide (line 4645) | function showHide( elements, show ) {
function getAll (line 4746) | function getAll( context, tag ) {
function setGlobalEval (line 4771) | function setGlobalEval( elems, refElements ) {
function buildFragment (line 4787) | function buildFragment( elems, context, scripts, selection, ignored ) {
function returnTrue (line 4908) | function returnTrue() {
function returnFalse (line 4912) | function returnFalse() {
function expectSync (line 4922) | function expectSync( elem, type ) {
function safeActiveElement (line 4929) | function safeActiveElement() {
function on (line 4935) | function on( elem, types, selector, data, fn, one ) {
function leverageNative (line 5420) | function leverageNative( el, type, expectSync ) {
function manipulationTarget (line 5791) | function manipulationTarget( elem, content ) {
function disableScript (line 5802) | function disableScript( elem ) {
function restoreScript (line 5806) | function restoreScript( elem ) {
function cloneCopyEvent (line 5816) | function cloneCopyEvent( src, dest ) {
function fixInput (line 5851) | function fixInput( src, dest ) {
function domManip (line 5864) | function domManip( collection, args, callback, ignored ) {
function remove (line 5956) | function remove( elem, selector, keepData ) {
function computeStyleTests (line 6249) | function computeStyleTests() {
function roundPixelMeasures (line 6293) | function roundPixelMeasures( measure ) {
function curCSS (line 6338) | function curCSS( elem, name, computed ) {
function addGetHookIf (line 6391) | function addGetHookIf( conditionFn, hookFn ) {
function vendorPropName (line 6416) | function vendorPropName( name ) {
function finalPropName (line 6431) | function finalPropName( name ) {
function setPositiveNumber (line 6457) | function setPositiveNumber( elem, value, subtract ) {
function boxModelAdjustment (line 6469) | function boxModelAdjustment( elem, dimension, box, isBorderBox, styles, ...
function getWidthOrHeight (line 6537) | function getWidthOrHeight( elem, dimension, extra ) {
function Tween (line 6904) | function Tween( elem, options, prop, end, easing ) {
function schedule (line 7027) | function schedule() {
function createFxNow (line 7040) | function createFxNow() {
function genFx (line 7048) | function genFx( type, includeWidth ) {
function createTween (line 7068) | function createTween( value, prop, animation ) {
function defaultPrefilter (line 7082) | function defaultPrefilter( elem, props, opts ) {
function propFilter (line 7254) | function propFilter( props, specialEasing ) {
function Animation (line 7291) | function Animation( elem, properties, options ) {
function stripAndCollapse (line 8006) | function stripAndCollapse( value ) {
function getClass (line 8012) | function getClass( elem ) {
function classesToArray (line 8016) | function classesToArray( value ) {
function buildParams (line 8638) | function buildParams( prefix, obj, traditional, add ) {
function addToPrefiltersOrTransports (line 8792) | function addToPrefiltersOrTransports( structure ) {
function inspectPrefiltersOrTransports (line 8826) | function inspectPrefiltersOrTransports( structure, options, originalOpti...
function ajaxExtend (line 8855) | function ajaxExtend( target, src ) {
function ajaxHandleResponses (line 8875) | function ajaxHandleResponses( s, jqXHR, responses ) {
function ajaxConvert (line 8933) | function ajaxConvert( s, response, jqXHR, isSuccess ) {
function done (line 9448) | function done( status, nativeStatusText, responses, headers ) {
FILE: docs/_build/html/_static/jquery.js
function b (line 2) | function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e...
function w (line 2) | function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof...
function d (line 2) | function d(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e...
function se (line 2) | function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeTy...
function ue (line 2) | function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cac...
function le (line 2) | function le(e){return e[k]=!0,e}
function ce (line 2) | function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(...
function fe (line 2) | function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[...
function pe (line 2) | function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourc...
function de (line 2) | function de(t){return function(e){return"input"===e.nodeName.toLowerCase...
function he (line 2) | function he(n){return function(e){var t=e.nodeName.toLowerCase();return(...
function ge (line 2) | function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e....
function ve (line 2) | function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,...
function ye (line 2) | function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}
function me (line 2) | function me(){}
function xe (line 2) | function xe(e){for(var t=0,n=e.length,r="";t<n;t++)r+=e[t].value;return r}
function be (line 2) | function be(s,e,t){var u=e.dir,l=e.next,c=l||u,f=t&&"parentNode"===c,p=r...
function we (line 2) | function we(i){return 1<i.length?function(e,t,n){var r=i.length;while(r-...
function Te (line 2) | function Te(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s<u;s++)(...
function Ce (line 2) | function Ce(d,h,g,v,y,e){return v&&!v[k]&&(v=Ce(v)),y&&!y[k]&&(y=Ce(y,e)...
function Ee (line 2) | function Ee(e){for(var i,t,n,r=e.length,o=b.relative[e[0].type],a=o||b.r...
function A (line 2) | function A(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerC...
function j (line 2) | function j(e,n,r){return m(n)?k.grep(e,function(e,t){return!!n.call(e,t,...
function P (line 2) | function P(e,t){while((e=e[t])&&1!==e.nodeType);return e}
function M (line 2) | function M(e){return e}
function I (line 2) | function I(e){throw e}
function W (line 2) | function W(e,t,n,r){var i;try{e&&m(i=e.promise)?i.call(e).done(t).fail(n...
function l (line 2) | function l(i,o,a,s){return function(){var n=this,r=arguments,e=function(...
function B (line 2) | function B(){E.removeEventListener("DOMContentLoaded",B),C.removeEventLi...
function X (line 2) | function X(e,t){return t.toUpperCase()}
function V (line 2) | function V(e){return e.replace(z,"ms-").replace(U,X)}
function Y (line 2) | function Y(){this.expando=k.expando+Y.uid++}
function ee (line 2) | function ee(e,t,n){var r,i;if(void 0===n&&1===e.nodeType)if(r="data-"+t....
function le (line 2) | function le(e,t,n,r){var i,o,a=20,s=r?function(){return r.cur()}:functio...
function fe (line 2) | function fe(e,t){for(var n,r,i,o,a,s,u,l=[],c=0,f=e.length;c<f;c++)(r=e[...
function ve (line 2) | function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagNa...
function ye (line 2) | function ye(e,t){for(var n=0,r=e.length;n<r;n++)Q.set(e[n],"globalEval",...
function we (line 2) | function we(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),...
function ke (line 2) | function ke(){return!0}
function Se (line 2) | function Se(){return!1}
function Ne (line 2) | function Ne(e,t){return e===function(){try{return E.activeElement}catch(...
function Ae (line 2) | function Ae(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"...
function De (line 2) | function De(e,i,o){o?(Q.set(e,i,!1),k.event.add(e,i,{namespace:!1,handle...
function Oe (line 2) | function Oe(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"...
function Pe (line 2) | function Pe(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}
function Re (line 2) | function Re(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.sli...
function Me (line 2) | function Me(e,t){var n,r,i,o,a,s,u,l;if(1===t.nodeType){if(Q.hasData(e)&...
function Ie (line 2) | function Ie(n,r,i,o){r=g.apply([],r);var e,t,a,s,u,l,c=0,f=n.length,p=f-...
function We (line 2) | function We(e,t,n){for(var r,i=t?k.filter(t,e):e,o=0;null!=(r=i[o]);o++)...
function _e (line 2) | function _e(e,t,n){var r,i,o,a,s=e.style;return(n=n||Fe(e))&&(""!==(a=n....
function ze (line 2) | function ze(e,t){return{get:function(){if(!e())return(this.get=t).apply(...
function e (line 2) | function e(){if(u){s.style.cssText="position:absolute;left:-11111px;widt...
function t (line 2) | function t(e){return Math.round(parseFloat(e))}
function Ge (line 2) | function Ge(e){var t=k.cssProps[e]||Ve[e];return t||(e in Xe?e:Ve[e]=fun...
function Ze (line 2) | function Ze(e,t,n){var r=ne.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[...
function et (line 2) | function et(e,t,n,r,i,o){var a="width"===t?1:0,s=0,u=0;if(n===(r?"border...
function tt (line 2) | function tt(e,t,n){var r=Fe(e),i=(!y.boxSizingReliable()||n)&&"border-bo...
function nt (line 2) | function nt(e,t,n,r,i){return new nt.prototype.init(e,t,n,r,i)}
function lt (line 2) | function lt(){it&&(!1===E.hidden&&C.requestAnimationFrame?C.requestAnima...
function ct (line 2) | function ct(){return C.setTimeout(function(){rt=void 0}),rt=Date.now()}
function ft (line 2) | function ft(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin...
function pt (line 2) | function pt(e,t,n){for(var r,i=(dt.tweeners[t]||[]).concat(dt.tweeners["...
function dt (line 2) | function dt(o,e,t){var n,a,r=0,i=dt.prefilters.length,s=k.Deferred().alw...
function mt (line 2) | function mt(e){return(e.match(R)||[]).join(" ")}
function xt (line 2) | function xt(e){return e.getAttribute&&e.getAttribute("class")||""}
function bt (line 2) | function bt(e){return Array.isArray(e)?e:"string"==typeof e&&e.match(R)|...
function qt (line 2) | function qt(n,e,r,i){var t;if(Array.isArray(e))k.each(e,function(e,t){r|...
function Bt (line 2) | function Bt(o){return function(e,t){"string"!=typeof e&&(t=e,e="*");var ...
function _t (line 2) | function _t(t,i,o,a){var s={},u=t===Wt;function l(e){var r;return s[e]=!...
function zt (line 2) | function zt(e,t){var n,r,i=k.ajaxSettings.flatOptions||{};for(n in t)voi...
function l (line 2) | function l(e,t,n,r){var i,o,a,s,u,l=t;h||(h=!0,d&&C.clearTimeout(d),c=vo...
FILE: docs/_build/html/_static/js/theme.js
function c (line 3) | function c(e,n){if(!a[e]){if(!s[e]){var i="function"==typeof require&&re...
FILE: docs/_build/html/_static/language_data.js
function splitQuery (line 278) | function splitQuery(query) {
FILE: docs/_build/html/_static/searchtools.js
function splitQuery (line 47) | function splitQuery(query) {
function pulse (line 112) | function pulse() {
function displayNextItem (line 243) | function displayNextItem() {
FILE: docs/_build/html/_static/underscore-1.3.1.js
function eq (line 669) | function eq(a, b, stack) {
FILE: docs/_build/html/_static/underscore.js
function q (line 8) | function q(a,c,d){if(a===c)return a!==0||1/a==1/c;if(a==null||c==null)re...
FILE: docs/_build/html/_static/websupport.js
function init (line 47) | function init() {
function initEvents (line 52) | function initEvents() {
function setComparator (line 107) | function setComparator() {
function initComparator (line 127) | function initComparator() {
function show (line 147) | function show(id) {
function hide (line 168) | function hide(id) {
function getComments (line 181) | function getComments(id) {
function addComment (line 216) | function addComment(form) {
function appendComments (line 275) | function appendComments(comments, ul) {
function insertComment (line 290) | function insertComment(comment) {
function acceptComment (line 320) | function acceptComment(id) {
function deleteComment (line 335) | function deleteComment(id) {
function showProposal (line 369) | function showProposal(id) {
function hideProposal (line 375) | function hideProposal(id) {
function showProposeChange (line 381) | function showProposeChange(id) {
function hideProposeChange (line 390) | function hideProposeChange(id) {
function toggleCommentMarkupBox (line 398) | function toggleCommentMarkupBox(id) {
function handleReSort (line 403) | function handleReSort(link) {
function handleVote (line 426) | function handleVote(link) {
function openReply (line 488) | function openReply(id) {
function closeReply (line 516) | function closeReply(id) {
function sortComments (line 530) | function sortComments(comments) {
function getChildren (line 542) | function getChildren(ul, recursive) {
function createCommentDiv (line 555) | function createCommentDiv(comment) {
function renderTemplate (line 593) | function renderTemplate(template, context) {
function showError (line 610) | function showError(message) {
FILE: script/convert_checkpoint_to_bert_model.py
function convert_checkpoint_to_bert_model (line 12) | def convert_checkpoint_to_bert_model(checkpoint_path, output_path):
FILE: script/convert_embedding_to_vocab_txt.py
function read_embedding_vocabs (line 5) | def read_embedding_vocabs(file_path):
function write_vocab (line 16) | def write_vocab(embedding_vocabs, output_path):
FILE: script/make_squad_synthetic_data.py
function make_squad_synthetic_data (line 9) | def make_squad_synthetic_data(output_path, max_context_length, question_...
function make_random_tokens (line 40) | def make_random_tokens(length, answer_token=""):
FILE: script/plot.py
function make_inference_latency_plot (line 14) | def make_inference_latency_plot(result_dir, max_elapsed_time=2000):
function make_summary_plot (line 88) | def make_summary_plot(result_dir, max_elapsed_time=100):
function make_scatter (line 149) | def make_scatter(
FILE: setup.py
class UploadCommand (line 49) | class UploadCommand(Command):
method status (line 56) | def status(s):
method initialize_options (line 60) | def initialize_options(self):
method finalize_options (line 63) | def finalize_options(self):
method run (line 66) | def run(self):
FILE: tests/claf/data/test_batch.py
function test_make_batch (line 5) | def test_make_batch():
function test_batch_sort_by_key (line 24) | def test_batch_sort_by_key():
FILE: tests/claf/machine/knowlege_base/test_docs.py
function test_read_wiki_articles (line 8) | def test_read_wiki_articles():
FILE: tests/claf/modules/test_functional.py
function test_add_masked_value (line 7) | def test_add_masked_value():
function test_add_masked_value_with_byte_tensor (line 24) | def test_add_masked_value_with_byte_tensor():
function test_get_mask_from_tokens_with_2_dim (line 41) | def test_get_mask_from_tokens_with_2_dim():
function test_get_mask_from_tokens_with_3_dim (line 55) | def test_get_mask_from_tokens_with_3_dim():
function test_last_dim_masked_softmax_with_2_dim (line 73) | def test_last_dim_masked_softmax_with_2_dim():
function test_masked_softmax (line 85) | def test_masked_softmax():
function test_masked_zero (line 101) | def test_masked_zero():
function test_get_sorted_seq_config (line 135) | def test_get_sorted_seq_config():
function test_forward_rnn_with_pack (line 148) | def test_forward_rnn_with_pack():
FILE: tests/claf/tokens/test_vocabulary.py
function test_init_vocab (line 8) | def test_init_vocab():
function test_init_vocab_with_special_token (line 15) | def test_init_vocab_with_special_token():
function test_from_texts (line 22) | def test_from_texts():
function test_from_texts_with_pad (line 31) | def test_from_texts_with_pad():
function test_from_texts_with_pad_but_not_define (line 40) | def test_from_texts_with_pad_but_not_define():
function test_build (line 49) | def test_build():
function test_build_with_max_vocab_size (line 59) | def test_build_with_max_vocab_size():
function test_build_with_min_count (line 69) | def test_build_with_min_count():
function test_get_token (line 79) | def test_get_token():
function test_save_and_load (line 89) | def test_save_and_load():
function test_build_with_pretrained_file_all (line 105) | def test_build_with_pretrained_file_all():
function test_build_with_pretrained_file_intersect (line 121) | def test_build_with_pretrained_file_intersect():
FILE: tests/integration/test_config.py
function test_train_argparse (line 9) | def test_train_argparse():
function test_train_base_config_argparse (line 15) | def test_train_base_config_argparse():
function test_eval_argparse (line 27) | def test_eval_argparse():
function test_predict_argparse (line 32) | def test_predict_argparse():
function test_machine_argparse (line 37) | def test_machine_argparse():
FILE: tests/integration/test_machine.py
function test_make_synthetic_data (line 22) | def test_make_synthetic_data():
function train_config (line 32) | def train_config(request):
function test_train_squad_bidaf_model (line 51) | def test_train_squad_bidaf_model(train_config):
function open_qa_config (line 57) | def open_qa_config(request):
function test_open_qa_with_bidaf_model (line 75) | def test_open_qa_with_bidaf_model(open_qa_config):
function test_remove_tested_directory (line 88) | def test_remove_tested_directory():
FILE: tests/integration/test_multi_task.py
function test_config (line 20) | def test_config(request):
function load_and_setting (line 24) | def load_and_setting(config_path):
function test_make_multi_task_synthetic_data (line 45) | def test_make_multi_task_synthetic_data():
function test_train_multi_task_bert_model (line 53) | def test_train_multi_task_bert_model(test_config):
FILE: tests/integration/test_reading_comprehension.py
function test_config (line 20) | def test_config(request):
function load_and_setting (line 24) | def load_and_setting(config_path):
function test_make_squad_synthetic_data (line 39) | def test_make_squad_synthetic_data():
function test_train_squad_bidaf_model (line 46) | def test_train_squad_bidaf_model(test_config):
function test_train_squad_bidaf_no_answer_model (line 53) | def test_train_squad_bidaf_no_answer_model(test_config):
function test_train_squad_bidaf_elmo_model (line 68) | def test_train_squad_bidaf_elmo_model(test_config):
function test_train_squad_drqa_model (line 75) | def test_train_squad_drqa_model(test_config):
function test_train_squad_drqa_model_with_sparse_to_embedding (line 82) | def test_train_squad_drqa_model_with_sparse_to_embedding(test_config):
function test_train_squad_docqa_model (line 89) | def test_train_squad_docqa_model(test_config):
function test_train_squad_docqa_no_answer_model (line 96) | def test_train_squad_docqa_no_answer_model(test_config):
function test_train_squad_qanet_model (line 103) | def test_train_squad_qanet_model(test_config):
function test_train_squad_bert_model (line 110) | def test_train_squad_bert_model(test_config):
function test_eval_squad_bidaf (line 124) | def test_eval_squad_bidaf():
function test_eval_infer_squad_bidaf (line 136) | def test_eval_infer_squad_bidaf():
function test_qa_predict_squad_bidaf_1_example (line 149) | def test_qa_predict_squad_bidaf_1_example():
function test_qa_predict_squad_bert_short_1_example (line 164) | def test_qa_predict_squad_bert_short_1_example():
function test_qa_predict_squad_bert_long_1_example (line 179) | def test_qa_predict_squad_bert_long_1_example():
function test_remove_tested_directory (line 194) | def test_remove_tested_directory():
FILE: tests/integration/test_semantic_parsing.py
function test_config (line 15) | def test_config(request):
function load_and_setting (line 19) | def load_and_setting(config_path):
function test_train_wikisql_sqlnet_model (line 33) | def test_train_wikisql_sqlnet_model(test_config):
function test_qa_predict_wikisql_sqlnet_1_example (line 41) | def test_qa_predict_wikisql_sqlnet_1_example():
function test_remove_tested_directory (line 58) | def test_remove_tested_directory():
FILE: tests/integration/test_sequence_classification.py
function test_config (line 19) | def test_config(request):
function load_and_setting (line 23) | def load_and_setting(config_path):
function test_make_synthetic_data (line 37) | def test_make_synthetic_data():
function test_train_nlu_ssa_model (line 43) | def test_train_nlu_ssa_model(test_config):
function test_train_nlu_bert_for_seq_cls_model (line 50) | def test_train_nlu_bert_for_seq_cls_model(test_config):
function test_eval_nlu_ssa (line 56) | def test_eval_nlu_ssa():
function test_eval_nlu_bert_for_seq_cls (line 68) | def test_eval_nlu_bert_for_seq_cls():
function test_predict_nlu_ssa_1_example (line 80) | def test_predict_nlu_ssa_1_example():
function test_predict_nlu_bert_for_seq_cls_1_example (line 94) | def test_predict_nlu_bert_for_seq_cls_1_example():
function test_remove_tested_directory (line 108) | def test_remove_tested_directory():
FILE: tests/integration/test_token_classification.py
function test_config (line 20) | def test_config(request):
function load_and_setting (line 24) | def load_and_setting(config_path):
function test_make_synthetic_data (line 38) | def test_make_synthetic_data():
function test_train_bert_tok_cls_model (line 44) | def test_train_bert_tok_cls_model(test_config):
function test_eval_nlu_bert_for_tok_cls (line 50) | def test_eval_nlu_bert_for_tok_cls():
function test_predict_nlu_bert_for_tok_cls_1_example (line 62) | def test_predict_nlu_bert_for_tok_cls_1_example():
function test_remove_tested_directory (line 76) | def test_remove_tested_directory():
FILE: tests/integration/test_tokenizers.py
function tokenizers (line 11) | def tokenizers(request):
function test_en_character_tokenize (line 40) | def test_en_character_tokenize(tokenizers):
function test_jamo_ko_tokenize (line 58) | def test_jamo_ko_tokenize(tokenizers):
function test_bert_uncased_en_tokenize (line 77) | def test_bert_uncased_en_tokenize(tokenizers):
function test_space_all_tokenize (line 94) | def test_space_all_tokenize(tokenizers):
function test_punkt_tokenize (line 111) | def test_punkt_tokenize(tokenizers):
function test_word_with_regex_example_tokenize (line 128) | def test_word_with_regex_example_tokenize(tokenizers):
function test_spacy_model_with_regex_example_tokenize (line 147) | def test_spacy_model_with_regex_example_tokenize(tokenizers):
function test_bpe_tokenize (line 189) | def test_bpe_tokenize(tokenizers):
FILE: tests/integration/utils.py
function make_bert_seq_cls_synthetic_data (line 13) | def make_bert_seq_cls_synthetic_data(output_path, remove_exist=True):
function make_bert_reg_synthetic_data (line 29) | def make_bert_reg_synthetic_data(output_path, remove_exist=True):
function make_squad_synthetic_data (line 45) | def make_squad_synthetic_data(output_path, remove_exist=True):
function make_directory (line 78) | def make_directory(output_path, remove_exist=True):
function make_wiki_article_synthetic_data (line 86) | def make_wiki_article_synthetic_data(output_dir):
function make_random_tokens (line 117) | def make_random_tokens(length, answer_token=""):
function make_seq_cls_synthetic_data (line 129) | def make_seq_cls_synthetic_data(output_path):
function make_tok_cls_synthetic_data (line 155) | def make_tok_cls_synthetic_data(output_path):
function make_dummy_tags (line 181) | def make_dummy_tags(sequence, dummy_tag_cands):
function write_embedding_txt (line 204) | def write_embedding_txt(output_path, dim):
Condensed preview — 660 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (8,765K chars).
[
{
"path": ".coveragerc",
"chars": 753,
"preview": "# .coveragerc to control coverage.py\n[run]\nbranch = True\n\n[report]\n# Regexes for lines to exclude from consideration\nexc"
},
{
"path": ".gitignore",
"chars": 1978,
"preview": "# Created by https://www.gitignore.io/api/macos,python,jupyternotebook\n\n### CLaF: Clova Language Framework ###\n\n/data/\nl"
},
{
"path": ".nojekyll",
"chars": 0,
"preview": ""
},
{
"path": ".readthedocs.yml",
"chars": 569,
"preview": "# .readthedocs.yml\n# Read the Docs configuration file\n# See https://docs.readthedocs.io/en/stable/config-file/v2.html fo"
},
{
"path": ".travis.yml",
"chars": 468,
"preview": "language: python\npython:\n - \"3.6\"\n# command to install dependencies\ninstall: \n - pip install --upgrade pip\n - pip ins"
},
{
"path": "CONTRIBUTING.md",
"chars": 1902,
"preview": "# Contributing to CLAF\n\nFirst of all, thank you for considering contributing to CLAF. It's people like you that make CLa"
},
{
"path": "Dockerfile",
"chars": 399,
"preview": "FROM jmin/pytorch:apex\nRUN git clone https://github.com/naver/claf && cd claf && pip install -r requirements.txt && pyth"
},
{
"path": "LICENSE",
"chars": 1074,
"preview": "Copyright (c) 2019-present NAVER Corp.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy \nof"
},
{
"path": "NOTICE",
"chars": 4022,
"preview": "CLaF\nCopyright 2019-present NAVER Corp.\n\nThis project contains subcomponents with separate copyright notices and license"
},
{
"path": "README.md",
"chars": 12583,
"preview": "<p align=\"center\">\n <img src=\"images/logo.png\" style=\"inline\" width=300>\n</p>\n\n<h4 align=\"center\">Clova Language Fram"
},
{
"path": "base_config/cola/bert_base_uncased.json",
"chars": 1854,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"cola_bert\",\n \"train_file_path\": \"<CoLA train.tsv path>\",\n "
},
{
"path": "base_config/cola/bert_large_uncased.json",
"chars": 1857,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"cola_bert\",\n \"train_file_path\": \"<CoLA train.tsv path>\",\n "
},
{
"path": "base_config/cola/structured_self_attention.json",
"chars": 2050,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"cola\",\n \"train_file_path\": \"<CoLA train.tsv path>\",\n \"valid_file_path\": \"<CoL"
},
{
"path": "base_config/conll2003/bert_large_cased.json",
"chars": 2058,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"conll2003_bert\",\n \"train_file_path\": \"https://raw.githubuserconten"
},
{
"path": "base_config/glue/cola_bert.json",
"chars": 1849,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"cola_bert\",\n \"train_file_path\": \"CoLA/train.tsv\",\n \"valid_"
},
{
"path": "base_config/glue/cola_roberta.json",
"chars": 2094,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"cola_bert\",\n \"train_file_path\": \"CoLA/train.tsv\",\n \"valid_"
},
{
"path": "base_config/glue/mnlim_bert.json",
"chars": 1854,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"mnli_bert\",\n \"train_file_path\": \"MNLI/train.tsv\",\n \"valid_"
},
{
"path": "base_config/glue/mnlim_roberta.json",
"chars": 2024,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"mnli_bert\",\n \"train_file_path\": \"MNLI/train.tsv\",\n \"valid_"
},
{
"path": "base_config/glue/mnlimm_bert.json",
"chars": 1792,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"mnli_bert\",\n \"train_file_path\": \"MNLI/train.tsv\",\n \"valid_file_"
},
{
"path": "base_config/glue/mnlimm_roberta.json",
"chars": 1960,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"mnli_bert\",\n \"train_file_path\": \"MNLI/train.tsv\",\n \"valid_file_"
},
{
"path": "base_config/glue/mrpc_bert.json",
"chars": 1837,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"mrpc_bert\",\n \"train_file_path\": \"MRPC/train.tsv\",\n \"valid_"
},
{
"path": "base_config/glue/mrpc_roberta.json",
"chars": 2008,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"mrpc_bert\",\n \"train_file_path\": \"MRPC/train.tsv\",\n \"valid_"
},
{
"path": "base_config/glue/qnli_bert.json",
"chars": 1845,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"qnli_bert\",\n \"train_file_path\": \"QNLI/train.tsv\",\n \"valid_"
},
{
"path": "base_config/glue/qnli_roberta.json",
"chars": 2015,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"qnli_bert\",\n \"train_file_path\": \"QNLI/train.tsv\",\n \"valid_"
},
{
"path": "base_config/glue/qqp_bert.json",
"chars": 1834,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"qqp_bert\",\n \"train_file_path\": \"QQP/train.tsv\",\n \"valid_fi"
},
{
"path": "base_config/glue/qqp_roberta.json",
"chars": 2004,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"qqp_bert\",\n \"train_file_path\": \"QQP/train.tsv\",\n \"valid_fi"
},
{
"path": "base_config/glue/rte_bert.json",
"chars": 1838,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"rte_bert\",\n \"train_file_path\": \"RTE/train.tsv\",\n \"valid_fi"
},
{
"path": "base_config/glue/rte_roberta.json",
"chars": 2008,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"rte_bert\",\n \"train_file_path\": \"RTE/train.tsv\",\n \"valid_fi"
},
{
"path": "base_config/glue/sst_bert.json",
"chars": 1844,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"sst_bert\",\n \"train_file_path\": \"SST-2/train.tsv\",\n \"valid_"
},
{
"path": "base_config/glue/sst_roberta.json",
"chars": 2014,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"sst_bert\",\n \"train_file_path\": \"SST-2/train.tsv\",\n \"valid_"
},
{
"path": "base_config/glue/stsb_bert.json",
"chars": 1851,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"stsb_bert\",\n \"train_file_path\": \"STS-B/train.tsv\",\n \"valid"
},
{
"path": "base_config/glue/stsb_roberta.json",
"chars": 2021,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"stsb_bert\",\n \"train_file_path\": \"STS-B/train.tsv\",\n \"valid"
},
{
"path": "base_config/glue/wnli_bert.json",
"chars": 1843,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"wnli_bert\",\n \"train_file_path\": \"WNLI/train.tsv\",\n \"valid_"
},
{
"path": "base_config/glue/wnli_roberta.json",
"chars": 2013,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"wnli_bert\",\n \"train_file_path\": \"WNLI/train.tsv\",\n \"valid_"
},
{
"path": "base_config/korquad/bert_base_multilingual_cased.yaml",
"chars": 1336,
"preview": "data_reader:\n dataset: \"squad_bert\"\n train_file_path: \"https://korquad.github.io/dataset/KorQuAD_v1.0_train.json\"\n va"
},
{
"path": "base_config/korquad/bert_base_multilingual_uncased.json",
"chars": 2098,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"squad_bert\",\n \"train_file_path\": \"https://korquad.github.io/datase"
},
{
"path": "base_config/korquad/bidaf.json",
"chars": 2180,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://korquad.github.io/dataset/Kor"
},
{
"path": "base_config/korquad/docqa.json",
"chars": 2309,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://korquad.github.io/dataset/KorQuAD"
},
{
"path": "base_config/multi_task/bert_base_glue+squad.json",
"chars": 4916,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"multitask_bert\",\n \"train_file_path\": \"train\",\n \"valid_file"
},
{
"path": "base_config/multi_task/bert_base_glue.json",
"chars": 4406,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"multitask_bert\",\n \"train_file_path\": \"train\",\n \"valid_file"
},
{
"path": "base_config/multi_task/bert_large_glue+squad.json",
"chars": 4918,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"multitask_bert\",\n \"train_file_path\": \"train\",\n \"valid_file"
},
{
"path": "base_config/multi_task/bert_large_glue.json",
"chars": 4409,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"multitask_bert\",\n \"train_file_path\": \"train\",\n \"valid_file"
},
{
"path": "base_config/squad/bert_base_uncased.json",
"chars": 2137,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"squad_bert\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuA"
},
{
"path": "base_config/squad/bert_large_uncased.json",
"chars": 2139,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"squad_bert\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuA"
},
{
"path": "base_config/squad/bidaf+elmo.json",
"chars": 2702,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-explore"
},
{
"path": "base_config/squad/bidaf.json",
"chars": 2378,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-exp"
},
{
"path": "base_config/squad/bidaf_no_answer.json",
"chars": 2059,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-exp"
},
{
"path": "base_config/squad/docqa+elmo.json",
"chars": 2665,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-explore"
},
{
"path": "base_config/squad/docqa.json",
"chars": 2354,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-explore"
},
{
"path": "base_config/squad/docqa_no_answer.json",
"chars": 2261,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-explore"
},
{
"path": "base_config/squad/drqa.json",
"chars": 2048,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-explore"
},
{
"path": "base_config/squad/drqa_paper.json",
"chars": 2292,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-explore"
},
{
"path": "base_config/squad/qanet.json",
"chars": 2849,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-expl"
},
{
"path": "base_config/squad/qanet_paper.json",
"chars": 2971,
"preview": "{\n\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-exp"
},
{
"path": "base_config/squad/roberta_base.json",
"chars": 2228,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"squad_bert\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuA"
},
{
"path": "base_config/squad/roberta_large.json",
"chars": 2232,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"squad_bert\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuA"
},
{
"path": "base_config/test/bert_for_multi_task.json",
"chars": 3076,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"multitask_bert\",\n \"train_file_path\": \"train\",\n \"valid_file"
},
{
"path": "base_config/test/bert_for_qa.yaml",
"chars": 1263,
"preview": "data_reader:\n dataset: \"squad_bert\"\n train_file_path: \"data/bert/squad_synthetic_data.json\"\n valid_file_path: \"data/b"
},
{
"path": "base_config/test/bert_for_seq_cls.json",
"chars": 1941,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"seq_cls_bert\",\n \"train_file_path\": \"logs/test/seq_cls/synthetic_da"
},
{
"path": "base_config/test/bert_for_tok_cls.json",
"chars": 1974,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"tok_cls_bert\",\n \"train_file_path\": \"logs/test/tok_cls/synthetic_da"
},
{
"path": "base_config/test/bidaf+bert.json",
"chars": 2805,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-exp"
},
{
"path": "base_config/test/bidaf+cove.json",
"chars": 2498,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-exp"
},
{
"path": "base_config/test/bidaf+elmo.json",
"chars": 2573,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-exp"
},
{
"path": "base_config/test/bidaf.yaml",
"chars": 1319,
"preview": "data_reader:\n dataset: \"squad\"\n train_file_path: \"https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json\"\n "
},
{
"path": "base_config/test/bidaf_no_answer.json",
"chars": 2064,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-exp"
},
{
"path": "base_config/test/cola_bert.json",
"chars": 1893,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"cola_bert\",\n \"train_file_path\": \"data/glue/CoLA/train.tsv\",\n "
},
{
"path": "base_config/test/cola_roberta.json",
"chars": 2076,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"cola_bert\",\n \"train_file_path\": \"data/glue/CoLA/train.tsv\",\n "
},
{
"path": "base_config/test/docqa.json",
"chars": 1993,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-explore"
},
{
"path": "base_config/test/docqa_no_answer.json",
"chars": 1856,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-explore"
},
{
"path": "base_config/test/drqa.json",
"chars": 2024,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-explorer/d"
},
{
"path": "base_config/test/drqa_sparse_to_embedding.json",
"chars": 2134,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-explorer/d"
},
{
"path": "base_config/test/mnlim_bert.json",
"chars": 1898,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"mnli_bert\",\n \"train_file_path\": \"data/glue/MNLI/train.tsv\",\n "
},
{
"path": "base_config/test/mrpc_bert.json",
"chars": 1882,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"mrpc_bert\",\n \"train_file_path\": \"data/glue/MRPC/train.tsv\",\n "
},
{
"path": "base_config/test/mt_bert.json",
"chars": 2949,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"multitask_bert\",\n \"train_file_path\": \"train\",\n \"valid_file"
},
{
"path": "base_config/test/open_qa.json",
"chars": 695,
"preview": " {\n \"name\": \"open_qa\",\n \"open_qa\": {\n \"tokenizers\": {\n \"sent\": {\n \"name\": \"punkt\"\n "
},
{
"path": "base_config/test/qanet.json",
"chars": 2542,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"squad\",\n \"train_file_path\": \"https://rajpurkar.github.io/SQuAD-expl"
},
{
"path": "base_config/test/qnli_bert.json",
"chars": 1889,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"qnli_bert\",\n \"train_file_path\": \"data/glue/QNLI/train.tsv\",\n "
},
{
"path": "base_config/test/qqp_bert.json",
"chars": 1883,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"qqp_bert\",\n \"train_file_path\": \"data/glue/QQP/train.tsv\",\n "
},
{
"path": "base_config/test/roberta_for_qa.json",
"chars": 2003,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"squad_bert\",\n \"train_file_path\": \"data/squad/dev-v1.1.json\",\n "
},
{
"path": "base_config/test/rte_bert.json",
"chars": 1882,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"rte_bert\",\n \"train_file_path\": \"data/glue/RTE/train.tsv\",\n "
},
{
"path": "base_config/test/rte_roberta.json",
"chars": 1990,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"rte_bert\",\n \"train_file_path\": \"data/glue/RTE/train.tsv\",\n "
},
{
"path": "base_config/test/sqlnet.json",
"chars": 1609,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"wikisql\",\n \"train_file_path\": \"data/wikisql/dev.jsonl\",\n \""
},
{
"path": "base_config/test/ssa.json",
"chars": 1675,
"preview": "{\n \"data_reader\": {\n \"dataset\": \"seq_cls\",\n \"train_file_path\": \"logs/test/seq_cls/synthetic_data.json\",\n \"vali"
},
{
"path": "base_config/test/sst_bert.json",
"chars": 1888,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"sst_bert\",\n \"train_file_path\": \"data/glue/SST-2/train.tsv\",\n "
},
{
"path": "base_config/test/stsb_bert.json",
"chars": 1896,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"stsb_bert\",\n \"train_file_path\": \"data/glue/STS-B/train.tsv\",\n "
},
{
"path": "base_config/test/stsb_roberta.json",
"chars": 2004,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"stsb_bert\",\n \"train_file_path\": \"data/glue/STS-B/train.tsv\",\n "
},
{
"path": "base_config/test/wnli_bert.json",
"chars": 1888,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"wnli_bert\",\n \"train_file_path\": \"data/glue/WNLI/train.tsv\",\n "
},
{
"path": "base_config/wikisql/sqlnet.json",
"chars": 1554,
"preview": " {\n \"data_reader\": {\n \"dataset\": \"wikisql\",\n \"train_file_path\": \"wikisql/train.jsonl\",\n \"val"
},
{
"path": "claf/__init__.py",
"chars": 168,
"preview": "# -*- coding: utf-8 -*-\n\n# register components\nfrom claf.data.reader import *\nfrom claf.machine import *\nfrom claf.machi"
},
{
"path": "claf/__version__.py",
"chars": 97,
"preview": "# CLaF: Clova Language Framework\n\nVERSION = (0, 2, 0)\n\n__version__ = \".\".join(map(str, VERSION))\n"
},
{
"path": "claf/config/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "claf/config/args.py",
"chars": 70141,
"preview": "\nimport argparse\nfrom argparse import RawTextHelpFormatter\nimport os\nimport sys\n\nimport torch\n\nfrom claf import nsml\nfro"
},
{
"path": "claf/config/namespace.py",
"chars": 2191,
"preview": "\nimport argparse\n\n\nclass NestedNamespace(argparse.Namespace):\n \"\"\"\n Nested Namespace\n (Simple class used by def"
},
{
"path": "claf/config/pattern.py",
"chars": 424,
"preview": "class Singleton(type):\n \"\"\"\n Design Pattern Base\n\n Singleton Meta Class\n the singleton pattern is a software"
},
{
"path": "claf/config/registry.py",
"chars": 1484,
"preview": "\nimport logging\n\nfrom claf.config.pattern import Singleton\n\nlogger = logging.getLogger(__name__)\n\n\nclass Registry(metacl"
},
{
"path": "claf/config/utils.py",
"chars": 2063,
"preview": "\nfrom argparse import Namespace\nimport copy\nimport json\nimport os\n\nimport jsbeautifier\nimport numpy as np\nimport random\n"
},
{
"path": "claf/data/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "claf/data/collate.py",
"chars": 3563,
"preview": "\nfrom overrides import overrides\n\nimport torch\nfrom torch.autograd import Variable\n\nfrom claf.data import utils\n\n\nclass "
},
{
"path": "claf/data/data_handler.py",
"chars": 5016,
"preview": "\nimport logging\nimport pickle\nimport os\nfrom pathlib import Path, PosixPath\nimport shutil\nimport tempfile\n\nimport msgpac"
},
{
"path": "claf/data/dataset/__init__.py",
"chars": 696,
"preview": "\nfrom claf.data.dataset.squad import SQuADDataset\nfrom claf.data.dataset.wikisql import WikiSQLDataset\nfrom claf.data.da"
},
{
"path": "claf/data/dataset/base.py",
"chars": 1768,
"preview": "\nfrom torch.utils.data.dataset import Dataset\n\nfrom claf.data import utils\n\n\nclass DatasetBase(Dataset):\n \"\"\"\n Dat"
},
{
"path": "claf/data/dataset/bert/__init__.py",
"chars": 1,
"preview": "\n"
},
{
"path": "claf/data/dataset/bert/multi_task.py",
"chars": 3063,
"preview": "\nimport json\nfrom overrides import overrides\nimport torch\nimport random\n\nfrom claf.factory.data_loader import make_data_"
},
{
"path": "claf/data/dataset/bert/regression.py",
"chars": 2911,
"preview": "\nimport json\nfrom overrides import overrides\n\nfrom claf.data import utils\nfrom claf.data.collate import PadCollator\nfrom"
},
{
"path": "claf/data/dataset/bert/seq_cls.py",
"chars": 3471,
"preview": "\nimport json\nfrom overrides import overrides\n\nfrom claf.data import utils\nfrom claf.data.collate import PadCollator\nfrom"
},
{
"path": "claf/data/dataset/bert/squad.py",
"chars": 5437,
"preview": "\nimport json\nfrom overrides import overrides\n\nfrom claf.data import utils\nfrom claf.data.collate import PadCollator\nfrom"
},
{
"path": "claf/data/dataset/bert/tok_cls.py",
"chars": 4262,
"preview": "\nimport json\nfrom overrides import overrides\n\nfrom claf.data import utils\nfrom claf.data.collate import FeatLabelPadColl"
},
{
"path": "claf/data/dataset/seq_cls.py",
"chars": 3229,
"preview": "\nimport json\nfrom overrides import overrides\nimport torch\n\nfrom claf.data import utils\nfrom claf.data.collate import Pad"
},
{
"path": "claf/data/dataset/squad.py",
"chars": 4788,
"preview": "\nimport json\nfrom overrides import overrides\n\nfrom claf.data import utils\nfrom claf.data.collate import PadCollator\nfrom"
},
{
"path": "claf/data/dataset/wikisql.py",
"chars": 3834,
"preview": "\nimport json\nfrom overrides import overrides\n\nimport torch\n\nfrom claf.data import utils\nfrom claf.data.collate import Pa"
},
{
"path": "claf/data/dto/__init__.py",
"chars": 214,
"preview": "\nfrom claf.data.dto.batch import Batch\nfrom claf.data.dto.bert_feature import BertFeature\nfrom claf.data.dto.helper impo"
},
{
"path": "claf/data/dto/batch.py",
"chars": 2467,
"preview": "\nimport logging\n\nlogger = logging.getLogger(__name__)\n\n\nclass Batch:\n \"\"\"\n Batch Data Transfer Object (DTO) Class\n"
},
{
"path": "claf/data/dto/bert_feature.py",
"chars": 864,
"preview": "\nfrom claf.data import utils\n\n\nclass BertFeature:\n \"\"\"\n BertFeature Data Transfer Object (DTO) Class\n\n dictiona"
},
{
"path": "claf/data/dto/helper.py",
"chars": 1126,
"preview": "\n\n\nclass Helper:\n \"\"\"\n Helper Data Transfer Object (DTO) Class\n (include model parameter - value defined by d"
},
{
"path": "claf/data/reader/__init__.py",
"chars": 1544,
"preview": "\nfrom claf.data.reader.seq_cls import SeqClsReader\nfrom claf.data.reader.cola import CoLAReader\n\nfrom claf.data.reader.s"
},
{
"path": "claf/data/reader/base.py",
"chars": 2334,
"preview": "\nimport logging\n\nfrom claf.data.data_handler import CachePath, DataHandler\nfrom claf import utils as common_utils\n\nlogge"
},
{
"path": "claf/data/reader/bert/__init__.py",
"chars": 1,
"preview": "\n"
},
{
"path": "claf/data/reader/bert/conll2003.py",
"chars": 1907,
"preview": "\nimport logging\nfrom itertools import chain\n\nfrom overrides import overrides\n\nfrom claf.data.reader import TokClsBertRea"
},
{
"path": "claf/data/reader/bert/glue/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "claf/data/reader/bert/glue/cola.py",
"chars": 1543,
"preview": "\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator impo"
},
{
"path": "claf/data/reader/bert/glue/mnli.py",
"chars": 1670,
"preview": "\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator impo"
},
{
"path": "claf/data/reader/bert/glue/mrpc.py",
"chars": 1627,
"preview": "\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator impo"
},
{
"path": "claf/data/reader/bert/glue/qnli.py",
"chars": 1660,
"preview": "\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator impo"
},
{
"path": "claf/data/reader/bert/glue/qqp.py",
"chars": 1672,
"preview": "\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator impo"
},
{
"path": "claf/data/reader/bert/glue/rte.py",
"chars": 1688,
"preview": "\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator impo"
},
{
"path": "claf/data/reader/bert/glue/sst.py",
"chars": 1641,
"preview": "\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator impo"
},
{
"path": "claf/data/reader/bert/glue/stsb.py",
"chars": 1670,
"preview": "\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import RegressionBertReader\nfrom claf.decorator "
},
{
"path": "claf/data/reader/bert/glue/wnli.py",
"chars": 1649,
"preview": "\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsBertReader\nfrom claf.decorator impo"
},
{
"path": "claf/data/reader/bert/multi_task.py",
"chars": 3808,
"preview": "\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.config.namespace import NestedNamespace\nfrom claf.config.reg"
},
{
"path": "claf/data/reader/bert/regression.py",
"chars": 4984,
"preview": "\nimport logging\nimport json\nimport uuid\n\nfrom overrides import overrides\nfrom tqdm import tqdm\n\nfrom claf.data.dataset i"
},
{
"path": "claf/data/reader/bert/seq_cls.py",
"chars": 6374,
"preview": "\nimport json\nimport logging\nimport uuid\n\nfrom overrides import overrides\nfrom tqdm import tqdm\n\nfrom claf.data.dataset i"
},
{
"path": "claf/data/reader/bert/squad.py",
"chars": 15422,
"preview": "\nfrom collections import Counter\nimport json\nimport logging\nimport re\n\nfrom overrides import overrides\nfrom tqdm import "
},
{
"path": "claf/data/reader/bert/tok_cls.py",
"chars": 8565,
"preview": "\nfrom itertools import chain\nimport json\nimport logging\nimport uuid\n\nfrom overrides import overrides\nfrom tqdm import tq"
},
{
"path": "claf/data/reader/cola.py",
"chars": 1373,
"preview": "\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.reader import SeqClsReader\nfrom claf.decorator import r"
},
{
"path": "claf/data/reader/seq_cls.py",
"chars": 4397,
"preview": "\nimport json\nimport logging\nimport uuid\n\nfrom overrides import overrides\nfrom tqdm import tqdm\n\nfrom claf.data.dataset.s"
},
{
"path": "claf/data/reader/squad.py",
"chars": 8067,
"preview": "\nfrom collections import Counter\nimport json\nimport logging\nimport re\n\nfrom overrides import overrides\nfrom tqdm import "
},
{
"path": "claf/data/reader/wikisql.py",
"chars": 5839,
"preview": "\nimport json\nimport logging\nfrom pathlib import Path\nimport uuid\n\nfrom overrides import overrides\nfrom tqdm import tqdm\n"
},
{
"path": "claf/data/utils.py",
"chars": 8524,
"preview": "\nfrom collections import defaultdict\n\nimport numpy as np\nimport torch\n\nfrom claf.data.dto.batch import Batch\n\n\ndef make_"
},
{
"path": "claf/decorator/__init__.py",
"chars": 149,
"preview": "\nfrom claf.decorator.arguments import arguments_required\nfrom claf.decorator.register import register\n\n\n__all__ = [\"argu"
},
{
"path": "claf/decorator/arguments.py",
"chars": 602,
"preview": "class arguments_required:\n \"\"\"\n Decorator Class\n check required arguments for predict function\n "
},
{
"path": "claf/decorator/register.py",
"chars": 382,
"preview": "\nfrom claf.config.registry import Registry\n\n\nclass register:\n \"\"\"\n Decorator Class\n register subclass w"
},
{
"path": "claf/factory/__init__.py",
"chars": 394,
"preview": "\nfrom claf.factory.data_reader import DataReaderFactory\nfrom claf.factory.data_loader import DataLoaderFactory\nfrom claf"
},
{
"path": "claf/factory/base.py",
"chars": 698,
"preview": "class Factory:\n \"\"\"\n Factory Base Class\n\n Factory method pattern\n\n In class-based programming, the factory m"
},
{
"path": "claf/factory/data_loader.py",
"chars": 1971,
"preview": "\nfrom overrides import overrides\nfrom torch.utils.data import DataLoader\n\nfrom .base import Factory\n\n\ndef make_data_load"
},
{
"path": "claf/factory/data_reader.py",
"chars": 1359,
"preview": "\nfrom overrides import overrides\n\nfrom claf.config.registry import Registry\n\nfrom .base import Factory\n\n\nclass DataReade"
},
{
"path": "claf/factory/model.py",
"chars": 1744,
"preview": "\nfrom overrides import overrides\n\nfrom claf.config.registry import Registry\nfrom claf.model.base import ModelWithTokenEm"
},
{
"path": "claf/factory/optimizer.py",
"chars": 5116,
"preview": "\nfrom overrides import overrides\nimport torch\n\nfrom claf.config.namespace import NestedNamespace\nfrom claf.learn.optimiz"
},
{
"path": "claf/factory/tokens.py",
"chars": 3202,
"preview": "\nfrom overrides import overrides\n\nfrom claf.config.registry import Registry\nfrom claf.config.utils import convert_config"
},
{
"path": "claf/learn/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "claf/learn/experiment.py",
"chars": 19274,
"preview": "\nimport atexit\nimport logging\nfrom pathlib import Path\n\nimport torch\n\nfrom claf import nsml\nfrom claf.factory import (\n "
},
{
"path": "claf/learn/mode.py",
"chars": 163,
"preview": "class Mode:\n \"\"\" Experiment Flag class \"\"\"\n\n TRAIN = \"train\"\n EVAL = \"eval\"\n INFER_EVAL = \"infer_eval\"\n P"
},
{
"path": "claf/learn/optimization/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "claf/learn/optimization/exponential_moving_avarage.py",
"chars": 762,
"preview": "class EMA:\n \"\"\"\n Exponential Moving Average\n\n the moving averages of all weights of the model are maintained\n "
},
{
"path": "claf/learn/optimization/learning_rate_scheduler.py",
"chars": 4486,
"preview": "\"\"\"\n https://github.com/allenai/allennlp/blob/master/allennlp/training/learning_rate_schedulers.py\n\"\"\"\n\nfrom override"
},
{
"path": "claf/learn/optimization/optimizer.py",
"chars": 615,
"preview": "\nfrom transformers import AdamW\nimport torch\n\n\ndef get_optimizer_by_name(name):\n optimizers = {\n \"adam\": torch"
},
{
"path": "claf/learn/tensorboard.py",
"chars": 1501,
"preview": "\nimport os\n\nfrom tensorboardX import SummaryWriter\n\nfrom claf import nsml\n\n\nclass TensorBoard:\n \"\"\" TensorBoard Wrapp"
},
{
"path": "claf/learn/trainer.py",
"chars": 20704,
"preview": "# -*- coding: utf-8 -*-\n\nimport json\nimport logging\nimport os\nimport time\nimport random\n\nimport torch\nfrom torch.nn.util"
},
{
"path": "claf/learn/utils.py",
"chars": 7088,
"preview": "\nfrom collections import OrderedDict\nimport json\nimport logging\nfrom pathlib import Path\nimport os\nimport re\n\nimport tor"
},
{
"path": "claf/machine/__init__.py",
"chars": 139,
"preview": "\nfrom claf.machine.open_qa import OpenQA\nfrom claf.machine.nlu import NLU\n\n\n# fmt: off\n\n__all__ = [\n \"OpenQA\",\n \"N"
},
{
"path": "claf/machine/base.py",
"chars": 2239,
"preview": "\nfrom argparse import Namespace\nimport json\n\nfrom claf.config.namespace import NestedNamespace\nfrom claf.config.registry"
},
{
"path": "claf/machine/components/__init__.py",
"chars": 123,
"preview": "\nfrom claf.machine.components.retrieval.tfidf import TFIDF\n\n# fmt: off\n\n__all__ = [\n \"TFIDF\", # Retrieval\n]\n\n# fmt: "
},
{
"path": "claf/machine/components/retrieval/__init__.py",
"chars": 1,
"preview": "\n"
},
{
"path": "claf/machine/components/retrieval/tfidf.py",
"chars": 2807,
"preview": "\nfrom pathlib import Path\n\nfrom gensim.corpora import Dictionary\nfrom gensim.models import TfidfModel\nfrom gensim.simila"
},
{
"path": "claf/machine/ensemble_topk.py",
"chars": 9446,
"preview": "\nfrom functools import reduce # Valid in Python 2.6+, required in Python 3\nimport logging\nimport json\nimport operator\n\n"
},
{
"path": "claf/machine/knowlege_base/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "claf/machine/knowlege_base/docs.py",
"chars": 2187,
"preview": "\nimport json\nimport logging\nimport os\n\nfrom tqdm import tqdm\n\n\nlogger = logging.getLogger(__name__)\n\n\ndef read_wiki_arti"
},
{
"path": "claf/machine/module.py",
"chars": 142,
"preview": "class Module:\n \"\"\" Machine Flag class \"\"\"\n\n KNOWLEDGE_BASE = \"knowledge_base\"\n COMPONENT = \"component\"\n EXPE"
},
{
"path": "claf/machine/nlu.py",
"chars": 1500,
"preview": "\nimport logging\n\nfrom overrides import overrides\n\nfrom claf.data.data_handler import CachePath, DataHandler\nfrom claf.de"
},
{
"path": "claf/machine/open_qa.py",
"chars": 3667,
"preview": "\nimport logging\nimport os\n\nfrom overrides import overrides\n\nfrom claf.config.utils import convert_config2dict\nfrom claf."
},
{
"path": "claf/metric/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "claf/metric/classification.py",
"chars": 650,
"preview": "\ndef recall(pycm_obj):\n return {key: pycm_obj.TPR[key] if pycm_obj.TPR[key] != \"None\" else 0. for key in pycm_obj.TPR"
},
{
"path": "claf/metric/glue.py",
"chars": 864,
"preview": "\nimport numpy as np\nfrom scipy.stats import pearsonr, spearmanr\nfrom sklearn.metrics import matthews_corrcoef, f1_score\n"
},
{
"path": "claf/metric/korquad_v1_official.py",
"chars": 4163,
"preview": "from __future__ import print_function\nfrom collections import Counter\nimport string\nimport re\nimport argparse\nimport jso"
},
{
"path": "claf/metric/regression.py",
"chars": 409,
"preview": "\nimport numpy as np\n\n\ndef mse(outputs, labels):\n if type(outputs) == list:\n outputs = np.array(outputs)\n if"
},
{
"path": "claf/metric/squad_v1_official.py",
"chars": 3521,
"preview": "\"\"\" Official evaluation script for v1.1 of the SQuAD dataset. \"\"\"\nfrom __future__ import print_function\nfrom collections"
},
{
"path": "claf/metric/squad_v2_official.py",
"chars": 12761,
"preview": "\"\"\"Official evaluation script for SQuAD version 2.0.\n\nIn addition to basic functionality, we also compute additional sta"
},
{
"path": "claf/metric/wikisql_lib/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "claf/metric/wikisql_lib/dbengine.py",
"chars": 2108,
"preview": "import records\nimport re\nfrom babel.numbers import parse_decimal, NumberFormatError\n\nfrom claf.metric.wikisql_lib.query "
},
{
"path": "claf/metric/wikisql_lib/query.py",
"chars": 10319,
"preview": "from collections import defaultdict\nfrom copy import deepcopy\nimport re\n\n\nre_whitespace = re.compile(r\"\\s+\", flags=re.UN"
},
{
"path": "claf/metric/wikisql_official.py",
"chars": 3001,
"preview": "\"\"\" Official evaluation script for WikiSQL dataset. \"\"\"\n\nimport json\nfrom argparse import ArgumentParser\nfrom tqdm impor"
},
{
"path": "claf/model/__init__.py",
"chars": 257,
"preview": "\nfrom claf.model.multi_task import *\nfrom claf.model.reading_comprehension import *\nfrom claf.model.regression import *\n"
},
{
"path": "claf/model/base.py",
"chars": 3234,
"preview": "\nimport json\nfrom pathlib import Path\n\nimport torch.nn as nn\n\n\nclass ModelBase(nn.Module):\n \"\"\"\n Model Base Class\n"
},
{
"path": "claf/model/cls_utils.py",
"chars": 1265,
"preview": "import csv\nfrom collections import defaultdict\n\nfrom seqeval.metrics.sequence_labeling import get_entities\n\n\n# pycm\ndef "
},
{
"path": "claf/model/multi_task/__init__.py",
"chars": 119,
"preview": "\nfrom claf.model.multi_task.bert import BertForMultiTask\n\n\n# fmt: off\n\n__all__ = [\n \"BertForMultiTask\"\n]\n\n# fmt: on\n"
},
{
"path": "claf/model/multi_task/bert.py",
"chars": 12005,
"preview": "\nfrom overrides import overrides\nimport torch.nn as nn\nfrom transformers import BertModel\n\nfrom claf.data.data_handler i"
},
{
"path": "claf/model/multi_task/category.py",
"chars": 246,
"preview": "\nclass TaskCategory:\n \"\"\" TaskCategory Flag class \"\"\"\n\n SEQUENCE_CLASSIFICATION = \"sequence_classification\"\n RE"
},
{
"path": "claf/model/multi_task/mixin.py",
"chars": 3862,
"preview": "\nimport logging\n\nfrom claf.model.multi_task.category import TaskCategory\nfrom claf.model.sequence_classification.mixin i"
},
{
"path": "claf/model/reading_comprehension/__init__.py",
"chars": 681,
"preview": "\nfrom claf.model.reading_comprehension.bert import BertForQA\nfrom claf.model.reading_comprehension.bidaf import BiDAF\nfr"
},
{
"path": "claf/model/reading_comprehension/bert.py",
"chars": 4163,
"preview": "\n\nfrom overrides import overrides\nimport torch.nn as nn\nfrom transformers import BertForQuestionAnswering\n\nfrom claf.dat"
},
{
"path": "claf/model/reading_comprehension/bidaf.py",
"chars": 9279,
"preview": "\nfrom overrides import overrides\nimport torch\nimport torch.nn as nn\n\nfrom claf.decorator import register\nfrom claf.model"
},
{
"path": "claf/model/reading_comprehension/bidaf_no_answer.py",
"chars": 9643,
"preview": "\nfrom overrides import overrides\nimport torch\nimport torch.nn as nn\n\nfrom claf.decorator import register\nfrom claf.model"
},
{
"path": "claf/model/reading_comprehension/docqa.py",
"chars": 10458,
"preview": "from overrides import overrides\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom claf.decorator "
},
{
"path": "claf/model/reading_comprehension/docqa_no_answer.py",
"chars": 12283,
"preview": "from overrides import overrides\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom claf.decorator "
},
{
"path": "claf/model/reading_comprehension/drqa.py",
"chars": 6171,
"preview": "from overrides import overrides\n\nimport torch.nn as nn\n\nfrom claf.decorator import register\nfrom claf.model.base import "
},
{
"path": "claf/model/reading_comprehension/mixin.py",
"chars": 15705,
"preview": "\nfrom collections import OrderedDict\n\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\n\nfrom claf.decorat"
},
{
"path": "claf/model/reading_comprehension/qanet.py",
"chars": 12794,
"preview": "import torch\nimport torch.nn as nn\n\nfrom overrides import overrides\n\nfrom claf.decorator import register\nfrom claf.model"
},
{
"path": "claf/model/reading_comprehension/roberta.py",
"chars": 4418,
"preview": "\nfrom overrides import overrides\nfrom transformers import RobertaModel\nimport torch.nn as nn\n\nfrom claf.data.data_handle"
},
{
"path": "claf/model/regression/__init__.py",
"chars": 207,
"preview": "\nfrom claf.model.regression.bert import BertForRegression\nfrom claf.model.regression.roberta import RobertaForRegression"
},
{
"path": "claf/model/regression/bert.py",
"chars": 4650,
"preview": "\nfrom overrides import overrides\nimport torch.nn as nn\nfrom transformers import BertForSequenceClassification\n\nfrom claf"
},
{
"path": "claf/model/regression/mixin.py",
"chars": 4588,
"preview": "\nimport logging\n\nfrom claf.metric.glue import pearson_and_spearman\nfrom claf.metric.regression import mse\nfrom claf.mode"
}
]
// ... and 460 more files (download for full content)
About this extraction
This page contains the full source code of the naver/claf GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 660 files (7.8 MB), approximately 2.1M tokens, and a symbol index with 1378 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.