Copy disabled (too large)
Download .txt
Showing preview only (26,547K chars total). Download the full file to get everything.
Repository: chenghuige/wenzheng
Branch: master
Commit: d57fd68610de
Files: 2780
Total size: 23.8 MB
Directory structure:
gitextract_426mla3a/
├── .gitignore
├── README.md
├── __init__.py
├── examples/
│ ├── fastai/
│ │ └── lm/
│ │ ├── lm.ipynb
│ │ └── lm.py
│ ├── pytorch/
│ │ ├── data_parallel_tutorial.py
│ │ └── sentiment/
│ │ ├── model.py
│ │ └── train.py
│ ├── shap/
│ │ └── explainer_1.py
│ └── tf/
│ └── eager/
│ ├── dynamic_dense.py
│ ├── rnn_ptb.py
│ └── rnn_ptb.sh
├── jupyter/
│ └── Untitled.ipynb
├── official/
│ ├── .gitignore
│ ├── Dockerfile.cpu
│ ├── Dockerfile.gpu
│ ├── README.md
│ ├── __init__.py
│ ├── benchmark/
│ │ ├── __init__.py
│ │ ├── benchmark_uploader.py
│ │ ├── benchmark_uploader_main.py
│ │ ├── benchmark_uploader_test.py
│ │ └── datastore/
│ │ └── schema/
│ │ ├── benchmark_metric.json
│ │ └── benchmark_run.json
│ ├── boosted_trees/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── data_download.py
│ │ ├── train_higgs.py
│ │ ├── train_higgs_test.csv
│ │ └── train_higgs_test.py
│ ├── mnist/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── dataset.py
│ │ ├── mnist.py
│ │ ├── mnist_eager.py
│ │ ├── mnist_eager_test.py
│ │ ├── mnist_test.py
│ │ └── mnist_tpu.py
│ ├── recommendation/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── constants.py
│ │ ├── data_download.py
│ │ ├── dataset.py
│ │ ├── dataset_test.py
│ │ ├── ncf_main.py
│ │ ├── neumf_model.py
│ │ └── unittest_data/
│ │ ├── test_eval_negative.csv
│ │ ├── test_eval_ratings.csv
│ │ └── test_train_ratings.csv
│ ├── resnet/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── cifar10_download_and_extract.py
│ │ ├── cifar10_main.py
│ │ ├── cifar10_test.py
│ │ ├── imagenet_main.py
│ │ ├── imagenet_preprocessing.py
│ │ ├── imagenet_test.py
│ │ ├── layer_test.py
│ │ ├── resnet_model.py
│ │ └── resnet_run_loop.py
│ ├── transformer/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── compute_bleu.py
│ │ ├── compute_bleu_test.py
│ │ ├── data_download.py
│ │ ├── model/
│ │ │ ├── __init__.py
│ │ │ ├── attention_layer.py
│ │ │ ├── beam_search.py
│ │ │ ├── beam_search_test.py
│ │ │ ├── embedding_layer.py
│ │ │ ├── ffn_layer.py
│ │ │ ├── model_params.py
│ │ │ ├── model_utils.py
│ │ │ ├── model_utils_test.py
│ │ │ └── transformer.py
│ │ ├── test_data/
│ │ │ ├── newstest2014.de
│ │ │ └── newstest2014.en
│ │ ├── transformer_main.py
│ │ ├── translate.py
│ │ └── utils/
│ │ ├── __init__.py
│ │ ├── dataset.py
│ │ ├── metrics.py
│ │ ├── tokenizer.py
│ │ └── tokenizer_test.py
│ ├── utils/
│ │ ├── __init__.py
│ │ ├── export/
│ │ │ ├── __init__.py
│ │ │ ├── export.py
│ │ │ └── export_test.py
│ │ ├── flags/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── _base.py
│ │ │ ├── _benchmark.py
│ │ │ ├── _conventions.py
│ │ │ ├── _misc.py
│ │ │ ├── _performance.py
│ │ │ ├── core.py
│ │ │ └── flags_test.py
│ │ ├── logs/
│ │ │ ├── __init__.py
│ │ │ ├── hooks.py
│ │ │ ├── hooks_helper.py
│ │ │ ├── hooks_helper_test.py
│ │ │ ├── hooks_test.py
│ │ │ ├── logger.py
│ │ │ ├── logger_test.py
│ │ │ ├── metric_hook.py
│ │ │ └── metric_hook_test.py
│ │ ├── misc/
│ │ │ ├── __init__.py
│ │ │ ├── model_helpers.py
│ │ │ └── model_helpers_test.py
│ │ └── testing/
│ │ ├── __init__.py
│ │ ├── integration.py
│ │ ├── mock_lib.py
│ │ ├── pylint.rcfile
│ │ ├── reference_data/
│ │ │ ├── reference_data_test/
│ │ │ │ ├── dense/
│ │ │ │ │ ├── expected_graph
│ │ │ │ │ ├── results.json
│ │ │ │ │ └── tf_version.json
│ │ │ │ └── uniform_random/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ └── resnet/
│ │ │ ├── batch-size-32_bottleneck_projection_version-1_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ ├── batch-size-32_bottleneck_projection_version-2_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ ├── batch-size-32_bottleneck_version-1_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ ├── batch-size-32_bottleneck_version-2_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ ├── batch-size-32_building_projection_version-1_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ ├── batch-size-32_building_projection_version-2_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ ├── batch-size-32_building_version-1_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ ├── batch-size-32_building_version-2_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ └── batch_norm/
│ │ │ ├── expected_graph
│ │ │ ├── results.json
│ │ │ └── tf_version.json
│ │ ├── reference_data.py
│ │ ├── reference_data_test.py
│ │ └── scripts/
│ │ └── presubmit.sh
│ └── wide_deep/
│ ├── README.md
│ ├── __init__.py
│ ├── data_download.py
│ ├── wide_deep.py
│ ├── wide_deep_test.csv
│ └── wide_deep_test.py
├── projects/
│ ├── __init__.py
│ ├── ai2018/
│ │ ├── __init__.py
│ │ ├── binary/
│ │ │ ├── __init__.py
│ │ │ ├── algos/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── config.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── model.py
│ │ │ │ └── weights.py
│ │ │ ├── dataset.py
│ │ │ ├── evaluate.py
│ │ │ ├── prepare/
│ │ │ │ ├── config.py
│ │ │ │ ├── gen-records.py
│ │ │ │ ├── run.sh
│ │ │ │ └── text2ids.py
│ │ │ ├── read-records.py
│ │ │ ├── torch-train.py
│ │ │ ├── torch_algos/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── loss.py
│ │ │ │ └── model.py
│ │ │ └── train/
│ │ │ └── train.sh
│ │ ├── reader/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── algos/
│ │ │ │ ├── README.md
│ │ │ │ ├── __init__.py
│ │ │ │ ├── baseline.py
│ │ │ │ ├── config.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── m_reader.py
│ │ │ │ ├── model.py
│ │ │ │ ├── qcatt.py
│ │ │ │ └── rnet.py
│ │ │ ├── dataset.py
│ │ │ ├── ensemble/
│ │ │ │ ├── ensemble-infer.py
│ │ │ │ ├── ensemble-valid.py
│ │ │ │ ├── ensemble.py
│ │ │ │ └── evaluate.py
│ │ │ ├── evaluate.py
│ │ │ ├── infer.py
│ │ │ ├── prepare/
│ │ │ │ ├── README.md
│ │ │ │ ├── gen-records.py
│ │ │ │ ├── gen-seg.py
│ │ │ │ ├── merge-emb.py
│ │ │ │ ├── pre-seg.py
│ │ │ │ ├── run-char-ft.sh
│ │ │ │ ├── run-char-glove.sh
│ │ │ │ ├── run-noemb.sh
│ │ │ │ ├── run-word-jieba-ft.sh
│ │ │ │ ├── run-word-jieba-glove.sh
│ │ │ │ ├── to-simplify-dureader.py
│ │ │ │ └── to-simplify.py
│ │ │ ├── prepare-unkvocab.sh
│ │ │ ├── prepare.sh
│ │ │ ├── prepare.v1/
│ │ │ │ ├── fix-vocab.py
│ │ │ │ ├── gen-char-vocab.py
│ │ │ │ ├── gen-content.py
│ │ │ │ ├── gen-records.py
│ │ │ │ ├── gen-seg.py
│ │ │ │ ├── gen-test-unkvocab.sh
│ │ │ │ ├── gen-test.sh
│ │ │ │ ├── gen-train-unkvocab.sh
│ │ │ │ ├── gen-train.sh
│ │ │ │ ├── gen-valid-unkvocab.sh
│ │ │ │ ├── gen-valid.sh
│ │ │ │ ├── gen-vocab.py
│ │ │ │ ├── gen-vocab.sh
│ │ │ │ ├── merge-emb.py
│ │ │ │ ├── run-char.sh
│ │ │ │ ├── run-unkvocab.sh
│ │ │ │ ├── run.sh
│ │ │ │ ├── text2ids.py
│ │ │ │ └── to-simplify.py
│ │ │ ├── read-records.py
│ │ │ ├── tools/
│ │ │ │ ├── check.py
│ │ │ │ ├── check.sh
│ │ │ │ ├── cp-best-epochs.py
│ │ │ │ ├── ensemble-infer.py
│ │ │ │ ├── ensemble-valid.py
│ │ │ │ ├── ensemble.sh
│ │ │ │ ├── evaluate.py
│ │ │ │ ├── fix-vocab.py
│ │ │ │ ├── gen-char-vocab.py
│ │ │ │ ├── show-best-epochs.py
│ │ │ │ ├── vocab-add-all-char.py
│ │ │ │ └── vocab-add-char.py
│ │ │ ├── torch-train.py
│ │ │ ├── torch_algos/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── baseline/
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── baseline.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── m_reader.py
│ │ │ │ ├── model.py
│ │ │ │ └── rnet.py
│ │ │ ├── train/
│ │ │ │ └── v1/
│ │ │ │ ├── torch.word.mreader.answeremb-split.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.att.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.last.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.maxatt.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.maxmean.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.mean.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.topk.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.topkatt.sh
│ │ │ │ └── torch.word.mreader.nopad.answeremb-split.topkattlast.sh
│ │ │ ├── train.py
│ │ │ └── train.v1/
│ │ │ ├── torch-decay.sh
│ │ │ ├── torch.sh
│ │ │ ├── v1/
│ │ │ │ ├── model2.1layer.keep07.sh
│ │ │ │ ├── qcatt.1layer.keep07.sh
│ │ │ │ ├── qcatt.sfu.1layer.keep07.sh
│ │ │ │ ├── rcontent.1layer.keep05.sh
│ │ │ │ ├── rcontent.1layer.keep07.sh
│ │ │ │ ├── rcontent.1layer.keep10.sh
│ │ │ │ ├── rcontent.emb.finetune.sh
│ │ │ │ ├── rcontent.emb.sh
│ │ │ │ ├── rcontent.keep07.sh
│ │ │ │ ├── rcontent.sh
│ │ │ │ ├── rcontent.split.sh
│ │ │ │ ├── rnet.1layer.keep07.emb.finetune.att.2.sh
│ │ │ │ ├── rnet.1layer.keep07.emb.finetune.att.finetune.type1only.sh
│ │ │ │ ├── rnet.1layer.keep07.emb.finetune.att.sh
│ │ │ │ ├── rnet.1layer.keep07.emb.finetune.att.type1only.sh
│ │ │ │ ├── rnet.1layer.keep07.emb.finetune.att.type1w2.sh
│ │ │ │ ├── rnet.1layer.keep07.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.att.2.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.att.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.att.type1w2.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.att.unkvocab.finetune.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.att.unkvocab.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.f07.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.f09.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.later.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.maxatt.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.nodecay.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.se.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.topk.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.unkvocab.finetune.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.unkvocab.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.nodecay.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.split.sh
│ │ │ │ └── rnet.sfu.2layer.keep07.emb.finetune.sh
│ │ │ ├── v2/
│ │ │ │ ├── rnet.2.sh
│ │ │ │ ├── rnet.decay.sh
│ │ │ │ ├── rnet.decay2.sh
│ │ │ │ ├── rnet.decay3.sh
│ │ │ │ ├── rnet.decay4.sh
│ │ │ │ ├── rnet.decay5.sh
│ │ │ │ ├── rnet.decay6.sh
│ │ │ │ ├── rnet.decay7.sh
│ │ │ │ ├── rnet.keep08.sh
│ │ │ │ ├── rnet.keep09.sh
│ │ │ │ ├── rnet.labelemb.sh
│ │ │ │ ├── rnet.max.sh
│ │ │ │ ├── rnet.sh
│ │ │ │ ├── rnet.split.finetune.type1only.sh
│ │ │ │ ├── rnet.split.sh
│ │ │ │ └── rnet.type0only.sh
│ │ │ ├── v3/
│ │ │ │ ├── bidaf.split.lr0002.sh
│ │ │ │ ├── bidaf.split.lrsmall.sh
│ │ │ │ ├── bidaf.split.sh
│ │ │ │ ├── bow.sh
│ │ │ │ ├── mreader.2.sh
│ │ │ │ ├── mreader.ae.sh
│ │ │ │ ├── mreader.epoch3.sh
│ │ │ │ ├── mreader.lr0002.sh
│ │ │ │ ├── mreader.max.nomask.sh
│ │ │ │ ├── mreader.max.sh
│ │ │ │ ├── mreader.noshare.max.sh
│ │ │ │ ├── mreader.sh
│ │ │ │ ├── mreader.split.sh
│ │ │ │ ├── mreader.v1.sh
│ │ │ │ ├── mreader.v2.sh
│ │ │ │ ├── mreader.v2.split.sh
│ │ │ │ ├── rcontent.3.sh
│ │ │ │ ├── rcontent.4.sh
│ │ │ │ ├── rcontent.5.sh
│ │ │ │ ├── rcontent.sh
│ │ │ │ ├── rnet.2.2.sh
│ │ │ │ ├── rnet.2.dec.sh
│ │ │ │ ├── rnet.2.sh
│ │ │ │ ├── rnet.2layer.sh
│ │ │ │ ├── rnet.3.sh
│ │ │ │ ├── rnet.4.sh
│ │ │ │ ├── rnet.answer.sh
│ │ │ │ ├── rnet.answer.split.batch64.sh
│ │ │ │ ├── rnet.answer.split.sh
│ │ │ │ ├── rnet.answer.split2.sh
│ │ │ │ ├── rnet.answer2.sh
│ │ │ │ ├── rnet.answer3.sh
│ │ │ │ ├── rnet.att2.sh
│ │ │ │ ├── rnet.batch64.sh
│ │ │ │ ├── rnet.gate.sh
│ │ │ │ ├── rnet.hidden200.sh
│ │ │ │ ├── rnet.keep075.sh
│ │ │ │ ├── rnet.labelatt.sh
│ │ │ │ ├── rnet.labelatt10.sh
│ │ │ │ ├── rnet.lr00005.sh
│ │ │ │ ├── rnet.max.sh
│ │ │ │ ├── rnet.nobucket.sh
│ │ │ │ ├── rnet.p2.batch64.sh
│ │ │ │ ├── rnet.p2.sh
│ │ │ │ ├── rnet.relu.sh
│ │ │ │ ├── rnet.sh
│ │ │ │ ├── rnet.split.batch64.sh
│ │ │ │ ├── rnet.split.cq.batch64.sh
│ │ │ │ ├── rnet.split.cq.sh
│ │ │ │ ├── rnet.split.p2.batch64.sh
│ │ │ │ ├── rnet.split.p2.sh
│ │ │ │ ├── rnet.split.reverse.sh
│ │ │ │ ├── rnet.split.sh
│ │ │ │ ├── rnet.split.test.sh
│ │ │ │ ├── rnet.split.type1c10.batch64.sh
│ │ │ │ ├── rnet.split.type1c2.batch64.sh
│ │ │ │ ├── rnet.split.type1c2.sh
│ │ │ │ ├── rnet.split.type1c5.batch64.sh
│ │ │ │ ├── rnet.split.type1w2.batch64.sh
│ │ │ │ ├── rnet.split.type1w2.sh
│ │ │ │ ├── rnet.tanh.sh
│ │ │ │ ├── torch-keep07.sh
│ │ │ │ ├── torch-mreader.emb.lr00005.sh
│ │ │ │ ├── torch-mreader.emb.lr0001.sh
│ │ │ │ ├── torch-mreader.emb.lr0001.split.sh
│ │ │ │ ├── torch-mreader.emb.lr0003.sh
│ │ │ │ ├── torch-mreader.emb.lr0004.sh
│ │ │ │ ├── torch-mreader.emb.sh
│ │ │ │ ├── torch-mreader.emb.split.sh
│ │ │ │ ├── torch-mreader.lr0001.sh
│ │ │ │ ├── torch-mreader.sh
│ │ │ │ ├── torch-mreader.v1.emb.sh
│ │ │ │ ├── torch-mwan.emb.2.sh
│ │ │ │ ├── torch-mwan.emb.batch64.sh
│ │ │ │ ├── torch-mwan.emb.sh
│ │ │ │ ├── torch-mwan.lr0001.sh
│ │ │ │ ├── torch-mwan.sh
│ │ │ │ ├── torch-rnet.2.sh
│ │ │ │ ├── torch-rnet.lr0001.emb.sh
│ │ │ │ ├── torch-rnet.lr0001.sh
│ │ │ │ ├── torch-rnet.sh
│ │ │ │ └── torch.sh
│ │ │ ├── v4/
│ │ │ │ ├── bow-emb.sh
│ │ │ │ ├── bow-nomask-emb.sh
│ │ │ │ ├── bow-nomask.sh
│ │ │ │ ├── bow.sh
│ │ │ │ ├── gru-emb-att.sh
│ │ │ │ ├── gru-emb-latt.sh
│ │ │ │ ├── gru-emb-nomask.2.sh
│ │ │ │ ├── gru-emb-nomask.sh
│ │ │ │ ├── gru-emb-simpledrop-bw.sh
│ │ │ │ ├── gru-emb-simpledrop.sh
│ │ │ │ ├── gru-emb-top2.sh
│ │ │ │ ├── gru-emb-top2mean.sh
│ │ │ │ ├── gru-emb-top2wmean.sh
│ │ │ │ ├── gru-emb-topkatt2.sh
│ │ │ │ ├── gru-emb-topkatt3.sh
│ │ │ │ ├── gru-emb.sh
│ │ │ │ ├── gru.sh
│ │ │ │ ├── mreader-v4-nopad.sh
│ │ │ │ ├── mreader-v4-split-nopad.sh
│ │ │ │ ├── mreader-v4-split.sh
│ │ │ │ ├── mreader-v4.sh
│ │ │ │ ├── mreader.max.nopad.sh
│ │ │ │ ├── mreader.max.sh
│ │ │ │ ├── mreader.max.split.nopad.sh
│ │ │ │ ├── mreader.max.split.sh
│ │ │ │ ├── mreader.nopad.sh
│ │ │ │ ├── mreader.sh
│ │ │ │ ├── mreader.split.nopad.sh
│ │ │ │ ├── mreader.split.sh
│ │ │ │ ├── torch-bow-emb.sh
│ │ │ │ ├── torch-bow.sh
│ │ │ │ ├── torch-gru-emb-att-nomask.sh
│ │ │ │ ├── torch-gru-emb-att-pad-typeemb.sh
│ │ │ │ ├── torch-gru-emb-att-pad.sh
│ │ │ │ ├── torch-gru-emb-att-type.sh
│ │ │ │ ├── torch-gru-emb-att-typeemb.sh
│ │ │ │ ├── torch-gru-emb-att-typernn.sh
│ │ │ │ ├── torch-gru-emb-att.sh
│ │ │ │ ├── torch-gru-emb-keep08.sh
│ │ │ │ ├── torch-gru-emb-latt.sh
│ │ │ │ ├── torch-gru-emb-mask.2.sh
│ │ │ │ ├── torch-gru-emb-mask.sh
│ │ │ │ ├── torch-gru-emb-nodrop.sh
│ │ │ │ ├── torch-gru-emb-nomask.2.sh
│ │ │ │ ├── torch-gru-emb-nomask.sh
│ │ │ │ ├── torch-gru-emb-topk2-nomask.sh
│ │ │ │ ├── torch-gru-emb-topk2.sh
│ │ │ │ ├── torch-gru-emb-topk3-nomask.sh
│ │ │ │ ├── torch-gru-emb-topk3.sh
│ │ │ │ ├── torch-gru-emb.sh
│ │ │ │ ├── torch-gru.2.sh
│ │ │ │ ├── torch-gru.sh
│ │ │ │ ├── torch-mreader-1hop-nopad.sh
│ │ │ │ ├── torch-mreader-1hop-split-nopad.sh
│ │ │ │ ├── torch-mreader-emb-att-nopad-split.sh
│ │ │ │ ├── torch-mreader-emb-att-nopad.sh
│ │ │ │ ├── torch-mreader-emb-att-pad.sh
│ │ │ │ ├── torch-mreader-emb-att-typeemb-nopad-split.sh
│ │ │ │ ├── torch-mreader-emb-att-typeemb-nopad.sh
│ │ │ │ ├── torch-mreader-emb-att-typeemb-pad.sh
│ │ │ │ ├── torch-mreader-emb-att-typeemb.sh
│ │ │ │ ├── torch-mreader-emb-att.sh
│ │ │ │ ├── torch-mreader-emb-nopad-split.sh
│ │ │ │ ├── torch-mreader-emb-nopad-typeemb-split.sh
│ │ │ │ ├── torch-mreader-emb-nopad-typeemb.sh
│ │ │ │ ├── torch-mreader-emb-nopad.sh
│ │ │ │ ├── torch-mreader-emb-pad-typeemb.sh
│ │ │ │ ├── torch-mreader-emb-pad.sh
│ │ │ │ ├── torch-mreader-emb-typeemb.sh
│ │ │ │ ├── torch-mreader-emb.sh
│ │ │ │ ├── torch-mreader-hop3-nopad.sh
│ │ │ │ ├── torch-mreader-nopad.sh
│ │ │ │ ├── torch-mreader-poolnopad.sh
│ │ │ │ ├── torch-mreader-split-hop3-nopad.sh
│ │ │ │ ├── torch-mreader-split-nopad.sh
│ │ │ │ ├── torch-mreader-split-poolnopad.sh
│ │ │ │ ├── torch-mreader-typeemb-hop3-nopad.sh
│ │ │ │ ├── torch-mreader-typeemb-nopad.sh
│ │ │ │ ├── torch-mreader-v3-decay09.sh
│ │ │ │ ├── torch-mreader-v3-nopad.sh
│ │ │ │ ├── torch-mreader-v3-poolnopad.sh
│ │ │ │ ├── torch-mreader-v3-split-decay09.sh
│ │ │ │ ├── torch-mreader-v3-split-nopad.sh
│ │ │ │ ├── torch-mreader-v3-split.sh
│ │ │ │ └── torch-mreader-v3.sh
│ │ │ └── v5/
│ │ │ ├── torch-mreader-split-answeremb-nopad.sh
│ │ │ ├── torch-mreader-split-nopad-ftwiki.sh
│ │ │ └── torch-mreader-split-nopad.sh
│ │ └── sentiment/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── algos/
│ │ │ ├── __init__.py
│ │ │ ├── config.py
│ │ │ ├── loss.py
│ │ │ ├── model.py
│ │ │ └── weights.py
│ │ ├── analysis/
│ │ │ ├── analysis.ipynb
│ │ │ ├── analyze.py
│ │ │ ├── beam-search.py
│ │ │ ├── beam_f.py
│ │ │ ├── beam_f_utils.py
│ │ │ ├── class-info.py
│ │ │ ├── class-info.train.csv
│ │ │ ├── class-info.valid.csv
│ │ │ ├── correlations-filter.py
│ │ │ ├── correlations.py
│ │ │ ├── diff.py
│ │ │ └── gen-class-weights.py
│ │ ├── dataset.py
│ │ ├── ensemble/
│ │ │ ├── README.md
│ │ │ ├── calibrate-cv.py
│ │ │ ├── ensemble-cv-parallel.py
│ │ │ ├── ensemble-cv-v1.py
│ │ │ ├── ensemble-cv.py
│ │ │ ├── ensemble-hillclimb.py
│ │ │ ├── ensemble-infer.py
│ │ │ ├── ensemble-v1.py
│ │ │ ├── ensemble.py
│ │ │ ├── evaluate.py
│ │ │ ├── gen-train.py
│ │ │ ├── hillclimb-ensembling.py
│ │ │ ├── lgb-adjust.py
│ │ │ ├── lgb-cv.py
│ │ │ └── lr-cv.py
│ │ ├── evaluate.py
│ │ ├── infer/
│ │ │ └── v11/
│ │ │ ├── README.md
│ │ │ ├── tf.char.rnet.nolatt.sh
│ │ │ ├── tf.char.rnet.sh
│ │ │ ├── tf.char.transformer.bert.cutfront.sh
│ │ │ ├── tf.char.transformer.bert.finetune.2.sh
│ │ │ ├── tf.char.transformer.bert.finetune.3.sh
│ │ │ ├── tf.char.transformer.bert.finetune.sh
│ │ │ ├── tf.char.transformer.bert.finetune4.sh
│ │ │ ├── tf.char.transformer.bert.rnn.finetune.sh
│ │ │ ├── tf.char.transformer.bert.sh
│ │ │ ├── tf.char.transformer.bert.topkatt.cutfront.sh
│ │ │ ├── tf.char.transformer.bert.topkatt.finetune.sh
│ │ │ ├── tf.char.transformer.bert.topkatt.sh
│ │ │ ├── tf.char.transformer.nbert.finetune.sh
│ │ │ ├── tf.mix.rnet.nolatt.sh
│ │ │ ├── tf.mix.rnet.sh
│ │ │ ├── tf.word.mreader.lm.2layer.sh
│ │ │ ├── tf.word.mreader.lm.2layer.unkaug.sh
│ │ │ ├── tf.word.mreader.lm.sh
│ │ │ ├── tf.word.mreader.lm.unkaug.sh
│ │ │ ├── tf.word.rnet.hidden600.sh
│ │ │ ├── tf.word.rnet.lm.sh
│ │ │ ├── tf.word.rnet.lm.unkaug.sh
│ │ │ ├── tf.word.rnet.nolatt.hidden600.sh
│ │ │ ├── tf.word.rnet.nolatt.sh
│ │ │ ├── tf.word.rnet.sh
│ │ │ ├── tf.word.rnet.unkaug.sh
│ │ │ ├── tf.word.rnetv2.lm.sh
│ │ │ ├── tf.word.rnetv2.lm.unkaug.sh
│ │ │ ├── torch.char.mreader.lm.nolatt.sh
│ │ │ ├── torch.char.mreader.lm.sh
│ │ │ ├── torch.char.mreader.nolatt.sh
│ │ │ ├── torch.char.mreader.sh
│ │ │ ├── torch.mix.mreader.lm.nolatt.sh
│ │ │ ├── torch.mix.mreader.lm.sh
│ │ │ ├── torch.mix.mreader.sh
│ │ │ ├── torch.word.mreader.lm.sh
│ │ │ ├── torch.word.mreader.nopad.bertopt.nolatt.unkaug.hidden400.sh
│ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.baike.sh
│ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden300.sh
│ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.sh
│ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.short.sh
│ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden768.sh
│ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.sh
│ │ │ ├── torch.word.mreader.nopad.lm.bertopt.unkaug.sh
│ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.hidden300.sh
│ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.hidden400.sh
│ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.hidden768.sh
│ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.sh
│ │ │ ├── torch.word.mreader.nopad.lm.sh
│ │ │ └── torch.word.mreader.nopad.lm.unkaug.sh
│ │ ├── infer.py
│ │ ├── lm-train.py
│ │ ├── lm_dataset.py
│ │ ├── prepare/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── config
│ │ │ ├── config.py
│ │ │ ├── embeddings/
│ │ │ │ ├── fasttext/
│ │ │ │ │ └── run.sh
│ │ │ │ └── glove/
│ │ │ │ └── run.sh
│ │ │ ├── filter.py
│ │ │ ├── find-chars.py
│ │ │ ├── fix-vocab.py
│ │ │ ├── gen-canyin.py
│ │ │ ├── gen-canyin.sh
│ │ │ ├── gen-char-vocab.py
│ │ │ ├── gen-content.py
│ │ │ ├── gen-dianping.py
│ │ │ ├── gen-dianping.sh
│ │ │ ├── gen-lm-records.py
│ │ │ ├── gen-mix-vocab.py
│ │ │ ├── gen-records.py
│ │ │ ├── gen-test.sh
│ │ │ ├── gen-train.sh
│ │ │ ├── gen-trans.py
│ │ │ ├── gen-trans.sh
│ │ │ ├── gen-valid.sh
│ │ │ ├── gen-vocab-v2.sh
│ │ │ ├── gen-vocab.py
│ │ │ ├── gen-vocab.sh
│ │ │ ├── merge-emb.py
│ │ │ ├── merge-vocabs.py
│ │ │ ├── pre-mix-seg-v1.py
│ │ │ ├── pre-mix-seg.py
│ │ │ ├── pre-seg-bert.py
│ │ │ ├── pre-seg.py
│ │ │ ├── run-char-bert.sh
│ │ │ ├── run-char-ft.sh
│ │ │ ├── run-char-glove.sh
│ │ │ ├── run-char-nbert.sh
│ │ │ ├── run-char.sh
│ │ │ ├── run-lm-char-len256.sh
│ │ │ ├── run-lm-char.sh
│ │ │ ├── run-lm-jieba-mix-len256.sh
│ │ │ ├── run-lm-mix.sh
│ │ │ ├── run-lm-word-baike-len20.sh
│ │ │ ├── run-lm-word-baike.sh
│ │ │ ├── run-lm-word-bseg-len40.sh
│ │ │ ├── run-lm-word-bseg-ner-len40.sh
│ │ │ ├── run-lm-word-bseg-ner2-len40.sh
│ │ │ ├── run-lm-word-jieba-tx-len40.sh
│ │ │ ├── run-lm-word-len40.sh
│ │ │ ├── run-lm-word-sp10w-len40.sh
│ │ │ ├── run-lm-word-sp10w.sh
│ │ │ ├── run-lm-word-sp1w-baike.sh
│ │ │ ├── run-lm-word-sp1w-len256.sh
│ │ │ ├── run-lm-word-sp1w.sh
│ │ │ ├── run-lm-word-sp20w-len40.sh
│ │ │ ├── run-lm-word-sp20w.sh
│ │ │ ├── run-lm-word.sh
│ │ │ ├── run-mix-bseg-ft.sh
│ │ │ ├── run-mix-bseg-glove-10epoch.sh
│ │ │ ├── run-mix-bseg-glove-40epoch.sh
│ │ │ ├── run-mix-bseg-glove.sh
│ │ │ ├── run-mix-bseg-tx.sh
│ │ │ ├── run-mix-glove.sh
│ │ │ ├── run-mix-jieba-ft.sh
│ │ │ ├── run-mix-tx.sh
│ │ │ ├── run-nchar-ft.sh
│ │ │ ├── run-noemb-aug.sh
│ │ │ ├── run-noemb-char-bert.sh
│ │ │ ├── run-noemb-char-nbert.sh
│ │ │ ├── run-noemb-char.sh
│ │ │ ├── run-noemb-mix-charaug.sh
│ │ │ ├── run-noemb-mix.sh
│ │ │ ├── run-noemb-pl.sh
│ │ │ ├── run-noemb.sh
│ │ │ ├── run-word-bseg-ft.sh
│ │ │ ├── run-word-bseg-glove.sh
│ │ │ ├── run-word-bseg-ner-ft-knowldege.sh
│ │ │ ├── run-word-bseg-ner-ft.sh
│ │ │ ├── run-word-bseg-ner-ft2.sh
│ │ │ ├── run-word-bseg-ner-ft3.sh
│ │ │ ├── run-word-bseg-ner-glove-10epoch.sh
│ │ │ ├── run-word-bseg-ner-glove-40epoch.sh
│ │ │ ├── run-word-bseg-ner-glove.sh
│ │ │ ├── run-word-bseg-ner-jiebapre-ft.sh
│ │ │ ├── run-word-bseg-subner-ft.sh
│ │ │ ├── run-word-bseg-subner-jiebapre-ft.sh
│ │ │ ├── run-word-jieba-ft.sh
│ │ │ ├── run-word-jieba-glove.sh
│ │ │ ├── run-word-jieba-pos-ft.sh
│ │ │ ├── run-word-jieba-pos-glove.sh
│ │ │ ├── run-word-jieba-tx.sh
│ │ │ ├── run-word-sp10w-ft.sh
│ │ │ ├── run-word-sp10w-glove.sh
│ │ │ ├── run-word-sp10w-tx.sh
│ │ │ ├── run-word-sp1w-ft.sh
│ │ │ ├── run-word-sp1w-glove.sh
│ │ │ ├── run-word-sp1w-tx.sh
│ │ │ ├── run-word-sp20w-ft-pl.sh
│ │ │ ├── run-word-sp20w-ft.sh
│ │ │ ├── run-word-sp20w-glove.sh
│ │ │ ├── run-word-stanford-ft.sh
│ │ │ ├── run-word-stanford-ft2.sh
│ │ │ ├── run-word-stanford-ft3.sh
│ │ │ ├── run-word-stanford-glove.sh
│ │ │ ├── run-word-stanford-tx-ftchar.sh
│ │ │ ├── run-word-stanford-tx-glovechar.sh
│ │ │ ├── run-word-tx-ftchar.sh
│ │ │ ├── run-word-tx-glovechar.sh
│ │ │ ├── text2ids.py
│ │ │ ├── tmp
│ │ │ ├── to-chars.py
│ │ │ ├── to-norm.py
│ │ │ └── to-simplify.py
│ │ ├── prepare.test/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── config
│ │ │ ├── config.py
│ │ │ ├── embeddings/
│ │ │ │ ├── fasttext/
│ │ │ │ │ └── run.sh
│ │ │ │ └── glove/
│ │ │ │ └── run.sh
│ │ │ ├── filter.py
│ │ │ ├── find-chars.py
│ │ │ ├── fix-vocab.py
│ │ │ ├── gen-canyin.py
│ │ │ ├── gen-canyin.sh
│ │ │ ├── gen-char-vocab.py
│ │ │ ├── gen-content.py
│ │ │ ├── gen-dianping.py
│ │ │ ├── gen-dianping.sh
│ │ │ ├── gen-lm-records.py
│ │ │ ├── gen-mix-vocab.py
│ │ │ ├── gen-records.py
│ │ │ ├── gen-test.sh
│ │ │ ├── gen-train.sh
│ │ │ ├── gen-trans.py
│ │ │ ├── gen-trans.sh
│ │ │ ├── gen-valid.sh
│ │ │ ├── gen-vocab-v2.sh
│ │ │ ├── gen-vocab.py
│ │ │ ├── gen-vocab.sh
│ │ │ ├── merge-emb.py
│ │ │ ├── merge-vocabs.py
│ │ │ ├── pre-mix-seg-v1.py
│ │ │ ├── pre-mix-seg.py
│ │ │ ├── pre-seg-bert.py
│ │ │ ├── pre-seg.py
│ │ │ ├── run-char-bert.sh
│ │ │ ├── run-char-ft.sh
│ │ │ ├── run-char-glove.sh
│ │ │ ├── run-char.sh
│ │ │ ├── run-lm-char-len256.sh
│ │ │ ├── run-lm-char.sh
│ │ │ ├── run-lm-jieba-mix-len256.sh
│ │ │ ├── run-lm-mix.sh
│ │ │ ├── run-lm-word-baike-len20.sh
│ │ │ ├── run-lm-word-baike.sh
│ │ │ ├── run-lm-word-bseg-len40.sh
│ │ │ ├── run-lm-word-bseg-ner-len40.sh
│ │ │ ├── run-lm-word-bseg-ner2-len40.sh
│ │ │ ├── run-lm-word-jieba-tx-len40.sh
│ │ │ ├── run-lm-word-len40.sh
│ │ │ ├── run-lm-word-sp10w-len40.sh
│ │ │ ├── run-lm-word-sp10w.sh
│ │ │ ├── run-lm-word-sp1w-baike.sh
│ │ │ ├── run-lm-word-sp1w-len256.sh
│ │ │ ├── run-lm-word-sp1w.sh
│ │ │ ├── run-lm-word.sh
│ │ │ ├── run-mix-bseg-ft.sh
│ │ │ ├── run-mix-bseg-glove-10epoch.sh
│ │ │ ├── run-mix-bseg-glove-40epoch.sh
│ │ │ ├── run-mix-bseg-glove.sh
│ │ │ ├── run-mix-bseg-tx.sh
│ │ │ ├── run-mix-glove.sh
│ │ │ ├── run-mix-jieba-ft.sh
│ │ │ ├── run-mix-tx.sh
│ │ │ ├── run-noemb-aug.sh
│ │ │ ├── run-noemb-char-bert.sh
│ │ │ ├── run-noemb-char.sh
│ │ │ ├── run-noemb-mix-charaug.sh
│ │ │ ├── run-noemb-mix.sh
│ │ │ ├── run-noemb-wordonly.sh
│ │ │ ├── run-noemb.sh
│ │ │ ├── run-word-bseg-ft.sh
│ │ │ ├── run-word-bseg-glove.sh
│ │ │ ├── run-word-bseg-ner-ft-knowldege.sh
│ │ │ ├── run-word-bseg-ner-ft.sh
│ │ │ ├── run-word-bseg-ner-ft2.sh
│ │ │ ├── run-word-bseg-ner-ft3.sh
│ │ │ ├── run-word-bseg-ner-glove-10epoch.sh
│ │ │ ├── run-word-bseg-ner-glove-40epoch.sh
│ │ │ ├── run-word-bseg-ner-glove.sh
│ │ │ ├── run-word-bseg-ner-jiebapre-ft.sh
│ │ │ ├── run-word-bseg-subner-ft.sh
│ │ │ ├── run-word-bseg-subner-jiebapre-ft.sh
│ │ │ ├── run-word-jieba-ft-short.sh
│ │ │ ├── run-word-jieba-ft.sh
│ │ │ ├── run-word-jieba-glove.sh
│ │ │ ├── run-word-jieba-pos-ft.sh
│ │ │ ├── run-word-jieba-pos-glove.sh
│ │ │ ├── run-word-jieba-tx.sh
│ │ │ ├── run-word-sp10w-ft.sh
│ │ │ ├── run-word-sp10w-glove.sh
│ │ │ ├── run-word-sp10w-tx.sh
│ │ │ ├── run-word-sp1w-ft.sh
│ │ │ ├── run-word-sp1w-glove.sh
│ │ │ ├── run-word-sp1w-tx.sh
│ │ │ ├── run-word-sp20w-ft.sh
│ │ │ ├── run-word-sp20w-glove.sh
│ │ │ ├── run-word-stanford-ft.sh
│ │ │ ├── run-word-stanford-ft2.sh
│ │ │ ├── run-word-stanford-ft3.sh
│ │ │ ├── run-word-stanford-glove.sh
│ │ │ ├── run-word-stanford-tx-ftchar.sh
│ │ │ ├── run-word-stanford-tx-glovechar.sh
│ │ │ ├── run-word-tx-ftchar.sh
│ │ │ ├── run-word-tx-glovechar.sh
│ │ │ ├── run.sh
│ │ │ ├── text2ids.py
│ │ │ ├── tmp
│ │ │ ├── to-chars.py
│ │ │ ├── to-norm.py
│ │ │ └── to-simplify.py
│ │ ├── prepare.testb/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── config
│ │ │ ├── config.py
│ │ │ ├── embeddings/
│ │ │ │ ├── fasttext/
│ │ │ │ │ └── run.sh
│ │ │ │ └── glove/
│ │ │ │ └── run.sh
│ │ │ ├── filter.py
│ │ │ ├── find-chars.py
│ │ │ ├── fix-vocab.py
│ │ │ ├── gen-canyin.py
│ │ │ ├── gen-canyin.sh
│ │ │ ├── gen-char-vocab.py
│ │ │ ├── gen-content.py
│ │ │ ├── gen-dianping.py
│ │ │ ├── gen-dianping.sh
│ │ │ ├── gen-lm-records.py
│ │ │ ├── gen-mix-vocab.py
│ │ │ ├── gen-records.py
│ │ │ ├── gen-test.sh
│ │ │ ├── gen-train.sh
│ │ │ ├── gen-trans.py
│ │ │ ├── gen-trans.sh
│ │ │ ├── gen-valid.sh
│ │ │ ├── gen-vocab-v2.sh
│ │ │ ├── gen-vocab.py
│ │ │ ├── gen-vocab.sh
│ │ │ ├── merge-emb.py
│ │ │ ├── merge-vocabs.py
│ │ │ ├── pre-mix-seg-v1.py
│ │ │ ├── pre-mix-seg.py
│ │ │ ├── pre-seg-bert.py
│ │ │ ├── pre-seg.py
│ │ │ ├── run-char-bert.sh
│ │ │ ├── run-char-ft.sh
│ │ │ ├── run-char-glove.sh
│ │ │ ├── run-char.sh
│ │ │ ├── run-lm-char-len256.sh
│ │ │ ├── run-lm-char.sh
│ │ │ ├── run-lm-jieba-mix-len256.sh
│ │ │ ├── run-lm-mix.sh
│ │ │ ├── run-lm-word-baike-len20.sh
│ │ │ ├── run-lm-word-baike.sh
│ │ │ ├── run-lm-word-bseg-len40.sh
│ │ │ ├── run-lm-word-bseg-ner-len40.sh
│ │ │ ├── run-lm-word-bseg-ner2-len40.sh
│ │ │ ├── run-lm-word-jieba-tx-len40.sh
│ │ │ ├── run-lm-word-len40.sh
│ │ │ ├── run-lm-word-sp10w-len40.sh
│ │ │ ├── run-lm-word-sp10w.sh
│ │ │ ├── run-lm-word-sp1w-baike.sh
│ │ │ ├── run-lm-word-sp1w-len256.sh
│ │ │ ├── run-lm-word-sp1w.sh
│ │ │ ├── run-lm-word.sh
│ │ │ ├── run-mix-bseg-ft.sh
│ │ │ ├── run-mix-bseg-glove-10epoch.sh
│ │ │ ├── run-mix-bseg-glove-40epoch.sh
│ │ │ ├── run-mix-bseg-glove.sh
│ │ │ ├── run-mix-bseg-tx.sh
│ │ │ ├── run-mix-glove.sh
│ │ │ ├── run-mix-jieba-ft.sh
│ │ │ ├── run-mix-tx.sh
│ │ │ ├── run-noemb-aug.sh
│ │ │ ├── run-noemb-char-bert.sh
│ │ │ ├── run-noemb-char.sh
│ │ │ ├── run-noemb-mix-charaug.sh
│ │ │ ├── run-noemb-mix.sh
│ │ │ ├── run-noemb-wordonly.sh
│ │ │ ├── run-noemb.sh
│ │ │ ├── run-word-bseg-ft.sh
│ │ │ ├── run-word-bseg-glove.sh
│ │ │ ├── run-word-bseg-ner-ft-knowldege.sh
│ │ │ ├── run-word-bseg-ner-ft.sh
│ │ │ ├── run-word-bseg-ner-ft2.sh
│ │ │ ├── run-word-bseg-ner-ft3.sh
│ │ │ ├── run-word-bseg-ner-glove-10epoch.sh
│ │ │ ├── run-word-bseg-ner-glove-40epoch.sh
│ │ │ ├── run-word-bseg-ner-glove.sh
│ │ │ ├── run-word-bseg-ner-jiebapre-ft.sh
│ │ │ ├── run-word-bseg-subner-ft.sh
│ │ │ ├── run-word-bseg-subner-jiebapre-ft.sh
│ │ │ ├── run-word-jieba-ft-short.sh
│ │ │ ├── run-word-jieba-ft.sh
│ │ │ ├── run-word-jieba-glove.sh
│ │ │ ├── run-word-jieba-pos-ft.sh
│ │ │ ├── run-word-jieba-pos-glove.sh
│ │ │ ├── run-word-jieba-tx.sh
│ │ │ ├── run-word-sp10w-ft.sh
│ │ │ ├── run-word-sp10w-glove.sh
│ │ │ ├── run-word-sp10w-tx.sh
│ │ │ ├── run-word-sp1w-ft.sh
│ │ │ ├── run-word-sp1w-glove.sh
│ │ │ ├── run-word-sp1w-tx.sh
│ │ │ ├── run-word-sp20w-ft.sh
│ │ │ ├── run-word-sp20w-glove.sh
│ │ │ ├── run-word-stanford-ft.sh
│ │ │ ├── run-word-stanford-ft2.sh
│ │ │ ├── run-word-stanford-ft3.sh
│ │ │ ├── run-word-stanford-glove.sh
│ │ │ ├── run-word-stanford-tx-ftchar.sh
│ │ │ ├── run-word-stanford-tx-glovechar.sh
│ │ │ ├── run-word-tx-ftchar.sh
│ │ │ ├── run-word-tx-glovechar.sh
│ │ │ ├── run.sh
│ │ │ ├── text2ids.py
│ │ │ ├── tmp
│ │ │ ├── to-chars.py
│ │ │ ├── to-norm.py
│ │ │ └── to-simplify.py
│ │ ├── prepare.v1/
│ │ │ ├── filter.py
│ │ │ ├── find-chars.py
│ │ │ ├── fix-vocab.py
│ │ │ ├── gen-canyin.py
│ │ │ ├── gen-canyin.sh
│ │ │ ├── gen-char-seg-canyin.py
│ │ │ ├── gen-char-seg-dianping.py
│ │ │ ├── gen-char-seg-train.py
│ │ │ ├── gen-char-seg.py
│ │ │ ├── gen-char-vocab.py
│ │ │ ├── gen-content.py
│ │ │ ├── gen-dianping.py
│ │ │ ├── gen-dianping.sh
│ │ │ ├── gen-mix-seg-canyin.py
│ │ │ ├── gen-mix-seg-dianping.py
│ │ │ ├── gen-records.py
│ │ │ ├── gen-seg-canyin.py
│ │ │ ├── gen-seg-dianping.py
│ │ │ ├── gen-seg-train.py
│ │ │ ├── gen-test.sh
│ │ │ ├── gen-train.sh
│ │ │ ├── gen-trans.py
│ │ │ ├── gen-trans.sh
│ │ │ ├── gen-valid.sh
│ │ │ ├── gen-vocab.py
│ │ │ ├── gen-vocab.sh
│ │ │ ├── merge-emb.py
│ │ │ ├── run-char.sh
│ │ │ ├── run-noemb.sh
│ │ │ ├── run.sh
│ │ │ └── text2ids.py
│ │ ├── prepare.v2/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── config.py
│ │ │ ├── embeddings/
│ │ │ │ ├── fasttext/
│ │ │ │ │ └── run.sh
│ │ │ │ └── glove/
│ │ │ │ └── run.sh
│ │ │ ├── filter.py
│ │ │ ├── find-chars.py
│ │ │ ├── fix-vocab.py
│ │ │ ├── gen-canyin.py
│ │ │ ├── gen-canyin.sh
│ │ │ ├── gen-char-seg-canyin.py
│ │ │ ├── gen-char-seg-dianping.py
│ │ │ ├── gen-char-seg-train.py
│ │ │ ├── gen-char-seg.py
│ │ │ ├── gen-char-vocab.py
│ │ │ ├── gen-content.py
│ │ │ ├── gen-dianping.py
│ │ │ ├── gen-dianping.sh
│ │ │ ├── gen-mix-seg-canyin.py
│ │ │ ├── gen-mix-seg-dianping.py
│ │ │ ├── gen-mix-seg-train.py
│ │ │ ├── gen-records.py
│ │ │ ├── gen-seg-canyin.py
│ │ │ ├── gen-seg-dianping.py
│ │ │ ├── gen-seg-train.py
│ │ │ ├── gen-test.sh
│ │ │ ├── gen-train.sh
│ │ │ ├── gen-trans.py
│ │ │ ├── gen-trans.sh
│ │ │ ├── gen-valid.sh
│ │ │ ├── gen-vocab-bseg-basic-v2.sh
│ │ │ ├── gen-vocab-bseg-basic.sh
│ │ │ ├── gen-vocab-bseg-phrase-v2.sh
│ │ │ ├── gen-vocab-bseg-phrase.sh
│ │ │ ├── gen-vocab-v2.sh
│ │ │ ├── gen-vocab.py
│ │ │ ├── gen-vocab.sh
│ │ │ ├── merge-emb.py
│ │ │ ├── run-char.sh
│ │ │ ├── run-noemb-bseg-basic-v2.sh
│ │ │ ├── run-noemb-bseg-basic.sh
│ │ │ ├── run-noemb-v2.sh
│ │ │ ├── run-noemb-v3.sh
│ │ │ ├── run-noemb.sh
│ │ │ ├── run.sh
│ │ │ └── text2ids.py
│ │ ├── read-records.py
│ │ ├── scripts/
│ │ │ ├── cp.py
│ │ │ ├── example.sh
│ │ │ ├── gen-infer.py
│ │ │ ├── infer-example.sh
│ │ │ ├── infer.py
│ │ │ ├── infer.sh_0
│ │ │ ├── infer.sh_1
│ │ │ ├── infer.sh_2
│ │ │ ├── infer.sh_3
│ │ │ ├── prepare-char.sh
│ │ │ ├── prepare-noemb.sh
│ │ │ └── prepare.sh
│ │ ├── tools/
│ │ │ ├── check-emb.py
│ │ │ ├── cp-best-epochs.py
│ │ │ ├── cp-best-epochs2.py
│ │ │ ├── evaluate.py
│ │ │ ├── find-best-epoch.py
│ │ │ ├── fix-emb.py
│ │ │ ├── lcp-loop.py
│ │ │ ├── lcp.py
│ │ │ ├── rename-finetune.sh
│ │ │ ├── rename-variables-finetune.py
│ │ │ ├── seg2corpus.py
│ │ │ ├── show-best-epochs.py
│ │ │ ├── show-metrics.py
│ │ │ └── slim.py
│ │ ├── torch-infer.py
│ │ ├── torch-lm-train.py
│ │ ├── torch-sim.py
│ │ ├── torch-train.py
│ │ ├── torch_algos/
│ │ │ ├── __init__.py
│ │ │ ├── loss.py
│ │ │ └── model.py
│ │ ├── train/
│ │ │ ├── README.md
│ │ │ ├── v1/
│ │ │ │ ├── bow.sh
│ │ │ │ ├── gru.char.bucket.sh
│ │ │ │ ├── gru.char.pos.ner.stanford.sh
│ │ │ │ ├── gru.char.pos.sh
│ │ │ │ ├── gru.char.pos.stanford.2.sh
│ │ │ │ ├── gru.char.pos.stanford.sh
│ │ │ │ ├── gru.char.sh
│ │ │ │ ├── gru.char.stanford.sh
│ │ │ │ ├── gru.emb.sh
│ │ │ │ └── gru.sh
│ │ │ ├── v10/
│ │ │ │ ├── README.md
│ │ │ │ ├── tf.char.bow.sh
│ │ │ │ ├── tf.char.bow.unkaug.sh
│ │ │ │ ├── tf.char.rnet.hidden300.sh
│ │ │ │ ├── tf.char.rnet.hidden400.sh
│ │ │ │ ├── tf.char.rnet.sh
│ │ │ │ ├── tf.char.transformer.2.sh
│ │ │ │ ├── tf.char.transformer.bert.cutfront.sh
│ │ │ │ ├── tf.char.transformer.bert.finetune.sh
│ │ │ │ ├── tf.char.transformer.bert.rnn.finetune.sh
│ │ │ │ ├── tf.char.transformer.bert.rnn.sh
│ │ │ │ ├── tf.char.transformer.bert.sh
│ │ │ │ ├── tf.char.transformer.sh
│ │ │ │ ├── tf.mix.rnet.hidden400.sh
│ │ │ │ ├── tf.word.bow.sh
│ │ │ │ ├── tf.word.lm.rnet.rand.sh
│ │ │ │ ├── tf.word.lm.rnet.sh
│ │ │ │ ├── tf.word.mreader.hidden300.sh
│ │ │ │ ├── tf.word.mreader.hidden400.sh
│ │ │ │ ├── tf.word.mreader.sh
│ │ │ │ ├── tf.word.rnet.3layer.sh
│ │ │ │ ├── tf.word.rnet.hidden300.3layer.sh
│ │ │ │ ├── tf.word.rnet.hidden300.sh
│ │ │ │ ├── tf.word.rnet.hidden400.decay6.sh
│ │ │ │ ├── tf.word.rnet.hidden400.lm.2.sh
│ │ │ │ ├── tf.word.rnet.hidden400.lm.sh
│ │ │ │ ├── tf.word.rnet.hidden400.lm.unkaug.sh
│ │ │ │ ├── tf.word.rnet.hidden400.sh
│ │ │ │ ├── tf.word.rnet.hidden600.sh
│ │ │ │ ├── tf.word.rnet.ner.sh
│ │ │ │ ├── tf.word.rnet.nolatt.hidden400.sh
│ │ │ │ ├── tf.word.rnet.nolatt.hidden600.sh
│ │ │ │ ├── tf.word.rnet.nolatt.sh
│ │ │ │ ├── tf.word.rnet.sh
│ │ │ │ ├── tf.word.rnet.unkaug.sh
│ │ │ │ ├── tf.word.rnetv2.hidden400.sh
│ │ │ │ ├── torch.char.mreader.nopad.bertopt.sh
│ │ │ │ ├── torch.char.mreader.nopad.hidden300.sh
│ │ │ │ ├── torch.char.mreader.nopad.hidden600.sh
│ │ │ │ ├── torch.char.mreader.nopad.lm.sh
│ │ │ │ ├── torch.char.mreader.nopad.sh
│ │ │ │ ├── torch.word.lm.hidden300.sh
│ │ │ │ ├── torch.word.lm.hidden400.b24.sh
│ │ │ │ ├── torch.word.lm.hidden400.sh
│ │ │ │ ├── torch.word.lm.hidden600.sh
│ │ │ │ ├── torch.word.lm.hidden768.sh
│ │ │ │ ├── torch.word.mreader.lm.bertopt.sh
│ │ │ │ ├── torch.word.mreader.lm.sh
│ │ │ │ ├── torch.word.mreader.nopad.bertopt.nolatt.unkaug.hidden400.sh
│ │ │ │ ├── torch.word.mreader.nopad.bertopt.nolatt.unkaug.hidden600.sh
│ │ │ │ ├── torch.word.mreader.nopad.bertopt.nolatt.unkaug.sh
│ │ │ │ ├── torch.word.mreader.nopad.hidden300.sh
│ │ │ │ ├── torch.word.mreader.nopad.hidden600.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.baike.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.baike.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden300.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden768.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.ner.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.nolatt.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.unkaug.sh
│ │ │ │ ├── torch.word.mreader.nopad.ner.sh
│ │ │ │ └── torch.word.mreader.nopad.sh
│ │ │ ├── v11/
│ │ │ │ ├── README.md
│ │ │ │ ├── tf.char.rnet.nolatt.sh
│ │ │ │ ├── tf.char.rnet.sh
│ │ │ │ ├── tf.char.transformer.bert.cutfront.finetune.2.sh
│ │ │ │ ├── tf.char.transformer.bert.cutfront.finetune.3.sh
│ │ │ │ ├── tf.char.transformer.bert.cutfront.finetune.sh
│ │ │ │ ├── tf.char.transformer.bert.cutfront.sh
│ │ │ │ ├── tf.char.transformer.bert.finetune.2.sh
│ │ │ │ ├── tf.char.transformer.bert.finetune.3.sh
│ │ │ │ ├── tf.char.transformer.bert.finetune.4.sh
│ │ │ │ ├── tf.char.transformer.bert.finetune.sh
│ │ │ │ ├── tf.char.transformer.bert.finetune4.sh
│ │ │ │ ├── tf.char.transformer.bert.rnn.finetune.sh
│ │ │ │ ├── tf.char.transformer.bert.sh
│ │ │ │ ├── tf.char.transformer.bert.topkatt.cutfront.sh
│ │ │ │ ├── tf.char.transformer.bert.topkatt.finetune.sh
│ │ │ │ ├── tf.char.transformer.bert.topkatt.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.2.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.topkatt.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune2.sh
│ │ │ │ ├── tf.char.transformer.nbert.sh
│ │ │ │ ├── tf.char.transformer.nbert.topkatt.sh
│ │ │ │ ├── tf.mix.rnet.nolatt.sh
│ │ │ │ ├── tf.mix.rnet.sh
│ │ │ │ ├── tf.word.mreader.lm.2layer.sh
│ │ │ │ ├── tf.word.mreader.lm.2layer.unkaug.sh
│ │ │ │ ├── tf.word.mreader.lm.sh
│ │ │ │ ├── tf.word.mreader.lm.unkaug.sh
│ │ │ │ ├── tf.word.rnet.hidden600.sh
│ │ │ │ ├── tf.word.rnet.lenemb.sh
│ │ │ │ ├── tf.word.rnet.lm.sh
│ │ │ │ ├── tf.word.rnet.lm.unkaug.sh
│ │ │ │ ├── tf.word.rnet.nolatt.hidden600.sh
│ │ │ │ ├── tf.word.rnet.nolatt.sh
│ │ │ │ ├── tf.word.rnet.sh
│ │ │ │ ├── tf.word.rnet.unkaug.lenemb.sh
│ │ │ │ ├── tf.word.rnet.unkaug.sh
│ │ │ │ ├── tf.word.rnetv2.lm.sh
│ │ │ │ ├── tf.word.rnetv2.lm.unkaug.sh
│ │ │ │ ├── tf.word.rnetv2.nolatt.unkaug.sh
│ │ │ │ ├── tf.word.rnetv2.unkaug.sh
│ │ │ │ ├── torch.char.mreader.lm.nolatt.sh
│ │ │ │ ├── torch.char.mreader.lm.sh
│ │ │ │ ├── torch.char.mreader.nolatt.sh
│ │ │ │ ├── torch.char.mreader.sh
│ │ │ │ ├── torch.mix.mreader.lm.nolatt.sh
│ │ │ │ ├── torch.mix.mreader.lm.sh
│ │ │ │ ├── torch.mix.mreader.sh
│ │ │ │ ├── torch.word.mreader.lm.sh
│ │ │ │ ├── torch.word.mreader.nopad.bertopt.nolatt.unkaug.hidden400.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.baike.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden300.es.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden300.ls.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden300.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.b24.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.es.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.ls.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.pl.b24.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.pl.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden768.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.unkaug.hidden300.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.unkaug.hidden400.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.unkaug.lenemb.hidden400.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.unkaug.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.hidden300.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.hidden400.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.hidden768.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.sh
│ │ │ │ └── torch.word.mreader.nopad.lm.unkaug.sh
│ │ │ ├── v12/
│ │ │ │ ├── tf.rnet.max_pooling.sh
│ │ │ │ ├── tf.rnet.max_pooling.unkaug.sh
│ │ │ │ ├── tf.rnet.sh
│ │ │ │ ├── tf.rnet.unkaug.sh
│ │ │ │ ├── tf.self_attention.max_pooling.sh
│ │ │ │ ├── tf.self_attention.max_pooling.unkaug.sh
│ │ │ │ ├── tf.self_attention.sh
│ │ │ │ ├── tf.self_attention.unkaug.sh
│ │ │ │ ├── tf.word.baseline.hidden100.sh
│ │ │ │ ├── tf.word.baseline.hidden200.sh
│ │ │ │ ├── tf.word.baseline.hidden400.attention_pooling.sh
│ │ │ │ ├── tf.word.baseline.hidden400.elmo.sh
│ │ │ │ ├── tf.word.baseline.hidden400.max_pooling.elmo.sh
│ │ │ │ ├── tf.word.baseline.hidden400.max_pooling.sh
│ │ │ │ ├── tf.word.baseline.hidden400.max_pooling.unkaug.sh
│ │ │ │ ├── tf.word.baseline.hidden400.sh
│ │ │ │ ├── tf.word.baseline.hidden400.unkaug.sh
│ │ │ │ ├── tf.word.baseline.sh
│ │ │ │ ├── tf.word.rnet.sh
│ │ │ │ ├── tf.word.rnet.unkaug.sh
│ │ │ │ ├── torch.mreader.elmo.sh
│ │ │ │ ├── torch.mreader.sh
│ │ │ │ ├── torch.mreader.unkaug.elmo.sh
│ │ │ │ ├── torch.mreader.unkaug.sh
│ │ │ │ ├── torch.self_attention.elmo.sh
│ │ │ │ ├── torch.self_attention.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.sh
│ │ │ │ ├── torch.self_attention.unkaug.sh
│ │ │ │ ├── torch.word.baseline.hidden100.nochar.sh
│ │ │ │ ├── torch.word.baseline.hidden100.sh
│ │ │ │ ├── torch.word.baseline.hidden200.sh
│ │ │ │ ├── torch.word.baseline.hidden400.attention_pooling.sh
│ │ │ │ ├── torch.word.baseline.hidden400.max_pooling.sh
│ │ │ │ ├── torch.word.baseline.hidden400.sh
│ │ │ │ ├── torch.word.baseline.hidden400.unkaug.elmo.sh
│ │ │ │ └── torch.word.baseline.hidden400.unkaug.sh
│ │ │ ├── v13/
│ │ │ │ ├── torch.self_attention.elmo.sh
│ │ │ │ ├── torch.self_attention.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.valid_decay.sh
│ │ │ │ ├── torch.self_attention.unkaug.sh
│ │ │ │ ├── torch.word.baseline.hidden400.unkaug.elmo.hack.sh
│ │ │ │ ├── torch.word.baseline.hidden400.unkaug.elmo.sh
│ │ │ │ └── torch.word.baseline.hidden400.unkaug.sh
│ │ │ ├── v14/
│ │ │ │ ├── test.sh
│ │ │ │ ├── test2.sh
│ │ │ │ ├── test3.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.1gpu.2.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.1gpu.3.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.1gpu.3epoch.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.1gpu.4.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.1gpu.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.2gpu.3epoch.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.2gpu.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.3epoch.lr2.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.3epoch.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.2.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.all_finetune.lr2.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.all_finetune.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.drop03.all_finetune.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.drop03.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr10.all.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr2.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr5.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr8.1w.ratio1.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr8.1w.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr8.1w.topkatt.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr8.all.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr8.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.no_finetune.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.ratio1.lr.2.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.ratio1.lr.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.ratio1.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.4epoch.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.5epoch.3layer.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.5epoch.all_finetune.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.5epoch.lr5.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.5epoch.lr8.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.5epoch.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.sh
│ │ │ │ ├── torch.self_attention.elmo.sh
│ │ │ │ ├── torch.self_attention.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.all_finetune.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_100k.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_10k.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_20k.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_50k.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_6k.batch32.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_6k.batch8.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_6k.exclusive_pooling.exclusive_fc.onlyatt.pad.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_6k.exclusive_pooling.exclusive_fc.onlyatt.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_6k.exclusive_pooling.exclusive_fc.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_6k.exclusive_pooling.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_6k.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.no_finetune.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.no_finetune_word.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.no_finetune_word2.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.nochar.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.valid_decay.sh
│ │ │ │ ├── torch.self_attention.unkaug.sh
│ │ │ │ ├── torch.word.baseline.hidden100.nobuckets.sh
│ │ │ │ ├── torch.word.baseline.hidden100.sh
│ │ │ │ ├── torch.word.baseline.hidden400.unkaug.elmo.hack.sh
│ │ │ │ ├── torch.word.baseline.hidden400.unkaug.elmo.sh
│ │ │ │ └── torch.word.baseline.hidden400.unkaug.sh
│ │ │ ├── v15/
│ │ │ │ ├── bert.sh
│ │ │ │ └── bert2.sh
│ │ │ ├── v2/
│ │ │ │ ├── rnet.3layer.sh
│ │ │ │ ├── rnet.sh
│ │ │ │ ├── rnet.wchar.3layer.sh
│ │ │ │ ├── rnet.wchar.pos.3layer.sh
│ │ │ │ ├── rnet.wchar.pos.ner.sh
│ │ │ │ ├── rnet.wchar.pos.sh
│ │ │ │ ├── rnet.wchar.sh
│ │ │ │ ├── torch.mreader.1hop.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.pos.ner.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.pos.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.sh
│ │ │ │ └── torch.mreader.2hop.wchar.pos.labelrnn.topkattlast.sh
│ │ │ ├── v3/
│ │ │ │ ├── rnet.3layer.sh
│ │ │ │ ├── rnet.sh
│ │ │ │ ├── rnet.wchar.3layer.sh
│ │ │ │ ├── rnet.wchar.pos.3layer.sh
│ │ │ │ ├── rnet.wchar.pos.ner.sh
│ │ │ │ ├── rnet.wchar.pos.sh
│ │ │ │ ├── rnet.wchar.sh
│ │ │ │ ├── torch.mreader.1hop.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.pos.ner.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.pos.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.sh
│ │ │ │ └── torch.mreader.2hop.wchar.pos.labelrnn.topkattlast.sh
│ │ │ ├── v4/
│ │ │ │ ├── rnet.3layer.sh
│ │ │ │ ├── rnet.sh
│ │ │ │ ├── rnet.wchar.3layer.sh
│ │ │ │ ├── rnet.wchar.pos.3layer.sh
│ │ │ │ ├── rnet.wchar.pos.ner.sh
│ │ │ │ ├── rnet.wchar.pos.sh
│ │ │ │ ├── rnet.wchar.sh
│ │ │ │ ├── torch.mreader.1hop.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.pos.ner.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.pos.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.sh
│ │ │ │ └── torch.mreader.2hop.wchar.pos.labelrnn.topkattlast.sh
│ │ │ ├── v5/
│ │ │ │ ├── README.md
│ │ │ │ ├── rnet.char.3layer.sh
│ │ │ │ ├── rnet.char.noconcat.residual.sh
│ │ │ │ ├── rnet.char.noconcat.sh
│ │ │ │ ├── rnet.char.residual.sh
│ │ │ │ ├── rnet.char.sh
│ │ │ │ ├── rnet.mix.3layer.sh
│ │ │ │ ├── rnet.mix.sh
│ │ │ │ ├── rnet.word.3layer.sh
│ │ │ │ ├── rnet.word.pos.3layer.sh
│ │ │ │ ├── rnet.word.pos.ner.sh
│ │ │ │ ├── rnet.word.pos.sh
│ │ │ │ ├── rnet.word.sh
│ │ │ │ ├── torch.mreader.1hop.char.sh
│ │ │ │ ├── torch.mreader.1hop.mix.sh
│ │ │ │ ├── torch.mreader.1hop.word.ner.fast.sh
│ │ │ │ ├── torch.mreader.1hop.word.ner.sh
│ │ │ │ ├── torch.mreader.1hop.word.pos.ner.sh
│ │ │ │ ├── torch.mreader.1hop.word.pos.sh
│ │ │ │ ├── torch.mreader.1hop.word.sh
│ │ │ │ ├── torch.mreader.1hop.word.subner.sh
│ │ │ │ └── torch.mreader.2hop.word.pos.labelrnn.topkattlast.sh
│ │ │ ├── v6/
│ │ │ │ ├── README.md
│ │ │ │ ├── rnet.char.3layer.sh
│ │ │ │ ├── rnet.char.residual.sh
│ │ │ │ ├── rnet.char.sh
│ │ │ │ ├── rnet.mix.3layer.sh
│ │ │ │ ├── rnet.mix.adafactor.sh
│ │ │ │ ├── rnet.mix.multistep.sh
│ │ │ │ ├── rnet.mix.natt.sh
│ │ │ │ ├── rnet.mix.sh
│ │ │ │ ├── rnet.mix.v1.sh
│ │ │ │ ├── rnet.mix.yellowfin.sh
│ │ │ │ ├── rnet.word.3layer.sh
│ │ │ │ ├── rnet.word.adamax.sh
│ │ │ │ ├── rnet.word.pos.3layer.sh
│ │ │ │ ├── rnet.word.pos.ner.sh
│ │ │ │ ├── rnet.word.pos.sh
│ │ │ │ ├── rnet.word.sh
│ │ │ │ ├── rnetv2.mix.natt.sh
│ │ │ │ ├── torch.mreader.1hop.char.sh
│ │ │ │ ├── torch.mreader.1hop.mix.sh
│ │ │ │ ├── torch.mreader.1hop.mix.v1.sh
│ │ │ │ ├── torch.mreader.1hop.mix.v2.sh
│ │ │ │ ├── torch.mreader.1hop.mix.v3.sh
│ │ │ │ ├── torch.mreader.1hop.word.latt.sh
│ │ │ │ ├── torch.mreader.1hop.word.ner.latt.sh
│ │ │ │ ├── torch.mreader.1hop.word.ner.sh
│ │ │ │ ├── torch.mreader.1hop.word.pos.ner.sh
│ │ │ │ ├── torch.mreader.1hop.word.pos.sh
│ │ │ │ ├── torch.mreader.1hop.word.sh
│ │ │ │ ├── torch.mreader.1hop.word.subner.sh
│ │ │ │ ├── torch.mreader.1hop.word.v2.sh
│ │ │ │ ├── torch.mreader.2hop.word.pos.labelrnn.topkattlast.sh
│ │ │ │ ├── torch.rnet.word.gate.sh
│ │ │ │ ├── torch.rnet.word.ner.sh
│ │ │ │ ├── torch.rnet.word.ner.v2.sh
│ │ │ │ ├── torch.rnet.word.ner.v3.sh
│ │ │ │ ├── torch.rnet.word.ner.v4.sh
│ │ │ │ ├── torch.rnet.word.ner.v5.sh
│ │ │ │ ├── torch.rnet.word.sh
│ │ │ │ ├── torch.rnet.word.v2.sh
│ │ │ │ └── torch.rnet.word.v4.sh
│ │ │ ├── v7/
│ │ │ │ ├── torch.mreader.1hop.word.latt.pad.sh
│ │ │ │ ├── torch.mreader.1hop.word.latt.sh
│ │ │ │ ├── torch.mreader.1hop.word.sh
│ │ │ │ ├── torch.mreader.1hop.word.v2.sh
│ │ │ │ ├── torch.mreader.1hop.word.v3.sh
│ │ │ │ └── torch.rnet.word.v2.sh
│ │ │ ├── v8/
│ │ │ │ ├── torch.lm.mreader.word.nopad.rand.sh
│ │ │ │ ├── torch.lm.mreader.word.nopad.rand2.sh
│ │ │ │ ├── torch.lm.mreader.word.nopad.sh
│ │ │ │ ├── torch.lm.mreader.word.nopad2.sh
│ │ │ │ ├── torch.lm.mreader.word.sh
│ │ │ │ ├── torch.mreader.word.aug.sh
│ │ │ │ ├── torch.mreader.word.chconv.sh
│ │ │ │ ├── torch.mreader.word.chpad.sh
│ │ │ │ ├── torch.mreader.word.dynamic.curnn.sh
│ │ │ │ ├── torch.mreader.word.dynamic.latt.nopad.sh
│ │ │ │ ├── torch.mreader.word.dynamic.latt.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.2layer.lm.rand.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.2layer.lm.rand2.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.2layer.lm.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.2layer.lm2.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.2layer.lm3.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.lm.rand.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.lm.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.lm2.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.sh
│ │ │ │ ├── torch.mreader.word.dynamic.sh
│ │ │ │ ├── torch.mreader.word.dynamic.transformer.sh
│ │ │ │ ├── torch.mreader.word.elmo.sh
│ │ │ │ ├── torch.mreader.word.finetune.sh
│ │ │ │ ├── torch.mreader.word.finetune2.sh
│ │ │ │ ├── torch.mreader.word.noam.sh
│ │ │ │ ├── torch.mreader.word.noam2.sh
│ │ │ │ ├── torch.mreader.word.rand.sh
│ │ │ │ └── torch.mreader.word.sh
│ │ │ └── v9/
│ │ │ ├── README.md
│ │ │ ├── tf.char.transformer.5epoch.sh
│ │ │ ├── tf.char.transformer.bert.sh
│ │ │ ├── tf.char.transformer.lm.10epoch.sh
│ │ │ ├── tf.char.transformer.lm.2.sh
│ │ │ ├── tf.char.transformer.lm.5epoch.finetune.sh
│ │ │ ├── tf.char.transformer.lm.5epoch.sh
│ │ │ ├── tf.char.transformer.lm.sh
│ │ │ ├── tf.char.transformer.rand.sh
│ │ │ ├── tf.char.transformer.sh
│ │ │ ├── tf.word.lm.2layer.sh
│ │ │ ├── tf.word.lm.rand.sh
│ │ │ ├── tf.word.lm.rnet.sh
│ │ │ ├── tf.word.lm.sh
│ │ │ ├── tf.word.mreader.sfu.sh
│ │ │ ├── tf.word.mreader.sh
│ │ │ ├── tf.word.rnet.attnodrop.sh
│ │ │ ├── tf.word.rnet.bertopt.10epoch.2.sh
│ │ │ ├── tf.word.rnet.bertopt.10epoch.sh
│ │ │ ├── tf.word.rnet.bertopt.15epoch.d5.sh
│ │ │ ├── tf.word.rnet.bertopt.5epoch.finetune.sh
│ │ │ ├── tf.word.rnet.bertopt.5epoch.sh
│ │ │ ├── tf.word.rnet.bertopt.sh
│ │ │ ├── tf.word.rnet.lm.sh
│ │ │ ├── tf.word.rnet.rand.sh
│ │ │ ├── tf.word.rnet.sh
│ │ │ ├── tf.word.transformer.5epoch.sh
│ │ │ ├── tf.word.transformer.sh
│ │ │ ├── torch.char.lm.lstm.nopad.rand.sh
│ │ │ ├── torch.char.lm.lstm.nopad.sh
│ │ │ ├── torch.char.lm.lstm.sh
│ │ │ ├── torch.char.lm.nopad.addbaike.sh
│ │ │ ├── torch.char.lm.nopad.rand.sh
│ │ │ ├── torch.char.lm.nopad.sh
│ │ │ ├── torch.char.lm.sh
│ │ │ ├── torch.char.mreader.2layer.nopad.sh
│ │ │ ├── torch.char.mreader.nopad.lm.sh
│ │ │ ├── torch.char.mreader.nopad.pos.sh
│ │ │ ├── torch.char.mreader.nopad.sh
│ │ │ ├── torch.word.lm.2layer.nopad.sh
│ │ │ ├── torch.word.lm.2layer.sh
│ │ │ ├── torch.word.lm.lstm.nopad.rand.sh
│ │ │ ├── torch.word.lm.lstm.nopad.sh
│ │ │ ├── torch.word.lm.lstm.sh
│ │ │ ├── torch.word.lm.nopad.addbaike.sh
│ │ │ ├── torch.word.lm.nopad.rand.sh
│ │ │ ├── torch.word.lm.nopad.sh
│ │ │ ├── torch.word.lm.sh
│ │ │ ├── torch.word.lstm.mreader.2layer.nopad.latt6.lm.sh
│ │ │ ├── torch.word.lstm.mreader.2layer.nopad.latt80.lm.sh
│ │ │ ├── torch.word.lstm.mreader.2layer.nopad.lm.sh
│ │ │ ├── torch.word.lstm.mreader.2layer.nopad.sh
│ │ │ ├── torch.word.mreader.2layer.lm.bertopt.sh
│ │ │ ├── torch.word.mreader.2layer.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nolatt.lm.bertopt.sh
│ │ │ ├── torch.word.mreader.2layer.nolatt.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.cw.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.labelrnn.2hop.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.labelrnn.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.latt6.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.latt80.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.2.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.3.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.4.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.5.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.beropt.d5.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.beropt.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.rand.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.v2.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.nolatt.lm.bertopt.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.nolatt.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.sh
│ │ │ ├── torch.word.mreader.3layer.nopad.lm.sh
│ │ │ ├── torch.word.mreader.3layer.nopad.nolatt.lm.sh
│ │ │ ├── torch.word.mreader.nopad.cw.sh
│ │ │ ├── torch.word.mreader.nopad.lm.sh
│ │ │ ├── torch.word.mreader.nopad.sh
│ │ │ ├── torch.word.mreader.sh
│ │ │ ├── torch.word.rnet.2layer.nopad.lm.sh
│ │ │ ├── torch.word.rnet.nopad.lm.sh
│ │ │ ├── torch.word.rnet.nopad.sh
│ │ │ ├── torch.word.rnet.sh
│ │ │ ├── torch.word.rnn.lm.sh
│ │ │ ├── torch.word.rnn.nopad.lm.sh
│ │ │ ├── torch.word.rnn.nopad.rand.sh
│ │ │ ├── torch.word.rnn.nopad.sh
│ │ │ └── torch.word.rnn.sh
│ │ ├── train.py
│ │ ├── train.v1/
│ │ │ ├── README.md
│ │ │ ├── v1/
│ │ │ │ ├── bow.emb.fix.sh
│ │ │ │ ├── bow.emb.sh
│ │ │ │ ├── bow.emb.warm.sh
│ │ │ │ ├── bow.labelatt.emb.sh
│ │ │ │ ├── bow.labelatt.sh
│ │ │ │ ├── bow.labelatt.simple.sh
│ │ │ │ ├── bow.mdecay.sh
│ │ │ │ ├── bow.sh
│ │ │ │ ├── bow.topk-3att.labelatt.emb.sh
│ │ │ │ ├── convnet.topk-3att.labelatt.emb.sh
│ │ │ │ ├── crnn.topk-3att.labelatt.emb.sh
│ │ │ │ ├── gru.2layer.emb.sh
│ │ │ │ ├── gru.2layer.sh
│ │ │ │ ├── gru.att.sh
│ │ │ │ ├── gru.att2.sh
│ │ │ │ ├── gru.emb.finetune.later.sh
│ │ │ │ ├── gru.emb.finetune.sh
│ │ │ │ ├── gru.emb.fix.sh
│ │ │ │ ├── gru.emb.sh
│ │ │ │ ├── gru.emb.warm.sh
│ │ │ │ ├── gru.f05.sh
│ │ │ │ ├── gru.f07.sh
│ │ │ │ ├── gru.f09.sh
│ │ │ │ ├── gru.focal.sh
│ │ │ │ ├── gru.focal2.sh
│ │ │ │ ├── gru.hidden200.sh
│ │ │ │ ├── gru.label.sh
│ │ │ │ ├── gru.maxatt.2.sh
│ │ │ │ ├── gru.maxatt.sh
│ │ │ │ ├── gru.maxmean.sh
│ │ │ │ ├── gru.mdecay.f09.sh
│ │ │ │ ├── gru.mdecay.p2.sh
│ │ │ │ ├── gru.mdecay.sh
│ │ │ │ ├── gru.mdecay2.f07.sh
│ │ │ │ ├── gru.mdecay2.f09.sh
│ │ │ │ ├── gru.mdecay2.sh
│ │ │ │ ├── gru.mean.sh
│ │ │ │ ├── gru.naonly.sh
│ │ │ │ ├── gru.naratio01.sh
│ │ │ │ ├── gru.naratio1.sh
│ │ │ │ ├── gru.nodecay.sh
│ │ │ │ ├── gru.p2.sh
│ │ │ │ ├── gru.sh
│ │ │ │ ├── gru.topk-3.2layer.sh
│ │ │ │ ├── gru.topk-3.cosine.10epoch.decay05.sh
│ │ │ │ ├── gru.topk-3.cosine.10epoch.sh
│ │ │ │ ├── gru.topk-3.cosine.5epoch.decay01.sh
│ │ │ │ ├── gru.topk-3.cosine.5epoch.decay04.sh
│ │ │ │ ├── gru.topk-3.cosine.5epoch.decay08.sh
│ │ │ │ ├── gru.topk-3.cosine.5epoch.sh
│ │ │ │ ├── gru.topk-3.cosine.decay05.sh
│ │ │ │ ├── gru.topk-3.cosine.decay08.sh
│ │ │ │ ├── gru.topk-3.cosine.sh
│ │ │ │ ├── gru.topk-3.earth.na.sh
│ │ │ │ ├── gru.topk-3.earth.sh
│ │ │ │ ├── gru.topk-3.hier.sh
│ │ │ │ ├── gru.topk-3.label.sh
│ │ │ │ ├── gru.topk-3.labelatt.emb.sh
│ │ │ │ ├── gru.topk-3.labelatt.sh
│ │ │ │ ├── gru.topk-3.mlp.2.sh
│ │ │ │ ├── gru.topk-3.mlp.sh
│ │ │ │ ├── gru.topk-3.mlp05.sh
│ │ │ │ ├── gru.topk-3.mlp1.sh
│ │ │ │ ├── gru.topk-3.na.sh
│ │ │ │ ├── gru.topk-3.sh
│ │ │ │ ├── gru.topk-3att.2layer.emb.sh
│ │ │ │ ├── gru.topk-3att.2layer.hier.sh
│ │ │ │ ├── gru.topk-3att.2layer.label.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.10w.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.1w.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.2.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.mdecay.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.naonly.finetune.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.naonly.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.selfmatch.2.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.selfmatch.naratio01.finetune.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.selfmatch.naratio01.finetune2.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.selfmatch.naratio01.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.selfmatch.naratio05.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.selfmatch.naratio1.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.selfmatch.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.mdecay.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.sfu.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.finetune.d05.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.finetune.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.sh
│ │ │ │ ├── gru.topk-3att.2layer.sh
│ │ │ │ ├── gru.topk-3att.sh
│ │ │ │ ├── gru.topk-4.sh
│ │ │ │ ├── gru.topk-5.sh
│ │ │ │ ├── gru.topk-6.sh
│ │ │ │ ├── gru.topk.sh
│ │ │ │ ├── gru.weights.0.naratio01.sh
│ │ │ │ ├── gru.weights.0.sh
│ │ │ │ ├── gru.weights.location.naratio01.sh
│ │ │ │ ├── gru.weights.location.sh
│ │ │ │ ├── gru.weights.service.sh
│ │ │ │ ├── gru.weights.service.topk.sh
│ │ │ │ ├── gru.weights.service0.sh
│ │ │ │ ├── gru.weights.service0.topk.sh
│ │ │ │ ├── qanet.topk-3att.labelatt.emb.sh
│ │ │ │ └── rcnn.topk-3att.labelatt.emb.sh
│ │ │ ├── v2/
│ │ │ │ ├── bow.sh
│ │ │ │ ├── gru.1w.2layer.topk3att.labelatt20.selfmatch.hidden200.sh
│ │ │ │ ├── gru.1w.2layer.topk3att.labelatt20.selfmatch.sh
│ │ │ │ ├── gru.1w.2layer.topk3att.labelatt80.selfmatch.hidden200.sh
│ │ │ │ ├── gru.1w.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.hidden200.sh
│ │ │ │ ├── gru.1w.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.sh
│ │ │ │ ├── gru.1w.wordpretrain_glove_dianping.2layer.topk3att.labelatt80.selfmatch.hidden200.sh
│ │ │ │ ├── gru.5k.2layer.topk3att.labelatt20.selfmatch.hidden200.sh
│ │ │ │ ├── gru.5k.2layer.topk3att.labelatt20.selfmatch.sh
│ │ │ │ ├── gru.5k.2layer.topk3att.labelatt80.selfmatch.hidden200.sh
│ │ │ │ ├── gru.5k.aspect.dish.sh
│ │ │ │ ├── gru.5k.aspect.environment.sh
│ │ │ │ ├── gru.5k.aspect.location.sh
│ │ │ │ ├── gru.5k.aspect.others.sh
│ │ │ │ ├── gru.5k.aspect.price.sh
│ │ │ │ ├── gru.5k.aspect.service.sh
│ │ │ │ ├── gru.5k.keep06.sh
│ │ │ │ ├── gru.5k.keep08.sh
│ │ │ │ ├── gru.5k.keep09.sh
│ │ │ │ ├── gru.5k.labelemb.sh
│ │ │ │ ├── gru.5k.labelrnn.sh
│ │ │ │ ├── gru.5k.sfu.sh
│ │ │ │ ├── gru.5k.sh
│ │ │ │ ├── gru.5k.tanh.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt10.selfmatch.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.hidden200.sfu.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.hidden200.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.hidden300.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt40.selfmatch.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt80.selfmatch.hidden200.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt80.selfmatch.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.3layer.topk3att.labelatt20.selfmatch.sh
│ │ │ │ ├── gru.char.2layer.topk3att.labelatt20.selfmatch.hidden200.sh
│ │ │ │ ├── gru.char.2layer.topk3att.labelatt20.selfmatch.sh
│ │ │ │ ├── gru.char.2layer.topk3att.labelatt80.selfmatch.hidden200.sh
│ │ │ │ ├── gru.char.aspect.dish.sh
│ │ │ │ ├── gru.char.aspect.environment.sh
│ │ │ │ ├── gru.char.aspect.location.sh
│ │ │ │ ├── gru.char.aspect.others.sh
│ │ │ │ ├── gru.char.aspect.price.sh
│ │ │ │ ├── gru.char.aspect.service.sh
│ │ │ │ ├── gru.char.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.hidden200.sh
│ │ │ │ ├── gru.char.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.hidden300.sh
│ │ │ │ ├── gru.char.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.sh
│ │ │ │ ├── gru.char.wordpretrain_glove_dianping.2layer.topk3att.labelatt80.selfmatch.hidden200.sh
│ │ │ │ ├── gru.char.wordpretrain_glove_dianping.3layer.topk3att.labelatt20.selfmatch.hidden200.sh
│ │ │ │ └── gru.char.wordpretrain_glove_dianping.3layer.topk3att.labelatt20.selfmatch.sh
│ │ │ ├── v3/
│ │ │ │ ├── gru-5k-ftwiki.sh
│ │ │ │ ├── gru-5k-labelrnn.sh
│ │ │ │ ├── gru-5k-nopad.sh
│ │ │ │ ├── gru-baseline-att.sh
│ │ │ │ ├── gru-baseline-ftwiki.sh
│ │ │ │ ├── gru-baseline.sh
│ │ │ │ ├── gru.5k.mreader.2hop.sh
│ │ │ │ ├── gru.5k.mreader.sfu.2hop.sh
│ │ │ │ ├── gru.5k.mreader.sfu.sh
│ │ │ │ ├── gru.5k.mreader.sh
│ │ │ │ ├── gru.5k.sfu.sh
│ │ │ │ ├── gru.5k.sh
│ │ │ │ ├── torch-gru-baseline-att-nopad.sh
│ │ │ │ ├── torch-gru-baseline-att-pad.sh
│ │ │ │ ├── torch-gru-baseline-att-v2.sh
│ │ │ │ ├── torch-gru-baseline-nopad.sh
│ │ │ │ ├── torch-gru-baseline-pad.sh
│ │ │ │ ├── torch-gru-baseline-v2-ftwiki.sh
│ │ │ │ ├── torch-gru-baseline-v2.sh
│ │ │ │ ├── torch-gru-baseline.sh
│ │ │ │ ├── torch-gru-labelatt-nopad.sh
│ │ │ │ ├── torch-gru-labelatt-sfu-nopad.sh
│ │ │ │ ├── torch-gru-rnet-curnn-nopad.sh
│ │ │ │ ├── torch-gru-rnet-curnn-sfu-nopad.sh
│ │ │ │ ├── torch-gru-rnet-curnn-sfu.sh
│ │ │ │ ├── torch-gru-rnet-curnn.sh
│ │ │ │ ├── torch-gru-rnet-nopad.sh
│ │ │ │ ├── torch-gru-rnet-sfu-nopad.sh
│ │ │ │ ├── torch-mreader-1hop.sh
│ │ │ │ ├── torch-mreader-2hop.sh
│ │ │ │ ├── torch-mreader-labelrnn-1hop.sh
│ │ │ │ └── torch-mreader-labelrnn-2hop.sh
│ │ │ └── v4/
│ │ │ ├── gru-5k-canyin-char-addneubinary-finetune.sh
│ │ │ ├── gru-5k-canyin-char-addneubinary-trans.sh
│ │ │ ├── gru-5k-canyin-char-addneubinary.sh
│ │ │ ├── gru-5k-canyin-char.sh
│ │ │ ├── gru-5k-canyin-mix-addbinaries-trans.sh
│ │ │ ├── gru-5k-canyin-mix-addbinaries.sh
│ │ │ ├── gru-5k-canyin-mix-addnabinary-finetune-epoch5.sh
│ │ │ ├── gru-5k-canyin-mix-addnabinary-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-addnabinary.sh
│ │ │ ├── gru-5k-canyin-mix-addnaneubinary-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-addneubinary-finetune-epoch5.sh
│ │ │ ├── gru-5k-canyin-mix-addneubinary-finetune-factor2.sh
│ │ │ ├── gru-5k-canyin-mix-addneubinary-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-addneubinary-trans.sh
│ │ │ ├── gru-5k-canyin-mix-addneubinary.2.sh
│ │ │ ├── gru-5k-canyin-mix-addneubinary.sh
│ │ │ ├── gru-5k-canyin-mix-binariesonly.sh
│ │ │ ├── gru-5k-canyin-mix-cotrain.sh
│ │ │ ├── gru-5k-canyin-mix-deformcanyin.sh
│ │ │ ├── gru-5k-canyin-mix-deformdianping-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-deformdianping.sh
│ │ │ ├── gru-5k-canyin-mix-finetune-dianpingemb.sh
│ │ │ ├── gru-5k-canyin-mix-hier-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-hier-trans.sh
│ │ │ ├── gru-5k-canyin-mix-hier.sh
│ │ │ ├── gru-5k-canyin-mix-hierneu-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-neubinary-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-neubinary-trans.sh
│ │ │ ├── gru-5k-canyin-mix-neubinary.sh
│ │ │ ├── gru-5k-canyin-mix-othersAgainOnly.sh
│ │ │ ├── gru-5k-canyin-mix-othersOnly.sh
│ │ │ ├── gru-5k-canyin-mix-othersOverallOnly.sh
│ │ │ ├── gru-5k-canyin-mix-othersOverallW2.sh
│ │ │ ├── gru-5k-canyin-mix-othersOverallW5.sh
│ │ │ ├── gru-5k-canyin-mix-othersW10.sh
│ │ │ ├── gru-5k-canyin-mix-othersW2.sh
│ │ │ ├── gru-5k-canyin-mix-othersW5.sh
│ │ │ ├── gru-5k-canyin-mix-regression-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-regression.sh
│ │ │ ├── gru-5k-canyin-mix-small.sh
│ │ │ ├── gru-5k-canyin-mix-trans.2.sh
│ │ │ ├── gru-5k-canyin-mix-trans.3.sh
│ │ │ ├── gru-5k-canyin-mix-trans.sh
│ │ │ ├── gru-5k-canyin-mix.sh
│ │ │ ├── gru-5k-canyin.sh
│ │ │ ├── gru-5k-hier-canyin-finetune.sh
│ │ │ ├── gru-5k-hier-canyin.sh
│ │ │ ├── gru-5k-lattonly-canyin-char.sh
│ │ │ ├── gru-5k-maxmean-canyin.sh
│ │ │ ├── gru-5k-rnetv2-canyin-char.sh
│ │ │ ├── gru-5k-rnetv2-canyin.sh
│ │ │ ├── gru-5k-rnetv2-sfu-canyin.sh
│ │ │ ├── gru-5k-rnetv3-canyin-char.sh
│ │ │ ├── gru-5k-rnetv3-canyin.sh
│ │ │ ├── gru-5k-rnetv3-selfmatchonly-canyin-char.sh
│ │ │ ├── gru-5k-rnetv4-canyin.sh
│ │ │ ├── gru-5k-selfmatchonly-canyin-char.sh
│ │ │ ├── gru-5k-topk3attmean-canyin-char.sh
│ │ │ ├── gru-5k-topk3attmean-canyin.sh
│ │ │ ├── gru-5k-topk3mean-canyin.sh
│ │ │ ├── gru.5k.sh
│ │ │ ├── rcnn-k5-canyin-mix.sh
│ │ │ ├── torch-mreader-trans.sh
│ │ │ └── torch-mreader.sh
│ │ └── train.v2/
│ │ ├── README.md
│ │ ├── v1/
│ │ │ ├── bow.emb.binary.sh
│ │ │ ├── bow.emb.sh
│ │ │ ├── bow.ori.emb.sh
│ │ │ ├── bow.ori.sh
│ │ │ ├── bow.sh
│ │ │ ├── bow.simplify.sh
│ │ │ ├── bow.testori.sh
│ │ │ ├── bow.testsimplify.sh
│ │ │ ├── gru.400.sh
│ │ │ ├── gru.char.sfu.sh
│ │ │ ├── gru.char.sh
│ │ │ ├── gru.emb.2.sh
│ │ │ ├── gru.emb.adjust.sh
│ │ │ ├── gru.emb.char.sh
│ │ │ ├── gru.emb.cw.sh
│ │ │ ├── gru.emb.decay.sh
│ │ │ ├── gru.emb.decay2.sh
│ │ │ ├── gru.emb.last.sh
│ │ │ ├── gru.emb.lastmax.sh
│ │ │ ├── gru.emb.lastmaxmean.sh
│ │ │ ├── gru.emb.loss.sh
│ │ │ ├── gru.emb.norec.sh
│ │ │ ├── gru.emb.rec.sh
│ │ │ ├── gru.emb.rec.topkattlast.sh
│ │ │ ├── gru.emb.sh
│ │ │ ├── gru.emb.smooth.sh
│ │ │ ├── gru.emb.trans.sh
│ │ │ ├── gru.lm.2.sh
│ │ │ ├── gru.lm.3.sh
│ │ │ ├── gru.lm.emb.2.sh
│ │ │ ├── gru.lm.emb.sh
│ │ │ ├── gru.lm.lr0002.sh
│ │ │ ├── gru.lm.pretrain.sh
│ │ │ ├── gru.lm.sh
│ │ │ ├── gru.ori.emb.sh
│ │ │ ├── gru.ori.sh
│ │ │ ├── gru.sh
│ │ │ ├── gru.simplify.emb.2.sh
│ │ │ ├── gru.simplify.emb.addneubinary.sh
│ │ │ ├── gru.simplify.emb.finetune.sh
│ │ │ ├── gru.simplify.emb.sh
│ │ │ ├── gru.simplify.sh
│ │ │ ├── gru.testsimplify.emb.finetune.sh
│ │ │ ├── gru.testsimplify.emb.sh
│ │ │ ├── gru.testsimplify.sh
│ │ │ ├── lstm.emb.norec.sh
│ │ │ ├── lstm.emb.rec.2layer.noconcat.sh
│ │ │ ├── lstm.emb.rec.2layer.noconcat.topkattlast.sh
│ │ │ ├── lstm.emb.rec.sh
│ │ │ ├── lstm.emb.rec.topkattlast.sh
│ │ │ ├── lstm.emb.sh
│ │ │ ├── rnet.char.ft.sh
│ │ │ ├── rnet.char.glove.sh
│ │ │ ├── torch.fastai.1layer.sh
│ │ │ ├── torch.fastai.2.sh
│ │ │ ├── torch.fastai.3.sh
│ │ │ ├── torch.fastai.4.sh
│ │ │ ├── torch.fastai.5.sh
│ │ │ ├── torch.fastai.sh
│ │ │ ├── torch.gru.2layer.sh
│ │ │ ├── torch.gru.sh
│ │ │ ├── torch.lstm.1layer.labelatt.2hop.topkattlast.sh
│ │ │ ├── torch.lstm.1layer.labelatt.topkattlast.sh
│ │ │ ├── torch.lstm.2.sh
│ │ │ ├── torch.lstm.2layer.noconcat.labelatt.gate.topkattlast.sh
│ │ │ ├── torch.lstm.2layer.noconcat.labelatt.topkattlast.sh
│ │ │ ├── torch.lstm.2layer.noconcat.sh
│ │ │ ├── torch.lstm.2layer.noconcat.topkattlast.sh
│ │ │ ├── torch.lstm.2layer.sh
│ │ │ ├── torch.lstm.evpadd.sh
│ │ │ ├── torch.lstm.max.2.sh
│ │ │ ├── torch.lstm.max.sh
│ │ │ ├── torch.lstm.padd.sh
│ │ │ ├── torch.lstm.recdrop.sh
│ │ │ ├── torch.lstm.sh
│ │ │ ├── torch.lstmV2.2.sh
│ │ │ ├── torch.lstmV2.bwdrop.sh
│ │ │ ├── torch.lstmV2.norec.2.sh
│ │ │ ├── torch.lstmV2.norec.nopad.sh
│ │ │ ├── torch.lstmV2.norec.sh
│ │ │ ├── torch.lstmV2.sh
│ │ │ ├── torch.rnet.2layer.noconcat.topkattlast.sh
│ │ │ └── torch.rnetV2.2layer.noconcat.topkattlast.sh
│ │ ├── v1.1/
│ │ │ ├── bow.emb.binary.sh
│ │ │ ├── bow.emb.sh
│ │ │ ├── bow.ori.emb.sh
│ │ │ ├── bow.ori.sh
│ │ │ ├── bow.sh
│ │ │ ├── bow.simplify.sh
│ │ │ ├── bow.testori.sh
│ │ │ ├── bow.testsimplify.sh
│ │ │ ├── gru.400.sh
│ │ │ ├── gru.char.sfu.sh
│ │ │ ├── gru.char.sh
│ │ │ ├── gru.emb.2.sh
│ │ │ ├── gru.emb.adjust.sh
│ │ │ ├── gru.emb.char.2.sh
│ │ │ ├── gru.emb.char.3.sh
│ │ │ ├── gru.emb.char.4.sh
│ │ │ ├── gru.emb.char.5.sh
│ │ │ ├── gru.emb.char.sh
│ │ │ ├── gru.emb.cw.sh
│ │ │ ├── gru.emb.decay.sh
│ │ │ ├── gru.emb.decay2.sh
│ │ │ ├── gru.emb.last.sh
│ │ │ ├── gru.emb.lastmax.sh
│ │ │ ├── gru.emb.lastmaxmean.sh
│ │ │ ├── gru.emb.loss.sh
│ │ │ ├── gru.emb.norec.sh
│ │ │ ├── gru.emb.rec.sh
│ │ │ ├── gru.emb.rec.topkattlast.sh
│ │ │ ├── gru.emb.sh
│ │ │ ├── gru.emb.smooth.sh
│ │ │ ├── gru.emb.trans.sh
│ │ │ ├── gru.lm.2.sh
│ │ │ ├── gru.lm.3.sh
│ │ │ ├── gru.lm.emb.2.sh
│ │ │ ├── gru.lm.emb.sh
│ │ │ ├── gru.lm.lr0002.sh
│ │ │ ├── gru.lm.pretrain.sh
│ │ │ ├── gru.lm.sh
│ │ │ ├── gru.ori.emb.sh
│ │ │ ├── gru.ori.sh
│ │ │ ├── gru.sh
│ │ │ ├── gru.simplify.emb.2.sh
│ │ │ ├── gru.simplify.emb.addneubinary.sh
│ │ │ ├── gru.simplify.emb.finetune.sh
│ │ │ ├── gru.simplify.emb.sh
│ │ │ ├── gru.simplify.sh
│ │ │ ├── gru.testsimplify.emb.finetune.sh
│ │ │ ├── gru.testsimplify.emb.sh
│ │ │ ├── gru.testsimplify.sh
│ │ │ ├── lstm.emb.norec.sh
│ │ │ ├── lstm.emb.rec.2layer.noconcat.sh
│ │ │ ├── lstm.emb.rec.2layer.noconcat.topkattlast.sh
│ │ │ ├── lstm.emb.rec.sh
│ │ │ ├── lstm.emb.rec.topkattlast.sh
│ │ │ ├── lstm.emb.sh
│ │ │ ├── rnet.char.ft.sh
│ │ │ ├── rnet.char.glove.sh
│ │ │ ├── torch.fastai.1layer.sh
│ │ │ ├── torch.fastai.2.sh
│ │ │ ├── torch.fastai.3.sh
│ │ │ ├── torch.fastai.4.sh
│ │ │ ├── torch.fastai.5.sh
│ │ │ ├── torch.fastai.sh
│ │ │ ├── torch.gru.2layer.sh
│ │ │ ├── torch.gru.sh
│ │ │ ├── torch.lstm.1layer.labelatt.2hop.topkattlast.sh
│ │ │ ├── torch.lstm.1layer.labelatt.topkattlast.sh
│ │ │ ├── torch.lstm.2.sh
│ │ │ ├── torch.lstm.2layer.noconcat.labelatt.gate.topkattlast.sh
│ │ │ ├── torch.lstm.2layer.noconcat.labelatt.topkattlast.sh
│ │ │ ├── torch.lstm.2layer.noconcat.sh
│ │ │ ├── torch.lstm.2layer.noconcat.topkattlast.sh
│ │ │ ├── torch.lstm.2layer.sh
│ │ │ ├── torch.lstm.evpadd.sh
│ │ │ ├── torch.lstm.max.2.sh
│ │ │ ├── torch.lstm.max.sh
│ │ │ ├── torch.lstm.padd.sh
│ │ │ ├── torch.lstm.recdrop.sh
│ │ │ ├── torch.lstm.sh
│ │ │ ├── torch.lstmV2.2.sh
│ │ │ ├── torch.lstmV2.bwdrop.sh
│ │ │ ├── torch.lstmV2.norec.2.sh
│ │ │ ├── torch.lstmV2.norec.nopad.sh
│ │ │ ├── torch.lstmV2.norec.sh
│ │ │ ├── torch.lstmV2.sh
│ │ │ ├── torch.rnet.2layer.noconcat.topkattlast.sh
│ │ │ └── torch.rnetV2.2layer.noconcat.topkattlast.sh
│ │ ├── v10/
│ │ │ ├── rnet.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ ├── v11/
│ │ │ ├── rnet.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ ├── v12/
│ │ │ ├── rnet.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ ├── v2/
│ │ │ ├── README.md
│ │ │ ├── rnet.3layer.noconcat.sh
│ │ │ ├── rnet.3layer.noconcat.topkattlast.sh
│ │ │ ├── rnet.3layer.sh
│ │ │ ├── rnet.3layer.topkattlast.sh
│ │ │ ├── rnet.noconcat.sh
│ │ │ ├── rnet.noconcat.topkattlast.sh
│ │ │ ├── rnet.sh
│ │ │ ├── rnet.topkattlast.sh
│ │ │ ├── torch.mreader.1hop.2layer.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.sh
│ │ │ ├── torch.mreader.1hop.decay09.sh
│ │ │ ├── torch.mreader.1hop.sh
│ │ │ ├── torch.mreader.1hop.topkattlast.sh
│ │ │ ├── torch.mreader.labelrnn.2hop.2layer.sh
│ │ │ ├── torch.mreader.labelrnn.2hop.2layer.topkattlast.sh
│ │ │ └── torch.mreaderV2.1hop.2layer.sh
│ │ ├── v2.old/
│ │ │ ├── README.md
│ │ │ ├── convnet.sh
│ │ │ ├── qanet.sh
│ │ │ ├── rnet.3layer.fine.lr10.sh
│ │ │ ├── rnet.3layer.fine.lr2.sh
│ │ │ ├── rnet.3layer.fine.lr4.sh
│ │ │ ├── rnet.3layer.fine.sh
│ │ │ ├── rnet.3layer.sh
│ │ │ ├── rnet.addneu.sh
│ │ │ ├── rnet.fine.addneu.sh
│ │ │ ├── rnet.fine.trans.sh
│ │ │ ├── rnet.fine.trans01.sh
│ │ │ ├── rnet.fine.trans05.sh
│ │ │ ├── rnet.fine2.addneu.sh
│ │ │ ├── rnet.hidden300.sh
│ │ │ ├── rnet.labelrnn.sh
│ │ │ ├── rnet.latt10.sh
│ │ │ ├── rnet.latt80.sh
│ │ │ ├── rnet.maxmeanatt.sh
│ │ │ ├── rnet.sh
│ │ │ ├── rnetv2.sh
│ │ │ ├── torch.mreader.1hop.labelrnn.addneu.sh
│ │ │ ├── torch.mreader.1hop.labelrnn.sh
│ │ │ ├── torch.mreader.1hop.sh
│ │ │ ├── torch.mreader.2hop.labelrnn.sh
│ │ │ └── torch.mreader.2hop.sh
│ │ ├── v3/
│ │ │ ├── README.md
│ │ │ ├── rnet.3layer.noconcat.sh
│ │ │ ├── rnet.3layer.noconcat.topkattlast.sh
│ │ │ ├── rnet.3layer.sh
│ │ │ ├── rnet.3layer.topkattlast.sh
│ │ │ ├── rnet.noconcat.lossesdecay.sh
│ │ │ ├── rnet.noconcat.sh
│ │ │ ├── rnet.noconcat.topkattlast.sh
│ │ │ ├── rnet.sh
│ │ │ ├── rnet.topkattlast.sh
│ │ │ ├── torch.mreader.1hop.2layer.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.adjf1sdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.aucsdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.f1sdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.lossesdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.sh
│ │ │ ├── torch.mreader.1hop.decay09.sh
│ │ │ ├── torch.mreader.1hop.sh
│ │ │ ├── torch.mreader.1hop.topkattlast.sh
│ │ │ ├── torch.mreader.labelrnn.2hop.2layer.sh
│ │ │ ├── torch.mreader.labelrnn.2hop.2layer.topkattlast.sh
│ │ │ └── torch.mreaderV2.1hop.2layer.sh
│ │ ├── v4/
│ │ │ ├── README.md
│ │ │ ├── rnet.3layer.noconcat.sh
│ │ │ ├── rnet.3layer.sh
│ │ │ ├── rnet.noconcat.lossesdecay.sh
│ │ │ ├── rnet.noconcat.sh
│ │ │ ├── rnet.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.adjf1sdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.aucsdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.f1sdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.lossesdecay.sh
│ │ │ ├── torch.mreader.1hop.decay09.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ ├── v5/
│ │ │ ├── README.md
│ │ │ ├── rnet.3layer.noconcat.sh
│ │ │ ├── rnet.3layer.sh
│ │ │ ├── rnet.noconcat.lossesdecay.sh
│ │ │ ├── rnet.noconcat.sh
│ │ │ ├── rnet.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.adjf1sdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.aucsdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.f1sdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.lossesdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.sh
│ │ │ ├── torch.mreader.1hop.decay09.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ ├── v6/
│ │ │ ├── README.md
│ │ │ ├── rnet.3layer.noconcat.sh
│ │ │ ├── rnet.3layer.sh
│ │ │ ├── rnet.noconcat.sh
│ │ │ ├── rnet.sh
│ │ │ ├── torch.mreader.1hop.decay09.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ ├── v7/
│ │ │ ├── rnet.finetune6k.sh
│ │ │ ├── rnet.fixword.sh
│ │ │ ├── rnet.sh
│ │ │ ├── torch.mreader.1hop.finetune6k.sh
│ │ │ ├── torch.mreader.1hop.fixword.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ ├── v8/
│ │ │ ├── rnet.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ └── v9/
│ │ ├── rnet.sh
│ │ └── torch.mreader.1hop.sh
│ ├── common/
│ │ └── lm/
│ │ ├── README.md
│ │ ├── algos/
│ │ │ ├── loss.py
│ │ │ └── model.py
│ │ ├── dataset.py
│ │ ├── prepare/
│ │ │ ├── to-ids.glove.word.sh
│ │ │ ├── to-ids.py
│ │ │ └── to-ids.sh
│ │ ├── read-records.py
│ │ ├── train/
│ │ │ ├── char.concat.sh
│ │ │ ├── char.emb.sh
│ │ │ ├── char.sh
│ │ │ ├── char.small.sh
│ │ │ ├── v1/
│ │ │ │ ├── train.concat.sh
│ │ │ │ ├── train.emb.sh
│ │ │ │ └── train.sh
│ │ │ ├── v2/
│ │ │ │ ├── train.concat.sh
│ │ │ │ ├── train.emb.sh
│ │ │ │ └── train.sh
│ │ │ ├── word.concat.sh
│ │ │ ├── word.emb.sh
│ │ │ └── word.sh
│ │ └── train.py
│ ├── feed/
│ │ └── rank/
│ │ ├── tf/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── config.py
│ │ │ ├── data/
│ │ │ │ └── train_0
│ │ │ ├── dataset.py
│ │ │ ├── eager-train/
│ │ │ │ └── best-v0.sh
│ │ │ ├── err/
│ │ │ │ └── torch-only-train.py
│ │ │ ├── evaluate.py
│ │ │ ├── gen-records.py
│ │ │ ├── loss.py
│ │ │ ├── model.py
│ │ │ ├── mpi.py
│ │ │ ├── prepare/
│ │ │ │ ├── gen-records.sh
│ │ │ │ ├── gen-train.sh
│ │ │ │ └── gen-valid.sh
│ │ │ ├── pyt/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── dataset.py
│ │ │ │ └── model.py
│ │ │ ├── read/
│ │ │ │ ├── README.md
│ │ │ │ ├── horovod.sh
│ │ │ │ ├── read-record.sh
│ │ │ │ └── read-text.sh
│ │ │ ├── read-record.py
│ │ │ ├── read-simple.py
│ │ │ ├── read-test.py
│ │ │ ├── read-test2.py
│ │ │ ├── read-test3.py
│ │ │ ├── read-text.py
│ │ │ ├── test/
│ │ │ │ ├── read-torch-dataset.py
│ │ │ │ └── test-torch-hvd.py
│ │ │ ├── text_dataset.py
│ │ │ ├── tfrecord_dataset.py
│ │ │ ├── torch-hvd-train/
│ │ │ │ ├── best-v0-small.sh
│ │ │ │ └── best-v0.sh
│ │ │ ├── torch-hvd-train.py
│ │ │ ├── torch-only-train/
│ │ │ │ ├── baseline.sh
│ │ │ │ ├── best-v0-small.sh
│ │ │ │ └── best-v0.sh
│ │ │ ├── torch-only-train-hvd/
│ │ │ │ ├── best-v0-small.sh
│ │ │ │ └── best-v0.sh
│ │ │ ├── torch-only-train-hvd.py
│ │ │ ├── torch-only-train.py
│ │ │ ├── torch-train/
│ │ │ │ ├── baseline.sh
│ │ │ │ ├── best-v0-small.sh
│ │ │ │ └── best-v0.sh
│ │ │ ├── torch-train.py
│ │ │ ├── train/
│ │ │ │ ├── README.md
│ │ │ │ ├── allsum.sh
│ │ │ │ ├── attention.sh
│ │ │ │ ├── baseline.sh
│ │ │ │ ├── baseline2.sh
│ │ │ │ ├── baseline3.sh
│ │ │ │ ├── baseline4.sh
│ │ │ │ ├── best-v0-horovod-lp.sh
│ │ │ │ ├── best-v0-horovod-lrnoscal-prerand.sh
│ │ │ │ ├── best-v0-horovod-lrnoscale-largebuffer.sh
│ │ │ │ ├── best-v0-horovod-lrnoscale-nbp.sh
│ │ │ │ ├── best-v0-horovod-lrnoscale.sh
│ │ │ │ ├── best-v0-horovod-noshard.sh
│ │ │ │ ├── best-v0-horovod-record-lrnoscale-largebuffer.sh
│ │ │ │ ├── best-v0-horovod-record-lrnoscale.sh
│ │ │ │ ├── best-v0-horovod-record.sh
│ │ │ │ ├── best-v0-horovod-sp.sh
│ │ │ │ ├── best-v0-horovod.sh
│ │ │ │ ├── best-v0-horovod2.sh
│ │ │ │ ├── best-v0-record.sh
│ │ │ │ ├── best-v0-sp.sh
│ │ │ │ ├── best-v0.sh
│ │ │ │ ├── deep-addval.sh
│ │ │ │ ├── deep.embact.sh
│ │ │ │ ├── deep.sh
│ │ │ │ ├── dw-add.sh
│ │ │ │ ├── dw-att-fieldemb.sh
│ │ │ │ ├── dw-bigbatch.sh
│ │ │ │ ├── dw-concat.sh
│ │ │ │ ├── dw-concat2.sh
│ │ │ │ ├── dw-concat3.sh
│ │ │ │ ├── dw-concat4.sh
│ │ │ │ ├── dw-field.sh
│ │ │ │ ├── dw-fieldemb.sh
│ │ │ │ ├── dw-hidden100.sh
│ │ │ │ ├── dw-hidden64.sh
│ │ │ │ ├── dw-max-fieldemb.sh
│ │ │ │ ├── dw-max.sh
│ │ │ │ ├── dw-max2.sh
│ │ │ │ ├── dw-mean.sh
│ │ │ │ ├── dw-mean2-2.sh
│ │ │ │ ├── dw-mean2-fieldemb-mlp1layer.sh
│ │ │ │ ├── dw-mean2-fieldemb-mlp1layer2.sh
│ │ │ │ ├── dw-mean2-fieldemb-mlp2layer.sh
│ │ │ │ ├── dw-mean2-fieldemb-rankloss.sh
│ │ │ │ ├── dw-mean2-fieldemb.sh
│ │ │ │ ├── dw-mean2-fieldemb2.sh
│ │ │ │ ├── dw-mean2-rankloss.sh
│ │ │ │ ├── dw-mean2.sh
│ │ │ │ ├── dw-mean3.sh
│ │ │ │ ├── dw-new-allsum.sh
│ │ │ │ ├── dw-new.sh
│ │ │ │ ├── dw-old-allsum.sh
│ │ │ │ ├── dw-old.sh
│ │ │ │ ├── dw-record.sh
│ │ │ │ ├── dw-sum-fconcat-fieldemb-mlp.sh
│ │ │ │ ├── dw-sum-fconcat-fieldemb-mlp2.sh
│ │ │ │ ├── dw-sum-fconcat-fieldemb-mlp3.sh
│ │ │ │ ├── dw-sum-fconcat-mlp.sh
│ │ │ │ ├── dw-sum-fconcat-mlp2.sh
│ │ │ │ ├── dw-sum-fconcat.sh
│ │ │ │ ├── dw-sum-fieldemb-rankloss.sh
│ │ │ │ ├── dw-sum-fieldemb.sh
│ │ │ │ ├── dw-sum-fieldemb2.sh
│ │ │ │ ├── dw-sum-horovod-noshard.sh
│ │ │ │ ├── dw-sum-horovod.sh
│ │ │ │ ├── dw-sum-record.sh
│ │ │ │ ├── dw-sum2-fieldemb-mlp1layer.sh
│ │ │ │ ├── dw-sum2.sh
│ │ │ │ ├── dw-text.sh
│ │ │ │ ├── dw-topk.sh
│ │ │ │ ├── fconcat-record.sh
│ │ │ │ ├── fconcat.sh
│ │ │ │ ├── fconcat2.sh
│ │ │ │ ├── fconcat3.sh
│ │ │ │ ├── horovod-batch128-test.sh
│ │ │ │ ├── horovod-batch256.sh
│ │ │ │ ├── horovod-batch64-2.sh
│ │ │ │ ├── horovod-batch64-test.sh
│ │ │ │ ├── horovod-batch64.sh
│ │ │ │ ├── horovod-scale.sh
│ │ │ │ ├── horovod-small.sh
│ │ │ │ ├── horovod-test-record.sh
│ │ │ │ ├── horovod-test.sh
│ │ │ │ ├── horovod.sh
│ │ │ │ ├── horovod2.sh
│ │ │ │ ├── rankloss.sh
│ │ │ │ ├── record.sh
│ │ │ │ ├── small.sh
│ │ │ │ ├── sum.sh
│ │ │ │ ├── test1.sh
│ │ │ │ ├── test10.sh
│ │ │ │ ├── test11.sh
│ │ │ │ ├── test12.sh
│ │ │ │ ├── test2.sh
│ │ │ │ ├── test3.sh
│ │ │ │ ├── test4.sh
│ │ │ │ ├── test5.sh
│ │ │ │ ├── test6-2.sh
│ │ │ │ ├── test6-3.sh
│ │ │ │ ├── test6.sh
│ │ │ │ ├── test7.sh
│ │ │ │ ├── test8.sh
│ │ │ │ ├── test9.sh
│ │ │ │ ├── wide-deep.sh
│ │ │ │ ├── wide-deep2-bert-2epoch.sh
│ │ │ │ ├── wide-deep2-bert.sh
│ │ │ │ ├── wide-deep2-bert2-2epoch.sh
│ │ │ │ ├── wide-deep2-bert2.sh
│ │ │ │ ├── wide-deep2-lazy.sh
│ │ │ │ ├── wide-deep2.sh
│ │ │ │ ├── wide-ftrl.sh
│ │ │ │ ├── wide-noval.sh
│ │ │ │ └── wide.sh
│ │ │ └── train.py
│ │ └── v1/
│ │ └── README.md
│ ├── kaggle/
│ │ ├── blindness/
│ │ │ ├── jupter/
│ │ │ │ └── tf-image.ipynb
│ │ │ ├── keras/
│ │ │ │ ├── READEME.md
│ │ │ │ ├── aug.py
│ │ │ │ ├── check.py
│ │ │ │ ├── config.py
│ │ │ │ ├── dataset.py
│ │ │ │ ├── evaluate.py
│ │ │ │ ├── evaluate2.py
│ │ │ │ ├── fake-infer.py
│ │ │ │ ├── folds.py
│ │ │ │ ├── gen-folds.py
│ │ │ │ ├── infer.py
│ │ │ │ ├── infer.sh
│ │ │ │ ├── jupter.ipynb
│ │ │ │ ├── loss.py
│ │ │ │ ├── lr.py
│ │ │ │ ├── model.py
│ │ │ │ ├── train.py
│ │ │ │ ├── train2.py
│ │ │ │ └── util.py
│ │ │ ├── keras2/
│ │ │ │ ├── READEME.md
│ │ │ │ ├── aug.py
│ │ │ │ ├── config.py
│ │ │ │ ├── dataset.py
│ │ │ │ ├── evaluate.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── model.py
│ │ │ │ └── train.py
│ │ │ ├── keras2tf/
│ │ │ │ ├── READEME.md
│ │ │ │ ├── aug.py
│ │ │ │ ├── config.py
│ │ │ │ ├── dataset.py
│ │ │ │ ├── evaluate.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── model.py
│ │ │ │ └── train.py
│ │ │ ├── keras3/
│ │ │ │ ├── READEME.md
│ │ │ │ ├── aug.py
│ │ │ │ ├── bak/
│ │ │ │ │ └── evaluate.py
│ │ │ │ ├── config.py
│ │ │ │ ├── dataset.py
│ │ │ │ ├── evaluate.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── model.py
│ │ │ │ └── train.py
│ │ │ ├── other/
│ │ │ │ ├── APTOS 2019_ Keras Baseline.ipynb
│ │ │ │ ├── Intro APTOS Diabetic Retinopathy (EDA & Sta e77c27.ipynb
│ │ │ │ ├── keras_baseline.py
│ │ │ │ └── training-mobilenet-v2-in-4-min.py
│ │ │ ├── prepare/
│ │ │ │ ├── gen-records.py
│ │ │ │ └── gen-records.sh
│ │ │ ├── tf/
│ │ │ │ ├── dataset.py
│ │ │ │ ├── evaluate.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── model.py
│ │ │ │ ├── model_base.py
│ │ │ │ ├── train/
│ │ │ │ │ └── train.sh
│ │ │ │ └── train.py
│ │ │ └── tf2/
│ │ │ ├── dataset.py
│ │ │ ├── evaluate.py
│ │ │ ├── loss.py
│ │ │ ├── model.py
│ │ │ ├── train/
│ │ │ │ └── train.sh
│ │ │ └── train.py
│ │ ├── cifar10/
│ │ │ ├── README.md
│ │ │ ├── baseline/
│ │ │ │ ├── keras/
│ │ │ │ │ ├── KerasT-master/
│ │ │ │ │ │ ├── .idea/
│ │ │ │ │ │ │ ├── KerasT.iml
│ │ │ │ │ │ │ ├── inspectionProfiles/
│ │ │ │ │ │ │ │ └── Project_Default.xml
│ │ │ │ │ │ │ ├── misc.xml
│ │ │ │ │ │ │ ├── modules.xml
│ │ │ │ │ │ │ ├── preferred-vcs.xml
│ │ │ │ │ │ │ └── vcs.xml
│ │ │ │ │ │ ├── 4layerCNN.py
│ │ │ │ │ │ ├── 6layerCNN.py
│ │ │ │ │ │ ├── ConfusionMatrixVisualization.py
│ │ │ │ │ │ └── README.md
│ │ │ │ │ ├── hands-on-deep-learning-master/
│ │ │ │ │ │ ├── .gitignore
│ │ │ │ │ │ ├── README.md
│ │ │ │ │ │ └── cifar_image_classification/
│ │ │ │ │ │ ├── aux/
│ │ │ │ │ │ │ ├── cifar100_meta
│ │ │ │ │ │ │ ├── cifar100_to_h5py.ipynb
│ │ │ │ │ │ │ └── cifar10_to_h5py.ipynb
│ │ │ │ │ │ ├── cnn_adv.py
│ │ │ │ │ │ ├── cnn_fchollet.py
│ │ │ │ │ │ ├── cnn_pkaur.py
│ │ │ │ │ │ ├── cnn_simple.py
│ │ │ │ │ │ ├── drawings.ipynb
│ │ │ │ │ │ ├── helpers.py
│ │ │ │ │ │ ├── lr.py
│ │ │ │ │ │ ├── mlp.py
│ │ │ │ │ │ └── neptune.yaml
│ │ │ │ │ └── simple/
│ │ │ │ │ ├── cifar10.py
│ │ │ │ │ └── result.csv
│ │ │ │ └── tf/
│ │ │ │ ├── cifar10.py
│ │ │ │ ├── cifar10_estimator/
│ │ │ │ │ ├── README.md
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── cifar10.py
│ │ │ │ │ ├── cifar10_main.py
│ │ │ │ │ ├── cifar10_model.py
│ │ │ │ │ ├── cifar10_multi_gpu_train.py
│ │ │ │ │ ├── cifar10_utils.py
│ │ │ │ │ ├── cmle_config.yaml
│ │ │ │ │ ├── evaluator.py
│ │ │ │ │ ├── gen-records.sh
│ │ │ │ │ ├── generate_cifar10_tfrecords.py
│ │ │ │ │ ├── inference.py
│ │ │ │ │ ├── model_base.py
│ │ │ │ │ ├── prepare.sh
│ │ │ │ │ ├── train/
│ │ │ │ │ │ ├── readme
│ │ │ │ │ │ ├── train-2gpu-2.sh
│ │ │ │ │ │ ├── train-2gpu.sh
│ │ │ │ │ │ ├── train-4gpu.sh
│ │ │ │ │ │ ├── train-melt-2gpu.sh
│ │ │ │ │ │ ├── train-melt-decay-2gpu.sh
│ │ │ │ │ │ ├── train-melt-decay.sh
│ │ │ │ │ │ ├── train-melt-decay98.sh
│ │ │ │ │ │ ├── train-melt-momentum-batch128.sh
│ │ │ │ │ │ ├── train-melt-momentum-decay-2gpu.sh
│ │ │ │ │ │ ├── train-melt-momentum-decay-batch64.sh
│ │ │ │ │ │ ├── train-melt-momentum-decay.sh
│ │ │ │ │ │ ├── train-melt-momentum.sh
│ │ │ │ │ │ ├── train-melt-momentum2-batch128-2gpu.sh
│ │ │ │ │ │ ├── train-melt-momentum2-batch128.sh
│ │ │ │ │ │ ├── train-melt-momentum2.sh
│ │ │ │ │ │ ├── train-melt.sh
│ │ │ │ │ │ └── train.sh
│ │ │ │ │ ├── train.py
│ │ │ │ │ └── train.tfrecords
│ │ │ │ └── cifar10_estimator.v1/
│ │ │ │ ├── README.md
│ │ │ │ ├── __init__.py
│ │ │ │ ├── cifar10.py
│ │ │ │ ├── cifar10_main.py
│ │ │ │ ├── cifar10_model.py
│ │ │ │ ├── cifar10_multi_gpu_train.py
│ │ │ │ ├── cifar10_utils.py
│ │ │ │ ├── cmle_config.yaml
│ │ │ │ ├── evaluator.py
│ │ │ │ ├── gen-records.sh
│ │ │ │ ├── generate_cifar10_tfrecords.py
│ │ │ │ ├── inference.py
│ │ │ │ ├── model_base.py
│ │ │ │ ├── prepare.sh
│ │ │ │ ├── train/
│ │ │ │ │ ├── readme
│ │ │ │ │ ├── train-2gpu-2.sh
│ │ │ │ │ ├── train-2gpu.sh
│ │ │ │ │ ├── train-4gpu.sh
│ │ │ │ │ ├── train-melt-2gpu.sh
│ │ │ │ │ ├── train-melt-decay-2gpu.sh
│ │ │ │ │ ├── train-melt-decay.sh
│ │ │ │ │ ├── train-melt-decay98.sh
│ │ │ │ │ ├── train-melt-momentum-batch128.sh
│ │ │ │ │ ├── train-melt-momentum-decay-2gpu.sh
│ │ │ │ │ ├── train-melt-momentum-decay-batch64.sh
│ │ │ │ │ ├── train-melt-momentum-decay.sh
│ │ │ │ │ ├── train-melt-momentum.sh
│ │ │ │ │ ├── train-melt-momentum2-batch128-2gpu.sh
│ │ │ │ │ ├── train-melt-momentum2-batch128.sh
│ │ │ │ │ ├── train-melt-momentum2.sh
│ │ │ │ │ ├── train-melt.sh
│ │ │ │ │ └── train.sh
│ │ │ │ ├── train.py
│ │ │ │ └── train.tfrecords
│ │ │ ├── fastai/
│ │ │ │ ├── cifar10-simplenet.ipynb
│ │ │ │ └── cifar10.ipynb
│ │ │ └── tf/
│ │ │ ├── cifar10.py
│ │ │ ├── cifar10_model.py
│ │ │ ├── evaluate.py
│ │ │ ├── evaluator.py
│ │ │ ├── loss.py
│ │ │ ├── model_base.py
│ │ │ ├── train/
│ │ │ │ ├── readme
│ │ │ │ ├── train-2gpu-2.sh
│ │ │ │ ├── train-2gpu.sh
│ │ │ │ ├── train-4gpu.sh
│ │ │ │ ├── train-melt-2gpu.sh
│ │ │ │ ├── train-melt-decay-2gpu.sh
│ │ │ │ ├── train-melt-decay.sh
│ │ │ │ ├── train-melt-decay98.sh
│ │ │ │ ├── train-melt-momentum-batch128.sh
│ │ │ │ ├── train-melt-momentum-decay-2gpu.sh
│ │ │ │ ├── train-melt-momentum-decay-batch64.sh
│ │ │ │ ├── train-melt-momentum-decay.sh
│ │ │ │ ├── train-melt-momentum-decay2-2gpu.sh
│ │ │ │ ├── train-melt-momentum-decay2.sh
│ │ │ │ ├── train-melt-momentum.sh
│ │ │ │ ├── train-melt-momentum2-batch128-2gpu.sh
│ │ │ │ ├── train-melt-momentum2-batch128.sh
│ │ │ │ ├── train-melt-momentum2.sh
│ │ │ │ ├── train-melt.sh
│ │ │ │ └── train.sh
│ │ │ ├── train.py
│ │ │ └── train2.py
│ │ └── toxic/
│ │ ├── algos/
│ │ │ ├── config.py
│ │ │ └── model.py
│ │ ├── checkpoint
│ │ ├── dataset.py
│ │ ├── evaluate.py
│ │ ├── prepare/
│ │ │ ├── __init__.py
│ │ │ ├── black.csv
│ │ │ ├── config.py
│ │ │ ├── count-unks.py
│ │ │ ├── extend-table.py
│ │ │ ├── filter-special.py
│ │ │ ├── gen-correction.py
│ │ │ ├── gen-en-lang-prob.py
│ │ │ ├── gen-full-vocab.sh
│ │ │ ├── gen-lang.py
│ │ │ ├── gen-ori-vocab.sh
│ │ │ ├── gen-records-parse.py
│ │ │ ├── gen-records.py
│ │ │ ├── gen-sentences-csv.py
│ │ │ ├── gen-sentences.py
│ │ │ ├── gen-tokens.py
│ │ │ ├── gen-twitter-ori-vocab.sh
│ │ │ ├── gen-vocab-parse.py
│ │ │ ├── gen-vocab.py
│ │ │ ├── gray.csv
│ │ │ ├── merge-2emb.py
│ │ │ ├── merge-charemb.py
│ │ │ ├── merge-emb.py
│ │ │ ├── merge-glove.py
│ │ │ ├── merge-ngram-emb.py
│ │ │ ├── merge-word-ngram-emb.py
│ │ │ ├── merge-wordemb.py
│ │ │ ├── post-deal.py
│ │ │ ├── prepare-test.sh
│ │ │ ├── prepare-train.sh
│ │ │ ├── preprocess.py
│ │ │ ├── readme
│ │ │ ├── run-gloveonly.sh
│ │ │ ├── run-scratch.sh
│ │ │ ├── run-v1.sh
│ │ │ ├── run-v10-fasttext.sh
│ │ │ ├── run-v10-fttalk.sh
│ │ │ ├── run-v10.sh
│ │ │ ├── run-v11-fasttext-lower.sh
│ │ │ ├── run-v11-fasttext.sh
│ │ │ ├── run-v11-ftngram.sh
│ │ │ ├── run-v11-fttalk.sh
│ │ │ ├── run-v11-fttalk2.sh
│ │ │ ├── run-v11-fttalk5.sh
│ │ │ ├── run-v11-glove-lower.sh
│ │ │ ├── run-v11-glove.sh
│ │ │ ├── run-v11-glovetalk.sh
│ │ │ ├── run-v11-glovetwitter.sh
│ │ │ ├── run-v11-lexvec.sh
│ │ │ ├── run-v11-lexvecc.sh
│ │ │ ├── run-v11-ngram-lower.sh
│ │ │ ├── run-v11.sh
│ │ │ ├── run-v13.sh
│ │ │ ├── run-v14.sh
│ │ │ ├── run-v15-clean-lower.sh
│ │ │ ├── run-v15-clean.sh
│ │ │ ├── run-v15-fasttext-lower-allngram.sh
│ │ │ ├── run-v15-fasttext-lower-lemma.sh
│ │ │ ├── run-v15-fasttext-lower.sh
│ │ │ ├── run-v15-fasttext.sh
│ │ │ ├── run-v15-fttalk3gram-lower.sh
│ │ │ ├── run-v15-fttalk3gramall-lower.sh
│ │ │ ├── run-v15-fttalk5-lower.sh
│ │ │ ├── run-v15-fttalk5.sh
│ │ │ ├── run-v15-glove-lower-allngram.sh
│ │ │ ├── run-v15-glove-lower-lem.sh
│ │ │ ├── run-v15-glove-lower.sh
│ │ │ ├── run-v15-gloveft-lower.sh
│ │ │ ├── run-v15-glovetalk-lower.sh
│ │ │ ├── run-v15-glovetalk.sh
│ │ │ ├── run-v15-lexvecc-clean.sh
│ │ │ ├── run-v15-lexvecc.sh
│ │ │ ├── run-v15.sh
│ │ │ ├── run-v16-fasttext-lower-lemma.sh
│ │ │ ├── run-v16-fasttext-lower.sh
│ │ │ ├── run-v16-fasttext.sh
│ │ │ ├── run-v16-ftfttalk-lower.sh
│ │ │ ├── run-v16-fttalk5-lower.sh
│ │ │ ├── run-v16-glove-lower-lemma.sh
│ │ │ ├── run-v16-glove-lower.sh
│ │ │ ├── run-v16-gloveft-lower-lemma.sh
│ │ │ ├── run-v16-gloveft-lower.sh
│ │ │ ├── run-v16-gloveglovetalk-lower.sh
│ │ │ ├── run-v16-lexveccfttalk-lower.sh
│ │ │ ├── run-v16-lm.sh
│ │ │ ├── run-v3-limit1000.sh
│ │ │ ├── run-v3-limit400.sh
│ │ │ ├── run-v3.sh
│ │ │ ├── run-v4-fasttext.sh
│ │ │ ├── run-v4-limmit400.sh
│ │ │ ├── run-v4.sh
│ │ │ ├── run-v7.sh
│ │ │ ├── run-v8.sh
│ │ │ ├── run-v9-full.sh
│ │ │ ├── run-v9-glovetwitter-full.sh
│ │ │ ├── run-v9-glovetwitter.sh
│ │ │ ├── run-v9-noparse.sh
│ │ │ ├── run-v9.sh
│ │ │ ├── run.sh
│ │ │ ├── test-tokenize.py
│ │ │ ├── test-tokenize2.py
│ │ │ ├── test-tokenize3.py
│ │ │ ├── test-tokenize4.py
│ │ │ ├── test-tokenize5.py
│ │ │ ├── tokenize-corpus.py
│ │ │ ├── tokenizer-v2.py
│ │ │ ├── tokenizer-v3.py
│ │ │ ├── tokenizer.py
│ │ │ ├── toxic_words.py
│ │ │ └── white.csv
│ │ ├── read-records.py
│ │ └── train.py
│ └── pyt/
│ └── README.md
├── tests/
│ ├── horovod/
│ │ ├── allgather.py
│ │ ├── allgather2.py
│ │ ├── allgather3.py
│ │ ├── allgather4.py
│ │ ├── allreduce.py
│ │ ├── allreduce2.py
│ │ ├── bcast.py
│ │ ├── shards.py
│ │ └── simple.py
│ └── sample-balance/
│ ├── README.md
│ ├── dataset.py
│ └── read-records.py
├── third/
│ ├── __init__.py
│ └── bert/
│ ├── CONTRIBUTING.md
│ ├── LICENSE
│ ├── README.md
│ ├── __init__.py
│ ├── create_pretraining_data.py
│ ├── extract_features.py
│ ├── modeling.py
│ ├── modeling_test.py
│ ├── multilingual.md
│ ├── optimization.py
│ ├── optimization_test.py
│ ├── run-char.sh
│ ├── run_classifier.py
│ ├── run_pretraining.py
│ ├── run_squad.py
│ ├── tokenization.py
│ └── tokenization_test.py
├── tools/
│ ├── dump-word-embedding.py
│ └── show-var-of-model.py
├── utils/
│ ├── README.md
│ ├── gezi/
│ │ ├── __init__.py
│ │ ├── avg_score.py
│ │ ├── bigdata_util.py
│ │ ├── bleu.py
│ │ ├── gezi_util.py
│ │ ├── hash.py
│ │ ├── libgezi_util.py
│ │ ├── melt/
│ │ │ ├── __init__.py
│ │ │ ├── logging.py
│ │ │ ├── tfrecords.py
│ │ │ └── util.py
│ │ ├── metrics/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── bleu/
│ │ │ │ ├── LICENSE
│ │ │ │ ├── __init__.py
│ │ │ │ ├── bleu.py
│ │ │ │ └── bleu_scorer.py
│ │ │ ├── cider/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── cider.py
│ │ │ │ └── cider_scorer.py
│ │ │ ├── ciderD/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── ciderD.py
│ │ │ │ └── ciderD_scorer.py
│ │ │ ├── correlation/
│ │ │ │ ├── __init__.py
│ │ │ │ └── correlation.py
│ │ │ ├── eval.py
│ │ │ ├── meteor/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── meteor-1.5.jar
│ │ │ │ └── meteor.py
│ │ │ ├── new_cider/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── cider.py
│ │ │ │ └── cider_scorer.py
│ │ │ ├── rouge/
│ │ │ │ ├── __init__.py
│ │ │ │ └── rouge.py
│ │ │ └── tokenizer/
│ │ │ ├── __init__.py
│ │ │ ├── ptbtokenizer.py
│ │ │ └── stanford-corenlp-3.4.1.jar
│ │ ├── ngram.py
│ │ ├── nowarning.py
│ │ ├── pydict.py
│ │ ├── rank_metrics.py
│ │ ├── segment.py
│ │ ├── setup.py
│ │ ├── summary.py
│ │ ├── test/
│ │ │ ├── test_get_single_cn.py
│ │ │ └── test_ngrams.py
│ │ ├── timer.py
│ │ ├── topn.py
│ │ ├── util.py
│ │ ├── vocabulary.py
│ │ ├── word_counter.py
│ │ └── zhtools/
│ │ ├── __init__.py
│ │ ├── chconv.py
│ │ ├── langconv.py
│ │ ├── test_langconv.py
│ │ ├── xpinyin.py
│ │ └── zh_wiki.py
│ ├── lele/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── apps/
│ │ │ ├── __init__.py
│ │ │ └── train.py
│ │ ├── fastai/
│ │ │ ├── __init__.py
│ │ │ ├── core.py
│ │ │ ├── imports/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── core.py
│ │ │ │ └── torch.py
│ │ │ ├── layers.py
│ │ │ ├── text/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── models.py
│ │ │ │ └── qrnn/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── forget_mult.py
│ │ │ │ └── qrnn.py
│ │ │ └── torch_core.py
│ │ ├── layers/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── classify_layer.py
│ │ │ ├── elmo/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── classify_layer.py
│ │ │ │ ├── elmo.py
│ │ │ │ ├── embedding_layer.py
│ │ │ │ ├── encoder_base.py
│ │ │ │ ├── highway.py
│ │ │ │ ├── lstm.py
│ │ │ │ ├── lstm_cell_with_projection.py
│ │ │ │ ├── token_embedder.py
│ │ │ │ └── util.py
│ │ │ ├── elmo.py
│ │ │ ├── embedding_layer.py
│ │ │ ├── encoder_base.py
│ │ │ ├── highway.py
│ │ │ ├── layers.py
│ │ │ ├── lstm.py
│ │ │ ├── lstm_cell_with_projection.py
│ │ │ ├── token_embedder.py
│ │ │ ├── transformer/
│ │ │ │ ├── __init__.py
│ │ │ │ └── transformer.py
│ │ │ └── util.py
│ │ ├── losses/
│ │ │ ├── __init__.py
│ │ │ └── losses.py
│ │ ├── ops/
│ │ │ ├── __init__.py
│ │ │ └── ops.py
│ │ ├── training/
│ │ │ ├── __init__.py
│ │ │ └── optimizers.py
│ │ └── util.py
│ └── melt/
│ ├── __init__.py
│ ├── apps/
│ │ ├── __init__.py
│ │ ├── image_processing.py
│ │ ├── read.py
│ │ └── train.py
│ ├── cnn/
│ │ ├── __init__.py
│ │ ├── cnn.py
│ │ ├── conv2d.py
│ │ └── qanet.py
│ ├── eager/
│ │ ├── __init__.py
│ │ ├── train.py
│ │ └── util.py
│ ├── encoder/
│ │ ├── __init__.py
│ │ ├── embedding_layer.py
│ │ └── transformer.py
│ ├── flow/
│ │ ├── __init__.py
│ │ ├── dataset_flow.py
│ │ ├── flow.py
│ │ ├── test.py
│ │ ├── test_once.py
│ │ ├── train.py
│ │ └── train_once.py
│ ├── image/
│ │ ├── __init__.py
│ │ ├── image_decoder.py
│ │ ├── image_embedding.py
│ │ ├── image_model.py
│ │ └── image_processing.py
│ ├── inference/
│ │ ├── __init__.py
│ │ ├── predictor.py
│ │ └── predictor_base.py
│ ├── layers/
│ │ ├── __init__.py
│ │ ├── cnn/
│ │ │ ├── __init__.py
│ │ │ ├── cnn.py
│ │ │ ├── convnet.py
│ │ │ └── qanet.py
│ │ ├── layers.py
│ │ ├── optimizers.py
│ │ ├── rnn/
│ │ │ ├── __init__.py
│ │ │ └── rnn.py
│ │ └── transformer/
│ │ └── __init__.py
│ ├── losses/
│ │ ├── __init__.py
│ │ └── losses.py
│ ├── metrics/
│ │ ├── __init__.py
│ │ └── rank_metrics.py
│ ├── models/
│ │ ├── __init__.py
│ │ └── mlp.py
│ ├── ops/
│ │ ├── __init__.py
│ │ ├── nn_impl.py
│ │ ├── ops.py
│ │ └── sparse_ops.py
│ ├── rnn/
│ │ ├── __init__.py
│ │ └── rnn.py
│ ├── seq2seq/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── attention_decoder_fn.py
│ │ ├── attention_wrapper.py
│ │ ├── basic_decoder.py
│ │ ├── beam_decoder.py
│ │ ├── beam_decoder_fn.py
│ │ ├── beam_search.py
│ │ ├── beam_search_decoder.py
│ │ ├── decoder.py
│ │ ├── decoder_fn.py
│ │ ├── exp/
│ │ │ └── beam_decoder.py
│ │ ├── helper.py
│ │ ├── legacy/
│ │ │ ├── beam_decoder.py
│ │ │ └── beam_search_decoder.py
│ │ ├── logprobs_decoder.py
│ │ ├── loss.py
│ │ ├── official/
│ │ │ └── beam_search_decoder.py
│ │ └── seq2seq.py
│ ├── setup.py
│ ├── slim2/
│ │ ├── __init__.py
│ │ ├── base_nets_factory.py
│ │ ├── layers.py
│ │ └── preprocessing_factory.py
│ ├── tfrecords/
│ │ ├── __init__.py
│ │ ├── dataset.py
│ │ ├── dataset_decode.py
│ │ ├── decode_then_shuffle.py
│ │ ├── libsvm_decode.py
│ │ ├── read.py
│ │ ├── shuffle_then_decode.py
│ │ └── write.py
│ ├── tools/
│ │ ├── count-records.py
│ │ ├── delete-old-models.py
│ │ ├── prepare-finetune-withhistory.py
│ │ ├── prepare-finetune.py
│ │ ├── rename-variables.py
│ │ ├── reset-model-top-scope.py
│ │ ├── show-records.py
│ │ └── show-var-of-model.py
│ ├── torch/
│ │ ├── __init__.py
│ │ └── train.py
│ ├── training/
│ │ ├── __init__.py
│ │ ├── adamax.py
│ │ ├── bert/
│ │ │ ├── __init__.py
│ │ │ └── optimization.py
│ │ ├── learning_rate_decay.py
│ │ └── training.py
│ ├── util.py
│ ├── utils/
│ │ ├── __init__.py
│ │ ├── embsim.py
│ │ ├── logging.py
│ │ ├── summary.py
│ │ └── weight_decay.py
│ └── variable/
│ ├── __init__.py
│ └── variable.py
└── wenzheng/
├── __init__.py
├── embedding.py
├── encoder.py
├── pyt/
│ ├── __init__.py
│ ├── embedding.py
│ └── encoder.py
└── utils/
├── __init__.py
├── conf.py
├── ids2text.py
├── input_flags.py
├── rnn_flags.py
├── text2ids.py
└── vocabulary.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
*.pyc
*.so
*.npy
*.txt
*.bin
*.html
*checkpoint.ipynb
*.swp
*.crc
*.jpg
*.jpeg
*.pdf
*.vshistory
.localhistory
.vs
.vscode
./vscode/*
.vs/*
model.ckpt*
/data
/build
/dist
/deepiu.egg-info
/util/build
/util/dist
/util/hasky_utils.egg-info
/examples/sparse-tensor-classification/model
/examples/classification/model
/examples/text-classification/model
/deepiu/image_caption/inference/flickr/tf-bow-predict
/scripts/log
*tmphws4so
/util/gezi/metrics/tokenizer/tmp*
util/gezi/metrics/tokenizer/tmp*
================================================
FILE: README.md
================================================
# wenzheng
Deep learning related projects using tf(eager), pytorch(fastai,allenlp) for contests like aichallenger, kaggle
# ai challenger 2018 细粒度用户评论情感分类第一名（后厂村静静团队）解决方案
wenzheng/project/ai2018/sentiment[https://github.com/chenghuige/wenzheng/tree/master/projects/ai2018/sentiment]
# ai challenger 2018 阅读理解解决方案（a榜 21 基于港科大rnet，mreader）
wenzheng/project/ai2018/reader[https://github.com/chenghuige/wenzheng/tree/master/projects/ai2018/reader]
================================================
FILE: __init__.py
================================================
import wenzheng
import third
import official
import projects
================================================
FILE: examples/fastai/lm/lm.ipynb
================================================
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"# http://docs.fast.ai/text.html\n",
"from fastai.text import *\n",
"import html\n",
"from pathlib import Path\n",
"import numpy as np\n",
"import pandas as pd\n",
"import sklearn\n",
"from sklearn import model_selection\n",
"import re\n",
"from functools import partial\n",
"from torch import optim"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"PosixPath('data/imdb_sample')"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"IMDB_PATH = Path('./data/imdb_sample/')\n",
"IMDB_PATH"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>Explanation\\nWhy the edits made under my usern...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>0</td>\n",
" <td>D'aww! He matches this background colour I'm s...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>0</td>\n",
" <td>Hey man, I'm really not trying to edit war. It...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>0</td>\n",
" <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>0</td>\n",
" <td>You, sir, are my hero. Any chance you remember...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" 0 1\n",
"0 0 Explanation\\nWhy the edits made under my usern...\n",
"1 0 D'aww! He matches this background colour I'm s...\n",
"2 0 Hey man, I'm really not trying to edit war. It...\n",
"3 0 \"\\nMore\\nI can't make any real suggestions on ...\n",
"4 0 You, sir, are my hero. Any chance you remember..."
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df = pd.read_csv(IMDB_PATH/'train.csv', header=None)\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"('negative', 'positive ')"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"classes = read_classes(IMDB_PATH/'classes.txt')\n",
"classes[0], classes[1]"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"#train_ds = TextDataset.from_csv(IMDB_PATH, name='train', classes=classes)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"data_lm = text_data_from_csv(Path(IMDB_PATH), data_func=lm_data)\n",
"data_clas = text_data_from_csv(Path(IMDB_PATH), data_func=classifier_data, vocab=data_lm.train_ds.vocab)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"#download_wt103_model()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"learn = RNNLearner.language_model(data_lm, pretrained_fnames=['lstm_wt103', 'itos_wt103'], drop_mult=0.5)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7dc9f9f007984489ba332496538bcdba",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(HBox(children=(IntProgress(value=0, max=1), HTML(value='0.00% [0/1 00:00<00:00]'))), HTML(value…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"ename": "RuntimeError",
"evalue": "CUDA error: out of memory",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-9-3ea49add0339>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_one_cycle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1e-2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai/train.py\u001b[0m in \u001b[0;36mfit_one_cycle\u001b[0;34m(learn, cyc_len, max_lr, moms, div_factor, pct_start, wd, **kwargs)\u001b[0m\n\u001b[1;32m 16\u001b[0m cbs = [OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor,\n\u001b[1;32m 17\u001b[0m pct_start=pct_start, **kwargs)]\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcyc_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_lr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcbs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mlr_find\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mLearner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_lr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_lr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_it\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[1;32m 131\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallback_fns\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 132\u001b[0m fit(epochs, self.model, self.loss_fn, opt=self.opt, data=self.data, metrics=self.metrics,\n\u001b[0;32m--> 133\u001b[0;31m callbacks=self.callbacks+callbacks)\n\u001b[0m\u001b[1;32m 134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 135\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(epochs, model, loss_fn, opt, data, callbacks, metrics)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[0mexception\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 87\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexception\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(epochs, model, loss_fn, opt, data, callbacks, metrics)\u001b[0m\n\u001b[1;32m 70\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0myb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 71\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 73\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/py3env/lib/python3.6/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mloss_batch\u001b[0;34m(model, xb, yb, loss_fn, opt, cb_handler, metrics)\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mopt\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_backward_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 31\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_backward_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 32\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/py3env/lib/python3.6/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \"\"\"\n\u001b[0;32m---> 96\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/py3env/lib/python3.6/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m 88\u001b[0m Variable._execution_engine.run_backward(\n\u001b[1;32m 89\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m allow_unreachable=True) # allow_unreachable flag\n\u001b[0m\u001b[1;32m 91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mRuntimeError\u001b[0m: CUDA error: out of memory"
]
}
],
"source": [
"learn.fit_one_cycle(1, 1e-2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
================================================
FILE: examples/fastai/lm/lm.py
================================================
# coding: utf-8
import melt
# http://docs.fast.ai/text.html
from fastai.text import *
import html
from pathlib import Path
import numpy as np
import pandas as pd
import sklearn
from sklearn import model_selection
import re
from functools import partial
from torch import optim
IMDB_PATH = Path('/home/gezi/data/imdb_sample/')
df = pd.read_csv(IMDB_PATH/'train.csv', header=None)
df.head()
classes = read_classes(IMDB_PATH/'classes.txt')
classes[0], classes[1]
data_lm = text_data_from_csv(Path(IMDB_PATH), data_func=lm_data)
data_clas = text_data_from_csv(Path(IMDB_PATH), data_func=classifier_data, vocab=data_lm.train_ds.vocab)
#download_wt103_model()
#learn = RNNLearner.language_model(data_lm, pretrained_fnames=['lstm_wt103', 'itos_wt103'], drop_mult=0.5)
#learn.fit_one_cycle(1, 1e-2)
# default bptt 70 will
learn = RNNLearner.language_model(data_lm, bptt=30, pretrained_fnames=['lstm_wt103', 'itos_wt103'])
learn.unfreeze()
learn.fit(2, slice(1e-4,1e-2))
learn.save_encoder('enc')
learn = RNNLearner.classifier(data_clas)
learn.load_encoder('enc')
learn.fit(3, 1e-3)
================================================
FILE: examples/pytorch/data_parallel_tutorial.py
================================================
"""
Optional: Data Parallelism
==========================
**Authors**: `Sung Kim <https://github.com/hunkim>`_ and `Jenny Kang <https://github.com/jennykang>`_
In this tutorial, we will learn how to use multiple GPUs using ``DataParallel``.
It's very easy to use GPUs with PyTorch. You can put the model on a GPU:
.. code:: python
device = torch.device("cuda:0")
model.to(device)
Then, you can copy all your tensors to the GPU:
.. code:: python
mytensor = my_tensor.to(device)
Please note that just calling ``my_tensor.to(device)`` returns a new copy of
``my_tensor`` on GPU instead of rewriting ``my_tensor``. You need to assign it to
a new tensor and use that tensor on the GPU.
It's natural to execute your forward, backward propagations on multiple GPUs.
However, Pytorch will only use one GPU by default. You can easily run your
operations on multiple GPUs by making your model run parallelly using
``DataParallel``:
.. code:: python
model = nn.DataParallel(model)
That's the core behind this tutorial. We will explore it in more detail below.
"""
######################################################################
# Imports and parameters
# ----------------------
#
# Import PyTorch modules and define parameters.
#
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
# Parameters and DataLoaders
input_size = 5
output_size = 2
batch_size = 30
data_size = 100
######################################################################
# Device
#
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
######################################################################
# Dummy DataSet
# -------------
#
# Make a dummy (random) dataset. You just need to implement the
# getitem
#
class RandomDataset(Dataset):
def __init__(self, size, length):
self.len = length
self.data = torch.randn(length, size)
def __getitem__(self, index):
return self.data[index]
def __len__(self):
return self.len
rand_loader = DataLoader(dataset=RandomDataset(input_size, data_size),
batch_size=batch_size, shuffle=True)
######################################################################
# Simple Model
# ------------
#
# For the demo, our model just gets an input, performs a linear operation, and
# gives an output. However, you can use ``DataParallel`` on any model (CNN, RNN,
# Capsule Net etc.)
#
# We've placed a print statement inside the model to monitor the size of input
# and output tensors.
# Please pay attention to what is printed at batch rank 0.
#
class Model(nn.Module):
# Our model
def __init__(self, input_size, output_size):
super(Model, self).__init__()
self.fc = nn.Linear(input_size, output_size)
def forward(self, input):
print('input\n', input)
output = self.fc(input)
print("\tIn Model: input size", input.size(),
"output size", output.size())
return output
######################################################################
# Create Model and DataParallel
# -----------------------------
#
# This is the core part of the tutorial. First, we need to make a model instance
# and check if we have multiple GPUs. If we have multiple GPUs, we can wrap
# our model using ``nn.DataParallel``. Then we can put our model on GPUs by
# ``model.to(device)``
#
model = Model(input_size, output_size)
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
# dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs
model = nn.DataParallel(model)
model.to(device)
######################################################################
# Run the Model
# -------------
#
# Now we can see the sizes of input and output tensors.
#
for data in rand_loader:
input = data.to(device)
output = model(input)
print("Outside: input size", input.size(),
"output_size", output.size())
######################################################################
# Results
# -------
#
# If you have no GPU or one GPU, when we batch 30 inputs and 30 outputs, the model gets 30 and outputs 30 as
# expected. But if you have multiple GPUs, then you can get results like this.
#
# 2 GPUs
# ~~~~~~
#
# If you have 2, you will see:
#
# .. code:: bash
#
# # on 2 GPUs
# Let's use 2 GPUs!
# In Model: input size torch.Size([15, 5]) output size torch.Size([15, 2])
# In Model: input size torch.Size([15, 5]) output size torch.Size([15, 2])
# Outside: input size torch.Size([30, 5]) output_size torch.Size([30, 2])
# In Model: input size torch.Size([15, 5]) output size torch.Size([15, 2])
# In Model: input size torch.Size([15, 5]) output size torch.Size([15, 2])
# Outside: input size torch.Size([30, 5]) output_size torch.Size([30, 2])
# In Model: input size torch.Size([15, 5]) output size torch.Size([15, 2])
# In Model: input size torch.Size([15, 5]) output size torch.Size([15, 2])
# Outside: input size torch.Size([30, 5]) output_size torch.Size([30, 2])
# In Model: input size torch.Size([5, 5]) output size torch.Size([5, 2])
# In Model: input size torch.Size([5, 5]) output size torch.Size([5, 2])
# Outside: input size torch.Size([10, 5]) output_size torch.Size([10, 2])
#
# 3 GPUs
# ~~~~~~
#
# If you have 3 GPUs, you will see:
#
# .. code:: bash
#
# Let's use 3 GPUs!
# In Model: input size torch.Size([10, 5]) output size torch.Size([10, 2])
# In Model: input size torch.Size([10, 5]) output size torch.Size([10, 2])
# In Model: input size torch.Size([10, 5]) output size torch.Size([10, 2])
# Outside: input size torch.Size([30, 5]) output_size torch.Size([30, 2])
# In Model: input size torch.Size([10, 5]) output size torch.Size([10, 2])
# In Model: input size torch.Size([10, 5]) output size torch.Size([10, 2])
# In Model: input size torch.Size([10, 5]) output size torch.Size([10, 2])
# Outside: input size torch.Size([30, 5]) output_size torch.Size([30, 2])
# In Model: input size torch.Size([10, 5]) output size torch.Size([10, 2])
# In Model: input size torch.Size([10, 5]) output size torch.Size([10, 2])
# In Model: input size torch.Size([10, 5]) output size torch.Size([10, 2])
# Outside: input size torch.Size([30, 5]) output_size torch.Size([30, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([2, 5]) output size torch.Size([2, 2])
# Outside: input size torch.Size([10, 5]) output_size torch.Size([10, 2])
#
# 8 GPUs
# ~~~~~~~~~~~~~~
#
# If you have 8, you will see:
#
# .. code:: bash
#
# Let's use 8 GPUs!
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([2, 5]) output size torch.Size([2, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# Outside: input size torch.Size([30, 5]) output_size torch.Size([30, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([2, 5]) output size torch.Size([2, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# Outside: input size torch.Size([30, 5]) output_size torch.Size([30, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([4, 5]) output size torch.Size([4, 2])
# In Model: input size torch.Size([2, 5]) output size torch.Size([2, 2])
# Outside: input size torch.Size([30, 5]) output_size torch.Size([30, 2])
# In Model: input size torch.Size([2, 5]) output size torch.Size([2, 2])
# In Model: input size torch.Size([2, 5]) output size torch.Size([2, 2])
# In Model: input size torch.Size([2, 5]) output size torch.Size([2, 2])
# In Model: input size torch.Size([2, 5]) output size torch.Size([2, 2])
# In Model: input size torch.Size([2, 5]) output size torch.Size([2, 2])
# Outside: input size torch.Size([10, 5]) output_size torch.Size([10, 2])
#
######################################################################
# Summary
# -------
#
# DataParallel splits your data automatically and sends job orders to multiple
# models on several GPUs. After each model finishes their job, DataParallel
# collects and merges the results before returning it to you.
#
# For more information, please check out
# https://pytorch.org/tutorials/beginner/former\_torchies/parallelism\_tutorial.html.
#
================================================
FILE: examples/pytorch/sentiment/model.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# ==============================================================================
# \file model.py
# \author chenghuige
# \date 2018-09-27 19:37:57.468263
# \Description
# ==============================================================================
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow as tf
flags = tf.app.flags
FLAGS = flags.FLAGS
#flags.DEFINE_string('input', '', '')
#flags.DEFINE_string('output', '', '')
import sys
import os
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
from torch.autograd import Variable
class SimpleGRU(nn.Module):
def __init__(self, vocab_size, embedding_dim, n_hidden, n_out):
super().__init__()
self.vocab_size,self.embedding_dim,self.n_hidden,self.n_out = vocab_size, embedding_dim, n_hidden, n_out
self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)
self.gru = nn.GRU(self.embedding_dim, self.n_hidden)
self.out = nn.Linear(self.n_hidden, self.n_out)
def forward(self, seq, lengths, gpu=True):
print('Sequence shape',seq.shape)
print('Lengths',lengths)
bs = seq.size(1) # batch size
print('batch size', bs)
self.h = self.init_hidden(bs, gpu) # initialize hidden state of GRU
print('Inititial hidden state shape', self.h.shape)
embs = self.emb(seq)
embs = pack_padded_sequence(embs, lengths) # unpad
gru_out, self.h = self.gru(embs, self.h) # gru returns hidden state of all timesteps as well as hidden state at last timestep
gru_out, lengths = pad_packed_sequence(gru_out) # pad the sequence to the max length in the batch
print('GRU output(all timesteps)', gru_out.shape)
print(gru_out)
print('GRU last timestep output')
print(gru_out[-1])
print('Last hidden state', self.h)
# since it is as classification problem, we will grab the last hidden state
outp = self.out(self.h[-1]) # self.h[-1] contains hidden state of last timestep
return F.log_softmax(outp, dim=-1)
def init_hidden(self, batch_size, gpu):
if gpu: return Variable(torch.zeros((1,batch_size,self.n_hidden)).cuda())
else: return Variable(torch.zeros((1,batch_size,self.n_hidden)))
================================================
FILE: examples/pytorch/sentiment/train.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# ==============================================================================
# \file train.py
# \author chenghuige
# \date 2018-09-27 19:37:53.455830
# \Description
# ==============================================================================
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow as tf
flags = tf.app.flags
FLAGS = flags.FLAGS
#flags.DEFINE_string('input', '', '')
#flags.DEFINE_string('output', '', '')
import sys
import os
import pathlib
import random
from collections import Counter, OrderedDict
import numpy as np
import pandas as pd
import spacy
from tqdm import tqdm, tnrange
from model import *
nlp = spacy.blank('en')
data_root = pathlib.Path('./data')
df = pd.read_csv(data_root/'Sentiment Analysis Dataset.csv', error_bad_lines=False)
df['SentimentText'] = df.SentimentText.apply(lambda x: x.strip())
words = Counter()
for sent in tqdm(df.SentimentText.values, ascii=True):
words.update(w.text.lower() for w in nlp(sent))
words = sorted(words, key=words.get, reverse=True)
words = ['_PAD','_UNK'] + words
word2idx = {o:i for i,o in enumerate(words)}
idx2word = {i:o for i,o in enumerate(words)}
def indexer(s):
return [word2idx[w.text.lower()] for w in nlp(s)]
df['sentimentidx'] = df.SentimentText.apply(indexer)
df['lengths'] = df.sentimentidx.apply(len)
class VectorizeData(Dataset):
def __init__(self, df_path, maxlen=10):
self.maxlen = maxlen
self.df = pd.read_csv(df_path, error_bad_lines=False)
self.df['SentimentText'] = self.df.SentimentText.apply(lambda x: x.strip())
print('Indexing...')
self.df['sentimentidx'] = self.df.SentimentText.apply(indexer)
print('Calculating lengths')
self.df['lengths'] = self.df.sentimentidx.apply(lambda x: self.maxlen if len(x) > self.maxlen else len(x))
print('Padding')
self.df['sentimentpadded'] = self.df.sentimentidx.apply(self.pad_data)
def __len__(self):
return self.df.shape[0]
def __getitem__(self, idx):
X = self.df.sentimentpadded[idx]
lens = self.df.lengths[idx]
y = self.df.Sentiment[idx]
return X,y,lens
def pad_data(self, s):
padded = np.zeros((self.maxlen,), dtype=np.int64)
if len(s) > self.maxlen: padded[:] = s[:self.maxlen]
else: padded[:len(s)] = s
return padded
ds = VectorizeData(data_root/'Sentiment Analysis Dataset.csv')
vocab_size = len(words)
embedding_dim = 4
n_hidden = 5
n_out = 2
m = SimpleGRU(vocab_size, embedding_dim, n_hidden, n_out)
def sort_batch(X, y, lengths):
lengths, indx = lengths.sort(dim=0, descending=True)
X = X[indx]
y = y[indx]
return X.transpose(0,1), y, lengths # transpose (batch x seq) to (seq x batch)
def fit(model, train_dl, val_dl, loss_fn, opt, epochs=3):
num_batch = len(train_dl)
for epoch in tnrange(epochs):
y_true_train = list()
y_pred_train = list()
total_loss_train = 0
if val_dl:
y_true_val = list()
y_pred_val = list()
total_loss_val = 0
t = tqdm(iter(train_dl), leave=False, total=num_batch, ascii=True)
for X,y, lengths in t:
t.set_description(f'Epoch {epoch}')
X,y,lengths = sort_batch(X,y,lengths)
X = Variable(X.cuda())
y = Variable(y.cuda())
lengths = lengths.numpy()
opt.zero_grad()
pred = model(X, lengths, gpu=True)
loss = loss_fn(pred, y)
loss.backward()
opt.step()
t.set_postfix(loss=loss.data[0])
pred_idx = torch.max(pred, dim=1)[1]
y_true_train += list(y.cpu().data.numpy())
y_pred_train += list(pred_idx.cpu().data.numpy())
total_loss_train += loss.data[0]
train_acc = accuracy_score(y_true_train, y_pred_train)
train_loss = total_loss_train/len(train_dl)
print(f' Epoch {epoch}: Train loss: {train_loss} acc: {train_acc}')
if val_dl:
for X,y,lengths in tqdm(valdl, leave=False, ascii=True):
X, y,lengths = sort_batch(X,y,lengths)
X = Variable(X.cuda())
y = Variable(y.cuda())
pred = model(X, lengths.numpy())
loss = loss_fn(pred, y)
pred_idx = torch.max(pred, 1)[1]
y_true_val += list(y.cpu().data.numpy())
y_pred_val += list(pred_idx.cpu().data.numpy())
total_loss_val += loss.data[0]
valacc = accuracy_score(y_true_val, y_pred_val)
valloss = total_loss_val/len(valdl)
print(f'Val loss: {valloss} acc: {valacc}')
train_dl = DataLoader(ds, batch_size=512)
m = SimpleGRU(vocab_size, embedding_dim, n_hidden, n_out).cuda()
opt = optim.Adam(m.parameters(), 1e-2)
fit(model=m, train_dl=train_dl, val_dl=None, loss_fn=F.nll_loss, opt=opt, epochs=4)
def main(_):
pass
if __name__ == '__main__':
tf.app.run()
================================================
FILE: examples/shap/explainer_1.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# ==============================================================================
# \file explainer_1.py
# \author chenghuige
# \date 2019-08-02 22:12:47.710013
# \Description
# ==============================================================================
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import sys
import os
import pandas as pd
import seaborn as sns
import keras
import shap
#let's load the diamonds dataset
df=sns.load_dataset(name='diamonds')
print(df.head())
print(df.describe())
================================================
FILE: examples/tf/eager/dynamic_dense.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# ==============================================================================
# \file dynamic_dense.py
# \author chenghuige
# \date 2018-09-12 10:53:31.858019
# \Description
# ==============================================================================
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import sys
import os
import tensorflow as tf
tfe = tf.contrib.eager
tf.enable_eager_execution()
keras = tf.keras
class Layer(keras.layers.Layer):
def __init__(self):
super(Layer, self).__init__()
self.abc = self.add_variable("abc", [1, 3], initializer=tf.ones_initializer(dtype=tf.float32))
def call(self, x):
result = x + self.abc
self.abc = self.abc * 5
return result
class Model(keras.Model):
def __init__(self):
super(Model, self).__init__()
#self.abc = self.add_variable("abc", [1, 3], initializer=tf.ones_initializer(dtype=tf.float32))
#self.encode = keras.layers.Dense(5, activation=None)
self.abc = tfe.Variable(tf.ones([1, 3], dtype=tf.float32), name='abc')
#self.abc = self.abc * 5
#self.abc = Layer()
self.inited = False
def call(self, x):
if not self.inited:
self.inited = True
print(tf.shape(x)[-1])
self.fc = keras.layers.Dense(tf.shape(x)[-1], activation=None)
print(x)
return self.fc(x)
#return self.encode(x + self.abc)
#result = x + self.abc
#self.abc = self.abc * 5
#return result
#return self.abc(x)
model = Model()
print(model(tf.constant([[1., 2., 3.]])))
#print(model.abc)
#m.save_weights('/home/gezi/tmp/model')
checkpoint = tf.train.Checkpoint(model=model)
ckpt_dir = '/home/gezi/tmp/model'
# latest_checkpoint = tf.train.latest_checkpoint(ckpt_dir)
# checkpoint.restore(latest_checkpoint)
# print(model([[1., 2., 3.]]))
checkpoint_prefix = os.path.join(ckpt_dir, 'ckpt')
checkpoint.save(checkpoint_prefix)
#model.abc = model.abc + 1.
#checkpoint.save(checkpoint_prefix)
#model.abc -= 1.
# latest_checkpoint = tf.train.latest_checkpoint(ckpt_dir)
# checkpoint.restore(latest_checkpoint)
# print(model([[1., 2., 3.]]))
================================================
FILE: examples/tf/eager/rnn_ptb.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Penn Treebank RNN model definition compatible with eager execution.
Model similar to
https://github.com/tensorflow/models/tree/master/tutorials/rnn/ptb
Usage: python ./rnn_ptb.py --data-path=<path_to_dataset>
Penn Treebank (PTB) dataset from:
http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
import os
import sys
import time
import numpy as np
import tensorflow as tf
from tensorflow.contrib.cudnn_rnn.python.layers import cudnn_rnn
from tensorflow.contrib.eager.python import tfe
layers = tf.keras.layers
class RNN(tf.keras.Model):
"""A static RNN.
Similar to tf.nn.static_rnn, implemented as a class.
"""
def __init__(self, hidden_dim, num_layers, keep_ratio):
super(RNN, self).__init__()
self.keep_ratio = keep_ratio
self.cells = tf.contrib.checkpoint.List([
tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_dim)
for _ in range(num_layers)
])
def call(self, input_seq, training):
batch_size = int(input_seq.shape[1])
for c in self.cells:
state = c.zero_state(batch_size, tf.float32)
outputs = []
input_seq = tf.unstack(input_seq, num=int(input_seq.shape[0]), axis=0)
for inp in input_seq:
output, state = c(inp, state)
outputs.append(output)
input_seq = tf.stack(outputs, axis=0)
if training:
input_seq = tf.nn.dropout(input_seq, self.keep_ratio)
# Returning a list instead of a single tensor so that the line:
# y = self.rnn(y, ...)[0]
# in PTBModel.call works for both this RNN and CudnnLSTM (which returns a
# tuple (output, output_states).
return [input_seq]
class Embedding(layers.Layer):
"""An Embedding layer."""
def __init__(self, vocab_size, embedding_dim, **kwargs):
super(Embedding, self).__init__(**kwargs)
self.vocab_size = vocab_size
self.embedding_dim = embedding_dim
def build(self, _):
self.embedding = self.add_variable(
"embedding_kernel",
shape=[self.vocab_size, self.embedding_dim],
dtype=tf.float32,
initializer=tf.random_uniform_initializer(-0.1, 0.1),
trainable=True)
def call(self, x):
return tf.nn.embedding_lookup(self.embedding, x)
# pylint: disable=not-callable
class PTBModel(tf.keras.Model):
"""LSTM for word language modeling.
Model described in:
(Zaremba, et. al.) Recurrent Neural Network Regularization
http://arxiv.org/abs/1409.2329
See also:
https://github.com/tensorflow/models/tree/master/tutorials/rnn/ptb
"""
def __init__(self,
vocab_size,
embedding_dim,
hidden_dim,
num_layers,
dropout_ratio,
use_cudnn_rnn=True):
super(PTBModel, self).__init__()
self.keep_ratio = 1 - dropout_ratio
self.use_cudnn_rnn = use_cudnn_rnn
self.embedding = Embedding(vocab_size, embedding_dim)
if self.use_cudnn_rnn:
self.rnn = cudnn_rnn.CudnnLSTM(
num_layers, hidden_dim, dropout=dropout_ratio)
else:
self.rnn = RNN(hidden_dim, num_layers, self.keep_ratio)
print('rnn:', self.rnn)
self.linear = layers.Dense(
vocab_size, kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1))
self._output_shape = [-1, embedding_dim]
def call(self, input_seq, training):
"""Run the forward pass of PTBModel.
Args:
input_seq: [length, batch] shape int64 tensor.
training: Is this a training call.
Returns:
outputs tensors of inference.
"""
y = self.embedding(input_seq)
if training:
y = tf.nn.dropout(y, self.keep_ratio)
y = self.rnn(y, training=training)[0]
return self.linear(tf.reshape(y, self._output_shape))
def clip_gradients(grads_and_vars, clip_ratio):
gradients, variables = zip(*grads_and_vars)
clipped, _ = tf.clip_by_global_norm(gradients, clip_ratio)
return zip(clipped, variables)
def loss_fn(model, inputs, targets, training):
labels = tf.reshape(targets, [-1])
outputs = model(inputs, training=training)
return tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=labels, logits=outputs))
def _divide_into_batches(data, batch_size):
"""Convert a sequence to a batch of sequences."""
nbatch = data.shape[0] // batch_size
data = data[:nbatch * batch_size]
data = data.reshape(batch_size, -1).transpose()
return data
def _get_batch(data, i, seq_len):
slen = min(seq_len, data.shape[0] - 1 - i)
inputs = data[i:i + slen, :]
target = data[i + 1:i + 1 + slen, :]
return tf.constant(inputs), tf.constant(target)
def evaluate(model, data):
"""evaluate an epoch."""
total_loss = 0.0
total_batches = 0
start = time.time()
for _, i in enumerate(range(0, data.shape[0] - 1, FLAGS.seq_len)):
inp, target = _get_batch(data, i, FLAGS.seq_len)
loss = loss_fn(model, inp, target, training=False)
total_loss += loss.numpy()
total_batches += 1
time_in_ms = (time.time() - start) * 1000
sys.stderr.write("eval loss %.2f (eval took %d ms)\n" %
(total_loss / total_batches, time_in_ms))
return total_loss
def train(model, optimizer, train_data, sequence_length, clip_ratio):
"""training an epoch."""
def model_loss(inputs, targets):
return loss_fn(model, inputs, targets, training=True)
grads = tfe.implicit_gradients(model_loss)
total_time = 0
for batch, i in enumerate(range(0, train_data.shape[0] - 1, sequence_length)):
train_seq, train_target = _get_batch(train_data, i, sequence_length)
start = time.time()
optimizer.apply_gradients(
clip_gradients(grads(train_seq, train_target), clip_ratio))
total_time += (time.time() - start)
if batch % 10 == 0:
time_in_ms = (total_time * 1000) / (batch + 1)
sys.stderr.write("batch %d: training loss %.2f, avg step time %d ms\n" %
(batch, model_loss(train_seq, train_target).numpy(),
time_in_ms))
class Datasets(object):
"""Processed form of the Penn Treebank dataset."""
def __init__(self, path):
"""Load the Penn Treebank dataset.
Args:
path: Path to the data/ directory of the dataset from Tomas Mikolov's
webpage - http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz
"""
self.word2idx = {} # string -> integer id
self.idx2word = [] # integer id -> word string
# Files represented as a list of integer ids (as opposed to list of string
# words).
self.train = self.tokenize(os.path.join(path, "ptb.train.txt"))
self.valid = self.tokenize(os.path.join(path, "ptb.valid.txt"))
def vocab_size(self):
return len(self.idx2word)
def add(self, word):
if word not in self.word2idx:
self.idx2word.append(word)
self.word2idx[word] = len(self.idx2word) - 1
def tokenize(self, path):
"""Read text file in path and return a list of integer token ids."""
tokens = 0
with tf.gfile.Open(path, "r") as f:
for line in f:
words = line.split() + ["<eos>"]
tokens += len(words)
for word in words:
self.add(word)
# Tokenize file content
with tf.gfile.Open(path, "r") as f:
ids = np.zeros(tokens).astype(np.int64)
token = 0
for line in f:
words = line.split() + ["<eos>"]
for word in words:
ids[token] = self.word2idx[word]
token += 1
return ids
def small_model(use_cudnn_rnn):
"""Returns a PTBModel with a 'small' configuration."""
return PTBModel(
vocab_size=10000,
embedding_dim=200,
hidden_dim=200,
num_layers=2,
dropout_ratio=0.,
use_cudnn_rnn=use_cudnn_rnn)
def large_model(use_cudnn_rnn):
"""Returns a PTBModel with a 'large' configuration."""
return PTBModel(
vocab_size=10000,
embedding_dim=650,
hidden_dim=650,
num_layers=2,
dropout_ratio=0.5,
use_cudnn_rnn=use_cudnn_rnn)
def test_model(use_cudnn_rnn):
"""Returns a tiny PTBModel for unit tests."""
return PTBModel(
vocab_size=100,
embedding_dim=20,
hidden_dim=20,
num_layers=2,
dropout_ratio=0.,
use_cudnn_rnn=use_cudnn_rnn)
def main(_):
tf.enable_eager_execution()
if not FLAGS.data_path:
raise ValueError("Must specify --data-path")
corpus = Datasets(FLAGS.data_path)
train_data = _divide_into_batches(corpus.train, FLAGS.batch_size)
eval_data = _divide_into_batches(corpus.valid, 10)
have_gpu = tfe.num_gpus() > 0
use_cudnn_rnn = not FLAGS.no_use_cudnn_rnn and have_gpu
with tf.device("/device:GPU:0" if have_gpu else None):
# Make learning_rate a Variable so it can be included in the checkpoint
# and we can resume training with the last saved learning_rate.
learning_rate = tfe.Variable(20.0, name="learning_rate")
model = PTBModel(corpus.vocab_size(), FLAGS.embedding_dim,
FLAGS.hidden_dim, FLAGS.num_layers, FLAGS.dropout,
use_cudnn_rnn)
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
checkpoint = tf.train.Checkpoint(
learning_rate=learning_rate, model=model,
# GradientDescentOptimizer has no state to checkpoint, but noting it
# here lets us swap in an optimizer that does.
optimizer=optimizer)
# Restore existing variables now (learning_rate), and restore new variables
# on creation if a checkpoint exists.
checkpoint.restore(tf.train.latest_checkpoint(FLAGS.logdir))
sys.stderr.write("learning_rate=%f\n" % learning_rate.numpy())
best_loss = None
for _ in range(FLAGS.epoch):
train(model, optimizer, train_data, FLAGS.seq_len, FLAGS.clip)
eval_loss = evaluate(model, eval_data)
if not best_loss or eval_loss < best_loss:
if FLAGS.logdir:
checkpoint.save(os.path.join(FLAGS.logdir, "ckpt"))
best_loss = eval_loss
else:
learning_rate.assign(learning_rate / 4.0)
sys.stderr.write("eval_loss did not reduce in this epoch, "
"changing learning rate to %f for the next epoch\n" %
learning_rate.numpy())
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"--data-path",
type=str,
default="",
help="Data directory of the Penn Treebank dataset from "
"http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz")
parser.add_argument(
"--logdir", type=str, default="", help="Directory for checkpoint.")
parser.add_argument("--epoch", type=int, default=20, help="Number of epochs.")
parser.add_argument("--batch-size", type=int, default=20, help="Batch size.")
parser.add_argument(
"--seq-len", type=int, default=35, help="Sequence length.")
parser.add_argument(
"--embedding-dim", type=int, default=200, help="Embedding dimension.")
parser.add_argument(
"--hidden-dim", type=int, default=200, help="Hidden layer dimension.")
parser.add_argument(
"--num-layers", type=int, default=2, help="Number of RNN layers.")
parser.add_argument(
"--dropout", type=float, default=0.2, help="Drop out ratio.")
parser.add_argument(
"--clip", type=float, default=0.25, help="Gradient clipping ratio.")
parser.add_argument(
"--no-use-cudnn-rnn",
action="store_true",
default=False,
help="Disable the fast CuDNN RNN (when no gpu)")
FLAGS, unparsed = parser.parse_known_args()
tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
================================================
FILE: examples/tf/eager/rnn_ptb.sh
================================================
python rnn_ptb.py --data-path /home/gezi/data/ptb/simple-examples/data
================================================
FILE: jupyter/Untitled.ipynb
================================================
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import torch "
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"tensorflow_version: 1.14.0\n",
"torch_version: 1.1.0\n",
"WARNING: Logging before flag parsing goes to stderr.\n",
"W0804 13:14:34.019139 139721067423552 deprecation_wrapper.py:119] From /search/odin/chenghuige/mine/wenzheng/utils/melt/training/bert/optimization.py:72: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
"\n",
"W0804 13:14:35.313520 139721067423552 lazy_loader.py:50] \n",
"The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
"For more information, please see:\n",
" * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
" * https://github.com/tensorflow/addons\n",
" * https://github.com/tensorflow/io (for I/O related ops)\n",
"If you depend on functionality not listed there, please file an issue.\n",
"\n",
"W0804 13:14:35.325613 139721067423552 deprecation_wrapper.py:119] From /search/odin/chenghuige/mine/wenzheng/utils/melt/util.py:297: The name tf.train.GradientDescentOptimizer is deprecated. Please use tf.compat.v1.train.GradientDescentOptimizer instead.\n",
"\n",
"W0804 13:14:35.326145 139721067423552 deprecation_wrapper.py:119] From /search/odin/chenghuige/mine/wenzheng/utils/melt/util.py:299: The name tf.train.AdagradOptimizer is deprecated. Please use tf.compat.v1.train.AdagradOptimizer instead.\n",
"\n",
"W0804 13:14:35.326648 139721067423552 deprecation_wrapper.py:119] From /search/odin/chenghuige/mine/wenzheng/utils/melt/util.py:314: The name tf.train.FtrlOptimizer is deprecated. Please use tf.compat.v1.train.FtrlOptimizer instead.\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/search/odin/chenghuige/mine/wenzheng/utils/melt/apps/train.py\", line 61, in <module>\n",
" import horovod.tensorflow as hvd\n",
"ModuleNotFoundError: No module named 'horovod'\n",
"\n",
"W0804 13:14:35.357142 139721067423552 deprecation_wrapper.py:119] From /search/odin/chenghuige/mine/wenzheng/official/transformer/model/attention_layer.py:24: The name tf.layers.Layer is deprecated. Please use tf.compat.v1.layers.Layer instead.\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"---------no horovod support for mutliple gpu, notice we use mpi4py for some allgather op for eval\n"
]
}
],
"source": [
"import melt"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"import tensorflow as tf"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"W0804 13:14:36.755970 139721067423552 deprecation_wrapper.py:119] From /search/odin/chenghuige/mine/wenzheng/utils/melt/util.py:335: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
"\n"
]
},
{
"ename": "UnrecognizedFlagError",
"evalue": "Unknown command line flag 'f'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mUnrecognizedFlagError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-4-557e387389a0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msess\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmelt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m/search/odin/chenghuige/mine/wenzheng/utils/melt/util.py\u001b[0m in \u001b[0;36mget_session\u001b[0;34m(log_device_placement, allow_soft_placement, debug, device_count)\u001b[0m\n\u001b[1;32m 350\u001b[0m \u001b[0;31m#config.operation_timeout_in_ms=50000 # terminate on long hangs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 351\u001b[0m \u001b[0;31m#https://github.com/tensorflow/tensorflow/issues/2292 allow_soft_placement=True\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 352\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mFLAGS\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_tpu\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 353\u001b[0m \u001b[0mtpu_cluster_resolver\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 354\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mFLAGS\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_tpu\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mFLAGS\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtpu_name\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/home/gezi/env/anaconda3/lib/python3.7/site-packages/tensorflow/python/platform/flags.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 82\u001b[0m \u001b[0;31m# a flag.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 83\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_parsed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 84\u001b[0;31m \u001b[0mwrapped\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_sys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 85\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/home/gezi/env/anaconda3/lib/python3.7/site-packages/absl/flags/_flagvalues.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, argv, known_only)\u001b[0m\n\u001b[1;32m 631\u001b[0m \u001b[0msuggestions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_helpers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_flag_suggestions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 632\u001b[0m raise _exceptions.UnrecognizedFlagError(\n\u001b[0;32m--> 633\u001b[0;31m name, value, suggestions=suggestions)\n\u001b[0m\u001b[1;32m 634\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 635\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmark_as_parsed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mUnrecognizedFlagError\u001b[0m: Unknown command line flag 'f'"
]
}
],
"source": [
"sess = melt.get_session()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"sess = tf.Session()"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import gezi"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\u001b[0;31mInit signature:\u001b[0m \u001b[0mgezi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAvgScore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mDocstring:\u001b[0m \n",
"Calcuatlate avg scores, input can be single value or list\n",
"Not using numpy, so may be much slower then numpy version, now mainly used for tensorflow testers\n",
"\u001b[0;31mFile:\u001b[0m /search/odin/chenghuige/mine/wenzheng/utils/gezi/avg_score.py\n",
"\u001b[0;31mType:\u001b[0m type\n",
"\u001b[0;31mSubclasses:\u001b[0m \n"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"gezi.AvgScore?"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
================================================
FILE: official/.gitignore
================================================
MNIST-data
labels.txt
================================================
FILE: official/Dockerfile.cpu
================================================
# Docker image for running examples in Tensorflow models.
# base_image depends on whether we are running on GPUs or non-GPUs
FROM ubuntu:latest
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
build-essential \
git \
python \
python-pip \
python-setuptools
RUN pip install tf-nightly
# Checkout tensorflow/models at HEAD
RUN git clone https://github.com/tensorflow/models.git /tensorflow_models
================================================
FILE: official/Dockerfile.gpu
================================================
# Docker image for running examples in Tensorflow models.
# base_image depends on whether we are running on GPUs or non-GPUs
FROM nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
build-essential \
git \
python \
python-pip \
python-setuptools
RUN pip install tf-nightly-gpu
# Checkout tensorflow/models at HEAD
RUN git clone https://github.com/tensorflow/models.git /tensorflow_models
================================================
FILE: official/README.md
================================================
# TensorFlow Official Models
The TensorFlow official models are a collection of example models that use TensorFlow's high-level APIs. They are intended to be well-maintained, tested, and kept up to date with the latest TensorFlow API. They should also be reasonably optimized for fast performance while still being easy to read.
The master branch of the models are **in development**, and they target the [nightly binaries](https://github.com/tensorflow/tensorflow#installation) built from the [master branch of TensorFlow](https://github.com/tensorflow/tensorflow/tree/master). We aim to keep them backwards compatible with the latest release when possible (currently TensorFlow 1.5), but we cannot always guarantee compatibility.
**Stable versions** of the official models targeting releases of TensorFlow are available as tagged branches or [downloadable releases](https://github.com/tensorflow/models/releases). Model repository version numbers match the target TensorFlow release, such that [branch r1.4.0](https://github.com/tensorflow/models/tree/r1.4.0) and [release v1.4.0](https://github.com/tensorflow/models/releases/tag/v1.4.0) are compatible with [TensorFlow v1.4.0](https://github.com/tensorflow/tensorflow/releases/tag/v1.4.0).
If you are on a version of TensorFlow earlier than 1.4, please [update your installation](https://www.tensorflow.org/install/).
---
Below is a list of the models available.
[boosted_trees](boosted_trees): A Gradient Boosted Trees model to classify higgs boson process from HIGGS Data Set.
[mnist](mnist): A basic model to classify digits from the MNIST dataset.
[resnet](resnet): A deep residual network that can be used to classify both CIFAR-10 and ImageNet's dataset of 1000 classes.
[wide_deep](wide_deep): A model that combines a wide model and deep network to classify census income data.
More models to come!
If you would like to make any fixes or improvements to the models, please [submit a pull request](https://github.com/tensorflow/models/compare).
---
## Running the models
The *Official Models* are made available as a Python module. To run the models and associated scripts, add the top-level ***/models*** folder to the Python path with the command: `export PYTHONPATH="$PYTHONPATH:/path/to/models"`
To install dependencies pass `-r official/requirements.txt` to pip. (i.e. `pip3 install --user -r official/requirements.txt`)
To make Official Models easier to use, we are planning to create a pip installable Official Models package. This is being tracked in [#917](https://github.com/tensorflow/models/issues/917).
================================================
FILE: official/__init__.py
================================================
================================================
FILE: official/benchmark/__init__.py
================================================
================================================
FILE: official/benchmark/benchmark_uploader.py
================================================
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Library to upload benchmark generated by BenchmarkLogger to remote repo.
This library require google cloud bigquery lib as dependency, which can be
installed with:
> pip install --upgrade google-cloud-bigquery
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import json
from google.cloud import bigquery
import tensorflow as tf
class BigQueryUploader(object):
"""Upload the benchmark and metric info from JSON input to BigQuery. """
def __init__(self, gcp_project=None, credentials=None):
"""Initialized BigQueryUploader with proper setting.
Args:
gcp_project: string, the name of the GCP project that the log will be
uploaded to. The default project name will be detected from local
environment if no value is provided.
credentials: google.auth.credentials. The credential to access the
BigQuery service. The default service account credential will be
detected from local environment if no value is provided. Please use
google.oauth2.service_account.Credentials to load credential from local
file for the case that the test is run out side of GCP.
"""
self._bq_client = bigquery.Client(
project=gcp_project, credentials=credentials)
def upload_benchmark_run_json(
self, dataset_name, table_name, run_id, run_json):
"""Upload benchmark run information to Bigquery.
Args:
dataset_name: string, the name of bigquery dataset where the data will be
uploaded.
table_name: string, the name of bigquery table under the dataset where
the data will be uploaded.
run_id: string, a unique ID that will be attached to the data, usually
this is a UUID4 format.
run_json: dict, the JSON data that contains the benchmark run info.
"""
run_json["model_id"] = run_id
self._upload_json(dataset_name, table_name, [run_json])
def upload_benchmark_metric_json(
self, dataset_name, table_name, run_id, metric_json_list):
"""Upload metric information to Bigquery.
Args:
dataset_name: string, the name of bigquery dataset where the data will be
uploaded.
table_name: string, the name of bigquery table under the dataset where
the metric data will be uploaded. This is different from the
benchmark_run table.
run_id: string, a unique ID that will be attached to the data, usually
this is a UUID4 format. This should be the same as the benchmark run_id.
metric_json_list: list, a list of JSON object that record the metric info.
"""
for m in metric_json_list:
m["run_id"] = run_id
self._upload_json(dataset_name, table_name, metric_json_list)
def upload_benchmark_run_file(
self, dataset_name, table_name, run_id, run_json_file):
"""Upload benchmark run information to Bigquery from input json file.
Args:
dataset_name: string, the name of bigquery dataset where the data will be
uploaded.
table_name: string, the name of bigquery table under the dataset where
the data will be uploaded.
run_id: string, a unique ID that will be attached to the data, usually
this is a UUID4 format.
run_json_file: string, the file path that contains the run JSON data.
"""
with tf.gfile.GFile(run_json_file) as f:
benchmark_json = json.load(f)
self.upload_benchmark_run_json(
dataset_name, table_name, run_id, benchmark_json)
def upload_metric_file(
self, dataset_name, table_name, run_id, metric_json_file):
"""Upload metric information to Bigquery from input json file.
Args:
dataset_name: string, the name of bigquery dataset where the data will be
uploaded.
table_name: string, the name of bigquery table under the dataset where
the metric data will be uploaded. This is different from the
benchmark_run table.
run_id: string, a unique ID that will be attached to the data, usually
this is a UUID4 format. This should be the same as the benchmark run_id.
metric_json_file: string, the file path that contains the metric JSON
data.
"""
with tf.gfile.GFile(metric_json_file) as f:
metrics = []
for line in f:
metrics.append(json.loads(line.strip()))
self.upload_benchmark_metric_json(
dataset_name, table_name, run_id, metrics)
def _upload_json(self, dataset_name, table_name, json_list):
# Find the unique table reference based on dataset and table name, so that
# the data can be inserted to it.
table_ref = self._bq_client.dataset(dataset_name).table(table_name)
errors = self._bq_client.insert_rows_json(table_ref, json_list)
if errors:
tf.logging.error(
"Failed to upload benchmark info to bigquery: {}".format(errors))
================================================
FILE: official/benchmark/benchmark_uploader_main.py
================================================
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Binary to upload benchmark generated by BenchmarkLogger to remote repo.
This library require google cloud bigquery lib as dependency, which can be
installed with:
> pip install --upgrade google-cloud-bigquery
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import sys
import uuid
from absl import app as absl_app
from absl import flags
from official.benchmark import benchmark_uploader
from official.utils.flags import core as flags_core
from official.utils.logs import logger
def main(_):
if not flags.FLAGS.benchmark_log_dir:
print("Usage: benchmark_uploader.py --benchmark_log_dir=/some/dir")
sys.exit(1)
uploader = benchmark_uploader.BigQueryUploader(
gcp_project=flags.FLAGS.gcp_project)
run_id = str(uuid.uuid4())
run_json_file = os.path.join(
flags.FLAGS.benchmark_log_dir, logger.BENCHMARK_RUN_LOG_FILE_NAME)
metric_json_file = os.path.join(
flags.FLAGS.benchmark_log_dir, logger.METRIC_LOG_FILE_NAME)
uploader.upload_benchmark_run_file(
flags.FLAGS.bigquery_data_set, flags.FLAGS.bigquery_run_table, run_id,
run_json_file)
uploader.upload_metric_file(
flags.FLAGS.bigquery_data_set, flags.FLAGS.bigquery_metric_table, run_id,
metric_json_file)
if __name__ == "__main__":
flags_core.define_benchmark()
flags.adopt_module_key_flags(flags_core)
absl_app.run(main=main)
================================================
FILE: official/benchmark/benchmark_uploader_test.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for benchmark_uploader."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import json
import os
import tempfile
import unittest
from mock import MagicMock
from mock import patch
import tensorflow as tf # pylint: disable=g-bad-import-order
try:
from google.cloud import bigquery
from official.benchmark import benchmark_uploader
except ImportError:
bigquery = None
benchmark_uploader = None
@unittest.skipIf(bigquery is None, 'Bigquery dependency is not installed.')
class BigQueryUploaderTest(tf.test.TestCase):
@patch.object(bigquery, 'Client')
def setUp(self, mock_bigquery):
self.mock_client = mock_bigquery.return_value
self.mock_dataset = MagicMock(name="dataset")
self.mock_table = MagicMock(name="table")
self.mock_client.dataset.return_value = self.mock_dataset
self.mock_dataset.table.return_value = self.mock_table
self.mock_client.insert_rows_json.return_value = []
self.benchmark_uploader = benchmark_uploader.BigQueryUploader()
self.benchmark_uploader._bq_client = self.mock_client
self.log_dir = tempfile.mkdtemp(dir=self.get_temp_dir())
with open(os.path.join(self.log_dir, 'metric.log'), 'a') as f:
json.dump({'name': 'accuracy', 'value': 1.0}, f)
f.write("\n")
json.dump({'name': 'loss', 'value': 0.5}, f)
f.write("\n")
with open(os.path.join(self.log_dir, 'run.log'), 'w') as f:
json.dump({'model_name': 'value'}, f)
def tearDown(self):
tf.gfile.DeleteRecursively(self.get_temp_dir())
def test_upload_benchmark_run_json(self):
self.benchmark_uploader.upload_benchmark_run_json(
'dataset', 'table', 'run_id', {'model_name': 'value'})
self.mock_client.insert_rows_json.assert_called_once_with(
self.mock_table, [{'model_name': 'value', 'model_id': 'run_id'}])
def test_upload_benchmark_metric_json(self):
metric_json_list = [
{'name': 'accuracy', 'value': 1.0},
{'name': 'loss', 'value': 0.5}
]
expected_params = [
{'run_id': 'run_id', 'name': 'accuracy', 'value': 1.0},
{'run_id': 'run_id', 'name': 'loss', 'value': 0.5}
]
self.benchmark_uploader.upload_benchmark_metric_json(
'dataset', 'table', 'run_id', metric_json_list)
self.mock_client.insert_rows_json.assert_called_once_with(
self.mock_table, expected_params)
def test_upload_benchmark_run_file(self):
self.benchmark_uploader.upload_benchmark_run_file(
'dataset', 'table', 'run_id', os.path.join(self.log_dir, 'run.log'))
self.mock_client.insert_rows_json.assert_called_once_with(
self.mock_table, [{'model_name': 'value', 'model_id': 'run_id'}])
def test_upload_metric_file(self):
self.benchmark_uploader.upload_metric_file(
'dataset', 'table', 'run_id',
os.path.join(self.log_dir, 'metric.log'))
expected_params = [
{'run_id': 'run_id', 'name': 'accuracy', 'value': 1.0},
{'run_id': 'run_id', 'name': 'loss', 'value': 0.5}
]
self.mock_client.insert_rows_json.assert_called_once_with(
self.mock_table, expected_params)
if __name__ == '__main__':
tf.test.main()
================================================
FILE: official/benchmark/datastore/schema/benchmark_metric.json
================================================
[
{
"description": "The ID of the benchmark run, where this metric should tie to.",
"mode": "REQUIRED",
"name": "run_id",
"type": "STRING"
},
{
"description": "The name of the metric, which should be descriptive. E.g. training_loss, accuracy.",
"mode": "REQUIRED",
"name": "name",
"type": "STRING"
},
{
"description": "The unit of the metric. E.g. MB per sec.",
"mode": "NULLABLE",
"name": "unit",
"type": "STRING"
},
{
"description": "The value of the metric.",
"mode": "NULLABLE",
"name": "value",
"type": "FLOAT"
},
{
"description": "The timestamp when the metric is recorded.",
"mode": "REQUIRED",
"name": "timestamp",
"type": "TIMESTAMP"
},
{
"description": "The global step when this metric is recorded.",
"mode": "NULLABLE",
"name": "global_step",
"type": "INTEGER"
},
{
"description": "Free format metadata for the extra information about the metric.",
"mode": "REPEATED",
"name": "extras",
"type": "RECORD",
"fields": [
{
"mode": "NULLABLE",
"name": "name",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "value",
"type": "STRING"
}
]
}
]
================================================
FILE: official/benchmark/datastore/schema/benchmark_run.json
================================================
[
{
"description": "The UUID of the run for the benchmark.",
"mode": "REQUIRED",
"name": "model_id",
"type": "STRING"
},
{
"description": "The status of the run for the benchmark. Eg, running, failed, success",
"mode": "NULLABLE",
"name": "status",
"type": "STRING"
},
{
"description": "The name of the model, E.g ResNet50, LeNet-5 etc.",
"mode": "REQUIRED",
"name": "model_name",
"type": "STRING"
},
{
"description": "The date when the test of the model is started",
"mode": "REQUIRED",
"name": "run_date",
"type": "TIMESTAMP"
},
{
"description": "The unique name for a test by the combination of key parameters, eg batch size, num of GPU, etc. It is hardware independent.",
"mode": "NULLABLE",
"name": "test_id",
"type": "STRING"
},
{
"description": "The tensorflow version information.",
"fields": [
{
"description": "Version of the tensorflow. E.g. 1.7.0-rc0",
"mode": "REQUIRED",
"name": "version",
"type": "STRING"
},
{
"description": "Git Hash of the tensorflow",
"mode": "NULLABLE",
"name": "git_hash",
"type": "STRING"
},
{
"description": "The channel of the tensorflow binary, eg, nightly, RC, final, custom.",
"mode": "NULLABLE",
"name": "channel",
"type": "STRING"
},
{
"description": "Identify anything special about the build, eg CUDA 10, NCCL, MKL, etc.",
"mode": "NULLABLE",
"name": "build_type",
"type": "STRING"
}
],
"mode": "REQUIRED",
"name": "tensorflow_version",
"type": "RECORD"
},
{
"description": "The arbitrary attribute of the model.",
"fields": [
{
"description": "The name of the attribute.",
"mode": "REQUIRED",
"name": "name",
"type": "STRING"
},
{
"description": "The value of the attribute.",
"mode": "NULLABLE",
"name": "value",
"type": "STRING"
}
],
"mode": "REPEATED",
"name": "attribute",
"type": "RECORD"
},
{
"description": "Environment variables when the benchmark run is executed.",
"fields": [
{
"description": "The name of the variable.",
"mode": "REQUIRED",
"name": "name",
"type": "STRING"
},
{
"description": "The value of the variable.",
"mode": "NULLABLE",
"name": "value",
"type": "STRING"
}
],
"mode": "REPEATED",
"name": "environment_variable",
"type": "RECORD"
},
{
"description": "TF Environment variables when the benchmark run is executed.",
"fields": [
{
"description": "The name of the variable.",
"mode": "REQUIRED",
"name": "name",
"type": "STRING"
},
{
"description": "The value of the variable.",
"mode": "NULLABLE",
"name": "value",
"type": "STRING"
}
],
"mode": "REPEATED",
"name": "tensorflow_environment_variables",
"type": "RECORD"
},
{
"description": "The list of parameters run with the model. It could contain hyperparameters or others.",
"fields": [
{
"description": "The name of the parameter.",
"mode": "REQUIRED",
"name": "name",
"type": "STRING"
},
{
"description": "The string value of the parameter.",
"mode": "NULLABLE",
"name": "string_value",
"type": "STRING"
},
{
"description": "The bool value of the parameter.",
"mode": "NULLABLE",
"name": "bool_value",
"type": "STRING"
},
{
"description": "The int/long value of the parameter.",
"mode": "NULLABLE",
"name": "long_value",
"type": "INTEGER"
},
{
"description": "The double/float value of parameter.",
"mode": "NULLABLE",
"name": "float_value",
"type": "FLOAT"
}
],
"mode": "REPEATED",
"name": "run_parameters",
"type": "RECORD"
},
{
"description": "The dataset that run with the benchmark.",
"mode": "NULLABLE",
"name": "dataset",
"type": "RECORD",
"fields": [
{
"description": "The name of the dataset that the model is trained/validated with. E.g ImageNet, mnist.",
"mode": "REQUIRED",
"name": "name",
"type": "STRING"
},
{
"description": "The arbitrary attribute of the dataset.",
"fields": [
{
"description": "The name of the attribute.",
"mode": "REQUIRED",
"name": "name",
"type": "STRING"
},
{
"description": "The value of the attribute.",
"mode": "NULLABLE",
"name": "value",
"type": "STRING"
}
],
"mode": "REPEATED",
"name": "attribute",
"type": "RECORD"
}
]
},
{
"description": "Used to differentiate from AWS, GCE or DGX-1 at a high level",
"mode": "NULLABLE",
"name": "test_environment",
"type": "STRING"
},
{
"description": "The machine configuration of the benchmark run.",
"mode": "NULLABLE",
"name": "machine_config",
"type": "RECORD",
"fields": [
{
"description": "The platform information of the benchmark run.",
"mode": "NULLABLE",
"name": "platform_info",
"type": "RECORD",
"fields": [
{
"description": "Eg: 64bit.",
"mode": "NULLABLE",
"name": "bits",
"type": "STRING"
},
{
"description": "Eg: ELF.",
"mode": "NULLABLE",
"name": "linkage",
"type": "STRING"
},
{
"description": "Eg: i386.",
"mode": "NULLABLE",
"name": "machine",
"type": "STRING"
},
{
"description": "Eg: 3.13.0-76-generic.",
"mode": "NULLABLE",
"name": "release",
"type": "STRING"
},
{
"description": "Eg: Linux.",
"mode": "NULLABLE",
"name": "system",
"type": "STRING"
},
{
"description": "Eg: #120-Ubuntu SMP Mon Jan 18 15:59:10 UTC 2016.",
"mode": "NULLABLE",
"name": "version",
"type": "STRING"
}
]
},
{
"description": "The CPU information of the benchmark run.",
"mode": "NULLABLE",
"name": "cpu_info",
"type": "RECORD",
"fields": [
{
"mode": "NULLABLE",
"name": "num_cores",
"type": "INTEGER"
},
{
"mode": "NULLABLE",
"name": "num_cores_allowed",
"type": "INTEGER"
},
{
"description" : "How fast are those CPUs.",
"mode": "NULLABLE",
"name": "mhz_per_cpu",
"type": "FLOAT"
},
{
"description" : "Additional CPU info, Eg: Intel Ivybridge with HyperThreading (24 cores).",
"mode": "NULLABLE",
"name": "cpu_info",
"type": "STRING"
},
{
"description" : "What kind of cpu scaling is enabled on the host. Eg performance, ondemand, conservative, mixed.",
"mode": "NULLABLE",
"name": "cpu_governor",
"type": "STRING"
},
{
"description": "Cache size of the CPUs.",
"mode": "NULLABLE",
"name": "cache_size",
"type": "RECORD",
"fields": [
{
"mode": "NULLABLE",
"name": "level",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "size",
"type": "INTEGER"
}
]
}
]
},
{
"mode": "NULLABLE",
"name": "gpu_info",
"type": "RECORD",
"fields": [
{
"mode": "NULLABLE",
"name": "count",
"type": "INTEGER"
},
{
"mode": "NULLABLE",
"name": "model",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "cuda_version",
"type": "STRING"
}
]
},
{
"description": "The cloud instance inforation if the benchmark run is executed on cloud",
"mode": "NULLABLE",
"name": "cloud_info",
"type": "RECORD",
"fields": [
{
"description": "The instance type, E.g. n1-standard-4.",
"mode": "NULLABLE",
"name": "instance_type",
"type": "STRING"
},
{
"description": "The arbitrary attribute of the cloud info.",
"fields": [
{
"description": "The name of the attribute.",
"mode": "REQUIRED",
"name": "name",
"type": "STRING"
},
{
"description": "The value of the attribute.",
"mode": "NULLABLE",
"name": "value",
"type": "STRING"
}
],
"mode": "REPEATED",
"name": "attribute",
"type": "RECORD"
}
]
},
{
"mode": "NULLABLE",
"name": "memory_total",
"type": "INTEGER"
},
{
"mode": "NULLABLE",
"name": "memory_available",
"type": "STRING"
}
]
}
]
================================================
FILE: official/boosted_trees/README.md
================================================
# Classifying Higgs boson processes in the HIGGS Data Set
## Overview
The [HIGGS Data Set](https://archive.ics.uci.edu/ml/datasets/HIGGS) contains 11 million samples with 28 features, and is for the classification problem to distinguish between a signal process which produces Higgs bosons and a background process which does not.
We use Gradient Boosted Trees algorithm to distinguish the two classes.
---
The code sample uses the high level `tf.estimator.Estimator` and `tf.data.Dataset`. These APIs are great for fast iteration and quickly adapting models to your own datasets without major code overhauls. It allows you to move from single-worker training to distributed training, and makes it easy to export model binaries for prediction. Here, for further simplicity and faster execution, we use a utility function `tf.contrib.estimator.boosted_trees_classifier_train_in_memory`. This utility function is especially effective when the input is provided as in-memory data sets like numpy arrays.
An input function for the `Estimator` typically uses `tf.data.Dataset` API, which can handle various data control like streaming, batching, transform and shuffling. However `boosted_trees_classifier_train_in_memory()` utility function requires that the entire data is provided as a single batch (i.e. without using `batch()` API). Thus in this practice, simply `Dataset.from_tensors()` is used to convert numpy arrays into structured tensors, and `Dataset.zip()` is used to put features and label together.
For further references of `Dataset`, [Read more here](https://www.tensorflow.org/programmers_guide/datasets).
## Running the code
First make sure you've [added the models folder to your Python path](/official/#running-the-models); otherwise you may encounter an error like `ImportError: No module named official.boosted_trees`.
### Setup
The [HIGGS Data Set](https://archive.ics.uci.edu/ml/datasets/HIGGS) that this sample uses for training is hosted by the [UC Irvine Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/). We have provided a script that downloads and cleans the necessary files.
```
python data_download.py
```
This will download a file and store the processed file under the directory designated by `--data_dir` (defaults to `/tmp/higgs_data/`). To change the target directory, set the `--data_dir` flag. The directory could be network storages that Tensorflow supports (like Google Cloud Storage, `gs://<bucket>/<path>/`).
The file downloaded to the local temporary folder is about 2.8 GB, and the processed file is about 0.8 GB, so there should be enough storage to handle them.
### Training
This example uses about 3 GB of RAM during training.
You can run the code locally as follows:
```
python train_higgs.py
```
The model is by default saved to `/tmp/higgs_model`, which can be changed using the `--model_dir` flag.
Note that the model_dir is cleaned up before every time training starts.
Model parameters can be adjusted by flags, like `--n_trees`, `--max_depth`, `--learning_rate` and so on. Check out the code for details.
The final accuracy will be around 74% and loss will be around 0.516 over the eval set, when trained with the default parameters.
By default, the first 1 million examples among 11 millions are used for training, and the last 1 million examples are used for evaluation.
The training/evaluation data can be selected as index ranges by flags `--train_start`, `--train_count`, `--eval_start`, `--eval_count`, etc.
### TensorBoard
Run TensorBoard to inspect the details about the graph and training progression.
```
tensorboard --logdir=/tmp/higgs_model # set logdir as --model_dir set during training.
```
## Inference with SavedModel
You can export the model into Tensorflow [SavedModel](https://www.tensorflow.org/programmers_guide/saved_model) format by using the argument `--export_dir`:
```
python train_higgs.py --export_dir /tmp/higgs_boosted_trees_saved_model
```
After the model finishes training, use [`saved_model_cli`](https://www.tensorflow.org/programmers_guide/saved_model#cli_to_inspect_and_execute_savedmodel) to inspect and execute the SavedModel.
Try the following commands to inspect the SavedModel:
**Replace `${TIMESTAMP}` with the folder produced (e.g. 1524249124)**
```
# List possible tag_sets. Only one metagraph is saved, so there will be one option.
saved_model_cli show --dir /tmp/higgs_boosted_trees_saved_model/${TIMESTAMP}/
# Show SignatureDefs for tag_set=serve. SignatureDefs define the outputs to show.
saved_model_cli show --dir /tmp/higgs_boosted_trees_saved_model/${TIMESTAMP}/ \
--tag_set serve --all
```
### Inference
Let's use the model to predict the income group of two examples.
Note that this model exports SavedModel with the custom parsing module that accepts csv lines as features. (Each line is an example with 28 columns; be careful to not add a label column, unlike in the training data.)
```
saved_model_cli run --dir /tmp/boosted_trees_higgs_saved_model/${TIMESTAMP}/ \
--tag_set serve --signature_def="predict" \
--input_exprs='inputs=["0.869293,-0.635082,0.225690,0.327470,-0.689993,0.754202,-0.248573,-1.092064,0.0,1.374992,-0.653674,0.930349,1.107436,1.138904,-1.578198,-1.046985,0.0,0.657930,-0.010455,-0.045767,3.101961,1.353760,0.979563,0.978076,0.920005,0.721657,0.988751,0.876678", "1.595839,-0.607811,0.007075,1.818450,-0.111906,0.847550,-0.566437,1.581239,2.173076,0.755421,0.643110,1.426367,0.0,0.921661,-1.190432,-1.615589,0.0,0.651114,-0.654227,-1.274345,3.101961,0.823761,0.938191,0.971758,0.789176,0.430553,0.961357,0.957818"]'
```
This will print out the predicted classes and class probabilities. Something like:
```
Result for output key class_ids:
[[1]
[0]]
Result for output key classes:
[['1']
['0']]
Result for output key logistic:
[[0.6440273 ]
[0.10902369]]
Result for output key logits:
[[ 0.59288704]
[-2.1007526 ]]
Result for output key probabilities:
[[0.3559727 0.6440273]
[0.8909763 0.1090237]]
```
Please note that "predict" signature_def gives out different (more detailed) results than "classification" or "serving_default".
## Additional Links
If you are interested in distributed training, take a look at [Distributed TensorFlow](https://www.tensorflow.org/deploy/distributed).
You can also [train models on Cloud ML Engine](https://cloud.google.com/ml-engine/docs/getting-started-training-prediction), which provides [hyperparameter tuning](https://cloud.google.com/ml-engine/docs/getting-started-training-prediction#hyperparameter_tuning) to maximize your model's results and enables [deploying your model for prediction](https://cloud.google.com/ml-engine/docs/getting-started-training-prediction#deploy_a_model_to_support_prediction).
================================================
FILE: official/boosted_trees/__init__.py
================================================
================================================
FILE: official/boosted_trees/data_download.py
================================================
"""Downloads the UCI HIGGS Dataset and prepares train data.
The details on the dataset are in https://archive.ics.uci.edu/ml/datasets/HIGGS
It takes a while as it needs to download 2.8 GB over the network, process, then
store it into the specified location as a compressed numpy file.
Usage:
$ python data_download.py --data_dir=/tmp/higgs_data
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import gzip
import os
import tempfile
# pylint: disable=g-bad-import-order
import numpy as np
import pandas as pd
from six.moves import urllib
from absl import app as absl_app
from absl import flags
import tensorflow as tf
from official.utils.flags import core as flags_core
URL_ROOT = "https://archive.ics.uci.edu/ml/machine-learning-databases/00280"
INPUT_FILE = "HIGGS.csv.gz"
NPZ_FILE = "HIGGS.csv.gz.npz" # numpy compressed file to contain "data" array.
def _download_higgs_data_and_save_npz(data_dir):
"""Download higgs data and store as a numpy compressed file."""
input_url = os.path.join(URL_ROOT, INPUT_FILE)
np_filename = os.path.join(data_dir, NPZ_FILE)
if tf.gfile.Exists(np_filename):
raise ValueError("data_dir already has the processed data file: {}".format(
np_filename))
if not tf.gfile.Exists(data_dir):
tf.gfile.MkDir(data_dir)
# 2.8 GB to download.
try:
tf.logging.info("Data downloading...")
temp_filename, _ = urllib.request.urlretrieve(input_url)
# Reading and parsing 11 million csv lines takes 2~3 minutes.
tf.logging.info("Data processing... taking multiple minutes...")
with gzip.open(temp_filename, "rb") as csv_file:
data = pd.read_csv(
csv_file,
dtype=np.float32,
names=["c%02d" % i for i in range(29)] # label + 28 features.
).as_matrix()
finally:
tf.gfile.Remove(temp_filename)
# Writing to temporary location then copy to the data_dir (0.8 GB).
f = tempfile.NamedTemporaryFile()
np.savez_compressed(f, data=data)
tf.gfile.Copy(f.name, np_filename)
tf.logging.info("Data saved to: {}".format(np_filename))
def main(unused_argv):
if not tf.gfile.Exists(FLAGS.data_dir):
tf.gfile.MkDir(FLAGS.data_dir)
_download_higgs_data_and_save_npz(FLAGS.data_dir)
def define_data_download_flags():
"""Add flags specifying data download arguments."""
flags.DEFINE_string(
name="data_dir", default="/tmp/higgs_data",
help=flags_core.help_wrap(
"Directory to download higgs dataset and store training/eval data."))
if __name__ == "__main__":
tf.logging.set_verbosity(tf.logging.INFO)
define_data_download_flags()
FLAGS = flags.FLAGS
absl_app.run(main)
================================================
FILE: official/boosted_trees/train_higgs.py
================================================
r"""A script that builds boosted trees over higgs data.
If you haven't, please run data_download.py beforehand to prepare the data.
For some more details on this example, please refer to README.md as well.
Note that the model_dir is cleaned up before starting the training.
Usage:
$ python train_higgs.py --n_trees=100 --max_depth=6 --learning_rate=0.1 \
--model_dir=/tmp/higgs_model
Note that BoostedTreesClassifier is available since Tensorflow 1.8.0.
So you need to install recent enough version of Tensorflow to use this example.
The training data is by default the first million examples out of 11M examples,
and eval data is by default the last million examples.
They are controlled by --train_start, --train_count, --eval_start, --eval_count.
e.g. to train over the first 10 million examples instead of 1 million:
$ python train_higgs.py --n_trees=100 --max_depth=6 --learning_rate=0.1 \
--model_dir=/tmp/higgs_model --train_count=10000000
Training history and metrics can be inspected using tensorboard.
Set --logdir as the --model_dir set by flag when training
(or the default /tmp/higgs_model).
$ tensorboard --logdir=/tmp/higgs_model
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
# pylint: disable=g-bad-import-order
import numpy as np
from absl import app as absl_app
from absl import flags
import tensorflow as tf
# pylint: enable=g-bad-import-order
from official.utils.flags import core as flags_core
from official.utils.flags._conventions import help_wrap
from official.utils.logs import logger
NPZ_FILE = "HIGGS.csv.gz.npz" # numpy compressed file containing "data" array
def read_higgs_data(data_dir, train_start, train_count, eval_start, eval_count):
"""Reads higgs data from csv and returns train and eval data.
Args:
data_dir: A string, the directory of higgs dataset.
train_start: An integer, the start index of train examples within the data.
train_count: An integer, the number of train examples within the data.
eval_start: An integer, the start index of eval examples within the data.
eval_count: An integer, the number of eval examples within the data.
Returns:
Numpy array of train data and eval data.
"""
npz_filename = os.path.join(data_dir, NPZ_FILE)
try:
# gfile allows numpy to read data from network data sources as well.
with tf.gfile.Open(npz_filename, "rb") as npz_file:
with np.load(npz_file) as npz:
data = npz["data"]
except tf.errors.NotFoundError as e:
raise RuntimeError(
"Error loading data; use data_download.py to prepare the data.\n{}: {}"
.format(type(e).__name__, e))
return (data[train_start:train_start+train_count],
data[eval_start:eval_start+eval_count])
# This showcases how to make input_fn when the input data is available in the
# form of numpy arrays.
def make_inputs_from_np_arrays(features_np, label_np):
"""Makes and returns input_fn and feature_columns from numpy arrays.
The generated input_fn will return tf.data.Dataset of feature dictionary and a
label, and feature_columns will consist of the list of
tf.feature_column.BucketizedColumn.
Note, for in-memory training, tf.data.Dataset should contain the whole data
as a single tensor. Don't use batch.
Args:
features_np: A numpy ndarray (shape=[batch_size, num_features]) for
float32 features.
label_np: A numpy ndarray (shape=[batch_size, 1]) for labels.
Returns:
input_fn: A function returning a Dataset of feature dict and label.
feature_names: A list of feature names.
feature_column: A list of tf.feature_column.BucketizedColumn.
"""
num_features = features_np.shape[1]
features_np_list = np.split(features_np, num_features, axis=1)
# 1-based feature names.
feature_names = ["feature_%02d" % (i + 1) for i in range(num_features)]
# Create source feature_columns and bucketized_columns.
def get_bucket_boundaries(feature):
"""Returns bucket boundaries for feature by percentiles."""
return np.unique(np.percentile(feature, range(0, 100))).tolist()
source_columns = [
tf.feature_column.numeric_column(
feature_name, dtype=tf.float32,
# Although higgs data have no missing values, in general, default
# could be set as 0 or some reasonable value for missing values.
default_value=0.0)
for feature_name in feature_names
]
bucketized_columns = [
tf.feature_column.bucketized_column(
source_columns[i],
boundaries=get_bucket_boundaries(features_np_list[i]))
for i in range(num_features)
]
# Make an input_fn that extracts source features.
def input_fn():
"""Returns features as a dictionary of numpy arrays, and a label."""
features = {
feature_name: tf.constant(features_np_list[i])
for i, feature_name in enumerate(feature_names)
}
return tf.data.Dataset.zip((tf.data.Dataset.from_tensors(features),
tf.data.Dataset.from_tensors(label_np),))
return input_fn, feature_names, bucketized_columns
def make_eval_inputs_from_np_arrays(features_np, label_np):
"""Makes eval input as streaming batches."""
num_features = features_np.shape[1]
features_np_list = np.split(features_np, num_features, axis=1)
# 1-based feature names.
feature_names = ["feature_%02d" % (i + 1) for i in range(num_features)]
def input_fn():
features = {
feature_name: tf.constant(features_np_list[i])
for i, feature_name in enumerate(feature_names)
}
return tf.data.Dataset.zip((
tf.data.Dataset.from_tensor_slices(features),
tf.data.Dataset.from_tensor_slices(label_np),)).batch(1000)
return input_fn
def _make_csv_serving_input_receiver_fn(column_names, column_defaults):
"""Returns serving_input_receiver_fn for csv.
The input arguments are relevant to `tf.decode_csv()`.
Args:
column_names: a list of column names in the order within input csv.
column_defaults: a list of default values with the same size of
column_names. Each entity must be either a list of one scalar, or an
empty list to denote the corresponding column is required.
e.g. [[""], [2.5], []] indicates the third column is required while
the first column must be string and the second must be float/double.
Returns:
a serving_input_receiver_fn that handles csv for serving.
"""
def serving_input_receiver_fn():
csv = tf.placeholder(dtype=tf.string, shape=[None], name="csv")
features = dict(zip(column_names, tf.decode_csv(csv, column_defaults)))
receiver_tensors = {"inputs": csv}
return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)
return serving_input_receiver_fn
def train_boosted_trees(flags_obj):
"""Train boosted_trees estimator on HIGGS data.
Args:
flags_obj: An object containing parsed flag values.
"""
# Clean up the model directory if present.
if tf.gfile.Exists(flags_obj.model_dir):
tf.gfile.DeleteRecursively(flags_obj.model_dir)
tf.logging.info("## Data loading...")
train_data, eval_data = read_higgs_data(
flags_obj.data_dir, flags_obj.train_start, flags_obj.train_count,
flags_obj.eval_start, flags_obj.eval_count)
tf.logging.info("## Data loaded; train: {}{}, eval: {}{}".format(
train_data.dtype, train_data.shape, eval_data.dtype, eval_data.shape))
# Data consists of one label column followed by 28 feature columns.
train_input_fn, feature_names, feature_columns = make_inputs_from_np_arrays(
features_np=train_data[:, 1:], label_np=train_data[:, 0:1])
eval_input_fn = make_eval_inputs_from_np_arrays(
features_np=eval_data[:, 1:], label_np=eval_data[:, 0:1])
tf.logging.info("## Features prepared. Training starts...")
# Create benchmark logger to log info about the training and metric values
run_params = {
"train_start": flags_obj.train_start,
"train_count": flags_obj.train_count,
"eval_start": flags_obj.eval_start,
"eval_count": flags_obj.eval_count,
"n_trees": flags_obj.n_trees,
"max_depth": flags_obj.max_depth,
}
benchmark_logger = logger.config_benchmark_logger(flags_obj)
benchmark_logger.log_run_info(
model_name="boosted_trees",
dataset_name="higgs",
run_params=run_params)
# Though BoostedTreesClassifier is under tf.estimator, faster in-memory
# training is yet provided as a contrib library.
classifier = tf.contrib.estimator.boosted_trees_classifier_train_in_memory(
train_input_fn,
feature_columns,
model_dir=flags_obj.model_dir or None,
n_trees=flags_obj.n_trees,
max_depth=flags_obj.max_depth,
learning_rate=flags_obj.learning_rate)
# Evaluation.
eval_results = classifier.evaluate(eval_input_fn)
# Benchmark the evaluation results
benchmark_logger.log_evaluation_result(eval_results)
# Exporting the savedmodel with csv parsing.
if flags_obj.export_dir is not None:
classifier.export_savedmodel(
flags_obj.export_dir,
_make_csv_serving_input_receiver_fn(
column_names=feature_names,
# columns are all floats.
column_defaults=[[0.0]] * len(feature_names)))
def main(_):
train_boosted_trees(flags.FLAGS)
def define_train_higgs_flags():
"""Add tree related flags as well as training/eval configuration."""
flags_core.define_base(stop_threshold=False, batch_size=False, num_gpu=False)
flags.adopt_module_key_flags(flags_core)
flags.DEFINE_integer(
name="train_start", default=0,
help=help_wrap("Start index of train examples within the data."))
flags.DEFINE_integer(
name="train_count", default=1000000,
help=help_wrap("Number of train examples within the data."))
flags.DEFINE_integer(
name="eval_start", default=10000000,
help=help_wrap("Start index of eval examples within the data."))
flags.DEFINE_integer(
name="eval_count", default=1000000,
help=help_wrap("Number of eval examples within the data."))
flags.DEFINE_integer(
"n_trees", default=100, help=help_wrap("Number of trees to build."))
flags.DEFINE_integer(
"max_depth", default=6, help=help_wrap("Maximum depths of each tree."))
flags.DEFINE_float(
"learning_rate", default=0.1,
help=help_wrap("The learning rate."))
flags_core.set_defaults(data_dir="/tmp/higgs_data",
model_dir="/tmp/higgs_model")
if __name__ == "__main__":
# Training progress and eval results are shown as logging.INFO; so enables it.
tf.logging.set_verbosity(tf.logging.INFO)
define_train_higgs_flags()
absl_app.run(main)
================================================
FILE: official/boosted_trees/train_higgs_test.csv
================================================
1.000000000000000000e+00,8.692932128906250000e-01,-6.350818276405334473e-01,2.256902605295181274e-01,3.274700641632080078e-01,-6.899932026863098145e-01,7.542022466659545898e-01,-2.485731393098831177e-01,-1.092063903808593750e+00,0.000000000000000000e+00,1.374992132186889648e+00,-6.536741852760314941e-01,9.303491115570068359e-01,1.107436060905456543e+00,1.138904333114624023e+00,-1.578198313713073730e+00,-1.046985387802124023e+00,0.000000000000000000e+00,6.579295396804809570e-01,-1.045456994324922562e-02,-4.576716944575309753e-02,3.101961374282836914e+00,1.353760004043579102e+00,9.795631170272827148e-01,9.780761599540710449e-01,9.200048446655273438e-01,7.216574549674987793e-01,9.887509346008300781e-01,8.766783475875854492e-01
1.000000000000000000e+00,9.075421094894409180e-01,3.291472792625427246e-01,3.594118654727935791e-01,1.497969865798950195e+00,-3.130095303058624268e-01,1.095530629158020020e+00,-5.575249195098876953e-01,-1.588229775428771973e+00,2.173076152801513672e+00,8.125811815261840820e-01,-2.136419266462326050e-01,1.271014571189880371e+00,2.214872121810913086e+00,4.999939501285552979e-01,-1.261431813240051270e+00,7.321561574935913086e-01,0.000000000000000000e+00,3.987008929252624512e-01,-1.138930082321166992e+00,-8.191101951524615288e-04,0.000000000000000000e+00,3.022198975086212158e-01,8.330481648445129395e-01,9.856996536254882812e-01,9.780983924865722656e-01,7.797321677207946777e-01,9.923557639122009277e-01,7.983425855636596680e-01
1.000000000000000000e+00,7.988347411155700684e-01,1.470638751983642578e+00,-1.635974764823913574e+00,4.537731707096099854e-01,4.256291687488555908e-01,1.104874610900878906e+00,1.282322287559509277e+00,1.381664276123046875e+00,0.000000000000000000e+00,8.517372012138366699e-01,1.540658950805664062e+00,-8.196895122528076172e-01,2.214872121810913086e+00,9.934899210929870605e-01,3.560801148414611816e-01,-2.087775468826293945e-01,2.548224449157714844e+00,1.256954550743103027e+00,1.128847599029541016e+00,9.004608392715454102e-01,0.000000000000000000e+00,9.097532629966735840e-01,1.108330488204956055e+00,9.856922030448913574e-01,9.513312578201293945e-01,8.032515048980712891e-01,8.659244179725646973e-01,7.801175713539123535e-01
0.000000000000000000e+00,1.344384789466857910e+00,-8.766260147094726562e-01,9.359127283096313477e-01,1.992050051689147949e+00,8.824543952941894531e-01,1.786065936088562012e+00,-1.646777749061584473e+00,-9.423825144767761230e-01,0.000000000000000000e+00,2.423264741897583008e+00,-6.760157942771911621e-01,7.361586689949035645e-01,2.214872121810913086e+00,1.298719763755798340e+00,-1.430738091468811035e+00,-3.646581768989562988e-01,0.000000000000000000e+00,7.453126907348632812e-01,-6.783788204193115234e-01,-1.360356330871582031e+00,0.000000000000000000e+00,9.466524720191955566e-01,1.028703689575195312e+00,9.986560940742492676e-01,7.282806038856506348e-01,8.692002296447753906e-01,1.026736497879028320e+00,9.579039812088012695e-01
1.000000000000000000e+00,1.105008959770202637e+00,3.213555514812469482e-01,1.522401213645935059e+00,8.828076124191284180e-01,-1.205349326133728027e+00,6.814661026000976562e-01,-1.070463895797729492e+00,-9.218706488609313965e-01,0.000000000000000000e+00,8.008721470832824707e-01,1.020974040031433105e+00,9.714065194129943848e-01,2.214872121810913086e+00,5.967612862586975098e-01,-3.502728641033172607e-01,6.311942934989929199e-01,0.000000000000000000e+00,4.799988865852355957e-01,-3.735655248165130615e-01,1.130406111478805542e-01,0.000000000000000000e+00,7.558564543724060059e-01,1.361057043075561523e+00,9.866096973419189453e-01,8.380846381187438965e-01,1.133295178413391113e+00,8.722448945045471191e-01,8.084865212440490723e-01
0.000000000000000000e+00,1.595839262008666992e+00,-6.078106760978698730e-01,7.074915803968906403e-03,1.818449616432189941e+00,-1.119059920310974121e-01,8.475499153137207031e-01,-5.664370059967041016e-01,1.581239342689514160e+00,2.173076152801513672e+00,7.554209828376770020e-01,6.431096196174621582e-01,1.426366806030273438e+00,0.000000000000000000e+00,9.216607809066772461e-01,-1.190432429313659668e+00,-1.615589022636413574e+00,0.000000000000000000e+00,6.511141061782836914e-01,-6.542269587516784668e-01,-1.274344921112060547e+00,3.101961374282836914e+00,8.237605690956115723e-01,9.381914138793945312e-01,9.717581868171691895e-01,7.891763448715209961e-01,4.305532872676849365e-01,9.613569378852844238e-01,9.578179121017456055e-01
1.000000000000000000e+00,4.093913435935974121e-01,-1.884683609008789062e+00,-1.027292013168334961e+00,1.672451734542846680e+00,-1.604598283767700195e+00,1.338014960289001465e+00,5.542744323611259460e-02,1.346588134765625000e-02,2.173076152801513672e+00,5.097832679748535156e-01,-1.038338065147399902e+00,7.078623175621032715e-01,0.000000000000000000e+00,7.469175457954406738e-01,-3.584651052951812744e-01,-1.646654248237609863e+00,0.000000000000000000e+00,3.670579791069030762e-01,6.949646025896072388e-02,1.377130270004272461e+00,3.101961374282836914e+00,8.694183826446533203e-01,1.222082972526550293e+00,1.000627398490905762e+00,5.450449585914611816e-01,6.986525058746337891e-01,9.773144721984863281e-01,8.287860751152038574e-01
1.000000000000000000e+00,9.338953495025634766e-01,6.291297078132629395e-01,5.275348424911499023e-01,2.380327433347702026e-01,-9.665691256523132324e-01,5.478111505508422852e-01,-5.943922698497772217e-02,-1.706866145133972168e+00,2.173076152801513672e+00,9.410027265548706055e-01,-2.653732776641845703e+00,-1.572199910879135132e-01,0.000000000000000000e+00,1.030370354652404785e+00,-1.755051016807556152e-01,5.230209231376647949e-01,2.548224449157714844e+00,1.373546600341796875e+00,1.291248083114624023e+00,-1.467454433441162109e+00,0.000000000000000000e+00,9.018372893333435059e-01,1.083671212196350098e+00,9.796960949897766113e-01,7.833003997802734375e-01,8.491951823234558105e-01,8.943563103675842285e-01,7.748793959617614746e-01
1.000000000000000000e+00,1.405143737792968750e+00,5.366026163101196289e-01,6.895543336868286133e-01,1.179567337036132812e+00,-1.100611537694931030e-01,3.202404975891113281e+00,-1.526960015296936035e+00,-1.576033473014831543e+00,0.000000000000000000e+00,2.931536912918090820e+00,5.673424601554870605e-01,-1.300333440303802490e-01,2.214872121810913086e+00,1.787122726440429688e+00,8.994985818862915039e-01,5.851513147354125977e-01,2.548224449157714844e+00,4.018652141094207764e-01,-1.512016952037811279e-01,1.163489103317260742e+00,0.000000000000000000e+00,1.667070508003234863e+00,4.039272785186767578e+00,1.175828456878662109e+00,1.045351743698120117e+00,1.542971968650817871e+00,3.534826755523681641e+00,2.740753889083862305e+00
1.000000000000000000e+00,1.176565527915954590e+00,1.041605025529861450e-01,1.397002458572387695e+00,4.797213077545166016e-01,2.655133903026580811e-01,1.135563015937805176e+00,1.534830927848815918e+00,-2.532912194728851318e-01,0.000000000000000000e+00,1.027246594429016113e+00,5.343157649040222168e-01,1.180022358894348145e+00,0.000000000000000000e+00,2.405661106109619141e+00,8.755676448345184326e-02,-9.765340685844421387e-01,2.548224449157714844e+00,1.250382542610168457e+00,2.685412168502807617e-01,5.303344726562500000e-01,0.000000000000000000e+00,8.331748843193054199e-01,7.739681005477905273e-01,9.857499599456787109e-01,1.103696346282958984e+00,8.491398692131042480e-01,9.371039867401123047e-01,8.123638033866882324e-01
1.000000000000000000e+00,9.459739923477172852e-01,1.111244320869445801e+00,1.218337059020996094e+00,9.076390862464904785e-01,8.215369582176208496e-01,1.153243303298950195e+00,-3.654202818870544434e-01,-1.566054821014404297e+00,0.000000000000000000e+00,7.447192072868347168e-01,7.208195328712463379e-01,-3.758229315280914307e-01,2.214872121810913086e+00,6.088791489601135254e-01,3.078369498252868652e-01,-1.281638383865356445e+00,0.000000000000000000e+00,1.597967982292175293e+00,-4.510180354118347168e-01,6.365344673395156860e-02,3.101961374282836914e+00,8.290241360664367676e-01,9.806482791900634766e-01,9.943597912788391113e-01,9.082478284835815430e-01,7.758789062500000000e-01,7.833113670349121094e-01,7.251217961311340332e-01
0.000000000000000000e+00,7.393567562103271484e-01,-1.782904267311096191e-01,8.299342393875122070e-01,5.045390725135803223e-01,-1.302167475223541260e-01,9.610513448715209961e-01,-3.555179834365844727e-01,-1.717399358749389648e+00,2.173076152801513672e+00,6.209560632705688477e-01,-4.817410409450531006e-01,-1.199193239212036133e+00,0.000000000000000000e+00,9.826014041900634766e-01,8.118502795696258545e-02,-2.903236448764801025e-01,0.000000000000000000e+00,1.064662933349609375e+00,7.740649580955505371e-01,3.988203406333923340e-01,3.101961374282836914e+00,9.445360302925109863e-01,1.026260614395141602e+00,9.821967482566833496e-01,5.421146750450134277e-01,1.250978946685791016e+00,8.300446271896362305e-01,7.613079547882080078e-01
1.000000000000000000e+00,1.384097695350646973e+00,1.168220937252044678e-01,-1.179878950119018555e+00,7.629125714302062988e-01,-7.978226989507675171e-02,1.019863128662109375e+00,8.773182630538940430e-01,1.276887178421020508e+00,2.173076152801513672e+00,3.312520980834960938e-01,1.409523487091064453e+00,-1.474388837814331055e+00,0.000000000000000000e+00,1.282738208770751953e+00,7.374743819236755371e-01,-2.254196107387542725e-01,0.000000000000000000e+00,1.559753060340881348e+00,8.465205430984497070e-01,5.048085451126098633e-01,3.101961374282836914e+00,9.593246579170227051e-01,8.073760271072387695e-01,1.191813588142395020e+00,1.221210360527038574e+00,8.611412644386291504e-01,9.293408989906311035e-01,8.383023738861083984e-01
1.000000000000000000e+00,1.383548736572265625e+00,8.891792893409729004e-01,6.185320615768432617e-01,1.081547021865844727e+00,3.446055650711059570e-01,9.563793540000915527e-01,8.545429706573486328e-01,-1.129207015037536621e+00,2.173076152801513672e+00,5.456657409667968750e-01,-3.078651726245880127e-01,-6.232798099517822266e-01,2.214872121810913086e+00,3.482571244239807129e-01,1.024202585220336914e+00,1.840776652097702026e-01,0.000000000000000000e+00,7.813369035720825195e-01,-1.636125564575195312e+00,1.144067287445068359e+00,0.000000000000000000e+00,5.222384929656982422e-01,7.376385331153869629e-01,9.861995577812194824e-01,1.349615693092346191e+00,8.127878904342651367e-01,9.534064531326293945e-01,7.797226309776306152e-01
1.000000000000000000e+00,1.343652725219726562e+00,8.385329246520996094e-01,-1.061138510704040527e+00,2.472015142440795898e+00,-5.726317167282104492e-01,1.512709975242614746e+00,1.143690109252929688e+00,8.555619716644287109e-01,0.000000000000000000e+00,8.842203021049499512e-01,1.474605560302734375e+00,-1.360648751258850098e+00,1.107436060905456543e+00,1.587265610694885254e+00,2.234833478927612305e+00,7.756848633289337158e-02,0.000000000000000000e+00,1.609408140182495117e+00,2.396404743194580078e+00,7.572935223579406738e-01,0.000000000000000000e+00,9.340201020240783691e-01,8.447072505950927734e-01,1.077844023704528809e+00,1.400183677673339844e+00,9.477745294570922852e-01,1.007614254951477051e+00,9.010174870491027832e-01
0.000000000000000000e+00,5.470141768455505371e-01,-3.497089445590972900e-01,-6.466571688652038574e-01,2.040462255477905273e+00,2.764569818973541260e-01,5.446965098381042480e-01,8.386992812156677246e-01,1.728703141212463379e+00,0.000000000000000000e+00,6.528096199035644531e-01,1.471691370010375977e+00,1.243273019790649414e+00,0.000000000000000000e+00,7.857298851013183594e-01,-4.442929103970527649e-02,-1.019803404808044434e+00,2.548224449157714844e+00,4.191471040248870850e-01,-6.292421817779541016e-01,1.570794582366943359e+00,3.101961374282836914e+00,6.894335746765136719e-01,8.672295808792114258e-01,1.082487821578979492e+00,6.641419529914855957e-01,3.541145622730255127e-01,5.799450278282165527e-01,8.172734379768371582e-01
1.000000000000000000e+00,1.484203696250915527e+00,1.699521422386169434e+00,-1.059473991394042969e+00,2.700195550918579102e+00,-1.055963873863220215e+00,2.409452915191650391e+00,4.574607908725738525e-01,3.449823260307312012e-01,0.000000000000000000e+00,1.414903521537780762e+00,1.114225864410400391e+00,-1.448866605758666992e+00,0.000000000000000000e+00,1.012983918190002441e+00,-2.056988954544067383e+00,1.131010890007019043e+00,0.000000000000000000e+00,9.054746031761169434e-01,2.182368993759155273e+00,1.043073177337646484e+00,0.000000000000000000e+00,1.653626322746276855e+00,9.935762286186218262e-01,9.833217859268188477e-01,7.413797974586486816e-01,1.633816361427307129e-01,5.923243165016174316e-01,7.451378703117370605e-01
0.000000000000000000e+00,1.057975649833679199e+00,-1.607590019702911377e-01,-1.949972510337829590e-01,2.705023050308227539e+00,-7.514767050743103027e-01,1.909918904304504395e+00,-1.031844973564147949e+00,8.649863600730895996e-01,0.000000000000000000e+00,1.300834894180297852e+00,1.467376798391342163e-01,-1.118742942810058594e+00,1.107436060905456543e+00,9.669710993766784668e-01,-3.666573464870452881e-01,1.108266711235046387e+00,0.000000000000000000e+00,5.547249317169189453e-01,-7.141901850700378418e-01,1.505314946174621582e+00,3.101961374282836914e+00,9.544943571090698242e-01,6.510385870933532715e-01,1.124949693679809570e+00,8.940010070800781250e-01,6.721734404563903809e-01,1.182358264923095703e+00,1.316304087638854980e+00
0.000000000000000000e+00,6.753035783767700195e-01,1.120983958244323730e+00,-2.804459035396575928e-01,1.539554953575134277e+00,7.345175743103027344e-01,6.146844029426574707e-01,-5.070231556892395020e-01,7.945806980133056641e-01,2.173076152801513672e+00,2.188202738761901855e-01,-1.894118309020996094e+00,-5.805578827857971191e-01,0.000000000000000000e+00,1.245682120323181152e+00,-3.475421071052551270e-01,-8.561564683914184570e-01,2.548224449157714844e+00,7.531017661094665527e-01,-1.145592689514160156e+00,-1.374783992767333984e+00,0.000000000000000000e+00,9.069401025772094727e-01,8.983390927314758301e-01,1.119651079177856445e+00,1.269073486328125000e+00,1.088765859603881836e+00,1.015413045883178711e+00,9.146358966827392578e-01
1.000000000000000000e+00,6.427279114723205566e-01,-1.429840326309204102e+00,1.519071936607360840e+00,9.409985542297363281e-01,8.872274160385131836e-01,1.615126848220825195e+00,-1.336835741996765137e+00,-2.665962278842926025e-01,1.086538076400756836e+00,1.667088270187377930e+00,6.557375192642211914e-01,-1.588128924369812012e+00,0.000000000000000000e+00,8.282302021980285645e-01,1.836144566535949707e+00,4.081907570362091064e-01,0.000000000000000000e+00,1.708718180656433105e+00,-3.469151556491851807e-01,-1.182784557342529297e+00,3.101961374282836914e+00,9.210902452468872070e-01,1.373361706733703613e+00,9.849172830581665039e-01,1.422878146171569824e+00,1.546551108360290527e+00,1.782585501670837402e+00,1.438173770904541016e+00
================================================
FILE: official/boosted_trees/train_higgs_test.py
================================================
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for boosted_tree."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import tempfile
import numpy as np
import pandas as pd
import tensorflow as tf
# pylint: disable=g-bad-import-order
from official.boosted_trees import train_higgs
from official.utils.testing import integration
TEST_CSV = os.path.join(os.path.dirname(__file__), "train_higgs_test.csv")
tf.logging.set_verbosity(tf.logging.ERROR)
class BaseTest(tf.test.TestCase):
"""Tests for Wide Deep model."""
@classmethod
def setUpClass(cls): # pylint: disable=invalid-name
super(BaseTest, cls).setUpClass()
train_higgs.define_train_higgs_flags()
def setUp(self):
# Create temporary CSV file
self.data_dir = self.get_temp_dir()
data = pd.read_csv(
TEST_CSV, dtype=np.float32, names=["c%02d" % i for i in range(29)]
).as_matrix()
self.input_npz = os.path.join(self.data_dir, train_higgs.NPZ_FILE)
# numpy.savez doesn't take gfile.Gfile, so need to write down and copy.
tmpfile = tempfile.NamedTemporaryFile()
np.savez_compressed(tmpfile, data=data)
tf.gfile.Copy(tmpfile.name, self.input_npz)
def test_read_higgs_data(self):
"""Tests read_higgs_data() function."""
# Error when a wrong data_dir is given.
with self.assertRaisesRegexp(RuntimeError, "Error loading data.*"):
train_data, eval_data = train_higgs.read_higgs_data(
self.data_dir + "non-existing-path",
train_start=0, train_count=15, eval_start=15, eval_count=5)
# Loading fine with the correct data_dir.
train_data, eval_data = train_higgs.read_higgs_data(
self.data_dir,
train_start=0, train_count=15, eval_start=15, eval_count=5)
self.assertEqual((15, 29), train_data.shape)
self.assertEqual((5, 29), eval_data.shape)
def test_make_inputs_from_np_arrays(self):
"""Tests make_inputs_from_np_arrays() function."""
train_data, _ = train_higgs.read_higgs_data(
self.data_dir,
train_start=0, train_count=15, eval_start=15, eval_count=5)
(input_fn, feature_names,
feature_columns) = train_higgs.make_inputs_from_np_arrays(
features_np=train_data[:, 1:], label_np=train_data[:, 0:1])
# Check feature_names.
self.assertAllEqual(feature_names,
["feature_%02d" % (i+1) for i in range(28)])
# Check feature columns.
self.assertEqual(28, len(feature_columns))
bucketized_column_type = type(
tf.feature_column.bucketized_column(
tf.feature_column.numeric_column("feature_01"),
boundaries=[0, 1, 2])) # dummy boundaries.
for feature_column in feature_columns:
self.assertIsInstance(feature_column, bucketized_column_type)
# At least 2 boundaries.
self.assertGreaterEqual(len(feature_column.boundaries), 2)
# Tests that the source column names of the bucketized columns match.
self.assertAllEqual(feature_names,
[col.source_column.name for col in feature_columns])
# Check features.
features, labels = input_fn().make_one_shot_iterator().get_next()
with tf.Session() as sess:
features, labels = sess.run((features, labels))
self.assertIsInstance(features, dict)
self.assertAllEqual(feature_names, sorted(features.keys()))
self.assertAllEqual([[15, 1]] * 28,
[features[name].shape for name in feature_names])
# Validate actual values of some features.
self.assertAllClose(
[0.869293, 0.907542, 0.798834, 1.344384, 1.105009, 1.595839,
0.409391, 0.933895, 1.405143, 1.176565, 0.945974, 0.739356,
1.384097, 1.383548, 1.343652],
np.squeeze(features[feature_names[0]], 1))
self.assertAllClose(
[-0.653674, -0.213641, 1.540659, -0.676015, 1.020974, 0.643109,
-1.038338, -2.653732, 0.567342, 0.534315, 0.720819, -0.481741,
1.409523, -0.307865, 1.474605],
np.squeeze(features[feature_names[10]], 1))
def test_end_to_end(self):
"""Tests end-to-end running."""
model_dir = os.path.join(self.get_temp_dir(), "model")
integration.run_synthetic(
main=train_higgs.main, tmp_root=self.get_temp_dir(), extra_flags=[
"--data_dir", self.data_dir,
"--model_dir", model_dir,
"--n_trees", "5",
"--train_start", "0",
"--train_count", "12",
"--eval_start", "12",
"--eval_count", "8",
],
synth=False, max_train=None)
self.assertTrue(tf.gfile.Exists(os.path.join(model_dir, "checkpoint")))
def test_end_to_end_with_export(self):
"""Tests end-to-end running."""
model_dir = os.path.join(self.get_temp_dir(), "model")
export_dir = os.path.join(self.get_temp_dir(), "export")
integration.run_synthetic(
main=train_higgs.main, tmp_root=self.get_temp_dir(), extra_flags=[
"--data_dir", self.data_dir,
"--model_dir", model_dir,
"--export_dir", export_dir,
"--n_trees", "5",
"--train_start", "0",
"--train_count", "12",
"--eval_start", "12",
"--eval_count", "8",
],
synth=False, max_train=None)
self.assertTrue(tf.gfile.Exists(os.path.join(model_dir, "checkpoint")))
self.assertTrue(tf.gfile.Exists(os.path.join(export_dir)))
if __name__ == "__main__":
tf.test.main()
================================================
FILE: official/mnist/README.md
================================================
# MNIST in TensorFlow
This directory builds a convolutional neural net to classify the [MNIST
dataset](http://yann.lecun.com/exdb/mnist/) using the
[tf.data](https://www.tensorflow.org/api_docs/python/tf/data),
[tf.estimator.Estimator](https://www.tensorflow.org/api_docs/python/tf/estimator/Estimator),
and
[tf.layers](https://www.tensorflow.org/api_docs/python/tf/layers)
APIs.
## Setup
To begin, you'll simply need the latest version of TensorFlow installed.
First make sure you've [added the models folder to your Python path](/official/#running-the-models); otherwise you may encounter an error like `ImportError: No module named official.mnist`.
Then to train the model, run the following:
```
python mnist.py
```
The model will begin training and will automatically evaluate itself on the
validation data.
Illustrative unit tests and benchmarks can be run with:
```
python mnist_test.py
python mnist_test.py --benchmarks=.
```
## Exporting the model
You can export the model into Tensorflow [SavedModel](https://www.tensorflow.org/programmers_guide/saved_model) format by using the argument `--export_dir`:
```
python mnist.py --export_dir /tmp/mnist_saved_model
```
The SavedModel will be saved in a timestamped directory under `/tmp/mnist_saved_model/` (e.g. `/tmp/mnist_saved_model/1513630966/`).
**Getting predictions with SavedModel**
Use [`saved_model_cli`](https://www.tensorflow.org/programmers_guide/saved_model#cli_to_inspect_and_execute_savedmodel) to inspect and execute the SavedModel.
```
saved_model_cli run --dir /tmp/mnist_saved_model/TIMESTAMP --tag_set serve --signature_def classify --inputs image=examples.npy
```
`examples.npy` contains the data from `example5.png` and `example3.png` in a numpy array, in that order. The array values are normalized to values between 0 and 1.
The output should look similar to below:
```
Result for output key classes:
[5 3]
Result for output key probabilities:
[[ 1.53558474e-07 1.95694142e-13 1.31193523e-09 5.47467265e-03
5.85711526e-22 9.94520664e-01 3.48423509e-06 2.65365645e-17
9.78631419e-07 3.15522470e-08]
[ 1.22413359e-04 5.87615965e-08 1.72251271e-06 9.39960718e-01
3.30306928e-11 2.87386645e-02 2.82353517e-02 8.21146413e-18
2.52568233e-03 4.15460236e-04]]
```
## Experimental: Eager Execution
[Eager execution](https://research.googleblog.com/2017/10/eager-execution-imperative-define-by.html)
(an preview feature in TensorFlow 1.5) is an imperative interface to TensorFlow.
The exact same model defined in `mnist.py` can be trained without creating a
TensorFlow graph using:
```
python mnist_eager.py
```
## Experimental: TPU Acceleration
`mnist.py` (and `mnist_eager.py`) demonstrate training a neural network to
classify digits on CPUs and GPUs. `mnist_tpu.py` can be used to train the
same model using TPUs for hardware acceleration. More information in
the [tensorflow/tpu](https://github.com/tensorflow/tpu) repository.
================================================
FILE: official/mnist/__init__.py
================================================
================================================
FILE: official/mnist/dataset.py
================================================
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""tf.data.Dataset interface to the MNIST dataset."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import gzip
import os
import shutil
import tempfile
import numpy as np
from six.moves import urllib
import tensorflow as tf
def read32(bytestream):
"""Read 4 bytes from bytestream as an unsigned 32-bit integer."""
dt = np.dtype(np.uint32).newbyteorder('>')
return np.frombuffer(bytestream.read(4), dtype=dt)[0]
def check_image_file_header(filename):
"""Validate that filename corresponds to images for the MNIST dataset."""
with tf.gfile.Open(filename, 'rb') as f:
magic = read32(f)
read32(f) # num_images, unused
rows = read32(f)
cols = read32(f)
if magic != 2051:
raise ValueError('Invalid magic number %d in MNIST file %s' % (magic,
f.name))
if rows != 28 or cols != 28:
raise ValueError(
'Invalid MNIST file %s: Expected 28x28 images, found %dx%d' %
(f.name, rows, cols))
def check_labels_file_header(filename):
"""Validate that filename corresponds to labels for the MNIST dataset."""
with tf.gfile.Open(filename, 'rb') as f:
magic = read32(f)
read32(f) # num_items, unused
if magic != 2049:
raise ValueError('Invalid magic number %d in MNIST file %s' % (magic,
f.name))
def download(directory, filename):
"""Download (and unzip) a file from the MNIST dataset if not already done."""
filepath = os.path.join(directory, filename)
if tf.gfile.Exists(filepath):
return filepath
if not tf.gfile.Exists(directory):
tf.gfile.MakeDirs(directory)
# CVDF mirror of http://yann.lecun.com/exdb/mnist/
url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz'
_, zipped_filepath = tempfile.mkstemp(suffix='.gz')
print('Downloading %s to %s' % (url, zipped_filepath))
urllib.request.urlretrieve(url, zipped_filepath)
with gzip.open(zipped_filepath, 'rb') as f_in, \
tf.gfile.Open(filepath, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
os.remove(zipped_filepath)
return filepath
def dataset(directory, images_file, labels_file):
"""Download and parse MNIST dataset."""
images_file = download(directory, images_file)
labels_file = download(directory, labels_file)
check_image_file_header(images_file)
check_labels_file_header(labels_file)
def decode_image(image):
# Normalize from [0, 255] to [0.0, 1.0]
image = tf.decode_raw(image, tf.uint8)
image = tf.cast(image, tf.float32)
image = tf.reshape(image, [784])
return image / 255.0
def decode_label(label):
label = tf.decode_raw(label, tf.uint8) # tf.string -> [tf.uint8]
label = tf.reshape(label, []) # label is a scalar
return tf.to_int32(label)
images = tf.data.FixedLengthRecordDataset(
images_file, 28 * 28, header_bytes=16).map(decode_image)
labels = tf.data.FixedLengthRecordDataset(
labels_file, 1, header_bytes=8).map(decode_label)
return tf.data.Dataset.zip((images, labels))
def train(directory):
"""tf.data.Dataset object for MNIST training data."""
return dataset(directory, 'train-images-idx3-ubyte',
'train-labels-idx1-ubyte')
def test(directory):
"""tf.data.Dataset object for MNIST test data."""
return dataset(directory, 't10k-images-idx3-ubyte', 't10k-labels-idx1-ubyte')
================================================
FILE: official/mnist/mnist.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Convolutional Neural Network Estimator for MNIST, built with tf.layers."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from absl import app as absl_app
from absl import flags
import tensorflow as tf # pylint: disable=g-bad-import-order
from official.mnist import dataset
from official.utils.flags import core as flags_core
from official.utils.logs import hooks_helper
from official.utils.misc import model_helpers
LEARNING_RATE = 1e-4
def create_model(data_format):
"""Model to recognize digits in the MNIST dataset.
Network structure is equivalent to:
https://github.com/tensorflow/tensorflow/blob/r1.5/tensorflow/examples/tutorials/mnist/mnist_deep.py
and
https://github.com/tensorflow/models/blob/master/tutorials/image/mnist/convolutional.py
But uses the tf.keras API.
Args:
data_format: Either 'channels_first' or 'channels_last'. 'channels_first' is
typically faster on GPUs while 'channels_last' is typically faster on
CPUs. See
https://www.tensorflow.org/performance/performance_guide#data_formats
Returns:
A tf.keras.Model.
"""
if data_format == 'channels_first':
input_shape = [1, 28, 28]
else:
assert data_format == 'channels_last'
input_shape = [28, 28, 1]
l = tf.keras.layers
max_pool = l.MaxPooling2D(
(2, 2), (2, 2), padding='same', data_format=data_format)
# The model consists of a sequential chain of layers, so tf.keras.Sequential
# (a subclass of tf.keras.Model) makes for a compact description.
return tf.keras.Sequential(
[
l.Reshape(
target_shape=input_shape,
input_shape=(28 * 28,)),
l.Conv2D(
32,
5,
padding='same',
data_format=data_format,
activation=tf.nn.relu),
max_pool,
l.Conv2D(
64,
5,
padding='same',
data_format=data_format,
activation=tf.nn.relu),
max_pool,
l.Flatten(),
l.Dense(1024, activation=tf.nn.relu),
l.Dropout(0.4),
l.Dense(10)
])
def define_mnist_flags():
flags_core.define_base(multi_gpu=True, num_gpu=False)
flags_core.define_image()
flags.adopt_module_key_flags(flags_core)
flags_core.set_defaults(data_dir='/tmp/mnist_data',
model_dir='/tmp/mnist_model',
batch_size=100,
train_epochs=40)
def model_fn(features, labels, mode, params):
"""The model_fn argument for creating an Estimator."""
model = create_model(params['data_format'])
image = features
if isinstance(image, dict):
image = features['image']
if mode == tf.estimator.ModeKeys.PREDICT:
logits = model(image, training=False)
predictions = {
'classes': tf.argmax(logits, axis=1),
'probabilities': tf.nn.softmax(logits),
}
return tf.estimator.EstimatorSpec(
mode=tf.estimator.ModeKeys.PREDICT,
predictions=predictions,
export_outputs={
'classify': tf.estimator.export.PredictOutput(predictions)
})
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE)
# If we are running multi-GPU, we need to wrap the optimizer.
if params.get('multi_gpu'):
optimizer = tf.contrib.estimator.TowerOptimizer(optimizer)
logits = model(image, training=True)
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
accuracy = tf.metrics.accuracy(
labels=labels, predictions=tf.argmax(logits, axis=1))
# Name tensors to be logged with LoggingTensorHook.
tf.identity(LEARNING_RATE, 'learning_rate')
tf.identity(loss, 'cross_entropy')
tf.identity(accuracy[1], name='train_accuracy')
# Save accuracy scalar to Tensorboard output.
tf.summary.scalar('train_accuracy', accuracy[1])
return tf.estimator.EstimatorSpec(
mode=tf.estimator.ModeKeys.TRAIN,
loss=loss,
train_op=optimizer.minimize(loss, tf.train.get_or_create_global_step()))
if mode == tf.estimator.ModeKeys.EVAL:
logits = model(image, training=False)
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
return tf.estimator.EstimatorSpec(
mode=tf.estimator.ModeKeys.EVAL,
loss=loss,
eval_metric_ops={
'accuracy':
tf.metrics.accuracy(
labels=labels, predictions=tf.argmax(logits, axis=1)),
})
def validate_batch_size_for_multi_gpu(batch_size):
"""For multi-gpu, batch-size must be a multiple of the number of GPUs.
Note that this should eventually be handled by replicate_model_fn
directly. Multi-GPU support is currently experimental, however,
so doing the work here until that feature is in place.
Args:
batch_size: the number of examples processed in each training batch.
Raises:
ValueError: if no GPUs are found, or selected batch_size is invalid.
"""
from tensorflow.python.client import device_lib # pylint: disable=g-import-not-at-top
local_device_protos = device_lib.list_local_devices()
num_gpus = sum([1 for d in local_device_protos if d.device_type == 'GPU'])
if not num_gpus:
raise ValueError('Multi-GPU mode was specified, but no GPUs '
'were found. To use CPU, run without --multi_gpu.')
remainder = batch_size % num_gpus
if remainder:
err = ('When running with multiple GPUs, batch size '
'must be a multiple of the number of available GPUs. '
'Found {} GPUs with a batch size of {}; try --batch_size={} instead.'
).format(num_gpus, batch_size, batch_size - remainder)
raise ValueError(err)
def run_mnist(flags_obj):
"""Run MNIST training and eval loop.
Args:
flags_obj: An object containing parsed flag values.
"""
model_function = model_fn
if flags_obj.multi_gpu:
validate_batch_size_for_multi_gpu(flags_obj.batch_size)
# There are two steps required if using multi-GPU: (1) wrap the model_fn,
# and (2) wrap the optimizer. The first happens here, and (2) happens
# in the model_fn itself when the optimizer is defined.
model_function = tf.contrib.estimator.replicate_model_fn(
model_fn, loss_reduction=tf.losses.Reduction.MEAN)
data_format = flags_obj.data_format
if data_format is None:
data_format = ('channels_first'
if tf.test.is_built_with_cuda() else 'channels_last')
mnist_classifier = tf.estimator.Estimator(
model_fn=model_function,
model_dir=flags_obj.model_dir,
params={
'data_format': data_format,
'multi_gpu': flags_obj.multi_gpu
})
# Set up training and evaluation input functions.
def train_input_fn():
"""Prepare data for training."""
# When choosing shuffle buffer sizes, larger sizes result in better
# randomness, while smaller sizes use less memory. MNIST is a small
# enough dataset that we can easily shuffle the full epoch.
ds = dataset.train(flags_obj.data_dir)
ds = ds.cache().shuffle(buffer_size=50000).batch(flags_obj.batch_size)
# Iterate through the dataset a set number (`epochs_between_evals`) of times
# during each training session.
ds = ds.repeat(flags_obj.epochs_between_evals)
return ds
def eval_input_fn():
return dataset.test(flags_obj.data_dir).batch(
flags_obj.batch_size).make_one_shot_iterator().get_next()
# Set up hook that outputs training logs every 100 steps.
train_hooks = hooks_helper.get_train_hooks(
flags_obj.hooks, batch_size=flags_obj.batch_size)
# Train and evaluate model.
for _ in range(flags_obj.train_epochs // flags_obj.epochs_between_evals):
mnist_classifier.train(input_fn=train_input_fn, hooks=train_hooks)
eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)
print('\nEvaluation results:\n\t%s\n' % eval_results)
if model_helpers.past_stop_threshold(flags_obj.stop_threshold,
eval_results['accuracy']):
break
# Export the model
if flags_obj.export_dir is not None:
image = tf.placeholder(tf.float32, [None, 28, 28])
input_fn = tf.estimator.export.build_raw_serving_input_receiver_fn({
'image': image,
})
mnist_classifier.export_savedmodel(flags_obj.export_dir, input_fn)
def main(_):
run_mnist(flags.FLAGS)
if __name__ == '__main__':
tf.logging.set_verbosity(tf.logging.INFO)
define_mnist_flags()
absl_app.run(main)
================================================
FILE: official/mnist/mnist_eager.py
================================================
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""MNIST model training with TensorFlow eager execution.
See:
https://research.googleblog.com/2017/10/eager-execution-imperative-define-by.html
This program demonstrates training of the convolutional neural network model
defined in mnist.py with eager execution enabled.
If you are not interested in eager execution, you should ignore this file.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import time
# pylint: disable=g-bad-import-order
from absl import app as absl_app
from absl import flags
import tensorflow as tf
import tensorflow.contrib.eager as tfe
# pylint: enable=g-bad-import-order
from official.mnist import dataset as mnist_dataset
from official.mnist import mnist
from official.utils.flags import core as flags_core
def loss(logits, labels):
return tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=labels))
def compute_accuracy(logits, labels):
predictions = tf.argmax(logits, axis=1, output_type=tf.int64)
labels = tf.cast(labels, tf.int64)
batch_size = int(logits.shape[0])
return tf.reduce_sum(
tf.cast(tf.equal(predictions, labels), dtype=tf.float32)) / batch_size
def train(model, optimizer, dataset, step_counter, log_interval=None):
"""Trains model on `dataset` using `optimizer`."""
start = time.time()
for (batch, (images, labels)) in enumerate(tfe.Iterator(dataset)):
with tf.contrib.summary.record_summaries_every_n_global_steps(
10, global_step=step_counter):
# Record the operations used to compute the loss given the input,
# so that the gradient of the loss with respect to the variables
# can be computed.
with tf.GradientTape() as tape:
logits = model(images, training=True)
loss_value = loss(logits, labels)
tf.contrib.summary.scalar('loss', loss_value)
tf.contrib.summary.scalar('accuracy', compute_accuracy(logits, labels))
grads = tape.gradient(loss_value, model.variables)
optimizer.apply_gradients(
zip(grads, model.variables), global_step=step_counter)
if log_interval and batch % log_interval == 0:
rate = log_interval / (time.time() - start)
print('Step #%d\tLoss: %.6f (%d steps/sec)' % (batch, loss_value, rate))
start = time.time()
def test(model, dataset):
"""Perform an evaluation of `model` on the examples from `dataset`."""
avg_loss = tfe.metrics.Mean('loss')
accuracy = tfe.metrics.Accuracy('accuracy')
for (images, labels) in tfe.Iterator(dataset):
logits = model(images, training=False)
avg_loss(loss(logits, labels))
accuracy(
tf.argmax(logits, axis=1, output_type=tf.int64),
tf.cast(labels, tf.int64))
print('Test set: Average loss: %.4f, Accuracy: %4f%%\n' %
(avg_loss.result(), 100 * accuracy.result()))
with tf.contrib.summary.always_record_summaries():
tf.contrib.summary.scalar('loss', avg_loss.result())
tf.contrib.summary.scalar('accuracy', accuracy.result())
def run_mnist_eager(flags_obj):
"""Run MNIST training and eval loop in eager mode.
Args:
flags_obj: An object containing parsed flag values.
"""
tf.enable_eager_execution()
# Automatically determine device and data_format
(device, data_format) = ('/gpu:0', 'channels_first')
if flags_obj.no_gpu or not tf.test.is_gpu_available():
(device, data_format) = ('/cpu:0', 'channels_last')
# If data_format is defined in FLAGS, overwrite automatically set value.
if flags_obj.data_format is not None:
data_format = flags_obj.data_format
print('Using device %s, and data format %s.' % (device, data_format))
# Load the datasets
train_ds = mnist_dataset.train(flags_obj.data_dir).shuffle(60000).batch(
flags_obj.batch_size)
test_ds = mnist_dataset.test(flags_obj.data_dir).batch(
flags_obj.batch_size)
# Create the model and optimizer
model = mnist.create_model(data_format)
optimizer = tf.train.MomentumOptimizer(flags_obj.lr, flags_obj.momentum)
# Create file writers for writing TensorBoard summaries.
if flags_obj.output_dir:
# Create directories to which summaries will be written
# tensorboard --logdir=<output_dir>
# can then be used to see the recorded summaries.
train_dir = os.path.join(flags_obj.output_dir, 'train')
test_dir = os.path.join(flags_obj.output_dir, 'eval')
tf.gfile.MakeDirs(flags_obj.output_dir)
else:
train_dir = None
test_dir = None
summary_writer = tf.contrib.summary.create_file_writer(
train_dir, flush_millis=10000)
test_summary_writer = tf.contrib.summary.create_file_writer(
test_dir, flush_millis=10000, name='test')
# Create and restore checkpoint (if one exists on the path)
checkpoint_prefix = os.path.join(flags_obj.model_dir, 'ckpt')
step_counter = tf.train.get_or_create_global_step()
checkpoint = tfe.Checkpoint(
model=model, optimizer=optimizer, step_counter=step_counter)
# Restore variables on creation if a checkpoint exists.
checkpoint.restore(tf.train.latest_checkpoint(flags_obj.model_dir))
# Train and evaluate for a set number of epochs.
with tf.device(device):
for _ in range(flags_obj.train_epochs):
start = time.time()
with summary_writer.as_default():
train(model, optimizer, train_ds, step_counter,
flags_obj.log_interval)
end = time.time()
print('\nTrain time for epoch #%d (%d total steps): %f' %
(checkpoint.save_counter.numpy() + 1,
step_counter.numpy(),
end - start))
with test_summary_writer.as_default():
test(model, test_ds)
checkpoint.save(checkpoint_prefix)
def define_mnist_eager_flags():
"""Defined flags and defaults for MNIST in eager mode."""
flags_core.define_base_eager()
flags_core.define_image()
flags.adopt_module_key_flags(flags_core)
flags.DEFINE_integer(
name='log_interval', short_name='li', default=10,
help=flags_core.help_wrap('batches between logging training status'))
flags.DEFINE_string(
name='output_dir', short_name='od', default=None,
help=flags_core.help_wrap('Directory to write TensorBoard summaries'))
flags.DEFINE_float(name='learning_rate', short_name='lr', default=0.01,
help=flags_core.help_wrap('Learning rate.'))
flags.DEFINE_float(name='momentum', short_name='m', default=0.5,
help=flags_core.help_wrap('SGD momentum.'))
flags.DEFINE_bool(name='no_gpu', short_name='nogpu', default=False,
help=flags_core.help_wrap(
'disables GPU usage even if a GPU is available'))
flags_core.set_defaults(
data_dir='/tmp/tensorflow/mnist/input_data',
model_dir='/tmp/tensorflow/mnist/checkpoints/',
batch_size=100,
train_epochs=10,
)
def main(_):
run_mnist_eager(flags.FLAGS)
if __name__ == '__main__':
define_mnist_eager_flags()
absl_app.run(main=main)
================================================
FILE: official/mnist/mnist_eager_test.py
================================================
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow as tf # pylint: disable=g-bad-import-order
import tensorflow.contrib.eager as tfe # pylint: disable=g-bad-import-order
from official.mnist import mnist
from official.mnist import mnist_eager
def device():
return "/device:GPU:0" if tfe.num_gpus() else "/device:CPU:0"
def data_format():
return "channels_first" if tfe.num_gpus() else "channels_last"
def random_dataset():
batch_size = 64
images = tf.random_normal([batch_size, 784])
labels = tf.random_uniform([batch_size], minval=0, maxval=10, dtype=tf.int32)
return tf.data.Dataset.from_tensors((images, labels))
def train(defun=False):
model = mnist.create_model(data_format())
if defun:
model.call = tfe.defun(model.call)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)
dataset = random_dataset()
with tf.device(device()):
mnist_eager.train(model, optimizer, dataset,
step_counter=tf.train.get_or_create_global_step())
def evaluate(defun=False):
model = mnist.create_model(data_format())
dataset = random_dataset()
if defun:
model.call = tfe.defun(model.call)
with tf.device(device()):
mnist_eager.test(model, dataset)
class MNISTTest(tf.test.TestCase):
"""Run tests for MNIST eager loop."""
def test_train(self):
train(defun=False)
def test_evaluate(self):
evaluate(defun=False)
def test_train_with_defun(self):
train(defun=True)
def test_evaluate_with_defun(self):
evaluate(defun=True)
if __name__ == "__main__":
tfe.enable_eager_execution()
tf.test.main()
================================================
FILE: official/mnist/mnist_test.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import time
import tensorflow as tf # pylint: disable=g-bad-import-order
from official.mnist import mnist
BATCH_SIZE = 100
def dummy_input_fn():
image = tf.random_uniform([BATCH_SIZE, 784])
labels = tf.random_uniform([BATCH_SIZE, 1], maxval=9, dtype=tf.int32)
return image, labels
def make_estimator():
data_format = 'channels_last'
if tf.test.is_built_with_cuda():
data_format = 'channels_first'
return tf.estimator.Estimator(
model_fn=mnist.model_fn, params={
'data_format': data_format
})
class Tests(tf.test.TestCase):
"""Run tests for MNIST model."""
def test_mnist(self):
classifier = make_estimator()
classifier.train(input_fn=dummy_input_fn, steps=2)
eval_results = classifier.evaluate(input_fn=dummy_input_fn, steps=1)
loss = eval_results['loss']
global_step = eval_results['global_step']
accuracy = eval_results['accuracy']
self.assertEqual(loss.shape, ())
self.assertEqual(2, global_step)
self.assertEqual(accuracy.shape, ())
input_fn = lambda: tf.random_uniform([3, 784])
predictions_generator = classifier.predict(input_fn)
for _ in range(3):
predictions = next(predictions_generator)
self.assertEqual(predictions['probabilities'].shape, (10,))
self.assertEqual(predictions['classes'].shape, ())
def mnist_model_fn_helper(self, mode, multi_gpu=False):
features, labels = dummy_input_fn()
image_count = features.shape[0]
spec = mnist.model_fn(features, labels, mode, {
'data_format': 'channels_last',
'multi_gpu': multi_gpu
})
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = spec.predictions
self.assertAllEqual(predictions['probabilities'].shape, (image_count, 10))
self.assertEqual(predictions['probabilities'].dtype, tf.float32)
self.assertAllEqual(predictions['classes'].shape, (image_count,))
self.assertEqual(predictions['classes'].dtype, tf.int64)
if mode != tf.estimator.ModeKeys.PREDICT:
loss = spec.loss
self.assertAllEqual(loss.shape, ())
self.assertEqual(loss.dtype, tf.float32)
if mode == tf.estimator.ModeKeys.EVAL:
eval_metric_ops = spec.eval_metric_ops
self.assertAllEqual(eval_metric_ops['accuracy'][0].shape, ())
self.assertAllEqual(eval_metric_ops['accuracy'][1].shape, ())
self.assertEqual(eval_metric_ops['accuracy'][0].dtype, tf.float32)
self.assertEqual(eval_metric_ops['accuracy'][1].dtype, tf.float32)
def test_mnist_model_fn_train_mode(self):
self.mnist_model_fn_helper(tf.estimator.ModeKeys.TRAIN)
def test_mnist_model_fn_train_mode_multi_gpu(self):
self.mnist_model_fn_helper(tf.estimator.ModeKeys.TRAIN, multi_gpu=True)
def test_mnist_model_fn_eval_mode(self):
self.mnist_model_fn_helper(tf.estimator.ModeKeys.EVAL)
def test_mnist_model_fn_predict_mode(self):
self.mnist_model_fn_helper(tf.estimator.ModeKeys.PREDICT)
class Benchmarks(tf.test.Benchmark):
"""Simple speed benchmarking for MNIST."""
def benchmark_train_step_time(self):
classifier = make_estimator()
# Run one step to warmup any use of the GPU.
classifier.train(input_fn=dummy_input_fn, steps=1)
have_gpu = tf.test.is_gpu_available()
num_steps = 1000 if have_gpu else 100
name = 'train_step_time_%s' % ('gpu' if have_gpu else 'cpu')
start = time.time()
classifier.train(input_fn=dummy_input_fn, steps=num_steps)
end = time.time()
wall_time = (end - start) / num_steps
self.report_benchmark(
iters=num_steps,
wall_time=wall_time,
name=name,
extras={
'examples_per_sec': BATCH_SIZE / wall_time
})
if __name__ == '__main__':
tf.logging.set_verbosity(tf.logging.ERROR)
tf.test.main()
================================================
FILE: official/mnist/mnist_tpu.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""MNIST model training using TPUs.
This program demonstrates training of the convolutional neural network model
defined in mnist.py on Google Cloud TPUs (https://cloud.google.com/tpu/).
If you are not interested in TPUs, you should ignore this file.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import sys
import tensorflow as tf # pylint: disable=g-bad-import-order
# For open source environment, add grandparent directory for import
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(sys.path[0]))))
from official.mnist import dataset # pylint: disable=wrong-import-position
from official.mnist import mnist # pylint: disable=wrong-import-position
# Cloud TPU Cluster Resolver flags
tf.flags.DEFINE_string(
"tpu", default=None,
help="The Cloud TPU to use for training. This should be either the name "
"used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 "
"url.")
tf.flags.DEFINE_string(
"tpu_zone", default=None,
help="[Optional] GCE zone where the Cloud TPU is located in. If not "
"specified, we will attempt to automatically detect the GCE project from "
"metadata.")
tf.flags.DEFINE_string(
"gcp_project", default=None,
help="[Optional] Project name for the Cloud TPU-enabled project. If not "
"specified, we will attempt to automatically detect the GCE project from "
"metadata.")
# Model specific parameters
tf.flags.DEFINE_string("data_dir", "",
"Path to directory containing the MNIST dataset")
tf.flags.DEFINE_string("model_dir", None, "Estimator model_dir")
tf.flags.DEFINE_integer("batch_size", 1024,
"Mini-batch size for the training. Note that this "
"is the global batch size and not the per-shard batch.")
tf.flags.DEFINE_integer("train_steps", 1000, "Total number of training steps.")
tf.flags.DEFINE_integer("eval_steps", 0,
"Total number of evaluation steps. If `0`, evaluation "
"after training is skipped.")
tf.flags.DEFINE_float("learning_rate", 0.05, "Learning rate.")
tf.flags.DEFINE_bool("use_tpu", True, "Use TPUs rather than plain CPUs")
tf.flags.DEFINE_integer("iterations", 50,
"Number of iterations per TPU training loop.")
tf.flags.DEFINE_integer("num_shards", 8, "Number of shards (TPU chips).")
FLAGS = tf.flags.FLAGS
def metric_fn(labels, logits):
accuracy = tf.metrics.accuracy(
labels=labels, predictions=tf.argmax(logits, axis=1))
return {"accuracy": accuracy}
def model_fn(features, labels, mode, params):
"""model_fn constructs the ML model used to predict handwritten digits."""
del params
if mode == tf.estimator.ModeKeys.PREDICT:
raise RuntimeError("mode {} is not supported yet".format(mode))
image = features
if isinstance(image, dict):
image = features["image"]
model = mnist.create_model("channels_last")
logits = model(image, training=(mode == tf.estimator.ModeKeys.TRAIN))
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
if mode == tf.estimator.ModeKeys.TRAIN:
learning_rate = tf.train.exponential_decay(
FLAGS.learning_rate,
tf.train.get_global_step(),
decay_steps=100000,
decay_rate=0.96)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
if FLAGS.use_tpu:
optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer)
return tf.contrib.tpu.TPUEstimatorSpec(
mode=mode,
loss=loss,
train_op=optimizer.minimize(loss, tf.train.get_global_step()))
if mode == tf.estimator.ModeKeys.EVAL:
return tf.contrib.tpu.TPUEstimatorSpec(
mode=mode, loss=loss, eval_metrics=(metric_fn, [labels, logits]))
def train_input_fn(params):
"""train_input_fn defines the input pipeline used for training."""
batch_size = params["batch_size"]
data_dir = params["data_dir"]
# Retrieves the batch size for the current shard. The # of shards is
# computed according to the input pipeline deployment. See
# `tf.contrib.tpu.RunConfig` for details.
ds = dataset.train(data_dir).cache().repeat().shuffle(
buffer_size=50000).apply(
tf.contrib.data.batch_and_drop_remainder(batch_size))
images, labels = ds.make_one_shot_iterator().get_next()
return images, labels
def eval_input_fn(params):
batch_size = params["batch_size"]
data_dir = params["data_dir"]
ds = dataset.test(data_dir).apply(
tf.contrib.data.batch_and_drop_remainder(batch_size))
images, labels = ds.make_one_shot_iterator().get_next()
return images, labels
def main(argv):
del argv # Unused.
tf.logging.set_verbosity(tf.logging.INFO)
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
FLAGS.tpu,
zone=FLAGS.tpu_zone,
project=FLAGS.gcp_project
)
run_config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
model_dir=FLAGS.model_dir,
session_config=tf.ConfigProto(
allow_soft_placement=True, log_device_placement=True),
tpu_config=tf.contrib.tpu.TPUConfig(FLAGS.iterations, FLAGS.num_shards),
)
estimator = tf.contrib.tpu.TPUEstimator(
model_fn=model_fn,
use_tpu=FLAGS.use_tpu,
train_batch_size=FLAGS.batch_size,
eval_batch_size=FLAGS.batch_size,
params={"data_dir": FLAGS.data_dir},
config=run_config)
# TPUEstimator.train *requires* a max_steps argument.
estimator.train(input_fn=train_input_fn, max_steps=FLAGS.train_steps)
# TPUEstimator.evaluate *requires* a steps argument.
# Note that the number of examples used during evaluation is
# --eval_steps * --batch_size.
# So if you change --batch_size then change --eval_steps too.
if FLAGS.eval_steps:
estimator.evaluate(input_fn=eval_input_fn, steps=FLAGS.eval_steps)
if __name__ == "__main__":
tf.app.run()
================================================
FILE: official/recommendation/README.md
================================================
# Recommendation Model
## Overview
This is an implementation of the Neural Collaborative Filtering (NCF) framework with Neural Matrix Factorization (NeuMF) model as described in the [Neural Collaborative Filtering](https://arxiv.org/abs/1708.05031) paper. Current implementation is based on the code from the authors' [NCF code](https://github.com/hexiangnan/neural_collaborative_filtering) and the Stanford implementation in the [MLPerf Repo](https://github.com/mlperf/reference/tree/master/recommendation/pytorch).
NCF is a general framework for collaborative filtering of recommendations in which a neural network architecture is used to model user-item interactions. Unlike traditional models, NCF does not resort to Matrix Factorization (MF) with an inner product on latent features of users and items. It replaces the inner product with a multi-layer perceptron that can learn an arbitrary function from data.
Two instantiations of NCF are Generalized Matrix Factorization (GMF) and Multi-Layer Perceptron (MLP). GMF applies a linear kernel to model the latent feature interactions, and and MLP uses a nonlinear kernel to learn the interaction function from data. NeuMF is a fused model of GMF and MLP to better model the complex user-item interactions, and unifies the strengths of linearity of MF and non-linearity of MLP for modeling the user-item latent structures. NeuMF allows GMF and MLP to learn separate embeddings, and combines the two models by concatenating their last hidden layer. [neumf_model.py](neumf_model.py) defines the architecture details.
Some abbreviations used the code base include:
- NCF: Neural Collaborative Filtering
- NeuMF: Neural Matrix Factorization
- GMF: Generalized Matrix Factorization
- MLP: Multi-Layer Perceptron
- HR: Hit Ratio (HR)
- NDCG: Normalized Discounted Cumulative Gain
- ml-1m: MovieLens 1 million dataset
- ml-20m: MovieLens 20 million dataset
## Dataset
The [MovieLens datasets](http://files.grouplens.org/datasets/movielens/) are used for model training and evaluation. Specifically, we use two datasets: **ml-1m** (short for MovieLens 1 million) and **ml-20m** (short for MovieLens 20 million).
### ml-1m
ml-1m dataset contains 1,000,209 anonymous ratings of approximately 3,706 movies made by 6,040 users who joined MovieLens in 2000. All ratings are contained in the file "ratings.dat" without header row, and are in the following format:
```
UserID::MovieID::Rating::Timestamp
```
- UserIDs range between 1 and 6040.
- MovieIDs range between 1 and 3952.
- Ratings are made on a 5-star scale (whole-star ratings only).
### ml-20m
ml-20m dataset contains 20,000,263 ratings of 26,744 movies by 138493 users. All ratings are contained in the file "ratings.csv". Each line of this file after the header row represents one rating of one movie by one user, and has the following format:
```
userId,movieId,rating,timestamp
```
- The lines within this file are ordered first by userId, then, within user, by movieId.
- Ratings are made on a 5-star scale, with half-star increments (0.5 stars - 5.0 stars).
In both datasets, the timestamp is represented in seconds since midnight Coordinated Universal Time (UTC) of January 1, 1970. Each user has at least 20 ratings.
## Running Code
### Download and preprocess dataset
To download the dataset, please install Pandas package first. Then issue the following command:
```
python data_download.py
```
Arguments:
* `--data_dir`: Directory where to download and save the preprocessed data. By default, it is `/tmp/movielens-data/`.
* `--dataset`: The dataset name to be downloaded and preprocessed. By default, it is `ml-1m`.
Use the `--help` or `-h` flag to get a full list of possible arguments.
Note the ml-20m dataset is large (the rating file is ~500 MB), and it may take several minutes (~10 mins) for data preprocessing.
### Train and evaluate model
To train and evaluate the model, issue the following command:
```
python ncf_main.py
```
Arguments:
* `--model_dir`: Directory to save model training checkpoints. By default, it is `/tmp/ncf/`.
* `--data_dir`: This should be set to the same directory given to the `data_download`'s `data_dir` argument.
* `--dataset`: The dataset name to be downloaded and preprocessed. By default, it is `ml-1m`.
There are other arguments about models and training process. Use the `--help` or `-h` flag to get a full list of possible arguments with detailed descriptions.
## Benchmarks (TODO)
### Training times
### Evaluation results
================================================
FILE: official/recommendation/__init__.py
================================================
================================================
FILE: official/recommendation/constants.py
================================================
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""NCF Constants."""
TRAIN_RATINGS_FILENAME = 'train-ratings.csv'
TEST_RATINGS_FILENAME = 'test-ratings.csv'
TEST_NEG_FILENAME = 'test-negative.csv'
USER = "user_id"
ITEM = "item_id"
RATING = "rating"
================================================
FILE: official/recommendation/data_download.py
================================================
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Download and extract the MovieLens dataset from GroupLens website.
Download the dataset, and perform data-preprocessing to convert the raw dataset
into csv file to be used in model training and evaluation.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import collections
import os
import sys
import time
import zipfile
# pylint: disable=g-bad-import-order
import numpy as np
import pandas as pd
from six.moves import urllib # pylint: disable=redefined-builtin
from absl import app as absl_app
from absl import flags
import tensorflow as tf
# pylint: enable=g-bad-import-order
from official.recommendation import constants
from official.utils.flags import core as flags_core
# URL to download dataset
_DATA_URL = "http://files.grouplens.org/datasets/movielens/"
_USER_COLUMN = "user_id"
_ITEM_COLUMN = "item_id"
_RATING_COLUMN = "rating"
_TIMESTAMP_COLUMN = "timestamp"
# The number of negative examples attached with a positive example
# in training dataset. It is set as 100 in the paper.
_NUMBER_NEGATIVES = 100
# In both datasets, each user has at least 20 ratings.
_MIN_NUM_RATINGS = 20
RatingData = collections.namedtuple(
"RatingData", ["items", "users", "ratings", "min_date", "max_date"])
def _print_ratings_description(ratings):
"""Describe the rating dataset information.
Args:
ratings: A pandas DataFrame of the rating dataset.
"""
info = RatingData(items=len(ratings[_ITEM_COLUMN].unique()),
users=len(ratings[_USER_COLUMN].unique()),
ratings=len(ratings),
min_date=ratings[_TIMESTAMP_COLUMN].min(),
max_date=ratings[_TIMESTAMP_COLUMN].max())
tf.logging.info("{ratings} ratings on {items} items from {users} users"
" from {min_date} to {max_date}".format(**(info._asdict())))
def process_movielens(ratings, sort=True):
"""Sort and convert timestamp of the MovieLens dataset.
Args:
ratings: A pandas DataFrame of the rating dataset.
sort: A boolean to indicate whether to sort the data based on timestamp.
Returns:
ratings: The processed pandas DataFrame.
"""
ratings[_TIMESTAMP_COLUMN] = pd.to_datetime(
ratings[_TIMESTAMP_COLUMN], unit="s")
if sort:
ratings.sort_values(by=_TIMESTAMP_COLUMN, inplace=True)
_print_ratings_description(ratings)
return ratings
def load_movielens_1_million(file_name, sort=True):
"""Load the MovieLens 1 million dataset.
The file has no header row, and each line is in the following format:
UserID::MovieID::Rating::Timestamp
- UserIDs range between 1 and 6040
- MovieIDs range between 1 and 3952
- Ratings are made on a 5-star scale (whole-star ratings only)
- Timestamp is represented in seconds since midnight Coordinated Universal
Time (UTC) of January 1, 1970.
- Each user has at least 20 ratings
Args:
file_name: A string of the file name to be loaded.
sort: A boolean to indicate whether to sort the data based on timestamp.
Returns:
A processed pandas DataFrame of the rating dataset.
"""
names = [_USER_COLUMN, _ITEM_COLUMN, _RATING_COLUMN, _TIMESTAMP_COLUMN]
ratings = pd.read_csv(file_name, sep="::", names=names, engine="python")
return process_movielens(ratings, sort=sort)
def load_movielens_20_million(file_name, sort=True):
"""Load the MovieLens 20 million dataset.
Each line of this file after the header row represents one rating of one movie
by one user, and has the following format:
userId,movieId,rating,timestamp
- The lines within this file are ordered first by userId, then, within user,
by movieId.
- Ratings are made on a 5-star scale, with half-star increments
(0.5 stars - 5.0 stars).
- Timestamps represent seconds since midnight Coordinated Universal Time
(UTC) of January 1, 1970.
- All the users had rated at least 20 movies.
Args:
file_name: A string of the file name to be loaded.
sort: A boolean to indicate whether to sort the data based on timestamp.
Returns:
A processed pandas DataFrame of the rating dataset.
"""
ratings = pd.read_csv(file_name)
names = {"userId": _USER_COLUMN, "movieId": _ITEM_COLUMN}
ratings.rename(columns=names, inplace=True)
return process_movielens(ratings, sort=sort)
def load_file_to_df(file_name, sort=True):
"""Load rating dataset into DataFrame.
Two data loading functions are defined to handle dataset ml-1m and ml-20m,
as they are provided with different formats.
Args:
file_name: A string of the file name to be loaded.
sort: A boolean to indicate whether to sort the data based on timestamp.
Returns:
A pandas DataFrame of the rating dataset.
"""
dataset_name = os.path.basename(file_name).split(".")[0]
# ml-1m with extension .dat
file_extension = ".dat"
func = load_movielens_1_million
if dataset_name == "ml-20m":
file_extension = ".csv"
func = load_movielens_20_million
ratings_file = os.path.join(file_name, "ratings" + file_extension)
return func(ratings_file, sort=sort)
def generate_train_eval_data(df, original_users, original_items):
"""Generate the dataset for model training and evaluation.
Given all user and item interaction information, for each user, first sort
the interactions based on timestamp. Then the latest one is taken out as
Test ratings (leave-one-out evaluation) and the remaining data for training.
The Test negatives are randomly sampled from all non-interacted items, and the
number of Test negatives is 100 by default (defined as _NUMBER_NEGATIVES).
Args:
df: The DataFrame of ratings data.
original_users: A list of the original unique user ids in the dataset.
original_items: A list of the original unique item ids in the dataset.
Returns:
all_ratings: A list of the [user_id, item_id] with interactions.
test_ratings: A list of [user_id, item_id], and each line is the latest
user_item interaction for the user.
test_negs: A list of item ids with shape [num_users, 100].
Each line consists of 100 item ids for the user with no interactions.
"""
# Need to sort before popping to get last item
tf.logging.info("Sorting user_item_map by timestamp...")
df.sort_values(by=_TIMESTAMP_COLUMN, inplace=True)
all_ratings = set(zip(df[_USER_COLUMN], df[_ITEM_COLUMN]))
user_to_items = collections.defaultdict(list)
# Generate user_item rating matrix for training
t1 = time.time()
row_count = 0
for row in df.itertuples():
user_to_items[getattr(row, _USER_COLUMN)].append(getattr(row, _ITEM_COLUMN))
row_count += 1
if row_count % 50000 == 0:
tf.logging.info("Processing user_to_items row: {}".format(row_count))
tf.logging.info(
"Process {} rows in [{:.1f}]s".format(row_count, time.time() - t1))
# Generate test ratings and test negatives
t2 = time.time()
test_ratings = []
test_negs = []
# Generate the 0-based index for each item, and put it into a set
all_items = set(range(len(original_items)))
for user in range(len(original_users)):
test_item = user_to_items[user].pop() # Get the latest item id
all_ratings.remove((user, test_item)) # Remove the test item
all_negs = all_items.difference(user_to_items[user])
all_negs = sorted(list(all_negs)) # determinism
test_ratings.append((user, test_item))
test_negs.append(list(np.random.choice(all_negs, _NUMBER_NEGATIVES)))
if user % 1000 == 0:
tf.logging.info("Processing user: {}".format(user))
tf.logging.info("Process {} users in {:.1f}s".format(
len(original_users), time.time() - t2))
all_ratings = list(all_ratings) # convert set to list
return all_ratings, test_ratings, test_negs
def parse_file_to_csv(data_dir, dataset_name):
"""Parse the raw data to csv file to be used in model training and evaluation.
ml-1m dataset is small in size (~25M), while ml-20m is large (~500M). It may
take several minutes to process ml-20m dataset.
Args:
data_dir: A string, the directory with the unzipped dataset.
dataset_name: A string, the dataset name to be processed.
"""
# Use random seed as parameter
np.random.seed(0)
# Load the file as DataFrame
file_path = os.path.join(data_dir, dataset_name)
df = load_file_to_df(file_path, sort=False)
# Get the info of users who have more than 20 ratings on items
grouped = df.groupby(_USER_COLUMN)
df = grouped.filter(lambda x: len(x) >= _MIN_NUM_RATINGS)
original_users = df[_USER_COLUMN].unique()
original_items = df[_ITEM_COLUMN].unique()
# Map the ids of user and item to 0 based index for following processing
tf.logging.info("Generating user_map and item_map...")
user_map = {user: index for index, user in enumerate(original_users)}
item_map = {item: index for index, item in enumerate(original_items)}
df[_USER_COLUMN] = df[_USER_COLUMN].apply(lambda user: user_map[user])
df[_ITEM_COLUMN] = df[_ITEM_COLUMN].apply(lambda item: item_map[item])
assert df[_USER_COLUMN].max() == len(original_users) - 1
assert df[_ITEM_COLUMN].max() == len(original_items) - 1
# Generate data for train and test
all_ratings, test_ratings, test_negs = generate_train_eval_data(
df, original_users, original_items)
# Serialize to csv file. Each csv file contains three columns
# (user_id, item_id, interaction)
# As there are only two fields (user_id, item_id) in all_ratings and
# test_ratings, we need to add a fake rating to make three columns
df_train_ratings = pd.DataFrame(all_ratings)
df_train_ratings["fake_rating"] = 1
train_ratings_file = os.path.join(
FLAGS.data_dir, dataset_name + "-" + constants.TRAIN_RATINGS_FILENAME)
df_train_ratings.to_csv(
train_ratings_file,
index=False, header=False, sep="\t")
tf.logging.info("Train ratings is {}".format(train_ratings_file))
df_test_ratings = pd.DataFrame(test_ratings)
df_test_ratings["fake_rating"] = 1
test_ratings_file = os.path.join(
FLAGS.data_dir, dataset_name + "-" + constants.TEST_RATINGS_FILENAME)
df_test_ratings.to_csv(
test_ratings_file,
index=False, header=False, sep="\t")
tf.logging.info("Test ratings is {}".format(test_ratings_file))
df_test_negs = pd.DataFrame(test_negs)
test_negs_file = os.path.join(
FLAGS.data_dir, dataset_name + "-" + constants.TEST_NEG_FILENAME)
df_test_negs.to_csv(
test_negs_file,
index=False, header=False, sep="\t")
tf.logging.info("Test negatives is {}".format(test_negs_file))
def make_dir(file_dir):
if not tf.gfile.Exists(file_dir):
tf.logging.info("Creating directory {}".format(file_dir))
tf.gfile.MakeDirs(file_dir)
def main(_):
"""Download and extract the data from GroupLens website."""
tf.logging.set_verbosity(tf.logging.INFO)
make_dir(FLAGS.data_dir)
assert FLAGS.dataset, (
"Please specify which dataset to download. "
"Two datasets are available: ml-1m and ml-20m.")
# Download the zip dataset
dataset_zip = FLAGS.dataset + ".zip"
file_path = os.path.join(FLAGS.data_dir, dataset_zip)
if not tf.gfile.Exists(file_path):
def _progress(count, block_size, total_size):
sys.stdout.write("\r>> Downloading {} {:.1f}%".format(
file_path, 100.0 * count * block_size / total_size))
sys.stdout.flush()
file_path, _ = urllib.request.urlretrieve(
_DATA_URL + dataset_zip, file_path, _progress)
statinfo = os.stat(file_path)
# A new line to clear the carriage return from download progress
# tf.logging.info is not applicable here
print()
tf.logging.info(
"Successfully downloaded {} {} bytes".format(
file_path, statinfo.st_size))
# Unzip the dataset
if not tf.gfile.Exists(os.path.join(FLAGS.data_dir, FLAGS.dataset)):
zipfile.ZipFile(file_path, "r").extractall(FLAGS.data_dir)
# Preprocess and parse the dataset to csv
train_ratings = FLAGS.dataset + "-" + constants.TRAIN_RATINGS_FILENAME
if not tf.gfile.Exists(os.path.join(FLAGS.data_dir, train_ratings)):
parse_file_to_csv(FLAGS.data_dir, FLAGS.dataset)
def define_data_download_flags():
"""Add flags specifying data download arguments."""
flags.DEFINE_string(
name="data_dir", default="/tmp/movielens-data/",
help=flags_core.help_wrap(
"Directory to download and extract data."))
flags.DEFINE_enum(
name="dataset", default=None,
enum_values=["ml-1m", "ml-20m"], case_sensitive=False,
help=flags_core.help_wrap(
"Dataset to be trained and evaluated. Two datasets are available "
": ml-1m and ml-20m."))
if __name__ == "__main__":
tf.logging.set_verbosity(tf.logging.INFO)
define_data_download_flags()
FLAGS = flags.FLAGS
absl_app.run(main)
================================================
FILE: official/recommendation/dataset.py
================================================
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Prepare dataset for NCF.
Load the training dataset and evaluation dataset from csv file into memory.
Prepare input for model training and evaluation.
"""
import numpy as np
from six.moves import xrange # pylint: disable=redefined-builtin
import tensorflow as tf
from official.recommendation import constants # pylint: disable=g-bad-import-order
# The buffer size for shuffling train dataset.
_SHUFFLE_BUFFER_SIZE = 1024
class NCFDataSet(object):
"""A class containing data information for model training and evaluation."""
def __init__(self, train_data, num_users, num_items, num_negatives,
true_items, all_items, all_eval_data):
"""Initialize NCFDataset class.
Args:
train_data: A list containing the positive training instances.
num_users: An integer, the number of users in training dataset.
num_items: An integer, the number of items in training dataset.
num_negatives: An integer, the number of negative instances for each user
in train dataset.
true_items: A list, the ground truth (positive) items of users for
evaluation. Each entry is a latest positive instance for one user.
all_items: A nested list, all items for evaluation, and each entry is the
evaluation items for one user.
all_eval_data: A numpy array of eval/test dataset.
"""
self.train_data = train_data
self.num_users = num_users
self.num_items = num_items
self.num_negatives = num_negatives
self.eval_true_items = true_items
self.eval_all_items = all_items
self.all_eval_data = all_eval_data
def load_data(file_name):
"""Load data from a csv file which splits on tab key."""
lines = tf.gfile.Open(file_name, "r").readlines()
# Process the file line by line
def _process_line(line):
return [int(col) for col in line.split("\t")]
data = [_process_line(line) for line in lines]
return data
def data_preprocessing(train_fname, test_fname, test_neg_fname, num_negatives):
"""Preprocess the train and test dataset.
In data preprocessing, the training positive instances are loaded into memory
for random negative instance generation in each training epoch. The test
dataset are generated from test positive and negative instances.
Args:
train_fname: A string, the file name of training positive dataset.
test_fname: A string, the file name of test positive dataset. Each user has
one positive instance.
test_neg_fname: A string, the file name of test negative dataset. Each user
has 100 negative instances by default.
num_negatives: An integer, the number of negative instances for each user
in train dataset.
Returns:
ncf_dataset: A NCFDataset object containing information about training and
evaluation/test dataset.
"""
# Load training positive instances into memory for later train data generation
train_data = load_data(train_fname)
# Get total number of users in the dataset
num_users = len(np.unique(np.array(train_data)[:, 0]))
# Process test dataset to csv file
test_ratings = load_data(test_fname)
test_negatives = load_data(test_neg_fname)
# Get the total number of items in both train dataset and test dataset (the
# whole dataset)
num_items = len(
set(np.array(train_data)[:, 1]) | set(np.array(test_ratings)[:, 1]))
# Generate test instances for each user
true_items, all_items = [], []
all_test_data = []
for idx in range(num_users):
items = test_negatives[idx]
rating = test_ratings[idx]
user = rating[0] # User
true_item = rating[1] # Positive item as ground truth
# All items with first 100 as negative and last one positive
items.append(true_item)
users = np.full(len(items), user, dtype=np.int32)
users_items = list(zip(users, items)) # User-item list
true_items.append(true_item) # all ground truth items
all_items.append(items) # All items (including positive and negative items)
all_test_data.extend(users_items) # Generate test dataset
# Create NCFDataset object
ncf_dataset = NCFDataSet(
train_data, num_users, num_items, num_negatives, true_items, all_items,
np.asarray(all_test_data)
)
return ncf_dataset
def generate_train_dataset(train_data, num_items, num_negatives):
"""Generate train dataset for each epoch.
Given positive training instances, randomly generate negative instances to
form the training dataset.
Args:
train_data: A list of positive training instances.
num_items: An integer, the number of items in positive training instances.
num_negatives: An integer, the number of negative training instances
following positive training instances. It is 4 by default.
Returns:
A numpy array of training dataset.
"""
all_train_data = []
# A set with user-item tuples
train_data_set = set((u, i) for u, i, _ in train_data)
for u, i, _ in train_data:
# Positive instance
all_train_data.append([u, i, 1])
# Negative instances, randomly generated
for _ in xrange(num_negatives):
j = np.random.randint(num_items)
while (u, j) in train_data_set:
j = np.random.randint(num_items)
all_train_data.append([u, j, 0])
return np.asarray(all_train_data)
def input_fn(training, batch_size, ncf_dataset, repeat=1):
"""Input function for model training and evaluation.
The train input consists of 1 positive instance (user and item have
interactions) followed by some number of negative instances in which the items
are randomly chosen. The number of negative instances is "num_negatives" which
is 4 by default. Note that for each epoch, we need to re-generate the negative
instances. Together with positive instances, they form a new train dataset.
Args:
training: A boolean flag for training mode.
batch_size: An integer, batch size for training and evaluation.
ncf_dataset: An NCFDataSet object, which contains the information about
training and test data.
repeat: An integer, how many times to repeat the dataset.
Returns:
dataset: A tf.data.Dataset object containing examples loaded from the files.
"""
# Generate random negative instances for training in each epoch
if training:
train_data = generate_train_dataset(
ncf_dataset.train_data, ncf_dataset.num_items,
ncf_dataset.num_negatives)
# Get train features and labels
train_features = [
(constants.USER, np.expand_dims(train_data[:, 0], axis=1)),
(constants.ITEM, np.expand_dims(train_data[:, 1], axis=1))
]
train_labels = [
(constants.RATING, np.expand_dims(train_data[:, 2], axis=1))]
dataset = tf.data.Dataset.from_tensor_slices(
(dict(train_features), dict(train_labels))
)
dataset = dataset.shuffle(buffer_size=_SHUFFLE_BUFFER_SIZE)
else:
# Create eval/test dataset
test_user = ncf_dataset.all_eval_data[:, 0]
test_item = ncf_dataset.all_eval_data[:, 1]
test_features = [
(constants.USER, np.expand_dims(test_user, axis=1)),
(constants.ITEM, np.expand_dims(test_item, axis=1))]
dataset = tf.data.Dataset.from_tensor_slices(dict(test_features))
# Repeat and batch the dataset
dataset = dataset.repeat(repeat)
dataset = dataset.batch(batch_size)
# Prefetch to improve speed of input pipeline.
dataset = dataset.prefetch(1)
return dataset
================================================
FILE: official/recommendation/dataset_test.py
================================================
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Unit tests for dataset.py."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import numpy as np
import tensorflow as tf # pylint: disable=g-bad-import-order
from official.recommendation import dataset
_TRAIN_FNAME = os.path.join(
os.path.dirname(__file__), "unittest_data/test_train_ratings.csv")
_TEST_FNAME = os.path.join(
os.path.dirname(__file__), "unittest_data/test_eval_ratings.csv")
_TEST_NEG_FNAME = os.path.join(
os.path.dirname(__file__), "unittest_data/test_eval_negative.csv")
_NUM_NEG = 4
class DatasetTest(tf.test.TestCase):
def test_load_data(self):
data = dataset.load_data(_TEST_FNAME)
self.assertEqual(len(data), 2)
self.assertEqual(data[0][0], 0)
self.assertEqual(data[0][2], 1)
self.assertEqual(data[-1][0], 1)
self.assertEqual(data[-1][2], 1)
def test_data_preprocessing(self):
ncf_dataset = dataset.data_preprocessing(
_TRAIN_FNAME, _TEST_FNAME, _TEST_NEG_FNAME, _NUM_N
gitextract_426mla3a/
├── .gitignore
├── README.md
├── __init__.py
├── examples/
│ ├── fastai/
│ │ └── lm/
│ │ ├── lm.ipynb
│ │ └── lm.py
│ ├── pytorch/
│ │ ├── data_parallel_tutorial.py
│ │ └── sentiment/
│ │ ├── model.py
│ │ └── train.py
│ ├── shap/
│ │ └── explainer_1.py
│ └── tf/
│ └── eager/
│ ├── dynamic_dense.py
│ ├── rnn_ptb.py
│ └── rnn_ptb.sh
├── jupyter/
│ └── Untitled.ipynb
├── official/
│ ├── .gitignore
│ ├── Dockerfile.cpu
│ ├── Dockerfile.gpu
│ ├── README.md
│ ├── __init__.py
│ ├── benchmark/
│ │ ├── __init__.py
│ │ ├── benchmark_uploader.py
│ │ ├── benchmark_uploader_main.py
│ │ ├── benchmark_uploader_test.py
│ │ └── datastore/
│ │ └── schema/
│ │ ├── benchmark_metric.json
│ │ └── benchmark_run.json
│ ├── boosted_trees/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── data_download.py
│ │ ├── train_higgs.py
│ │ ├── train_higgs_test.csv
│ │ └── train_higgs_test.py
│ ├── mnist/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── dataset.py
│ │ ├── mnist.py
│ │ ├── mnist_eager.py
│ │ ├── mnist_eager_test.py
│ │ ├── mnist_test.py
│ │ └── mnist_tpu.py
│ ├── recommendation/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── constants.py
│ │ ├── data_download.py
│ │ ├── dataset.py
│ │ ├── dataset_test.py
│ │ ├── ncf_main.py
│ │ ├── neumf_model.py
│ │ └── unittest_data/
│ │ ├── test_eval_negative.csv
│ │ ├── test_eval_ratings.csv
│ │ └── test_train_ratings.csv
│ ├── resnet/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── cifar10_download_and_extract.py
│ │ ├── cifar10_main.py
│ │ ├── cifar10_test.py
│ │ ├── imagenet_main.py
│ │ ├── imagenet_preprocessing.py
│ │ ├── imagenet_test.py
│ │ ├── layer_test.py
│ │ ├── resnet_model.py
│ │ └── resnet_run_loop.py
│ ├── transformer/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── compute_bleu.py
│ │ ├── compute_bleu_test.py
│ │ ├── data_download.py
│ │ ├── model/
│ │ │ ├── __init__.py
│ │ │ ├── attention_layer.py
│ │ │ ├── beam_search.py
│ │ │ ├── beam_search_test.py
│ │ │ ├── embedding_layer.py
│ │ │ ├── ffn_layer.py
│ │ │ ├── model_params.py
│ │ │ ├── model_utils.py
│ │ │ ├── model_utils_test.py
│ │ │ └── transformer.py
│ │ ├── test_data/
│ │ │ ├── newstest2014.de
│ │ │ └── newstest2014.en
│ │ ├── transformer_main.py
│ │ ├── translate.py
│ │ └── utils/
│ │ ├── __init__.py
│ │ ├── dataset.py
│ │ ├── metrics.py
│ │ ├── tokenizer.py
│ │ └── tokenizer_test.py
│ ├── utils/
│ │ ├── __init__.py
│ │ ├── export/
│ │ │ ├── __init__.py
│ │ │ ├── export.py
│ │ │ └── export_test.py
│ │ ├── flags/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── _base.py
│ │ │ ├── _benchmark.py
│ │ │ ├── _conventions.py
│ │ │ ├── _misc.py
│ │ │ ├── _performance.py
│ │ │ ├── core.py
│ │ │ └── flags_test.py
│ │ ├── logs/
│ │ │ ├── __init__.py
│ │ │ ├── hooks.py
│ │ │ ├── hooks_helper.py
│ │ │ ├── hooks_helper_test.py
│ │ │ ├── hooks_test.py
│ │ │ ├── logger.py
│ │ │ ├── logger_test.py
│ │ │ ├── metric_hook.py
│ │ │ └── metric_hook_test.py
│ │ ├── misc/
│ │ │ ├── __init__.py
│ │ │ ├── model_helpers.py
│ │ │ └── model_helpers_test.py
│ │ └── testing/
│ │ ├── __init__.py
│ │ ├── integration.py
│ │ ├── mock_lib.py
│ │ ├── pylint.rcfile
│ │ ├── reference_data/
│ │ │ ├── reference_data_test/
│ │ │ │ ├── dense/
│ │ │ │ │ ├── expected_graph
│ │ │ │ │ ├── results.json
│ │ │ │ │ └── tf_version.json
│ │ │ │ └── uniform_random/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ └── resnet/
│ │ │ ├── batch-size-32_bottleneck_projection_version-1_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ ├── batch-size-32_bottleneck_projection_version-2_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ ├── batch-size-32_bottleneck_version-1_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ ├── batch-size-32_bottleneck_version-2_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ ├── batch-size-32_building_projection_version-1_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ ├── batch-size-32_building_projection_version-2_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ ├── batch-size-32_building_version-1_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ ├── batch-size-32_building_version-2_width-8_channels-4/
│ │ │ │ ├── expected_graph
│ │ │ │ ├── results.json
│ │ │ │ └── tf_version.json
│ │ │ └── batch_norm/
│ │ │ ├── expected_graph
│ │ │ ├── results.json
│ │ │ └── tf_version.json
│ │ ├── reference_data.py
│ │ ├── reference_data_test.py
│ │ └── scripts/
│ │ └── presubmit.sh
│ └── wide_deep/
│ ├── README.md
│ ├── __init__.py
│ ├── data_download.py
│ ├── wide_deep.py
│ ├── wide_deep_test.csv
│ └── wide_deep_test.py
├── projects/
│ ├── __init__.py
│ ├── ai2018/
│ │ ├── __init__.py
│ │ ├── binary/
│ │ │ ├── __init__.py
│ │ │ ├── algos/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── config.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── model.py
│ │ │ │ └── weights.py
│ │ │ ├── dataset.py
│ │ │ ├── evaluate.py
│ │ │ ├── prepare/
│ │ │ │ ├── config.py
│ │ │ │ ├── gen-records.py
│ │ │ │ ├── run.sh
│ │ │ │ └── text2ids.py
│ │ │ ├── read-records.py
│ │ │ ├── torch-train.py
│ │ │ ├── torch_algos/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── loss.py
│ │ │ │ └── model.py
│ │ │ └── train/
│ │ │ └── train.sh
│ │ ├── reader/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── algos/
│ │ │ │ ├── README.md
│ │ │ │ ├── __init__.py
│ │ │ │ ├── baseline.py
│ │ │ │ ├── config.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── m_reader.py
│ │ │ │ ├── model.py
│ │ │ │ ├── qcatt.py
│ │ │ │ └── rnet.py
│ │ │ ├── dataset.py
│ │ │ ├── ensemble/
│ │ │ │ ├── ensemble-infer.py
│ │ │ │ ├── ensemble-valid.py
│ │ │ │ ├── ensemble.py
│ │ │ │ └── evaluate.py
│ │ │ ├── evaluate.py
│ │ │ ├── infer.py
│ │ │ ├── prepare/
│ │ │ │ ├── README.md
│ │ │ │ ├── gen-records.py
│ │ │ │ ├── gen-seg.py
│ │ │ │ ├── merge-emb.py
│ │ │ │ ├── pre-seg.py
│ │ │ │ ├── run-char-ft.sh
│ │ │ │ ├── run-char-glove.sh
│ │ │ │ ├── run-noemb.sh
│ │ │ │ ├── run-word-jieba-ft.sh
│ │ │ │ ├── run-word-jieba-glove.sh
│ │ │ │ ├── to-simplify-dureader.py
│ │ │ │ └── to-simplify.py
│ │ │ ├── prepare-unkvocab.sh
│ │ │ ├── prepare.sh
│ │ │ ├── prepare.v1/
│ │ │ │ ├── fix-vocab.py
│ │ │ │ ├── gen-char-vocab.py
│ │ │ │ ├── gen-content.py
│ │ │ │ ├── gen-records.py
│ │ │ │ ├── gen-seg.py
│ │ │ │ ├── gen-test-unkvocab.sh
│ │ │ │ ├── gen-test.sh
│ │ │ │ ├── gen-train-unkvocab.sh
│ │ │ │ ├── gen-train.sh
│ │ │ │ ├── gen-valid-unkvocab.sh
│ │ │ │ ├── gen-valid.sh
│ │ │ │ ├── gen-vocab.py
│ │ │ │ ├── gen-vocab.sh
│ │ │ │ ├── merge-emb.py
│ │ │ │ ├── run-char.sh
│ │ │ │ ├── run-unkvocab.sh
│ │ │ │ ├── run.sh
│ │ │ │ ├── text2ids.py
│ │ │ │ └── to-simplify.py
│ │ │ ├── read-records.py
│ │ │ ├── tools/
│ │ │ │ ├── check.py
│ │ │ │ ├── check.sh
│ │ │ │ ├── cp-best-epochs.py
│ │ │ │ ├── ensemble-infer.py
│ │ │ │ ├── ensemble-valid.py
│ │ │ │ ├── ensemble.sh
│ │ │ │ ├── evaluate.py
│ │ │ │ ├── fix-vocab.py
│ │ │ │ ├── gen-char-vocab.py
│ │ │ │ ├── show-best-epochs.py
│ │ │ │ ├── vocab-add-all-char.py
│ │ │ │ └── vocab-add-char.py
│ │ │ ├── torch-train.py
│ │ │ ├── torch_algos/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── baseline/
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── baseline.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── m_reader.py
│ │ │ │ ├── model.py
│ │ │ │ └── rnet.py
│ │ │ ├── train/
│ │ │ │ └── v1/
│ │ │ │ ├── torch.word.mreader.answeremb-split.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.att.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.last.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.maxatt.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.maxmean.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.mean.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.topk.sh
│ │ │ │ ├── torch.word.mreader.nopad.answeremb-split.topkatt.sh
│ │ │ │ └── torch.word.mreader.nopad.answeremb-split.topkattlast.sh
│ │ │ ├── train.py
│ │ │ └── train.v1/
│ │ │ ├── torch-decay.sh
│ │ │ ├── torch.sh
│ │ │ ├── v1/
│ │ │ │ ├── model2.1layer.keep07.sh
│ │ │ │ ├── qcatt.1layer.keep07.sh
│ │ │ │ ├── qcatt.sfu.1layer.keep07.sh
│ │ │ │ ├── rcontent.1layer.keep05.sh
│ │ │ │ ├── rcontent.1layer.keep07.sh
│ │ │ │ ├── rcontent.1layer.keep10.sh
│ │ │ │ ├── rcontent.emb.finetune.sh
│ │ │ │ ├── rcontent.emb.sh
│ │ │ │ ├── rcontent.keep07.sh
│ │ │ │ ├── rcontent.sh
│ │ │ │ ├── rcontent.split.sh
│ │ │ │ ├── rnet.1layer.keep07.emb.finetune.att.2.sh
│ │ │ │ ├── rnet.1layer.keep07.emb.finetune.att.finetune.type1only.sh
│ │ │ │ ├── rnet.1layer.keep07.emb.finetune.att.sh
│ │ │ │ ├── rnet.1layer.keep07.emb.finetune.att.type1only.sh
│ │ │ │ ├── rnet.1layer.keep07.emb.finetune.att.type1w2.sh
│ │ │ │ ├── rnet.1layer.keep07.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.att.2.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.att.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.att.type1w2.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.att.unkvocab.finetune.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.att.unkvocab.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.f07.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.f09.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.later.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.maxatt.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.nodecay.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.se.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.topk.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.unkvocab.finetune.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.finetune.unkvocab.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.nodecay.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.emb.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.sh
│ │ │ │ ├── rnet.sfu.1layer.keep07.split.sh
│ │ │ │ └── rnet.sfu.2layer.keep07.emb.finetune.sh
│ │ │ ├── v2/
│ │ │ │ ├── rnet.2.sh
│ │ │ │ ├── rnet.decay.sh
│ │ │ │ ├── rnet.decay2.sh
│ │ │ │ ├── rnet.decay3.sh
│ │ │ │ ├── rnet.decay4.sh
│ │ │ │ ├── rnet.decay5.sh
│ │ │ │ ├── rnet.decay6.sh
│ │ │ │ ├── rnet.decay7.sh
│ │ │ │ ├── rnet.keep08.sh
│ │ │ │ ├── rnet.keep09.sh
│ │ │ │ ├── rnet.labelemb.sh
│ │ │ │ ├── rnet.max.sh
│ │ │ │ ├── rnet.sh
│ │ │ │ ├── rnet.split.finetune.type1only.sh
│ │ │ │ ├── rnet.split.sh
│ │ │ │ └── rnet.type0only.sh
│ │ │ ├── v3/
│ │ │ │ ├── bidaf.split.lr0002.sh
│ │ │ │ ├── bidaf.split.lrsmall.sh
│ │ │ │ ├── bidaf.split.sh
│ │ │ │ ├── bow.sh
│ │ │ │ ├── mreader.2.sh
│ │ │ │ ├── mreader.ae.sh
│ │ │ │ ├── mreader.epoch3.sh
│ │ │ │ ├── mreader.lr0002.sh
│ │ │ │ ├── mreader.max.nomask.sh
│ │ │ │ ├── mreader.max.sh
│ │ │ │ ├── mreader.noshare.max.sh
│ │ │ │ ├── mreader.sh
│ │ │ │ ├── mreader.split.sh
│ │ │ │ ├── mreader.v1.sh
│ │ │ │ ├── mreader.v2.sh
│ │ │ │ ├── mreader.v2.split.sh
│ │ │ │ ├── rcontent.3.sh
│ │ │ │ ├── rcontent.4.sh
│ │ │ │ ├── rcontent.5.sh
│ │ │ │ ├── rcontent.sh
│ │ │ │ ├── rnet.2.2.sh
│ │ │ │ ├── rnet.2.dec.sh
│ │ │ │ ├── rnet.2.sh
│ │ │ │ ├── rnet.2layer.sh
│ │ │ │ ├── rnet.3.sh
│ │ │ │ ├── rnet.4.sh
│ │ │ │ ├── rnet.answer.sh
│ │ │ │ ├── rnet.answer.split.batch64.sh
│ │ │ │ ├── rnet.answer.split.sh
│ │ │ │ ├── rnet.answer.split2.sh
│ │ │ │ ├── rnet.answer2.sh
│ │ │ │ ├── rnet.answer3.sh
│ │ │ │ ├── rnet.att2.sh
│ │ │ │ ├── rnet.batch64.sh
│ │ │ │ ├── rnet.gate.sh
│ │ │ │ ├── rnet.hidden200.sh
│ │ │ │ ├── rnet.keep075.sh
│ │ │ │ ├── rnet.labelatt.sh
│ │ │ │ ├── rnet.labelatt10.sh
│ │ │ │ ├── rnet.lr00005.sh
│ │ │ │ ├── rnet.max.sh
│ │ │ │ ├── rnet.nobucket.sh
│ │ │ │ ├── rnet.p2.batch64.sh
│ │ │ │ ├── rnet.p2.sh
│ │ │ │ ├── rnet.relu.sh
│ │ │ │ ├── rnet.sh
│ │ │ │ ├── rnet.split.batch64.sh
│ │ │ │ ├── rnet.split.cq.batch64.sh
│ │ │ │ ├── rnet.split.cq.sh
│ │ │ │ ├── rnet.split.p2.batch64.sh
│ │ │ │ ├── rnet.split.p2.sh
│ │ │ │ ├── rnet.split.reverse.sh
│ │ │ │ ├── rnet.split.sh
│ │ │ │ ├── rnet.split.test.sh
│ │ │ │ ├── rnet.split.type1c10.batch64.sh
│ │ │ │ ├── rnet.split.type1c2.batch64.sh
│ │ │ │ ├── rnet.split.type1c2.sh
│ │ │ │ ├── rnet.split.type1c5.batch64.sh
│ │ │ │ ├── rnet.split.type1w2.batch64.sh
│ │ │ │ ├── rnet.split.type1w2.sh
│ │ │ │ ├── rnet.tanh.sh
│ │ │ │ ├── torch-keep07.sh
│ │ │ │ ├── torch-mreader.emb.lr00005.sh
│ │ │ │ ├── torch-mreader.emb.lr0001.sh
│ │ │ │ ├── torch-mreader.emb.lr0001.split.sh
│ │ │ │ ├── torch-mreader.emb.lr0003.sh
│ │ │ │ ├── torch-mreader.emb.lr0004.sh
│ │ │ │ ├── torch-mreader.emb.sh
│ │ │ │ ├── torch-mreader.emb.split.sh
│ │ │ │ ├── torch-mreader.lr0001.sh
│ │ │ │ ├── torch-mreader.sh
│ │ │ │ ├── torch-mreader.v1.emb.sh
│ │ │ │ ├── torch-mwan.emb.2.sh
│ │ │ │ ├── torch-mwan.emb.batch64.sh
│ │ │ │ ├── torch-mwan.emb.sh
│ │ │ │ ├── torch-mwan.lr0001.sh
│ │ │ │ ├── torch-mwan.sh
│ │ │ │ ├── torch-rnet.2.sh
│ │ │ │ ├── torch-rnet.lr0001.emb.sh
│ │ │ │ ├── torch-rnet.lr0001.sh
│ │ │ │ ├── torch-rnet.sh
│ │ │ │ └── torch.sh
│ │ │ ├── v4/
│ │ │ │ ├── bow-emb.sh
│ │ │ │ ├── bow-nomask-emb.sh
│ │ │ │ ├── bow-nomask.sh
│ │ │ │ ├── bow.sh
│ │ │ │ ├── gru-emb-att.sh
│ │ │ │ ├── gru-emb-latt.sh
│ │ │ │ ├── gru-emb-nomask.2.sh
│ │ │ │ ├── gru-emb-nomask.sh
│ │ │ │ ├── gru-emb-simpledrop-bw.sh
│ │ │ │ ├── gru-emb-simpledrop.sh
│ │ │ │ ├── gru-emb-top2.sh
│ │ │ │ ├── gru-emb-top2mean.sh
│ │ │ │ ├── gru-emb-top2wmean.sh
│ │ │ │ ├── gru-emb-topkatt2.sh
│ │ │ │ ├── gru-emb-topkatt3.sh
│ │ │ │ ├── gru-emb.sh
│ │ │ │ ├── gru.sh
│ │ │ │ ├── mreader-v4-nopad.sh
│ │ │ │ ├── mreader-v4-split-nopad.sh
│ │ │ │ ├── mreader-v4-split.sh
│ │ │ │ ├── mreader-v4.sh
│ │ │ │ ├── mreader.max.nopad.sh
│ │ │ │ ├── mreader.max.sh
│ │ │ │ ├── mreader.max.split.nopad.sh
│ │ │ │ ├── mreader.max.split.sh
│ │ │ │ ├── mreader.nopad.sh
│ │ │ │ ├── mreader.sh
│ │ │ │ ├── mreader.split.nopad.sh
│ │ │ │ ├── mreader.split.sh
│ │ │ │ ├── torch-bow-emb.sh
│ │ │ │ ├── torch-bow.sh
│ │ │ │ ├── torch-gru-emb-att-nomask.sh
│ │ │ │ ├── torch-gru-emb-att-pad-typeemb.sh
│ │ │ │ ├── torch-gru-emb-att-pad.sh
│ │ │ │ ├── torch-gru-emb-att-type.sh
│ │ │ │ ├── torch-gru-emb-att-typeemb.sh
│ │ │ │ ├── torch-gru-emb-att-typernn.sh
│ │ │ │ ├── torch-gru-emb-att.sh
│ │ │ │ ├── torch-gru-emb-keep08.sh
│ │ │ │ ├── torch-gru-emb-latt.sh
│ │ │ │ ├── torch-gru-emb-mask.2.sh
│ │ │ │ ├── torch-gru-emb-mask.sh
│ │ │ │ ├── torch-gru-emb-nodrop.sh
│ │ │ │ ├── torch-gru-emb-nomask.2.sh
│ │ │ │ ├── torch-gru-emb-nomask.sh
│ │ │ │ ├── torch-gru-emb-topk2-nomask.sh
│ │ │ │ ├── torch-gru-emb-topk2.sh
│ │ │ │ ├── torch-gru-emb-topk3-nomask.sh
│ │ │ │ ├── torch-gru-emb-topk3.sh
│ │ │ │ ├── torch-gru-emb.sh
│ │ │ │ ├── torch-gru.2.sh
│ │ │ │ ├── torch-gru.sh
│ │ │ │ ├── torch-mreader-1hop-nopad.sh
│ │ │ │ ├── torch-mreader-1hop-split-nopad.sh
│ │ │ │ ├── torch-mreader-emb-att-nopad-split.sh
│ │ │ │ ├── torch-mreader-emb-att-nopad.sh
│ │ │ │ ├── torch-mreader-emb-att-pad.sh
│ │ │ │ ├── torch-mreader-emb-att-typeemb-nopad-split.sh
│ │ │ │ ├── torch-mreader-emb-att-typeemb-nopad.sh
│ │ │ │ ├── torch-mreader-emb-att-typeemb-pad.sh
│ │ │ │ ├── torch-mreader-emb-att-typeemb.sh
│ │ │ │ ├── torch-mreader-emb-att.sh
│ │ │ │ ├── torch-mreader-emb-nopad-split.sh
│ │ │ │ ├── torch-mreader-emb-nopad-typeemb-split.sh
│ │ │ │ ├── torch-mreader-emb-nopad-typeemb.sh
│ │ │ │ ├── torch-mreader-emb-nopad.sh
│ │ │ │ ├── torch-mreader-emb-pad-typeemb.sh
│ │ │ │ ├── torch-mreader-emb-pad.sh
│ │ │ │ ├── torch-mreader-emb-typeemb.sh
│ │ │ │ ├── torch-mreader-emb.sh
│ │ │ │ ├── torch-mreader-hop3-nopad.sh
│ │ │ │ ├── torch-mreader-nopad.sh
│ │ │ │ ├── torch-mreader-poolnopad.sh
│ │ │ │ ├── torch-mreader-split-hop3-nopad.sh
│ │ │ │ ├── torch-mreader-split-nopad.sh
│ │ │ │ ├── torch-mreader-split-poolnopad.sh
│ │ │ │ ├── torch-mreader-typeemb-hop3-nopad.sh
│ │ │ │ ├── torch-mreader-typeemb-nopad.sh
│ │ │ │ ├── torch-mreader-v3-decay09.sh
│ │ │ │ ├── torch-mreader-v3-nopad.sh
│ │ │ │ ├── torch-mreader-v3-poolnopad.sh
│ │ │ │ ├── torch-mreader-v3-split-decay09.sh
│ │ │ │ ├── torch-mreader-v3-split-nopad.sh
│ │ │ │ ├── torch-mreader-v3-split.sh
│ │ │ │ └── torch-mreader-v3.sh
│ │ │ └── v5/
│ │ │ ├── torch-mreader-split-answeremb-nopad.sh
│ │ │ ├── torch-mreader-split-nopad-ftwiki.sh
│ │ │ └── torch-mreader-split-nopad.sh
│ │ └── sentiment/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── algos/
│ │ │ ├── __init__.py
│ │ │ ├── config.py
│ │ │ ├── loss.py
│ │ │ ├── model.py
│ │ │ └── weights.py
│ │ ├── analysis/
│ │ │ ├── analysis.ipynb
│ │ │ ├── analyze.py
│ │ │ ├── beam-search.py
│ │ │ ├── beam_f.py
│ │ │ ├── beam_f_utils.py
│ │ │ ├── class-info.py
│ │ │ ├── class-info.train.csv
│ │ │ ├── class-info.valid.csv
│ │ │ ├── correlations-filter.py
│ │ │ ├── correlations.py
│ │ │ ├── diff.py
│ │ │ └── gen-class-weights.py
│ │ ├── dataset.py
│ │ ├── ensemble/
│ │ │ ├── README.md
│ │ │ ├── calibrate-cv.py
│ │ │ ├── ensemble-cv-parallel.py
│ │ │ ├── ensemble-cv-v1.py
│ │ │ ├── ensemble-cv.py
│ │ │ ├── ensemble-hillclimb.py
│ │ │ ├── ensemble-infer.py
│ │ │ ├── ensemble-v1.py
│ │ │ ├── ensemble.py
│ │ │ ├── evaluate.py
│ │ │ ├── gen-train.py
│ │ │ ├── hillclimb-ensembling.py
│ │ │ ├── lgb-adjust.py
│ │ │ ├── lgb-cv.py
│ │ │ └── lr-cv.py
│ │ ├── evaluate.py
│ │ ├── infer/
│ │ │ └── v11/
│ │ │ ├── README.md
│ │ │ ├── tf.char.rnet.nolatt.sh
│ │ │ ├── tf.char.rnet.sh
│ │ │ ├── tf.char.transformer.bert.cutfront.sh
│ │ │ ├── tf.char.transformer.bert.finetune.2.sh
│ │ │ ├── tf.char.transformer.bert.finetune.3.sh
│ │ │ ├── tf.char.transformer.bert.finetune.sh
│ │ │ ├── tf.char.transformer.bert.finetune4.sh
│ │ │ ├── tf.char.transformer.bert.rnn.finetune.sh
│ │ │ ├── tf.char.transformer.bert.sh
│ │ │ ├── tf.char.transformer.bert.topkatt.cutfront.sh
│ │ │ ├── tf.char.transformer.bert.topkatt.finetune.sh
│ │ │ ├── tf.char.transformer.bert.topkatt.sh
│ │ │ ├── tf.char.transformer.nbert.finetune.sh
│ │ │ ├── tf.mix.rnet.nolatt.sh
│ │ │ ├── tf.mix.rnet.sh
│ │ │ ├── tf.word.mreader.lm.2layer.sh
│ │ │ ├── tf.word.mreader.lm.2layer.unkaug.sh
│ │ │ ├── tf.word.mreader.lm.sh
│ │ │ ├── tf.word.mreader.lm.unkaug.sh
│ │ │ ├── tf.word.rnet.hidden600.sh
│ │ │ ├── tf.word.rnet.lm.sh
│ │ │ ├── tf.word.rnet.lm.unkaug.sh
│ │ │ ├── tf.word.rnet.nolatt.hidden600.sh
│ │ │ ├── tf.word.rnet.nolatt.sh
│ │ │ ├── tf.word.rnet.sh
│ │ │ ├── tf.word.rnet.unkaug.sh
│ │ │ ├── tf.word.rnetv2.lm.sh
│ │ │ ├── tf.word.rnetv2.lm.unkaug.sh
│ │ │ ├── torch.char.mreader.lm.nolatt.sh
│ │ │ ├── torch.char.mreader.lm.sh
│ │ │ ├── torch.char.mreader.nolatt.sh
│ │ │ ├── torch.char.mreader.sh
│ │ │ ├── torch.mix.mreader.lm.nolatt.sh
│ │ │ ├── torch.mix.mreader.lm.sh
│ │ │ ├── torch.mix.mreader.sh
│ │ │ ├── torch.word.mreader.lm.sh
│ │ │ ├── torch.word.mreader.nopad.bertopt.nolatt.unkaug.hidden400.sh
│ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.baike.sh
│ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden300.sh
│ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.sh
│ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.short.sh
│ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden768.sh
│ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.sh
│ │ │ ├── torch.word.mreader.nopad.lm.bertopt.unkaug.sh
│ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.hidden300.sh
│ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.hidden400.sh
│ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.hidden768.sh
│ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.sh
│ │ │ ├── torch.word.mreader.nopad.lm.sh
│ │ │ └── torch.word.mreader.nopad.lm.unkaug.sh
│ │ ├── infer.py
│ │ ├── lm-train.py
│ │ ├── lm_dataset.py
│ │ ├── prepare/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── config
│ │ │ ├── config.py
│ │ │ ├── embeddings/
│ │ │ │ ├── fasttext/
│ │ │ │ │ └── run.sh
│ │ │ │ └── glove/
│ │ │ │ └── run.sh
│ │ │ ├── filter.py
│ │ │ ├── find-chars.py
│ │ │ ├── fix-vocab.py
│ │ │ ├── gen-canyin.py
│ │ │ ├── gen-canyin.sh
│ │ │ ├── gen-char-vocab.py
│ │ │ ├── gen-content.py
│ │ │ ├── gen-dianping.py
│ │ │ ├── gen-dianping.sh
│ │ │ ├── gen-lm-records.py
│ │ │ ├── gen-mix-vocab.py
│ │ │ ├── gen-records.py
│ │ │ ├── gen-test.sh
│ │ │ ├── gen-train.sh
│ │ │ ├── gen-trans.py
│ │ │ ├── gen-trans.sh
│ │ │ ├── gen-valid.sh
│ │ │ ├── gen-vocab-v2.sh
│ │ │ ├── gen-vocab.py
│ │ │ ├── gen-vocab.sh
│ │ │ ├── merge-emb.py
│ │ │ ├── merge-vocabs.py
│ │ │ ├── pre-mix-seg-v1.py
│ │ │ ├── pre-mix-seg.py
│ │ │ ├── pre-seg-bert.py
│ │ │ ├── pre-seg.py
│ │ │ ├── run-char-bert.sh
│ │ │ ├── run-char-ft.sh
│ │ │ ├── run-char-glove.sh
│ │ │ ├── run-char-nbert.sh
│ │ │ ├── run-char.sh
│ │ │ ├── run-lm-char-len256.sh
│ │ │ ├── run-lm-char.sh
│ │ │ ├── run-lm-jieba-mix-len256.sh
│ │ │ ├── run-lm-mix.sh
│ │ │ ├── run-lm-word-baike-len20.sh
│ │ │ ├── run-lm-word-baike.sh
│ │ │ ├── run-lm-word-bseg-len40.sh
│ │ │ ├── run-lm-word-bseg-ner-len40.sh
│ │ │ ├── run-lm-word-bseg-ner2-len40.sh
│ │ │ ├── run-lm-word-jieba-tx-len40.sh
│ │ │ ├── run-lm-word-len40.sh
│ │ │ ├── run-lm-word-sp10w-len40.sh
│ │ │ ├── run-lm-word-sp10w.sh
│ │ │ ├── run-lm-word-sp1w-baike.sh
│ │ │ ├── run-lm-word-sp1w-len256.sh
│ │ │ ├── run-lm-word-sp1w.sh
│ │ │ ├── run-lm-word-sp20w-len40.sh
│ │ │ ├── run-lm-word-sp20w.sh
│ │ │ ├── run-lm-word.sh
│ │ │ ├── run-mix-bseg-ft.sh
│ │ │ ├── run-mix-bseg-glove-10epoch.sh
│ │ │ ├── run-mix-bseg-glove-40epoch.sh
│ │ │ ├── run-mix-bseg-glove.sh
│ │ │ ├── run-mix-bseg-tx.sh
│ │ │ ├── run-mix-glove.sh
│ │ │ ├── run-mix-jieba-ft.sh
│ │ │ ├── run-mix-tx.sh
│ │ │ ├── run-nchar-ft.sh
│ │ │ ├── run-noemb-aug.sh
│ │ │ ├── run-noemb-char-bert.sh
│ │ │ ├── run-noemb-char-nbert.sh
│ │ │ ├── run-noemb-char.sh
│ │ │ ├── run-noemb-mix-charaug.sh
│ │ │ ├── run-noemb-mix.sh
│ │ │ ├── run-noemb-pl.sh
│ │ │ ├── run-noemb.sh
│ │ │ ├── run-word-bseg-ft.sh
│ │ │ ├── run-word-bseg-glove.sh
│ │ │ ├── run-word-bseg-ner-ft-knowldege.sh
│ │ │ ├── run-word-bseg-ner-ft.sh
│ │ │ ├── run-word-bseg-ner-ft2.sh
│ │ │ ├── run-word-bseg-ner-ft3.sh
│ │ │ ├── run-word-bseg-ner-glove-10epoch.sh
│ │ │ ├── run-word-bseg-ner-glove-40epoch.sh
│ │ │ ├── run-word-bseg-ner-glove.sh
│ │ │ ├── run-word-bseg-ner-jiebapre-ft.sh
│ │ │ ├── run-word-bseg-subner-ft.sh
│ │ │ ├── run-word-bseg-subner-jiebapre-ft.sh
│ │ │ ├── run-word-jieba-ft.sh
│ │ │ ├── run-word-jieba-glove.sh
│ │ │ ├── run-word-jieba-pos-ft.sh
│ │ │ ├── run-word-jieba-pos-glove.sh
│ │ │ ├── run-word-jieba-tx.sh
│ │ │ ├── run-word-sp10w-ft.sh
│ │ │ ├── run-word-sp10w-glove.sh
│ │ │ ├── run-word-sp10w-tx.sh
│ │ │ ├── run-word-sp1w-ft.sh
│ │ │ ├── run-word-sp1w-glove.sh
│ │ │ ├── run-word-sp1w-tx.sh
│ │ │ ├── run-word-sp20w-ft-pl.sh
│ │ │ ├── run-word-sp20w-ft.sh
│ │ │ ├── run-word-sp20w-glove.sh
│ │ │ ├── run-word-stanford-ft.sh
│ │ │ ├── run-word-stanford-ft2.sh
│ │ │ ├── run-word-stanford-ft3.sh
│ │ │ ├── run-word-stanford-glove.sh
│ │ │ ├── run-word-stanford-tx-ftchar.sh
│ │ │ ├── run-word-stanford-tx-glovechar.sh
│ │ │ ├── run-word-tx-ftchar.sh
│ │ │ ├── run-word-tx-glovechar.sh
│ │ │ ├── text2ids.py
│ │ │ ├── tmp
│ │ │ ├── to-chars.py
│ │ │ ├── to-norm.py
│ │ │ └── to-simplify.py
│ │ ├── prepare.test/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── config
│ │ │ ├── config.py
│ │ │ ├── embeddings/
│ │ │ │ ├── fasttext/
│ │ │ │ │ └── run.sh
│ │ │ │ └── glove/
│ │ │ │ └── run.sh
│ │ │ ├── filter.py
│ │ │ ├── find-chars.py
│ │ │ ├── fix-vocab.py
│ │ │ ├── gen-canyin.py
│ │ │ ├── gen-canyin.sh
│ │ │ ├── gen-char-vocab.py
│ │ │ ├── gen-content.py
│ │ │ ├── gen-dianping.py
│ │ │ ├── gen-dianping.sh
│ │ │ ├── gen-lm-records.py
│ │ │ ├── gen-mix-vocab.py
│ │ │ ├── gen-records.py
│ │ │ ├── gen-test.sh
│ │ │ ├── gen-train.sh
│ │ │ ├── gen-trans.py
│ │ │ ├── gen-trans.sh
│ │ │ ├── gen-valid.sh
│ │ │ ├── gen-vocab-v2.sh
│ │ │ ├── gen-vocab.py
│ │ │ ├── gen-vocab.sh
│ │ │ ├── merge-emb.py
│ │ │ ├── merge-vocabs.py
│ │ │ ├── pre-mix-seg-v1.py
│ │ │ ├── pre-mix-seg.py
│ │ │ ├── pre-seg-bert.py
│ │ │ ├── pre-seg.py
│ │ │ ├── run-char-bert.sh
│ │ │ ├── run-char-ft.sh
│ │ │ ├── run-char-glove.sh
│ │ │ ├── run-char.sh
│ │ │ ├── run-lm-char-len256.sh
│ │ │ ├── run-lm-char.sh
│ │ │ ├── run-lm-jieba-mix-len256.sh
│ │ │ ├── run-lm-mix.sh
│ │ │ ├── run-lm-word-baike-len20.sh
│ │ │ ├── run-lm-word-baike.sh
│ │ │ ├── run-lm-word-bseg-len40.sh
│ │ │ ├── run-lm-word-bseg-ner-len40.sh
│ │ │ ├── run-lm-word-bseg-ner2-len40.sh
│ │ │ ├── run-lm-word-jieba-tx-len40.sh
│ │ │ ├── run-lm-word-len40.sh
│ │ │ ├── run-lm-word-sp10w-len40.sh
│ │ │ ├── run-lm-word-sp10w.sh
│ │ │ ├── run-lm-word-sp1w-baike.sh
│ │ │ ├── run-lm-word-sp1w-len256.sh
│ │ │ ├── run-lm-word-sp1w.sh
│ │ │ ├── run-lm-word.sh
│ │ │ ├── run-mix-bseg-ft.sh
│ │ │ ├── run-mix-bseg-glove-10epoch.sh
│ │ │ ├── run-mix-bseg-glove-40epoch.sh
│ │ │ ├── run-mix-bseg-glove.sh
│ │ │ ├── run-mix-bseg-tx.sh
│ │ │ ├── run-mix-glove.sh
│ │ │ ├── run-mix-jieba-ft.sh
│ │ │ ├── run-mix-tx.sh
│ │ │ ├── run-noemb-aug.sh
│ │ │ ├── run-noemb-char-bert.sh
│ │ │ ├── run-noemb-char.sh
│ │ │ ├── run-noemb-mix-charaug.sh
│ │ │ ├── run-noemb-mix.sh
│ │ │ ├── run-noemb-wordonly.sh
│ │ │ ├── run-noemb.sh
│ │ │ ├── run-word-bseg-ft.sh
│ │ │ ├── run-word-bseg-glove.sh
│ │ │ ├── run-word-bseg-ner-ft-knowldege.sh
│ │ │ ├── run-word-bseg-ner-ft.sh
│ │ │ ├── run-word-bseg-ner-ft2.sh
│ │ │ ├── run-word-bseg-ner-ft3.sh
│ │ │ ├── run-word-bseg-ner-glove-10epoch.sh
│ │ │ ├── run-word-bseg-ner-glove-40epoch.sh
│ │ │ ├── run-word-bseg-ner-glove.sh
│ │ │ ├── run-word-bseg-ner-jiebapre-ft.sh
│ │ │ ├── run-word-bseg-subner-ft.sh
│ │ │ ├── run-word-bseg-subner-jiebapre-ft.sh
│ │ │ ├── run-word-jieba-ft-short.sh
│ │ │ ├── run-word-jieba-ft.sh
│ │ │ ├── run-word-jieba-glove.sh
│ │ │ ├── run-word-jieba-pos-ft.sh
│ │ │ ├── run-word-jieba-pos-glove.sh
│ │ │ ├── run-word-jieba-tx.sh
│ │ │ ├── run-word-sp10w-ft.sh
│ │ │ ├── run-word-sp10w-glove.sh
│ │ │ ├── run-word-sp10w-tx.sh
│ │ │ ├── run-word-sp1w-ft.sh
│ │ │ ├── run-word-sp1w-glove.sh
│ │ │ ├── run-word-sp1w-tx.sh
│ │ │ ├── run-word-sp20w-ft.sh
│ │ │ ├── run-word-sp20w-glove.sh
│ │ │ ├── run-word-stanford-ft.sh
│ │ │ ├── run-word-stanford-ft2.sh
│ │ │ ├── run-word-stanford-ft3.sh
│ │ │ ├── run-word-stanford-glove.sh
│ │ │ ├── run-word-stanford-tx-ftchar.sh
│ │ │ ├── run-word-stanford-tx-glovechar.sh
│ │ │ ├── run-word-tx-ftchar.sh
│ │ │ ├── run-word-tx-glovechar.sh
│ │ │ ├── run.sh
│ │ │ ├── text2ids.py
│ │ │ ├── tmp
│ │ │ ├── to-chars.py
│ │ │ ├── to-norm.py
│ │ │ └── to-simplify.py
│ │ ├── prepare.testb/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── config
│ │ │ ├── config.py
│ │ │ ├── embeddings/
│ │ │ │ ├── fasttext/
│ │ │ │ │ └── run.sh
│ │ │ │ └── glove/
│ │ │ │ └── run.sh
│ │ │ ├── filter.py
│ │ │ ├── find-chars.py
│ │ │ ├── fix-vocab.py
│ │ │ ├── gen-canyin.py
│ │ │ ├── gen-canyin.sh
│ │ │ ├── gen-char-vocab.py
│ │ │ ├── gen-content.py
│ │ │ ├── gen-dianping.py
│ │ │ ├── gen-dianping.sh
│ │ │ ├── gen-lm-records.py
│ │ │ ├── gen-mix-vocab.py
│ │ │ ├── gen-records.py
│ │ │ ├── gen-test.sh
│ │ │ ├── gen-train.sh
│ │ │ ├── gen-trans.py
│ │ │ ├── gen-trans.sh
│ │ │ ├── gen-valid.sh
│ │ │ ├── gen-vocab-v2.sh
│ │ │ ├── gen-vocab.py
│ │ │ ├── gen-vocab.sh
│ │ │ ├── merge-emb.py
│ │ │ ├── merge-vocabs.py
│ │ │ ├── pre-mix-seg-v1.py
│ │ │ ├── pre-mix-seg.py
│ │ │ ├── pre-seg-bert.py
│ │ │ ├── pre-seg.py
│ │ │ ├── run-char-bert.sh
│ │ │ ├── run-char-ft.sh
│ │ │ ├── run-char-glove.sh
│ │ │ ├── run-char.sh
│ │ │ ├── run-lm-char-len256.sh
│ │ │ ├── run-lm-char.sh
│ │ │ ├── run-lm-jieba-mix-len256.sh
│ │ │ ├── run-lm-mix.sh
│ │ │ ├── run-lm-word-baike-len20.sh
│ │ │ ├── run-lm-word-baike.sh
│ │ │ ├── run-lm-word-bseg-len40.sh
│ │ │ ├── run-lm-word-bseg-ner-len40.sh
│ │ │ ├── run-lm-word-bseg-ner2-len40.sh
│ │ │ ├── run-lm-word-jieba-tx-len40.sh
│ │ │ ├── run-lm-word-len40.sh
│ │ │ ├── run-lm-word-sp10w-len40.sh
│ │ │ ├── run-lm-word-sp10w.sh
│ │ │ ├── run-lm-word-sp1w-baike.sh
│ │ │ ├── run-lm-word-sp1w-len256.sh
│ │ │ ├── run-lm-word-sp1w.sh
│ │ │ ├── run-lm-word.sh
│ │ │ ├── run-mix-bseg-ft.sh
│ │ │ ├── run-mix-bseg-glove-10epoch.sh
│ │ │ ├── run-mix-bseg-glove-40epoch.sh
│ │ │ ├── run-mix-bseg-glove.sh
│ │ │ ├── run-mix-bseg-tx.sh
│ │ │ ├── run-mix-glove.sh
│ │ │ ├── run-mix-jieba-ft.sh
│ │ │ ├── run-mix-tx.sh
│ │ │ ├── run-noemb-aug.sh
│ │ │ ├── run-noemb-char-bert.sh
│ │ │ ├── run-noemb-char.sh
│ │ │ ├── run-noemb-mix-charaug.sh
│ │ │ ├── run-noemb-mix.sh
│ │ │ ├── run-noemb-wordonly.sh
│ │ │ ├── run-noemb.sh
│ │ │ ├── run-word-bseg-ft.sh
│ │ │ ├── run-word-bseg-glove.sh
│ │ │ ├── run-word-bseg-ner-ft-knowldege.sh
│ │ │ ├── run-word-bseg-ner-ft.sh
│ │ │ ├── run-word-bseg-ner-ft2.sh
│ │ │ ├── run-word-bseg-ner-ft3.sh
│ │ │ ├── run-word-bseg-ner-glove-10epoch.sh
│ │ │ ├── run-word-bseg-ner-glove-40epoch.sh
│ │ │ ├── run-word-bseg-ner-glove.sh
│ │ │ ├── run-word-bseg-ner-jiebapre-ft.sh
│ │ │ ├── run-word-bseg-subner-ft.sh
│ │ │ ├── run-word-bseg-subner-jiebapre-ft.sh
│ │ │ ├── run-word-jieba-ft-short.sh
│ │ │ ├── run-word-jieba-ft.sh
│ │ │ ├── run-word-jieba-glove.sh
│ │ │ ├── run-word-jieba-pos-ft.sh
│ │ │ ├── run-word-jieba-pos-glove.sh
│ │ │ ├── run-word-jieba-tx.sh
│ │ │ ├── run-word-sp10w-ft.sh
│ │ │ ├── run-word-sp10w-glove.sh
│ │ │ ├── run-word-sp10w-tx.sh
│ │ │ ├── run-word-sp1w-ft.sh
│ │ │ ├── run-word-sp1w-glove.sh
│ │ │ ├── run-word-sp1w-tx.sh
│ │ │ ├── run-word-sp20w-ft.sh
│ │ │ ├── run-word-sp20w-glove.sh
│ │ │ ├── run-word-stanford-ft.sh
│ │ │ ├── run-word-stanford-ft2.sh
│ │ │ ├── run-word-stanford-ft3.sh
│ │ │ ├── run-word-stanford-glove.sh
│ │ │ ├── run-word-stanford-tx-ftchar.sh
│ │ │ ├── run-word-stanford-tx-glovechar.sh
│ │ │ ├── run-word-tx-ftchar.sh
│ │ │ ├── run-word-tx-glovechar.sh
│ │ │ ├── run.sh
│ │ │ ├── text2ids.py
│ │ │ ├── tmp
│ │ │ ├── to-chars.py
│ │ │ ├── to-norm.py
│ │ │ └── to-simplify.py
│ │ ├── prepare.v1/
│ │ │ ├── filter.py
│ │ │ ├── find-chars.py
│ │ │ ├── fix-vocab.py
│ │ │ ├── gen-canyin.py
│ │ │ ├── gen-canyin.sh
│ │ │ ├── gen-char-seg-canyin.py
│ │ │ ├── gen-char-seg-dianping.py
│ │ │ ├── gen-char-seg-train.py
│ │ │ ├── gen-char-seg.py
│ │ │ ├── gen-char-vocab.py
│ │ │ ├── gen-content.py
│ │ │ ├── gen-dianping.py
│ │ │ ├── gen-dianping.sh
│ │ │ ├── gen-mix-seg-canyin.py
│ │ │ ├── gen-mix-seg-dianping.py
│ │ │ ├── gen-records.py
│ │ │ ├── gen-seg-canyin.py
│ │ │ ├── gen-seg-dianping.py
│ │ │ ├── gen-seg-train.py
│ │ │ ├── gen-test.sh
│ │ │ ├── gen-train.sh
│ │ │ ├── gen-trans.py
│ │ │ ├── gen-trans.sh
│ │ │ ├── gen-valid.sh
│ │ │ ├── gen-vocab.py
│ │ │ ├── gen-vocab.sh
│ │ │ ├── merge-emb.py
│ │ │ ├── run-char.sh
│ │ │ ├── run-noemb.sh
│ │ │ ├── run.sh
│ │ │ └── text2ids.py
│ │ ├── prepare.v2/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── config.py
│ │ │ ├── embeddings/
│ │ │ │ ├── fasttext/
│ │ │ │ │ └── run.sh
│ │ │ │ └── glove/
│ │ │ │ └── run.sh
│ │ │ ├── filter.py
│ │ │ ├── find-chars.py
│ │ │ ├── fix-vocab.py
│ │ │ ├── gen-canyin.py
│ │ │ ├── gen-canyin.sh
│ │ │ ├── gen-char-seg-canyin.py
│ │ │ ├── gen-char-seg-dianping.py
│ │ │ ├── gen-char-seg-train.py
│ │ │ ├── gen-char-seg.py
│ │ │ ├── gen-char-vocab.py
│ │ │ ├── gen-content.py
│ │ │ ├── gen-dianping.py
│ │ │ ├── gen-dianping.sh
│ │ │ ├── gen-mix-seg-canyin.py
│ │ │ ├── gen-mix-seg-dianping.py
│ │ │ ├── gen-mix-seg-train.py
│ │ │ ├── gen-records.py
│ │ │ ├── gen-seg-canyin.py
│ │ │ ├── gen-seg-dianping.py
│ │ │ ├── gen-seg-train.py
│ │ │ ├── gen-test.sh
│ │ │ ├── gen-train.sh
│ │ │ ├── gen-trans.py
│ │ │ ├── gen-trans.sh
│ │ │ ├── gen-valid.sh
│ │ │ ├── gen-vocab-bseg-basic-v2.sh
│ │ │ ├── gen-vocab-bseg-basic.sh
│ │ │ ├── gen-vocab-bseg-phrase-v2.sh
│ │ │ ├── gen-vocab-bseg-phrase.sh
│ │ │ ├── gen-vocab-v2.sh
│ │ │ ├── gen-vocab.py
│ │ │ ├── gen-vocab.sh
│ │ │ ├── merge-emb.py
│ │ │ ├── run-char.sh
│ │ │ ├── run-noemb-bseg-basic-v2.sh
│ │ │ ├── run-noemb-bseg-basic.sh
│ │ │ ├── run-noemb-v2.sh
│ │ │ ├── run-noemb-v3.sh
│ │ │ ├── run-noemb.sh
│ │ │ ├── run.sh
│ │ │ └── text2ids.py
│ │ ├── read-records.py
│ │ ├── scripts/
│ │ │ ├── cp.py
│ │ │ ├── example.sh
│ │ │ ├── gen-infer.py
│ │ │ ├── infer-example.sh
│ │ │ ├── infer.py
│ │ │ ├── infer.sh_0
│ │ │ ├── infer.sh_1
│ │ │ ├── infer.sh_2
│ │ │ ├── infer.sh_3
│ │ │ ├── prepare-char.sh
│ │ │ ├── prepare-noemb.sh
│ │ │ └── prepare.sh
│ │ ├── tools/
│ │ │ ├── check-emb.py
│ │ │ ├── cp-best-epochs.py
│ │ │ ├── cp-best-epochs2.py
│ │ │ ├── evaluate.py
│ │ │ ├── find-best-epoch.py
│ │ │ ├── fix-emb.py
│ │ │ ├── lcp-loop.py
│ │ │ ├── lcp.py
│ │ │ ├── rename-finetune.sh
│ │ │ ├── rename-variables-finetune.py
│ │ │ ├── seg2corpus.py
│ │ │ ├── show-best-epochs.py
│ │ │ ├── show-metrics.py
│ │ │ └── slim.py
│ │ ├── torch-infer.py
│ │ ├── torch-lm-train.py
│ │ ├── torch-sim.py
│ │ ├── torch-train.py
│ │ ├── torch_algos/
│ │ │ ├── __init__.py
│ │ │ ├── loss.py
│ │ │ └── model.py
│ │ ├── train/
│ │ │ ├── README.md
│ │ │ ├── v1/
│ │ │ │ ├── bow.sh
│ │ │ │ ├── gru.char.bucket.sh
│ │ │ │ ├── gru.char.pos.ner.stanford.sh
│ │ │ │ ├── gru.char.pos.sh
│ │ │ │ ├── gru.char.pos.stanford.2.sh
│ │ │ │ ├── gru.char.pos.stanford.sh
│ │ │ │ ├── gru.char.sh
│ │ │ │ ├── gru.char.stanford.sh
│ │ │ │ ├── gru.emb.sh
│ │ │ │ └── gru.sh
│ │ │ ├── v10/
│ │ │ │ ├── README.md
│ │ │ │ ├── tf.char.bow.sh
│ │ │ │ ├── tf.char.bow.unkaug.sh
│ │ │ │ ├── tf.char.rnet.hidden300.sh
│ │ │ │ ├── tf.char.rnet.hidden400.sh
│ │ │ │ ├── tf.char.rnet.sh
│ │ │ │ ├── tf.char.transformer.2.sh
│ │ │ │ ├── tf.char.transformer.bert.cutfront.sh
│ │ │ │ ├── tf.char.transformer.bert.finetune.sh
│ │ │ │ ├── tf.char.transformer.bert.rnn.finetune.sh
│ │ │ │ ├── tf.char.transformer.bert.rnn.sh
│ │ │ │ ├── tf.char.transformer.bert.sh
│ │ │ │ ├── tf.char.transformer.sh
│ │ │ │ ├── tf.mix.rnet.hidden400.sh
│ │ │ │ ├── tf.word.bow.sh
│ │ │ │ ├── tf.word.lm.rnet.rand.sh
│ │ │ │ ├── tf.word.lm.rnet.sh
│ │ │ │ ├── tf.word.mreader.hidden300.sh
│ │ │ │ ├── tf.word.mreader.hidden400.sh
│ │ │ │ ├── tf.word.mreader.sh
│ │ │ │ ├── tf.word.rnet.3layer.sh
│ │ │ │ ├── tf.word.rnet.hidden300.3layer.sh
│ │ │ │ ├── tf.word.rnet.hidden300.sh
│ │ │ │ ├── tf.word.rnet.hidden400.decay6.sh
│ │ │ │ ├── tf.word.rnet.hidden400.lm.2.sh
│ │ │ │ ├── tf.word.rnet.hidden400.lm.sh
│ │ │ │ ├── tf.word.rnet.hidden400.lm.unkaug.sh
│ │ │ │ ├── tf.word.rnet.hidden400.sh
│ │ │ │ ├── tf.word.rnet.hidden600.sh
│ │ │ │ ├── tf.word.rnet.ner.sh
│ │ │ │ ├── tf.word.rnet.nolatt.hidden400.sh
│ │ │ │ ├── tf.word.rnet.nolatt.hidden600.sh
│ │ │ │ ├── tf.word.rnet.nolatt.sh
│ │ │ │ ├── tf.word.rnet.sh
│ │ │ │ ├── tf.word.rnet.unkaug.sh
│ │ │ │ ├── tf.word.rnetv2.hidden400.sh
│ │ │ │ ├── torch.char.mreader.nopad.bertopt.sh
│ │ │ │ ├── torch.char.mreader.nopad.hidden300.sh
│ │ │ │ ├── torch.char.mreader.nopad.hidden600.sh
│ │ │ │ ├── torch.char.mreader.nopad.lm.sh
│ │ │ │ ├── torch.char.mreader.nopad.sh
│ │ │ │ ├── torch.word.lm.hidden300.sh
│ │ │ │ ├── torch.word.lm.hidden400.b24.sh
│ │ │ │ ├── torch.word.lm.hidden400.sh
│ │ │ │ ├── torch.word.lm.hidden600.sh
│ │ │ │ ├── torch.word.lm.hidden768.sh
│ │ │ │ ├── torch.word.mreader.lm.bertopt.sh
│ │ │ │ ├── torch.word.mreader.lm.sh
│ │ │ │ ├── torch.word.mreader.nopad.bertopt.nolatt.unkaug.hidden400.sh
│ │ │ │ ├── torch.word.mreader.nopad.bertopt.nolatt.unkaug.hidden600.sh
│ │ │ │ ├── torch.word.mreader.nopad.bertopt.nolatt.unkaug.sh
│ │ │ │ ├── torch.word.mreader.nopad.hidden300.sh
│ │ │ │ ├── torch.word.mreader.nopad.hidden600.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.baike.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.baike.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden300.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden768.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.ner.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.nolatt.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.unkaug.sh
│ │ │ │ ├── torch.word.mreader.nopad.ner.sh
│ │ │ │ └── torch.word.mreader.nopad.sh
│ │ │ ├── v11/
│ │ │ │ ├── README.md
│ │ │ │ ├── tf.char.rnet.nolatt.sh
│ │ │ │ ├── tf.char.rnet.sh
│ │ │ │ ├── tf.char.transformer.bert.cutfront.finetune.2.sh
│ │ │ │ ├── tf.char.transformer.bert.cutfront.finetune.3.sh
│ │ │ │ ├── tf.char.transformer.bert.cutfront.finetune.sh
│ │ │ │ ├── tf.char.transformer.bert.cutfront.sh
│ │ │ │ ├── tf.char.transformer.bert.finetune.2.sh
│ │ │ │ ├── tf.char.transformer.bert.finetune.3.sh
│ │ │ │ ├── tf.char.transformer.bert.finetune.4.sh
│ │ │ │ ├── tf.char.transformer.bert.finetune.sh
│ │ │ │ ├── tf.char.transformer.bert.finetune4.sh
│ │ │ │ ├── tf.char.transformer.bert.rnn.finetune.sh
│ │ │ │ ├── tf.char.transformer.bert.sh
│ │ │ │ ├── tf.char.transformer.bert.topkatt.cutfront.sh
│ │ │ │ ├── tf.char.transformer.bert.topkatt.finetune.sh
│ │ │ │ ├── tf.char.transformer.bert.topkatt.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.2.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.topkatt.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune2.sh
│ │ │ │ ├── tf.char.transformer.nbert.sh
│ │ │ │ ├── tf.char.transformer.nbert.topkatt.sh
│ │ │ │ ├── tf.mix.rnet.nolatt.sh
│ │ │ │ ├── tf.mix.rnet.sh
│ │ │ │ ├── tf.word.mreader.lm.2layer.sh
│ │ │ │ ├── tf.word.mreader.lm.2layer.unkaug.sh
│ │ │ │ ├── tf.word.mreader.lm.sh
│ │ │ │ ├── tf.word.mreader.lm.unkaug.sh
│ │ │ │ ├── tf.word.rnet.hidden600.sh
│ │ │ │ ├── tf.word.rnet.lenemb.sh
│ │ │ │ ├── tf.word.rnet.lm.sh
│ │ │ │ ├── tf.word.rnet.lm.unkaug.sh
│ │ │ │ ├── tf.word.rnet.nolatt.hidden600.sh
│ │ │ │ ├── tf.word.rnet.nolatt.sh
│ │ │ │ ├── tf.word.rnet.sh
│ │ │ │ ├── tf.word.rnet.unkaug.lenemb.sh
│ │ │ │ ├── tf.word.rnet.unkaug.sh
│ │ │ │ ├── tf.word.rnetv2.lm.sh
│ │ │ │ ├── tf.word.rnetv2.lm.unkaug.sh
│ │ │ │ ├── tf.word.rnetv2.nolatt.unkaug.sh
│ │ │ │ ├── tf.word.rnetv2.unkaug.sh
│ │ │ │ ├── torch.char.mreader.lm.nolatt.sh
│ │ │ │ ├── torch.char.mreader.lm.sh
│ │ │ │ ├── torch.char.mreader.nolatt.sh
│ │ │ │ ├── torch.char.mreader.sh
│ │ │ │ ├── torch.mix.mreader.lm.nolatt.sh
│ │ │ │ ├── torch.mix.mreader.lm.sh
│ │ │ │ ├── torch.mix.mreader.sh
│ │ │ │ ├── torch.word.mreader.lm.sh
│ │ │ │ ├── torch.word.mreader.nopad.bertopt.nolatt.unkaug.hidden400.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.baike.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden300.es.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden300.ls.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden300.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.b24.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.es.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.ls.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.pl.b24.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.pl.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden400.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.hidden768.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.nolatt.unkaug.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.unkaug.hidden300.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.unkaug.hidden400.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.unkaug.lenemb.hidden400.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.bertopt.unkaug.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.hidden300.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.hidden400.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.hidden768.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.nolatt.unkaug.sh
│ │ │ │ ├── torch.word.mreader.nopad.lm.sh
│ │ │ │ └── torch.word.mreader.nopad.lm.unkaug.sh
│ │ │ ├── v12/
│ │ │ │ ├── tf.rnet.max_pooling.sh
│ │ │ │ ├── tf.rnet.max_pooling.unkaug.sh
│ │ │ │ ├── tf.rnet.sh
│ │ │ │ ├── tf.rnet.unkaug.sh
│ │ │ │ ├── tf.self_attention.max_pooling.sh
│ │ │ │ ├── tf.self_attention.max_pooling.unkaug.sh
│ │ │ │ ├── tf.self_attention.sh
│ │ │ │ ├── tf.self_attention.unkaug.sh
│ │ │ │ ├── tf.word.baseline.hidden100.sh
│ │ │ │ ├── tf.word.baseline.hidden200.sh
│ │ │ │ ├── tf.word.baseline.hidden400.attention_pooling.sh
│ │ │ │ ├── tf.word.baseline.hidden400.elmo.sh
│ │ │ │ ├── tf.word.baseline.hidden400.max_pooling.elmo.sh
│ │ │ │ ├── tf.word.baseline.hidden400.max_pooling.sh
│ │ │ │ ├── tf.word.baseline.hidden400.max_pooling.unkaug.sh
│ │ │ │ ├── tf.word.baseline.hidden400.sh
│ │ │ │ ├── tf.word.baseline.hidden400.unkaug.sh
│ │ │ │ ├── tf.word.baseline.sh
│ │ │ │ ├── tf.word.rnet.sh
│ │ │ │ ├── tf.word.rnet.unkaug.sh
│ │ │ │ ├── torch.mreader.elmo.sh
│ │ │ │ ├── torch.mreader.sh
│ │ │ │ ├── torch.mreader.unkaug.elmo.sh
│ │ │ │ ├── torch.mreader.unkaug.sh
│ │ │ │ ├── torch.self_attention.elmo.sh
│ │ │ │ ├── torch.self_attention.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.sh
│ │ │ │ ├── torch.self_attention.unkaug.sh
│ │ │ │ ├── torch.word.baseline.hidden100.nochar.sh
│ │ │ │ ├── torch.word.baseline.hidden100.sh
│ │ │ │ ├── torch.word.baseline.hidden200.sh
│ │ │ │ ├── torch.word.baseline.hidden400.attention_pooling.sh
│ │ │ │ ├── torch.word.baseline.hidden400.max_pooling.sh
│ │ │ │ ├── torch.word.baseline.hidden400.sh
│ │ │ │ ├── torch.word.baseline.hidden400.unkaug.elmo.sh
│ │ │ │ └── torch.word.baseline.hidden400.unkaug.sh
│ │ │ ├── v13/
│ │ │ │ ├── torch.self_attention.elmo.sh
│ │ │ │ ├── torch.self_attention.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.valid_decay.sh
│ │ │ │ ├── torch.self_attention.unkaug.sh
│ │ │ │ ├── torch.word.baseline.hidden400.unkaug.elmo.hack.sh
│ │ │ │ ├── torch.word.baseline.hidden400.unkaug.elmo.sh
│ │ │ │ └── torch.word.baseline.hidden400.unkaug.sh
│ │ │ ├── v14/
│ │ │ │ ├── test.sh
│ │ │ │ ├── test2.sh
│ │ │ │ ├── test3.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.1gpu.2.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.1gpu.3.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.1gpu.3epoch.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.1gpu.4.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.1gpu.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.2gpu.3epoch.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.2gpu.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.3epoch.lr2.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.3epoch.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.2.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.all_finetune.lr2.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.all_finetune.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.drop03.all_finetune.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.drop03.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr10.all.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr2.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr5.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr8.1w.ratio1.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr8.1w.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr8.1w.topkatt.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr8.all.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.lr8.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.no_finetune.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.ratio1.lr.2.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.ratio1.lr.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.ratio1.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.3epoch.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.4epoch.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.5epoch.3layer.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.5epoch.all_finetune.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.5epoch.lr5.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.5epoch.lr8.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.4gpu.5epoch.sh
│ │ │ │ ├── tf.char.transformer.nbert.finetune.sh
│ │ │ │ ├── torch.self_attention.elmo.sh
│ │ │ │ ├── torch.self_attention.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.all_finetune.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_100k.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_10k.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_20k.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_50k.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_6k.batch32.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_6k.batch8.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_6k.exclusive_pooling.exclusive_fc.onlyatt.pad.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_6k.exclusive_pooling.exclusive_fc.onlyatt.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_6k.exclusive_pooling.exclusive_fc.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_6k.exclusive_pooling.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.finetune_6k.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.no_finetune.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.no_finetune_word.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.no_finetune_word2.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.nochar.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.sh
│ │ │ │ ├── torch.self_attention.unkaug.elmo.valid_decay.sh
│ │ │ │ ├── torch.self_attention.unkaug.sh
│ │ │ │ ├── torch.word.baseline.hidden100.nobuckets.sh
│ │ │ │ ├── torch.word.baseline.hidden100.sh
│ │ │ │ ├── torch.word.baseline.hidden400.unkaug.elmo.hack.sh
│ │ │ │ ├── torch.word.baseline.hidden400.unkaug.elmo.sh
│ │ │ │ └── torch.word.baseline.hidden400.unkaug.sh
│ │ │ ├── v15/
│ │ │ │ ├── bert.sh
│ │ │ │ └── bert2.sh
│ │ │ ├── v2/
│ │ │ │ ├── rnet.3layer.sh
│ │ │ │ ├── rnet.sh
│ │ │ │ ├── rnet.wchar.3layer.sh
│ │ │ │ ├── rnet.wchar.pos.3layer.sh
│ │ │ │ ├── rnet.wchar.pos.ner.sh
│ │ │ │ ├── rnet.wchar.pos.sh
│ │ │ │ ├── rnet.wchar.sh
│ │ │ │ ├── torch.mreader.1hop.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.pos.ner.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.pos.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.sh
│ │ │ │ └── torch.mreader.2hop.wchar.pos.labelrnn.topkattlast.sh
│ │ │ ├── v3/
│ │ │ │ ├── rnet.3layer.sh
│ │ │ │ ├── rnet.sh
│ │ │ │ ├── rnet.wchar.3layer.sh
│ │ │ │ ├── rnet.wchar.pos.3layer.sh
│ │ │ │ ├── rnet.wchar.pos.ner.sh
│ │ │ │ ├── rnet.wchar.pos.sh
│ │ │ │ ├── rnet.wchar.sh
│ │ │ │ ├── torch.mreader.1hop.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.pos.ner.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.pos.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.sh
│ │ │ │ └── torch.mreader.2hop.wchar.pos.labelrnn.topkattlast.sh
│ │ │ ├── v4/
│ │ │ │ ├── rnet.3layer.sh
│ │ │ │ ├── rnet.sh
│ │ │ │ ├── rnet.wchar.3layer.sh
│ │ │ │ ├── rnet.wchar.pos.3layer.sh
│ │ │ │ ├── rnet.wchar.pos.ner.sh
│ │ │ │ ├── rnet.wchar.pos.sh
│ │ │ │ ├── rnet.wchar.sh
│ │ │ │ ├── torch.mreader.1hop.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.pos.ner.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.pos.sh
│ │ │ │ ├── torch.mreader.1hop.wchar.sh
│ │ │ │ └── torch.mreader.2hop.wchar.pos.labelrnn.topkattlast.sh
│ │ │ ├── v5/
│ │ │ │ ├── README.md
│ │ │ │ ├── rnet.char.3layer.sh
│ │ │ │ ├── rnet.char.noconcat.residual.sh
│ │ │ │ ├── rnet.char.noconcat.sh
│ │ │ │ ├── rnet.char.residual.sh
│ │ │ │ ├── rnet.char.sh
│ │ │ │ ├── rnet.mix.3layer.sh
│ │ │ │ ├── rnet.mix.sh
│ │ │ │ ├── rnet.word.3layer.sh
│ │ │ │ ├── rnet.word.pos.3layer.sh
│ │ │ │ ├── rnet.word.pos.ner.sh
│ │ │ │ ├── rnet.word.pos.sh
│ │ │ │ ├── rnet.word.sh
│ │ │ │ ├── torch.mreader.1hop.char.sh
│ │ │ │ ├── torch.mreader.1hop.mix.sh
│ │ │ │ ├── torch.mreader.1hop.word.ner.fast.sh
│ │ │ │ ├── torch.mreader.1hop.word.ner.sh
│ │ │ │ ├── torch.mreader.1hop.word.pos.ner.sh
│ │ │ │ ├── torch.mreader.1hop.word.pos.sh
│ │ │ │ ├── torch.mreader.1hop.word.sh
│ │ │ │ ├── torch.mreader.1hop.word.subner.sh
│ │ │ │ └── torch.mreader.2hop.word.pos.labelrnn.topkattlast.sh
│ │ │ ├── v6/
│ │ │ │ ├── README.md
│ │ │ │ ├── rnet.char.3layer.sh
│ │ │ │ ├── rnet.char.residual.sh
│ │ │ │ ├── rnet.char.sh
│ │ │ │ ├── rnet.mix.3layer.sh
│ │ │ │ ├── rnet.mix.adafactor.sh
│ │ │ │ ├── rnet.mix.multistep.sh
│ │ │ │ ├── rnet.mix.natt.sh
│ │ │ │ ├── rnet.mix.sh
│ │ │ │ ├── rnet.mix.v1.sh
│ │ │ │ ├── rnet.mix.yellowfin.sh
│ │ │ │ ├── rnet.word.3layer.sh
│ │ │ │ ├── rnet.word.adamax.sh
│ │ │ │ ├── rnet.word.pos.3layer.sh
│ │ │ │ ├── rnet.word.pos.ner.sh
│ │ │ │ ├── rnet.word.pos.sh
│ │ │ │ ├── rnet.word.sh
│ │ │ │ ├── rnetv2.mix.natt.sh
│ │ │ │ ├── torch.mreader.1hop.char.sh
│ │ │ │ ├── torch.mreader.1hop.mix.sh
│ │ │ │ ├── torch.mreader.1hop.mix.v1.sh
│ │ │ │ ├── torch.mreader.1hop.mix.v2.sh
│ │ │ │ ├── torch.mreader.1hop.mix.v3.sh
│ │ │ │ ├── torch.mreader.1hop.word.latt.sh
│ │ │ │ ├── torch.mreader.1hop.word.ner.latt.sh
│ │ │ │ ├── torch.mreader.1hop.word.ner.sh
│ │ │ │ ├── torch.mreader.1hop.word.pos.ner.sh
│ │ │ │ ├── torch.mreader.1hop.word.pos.sh
│ │ │ │ ├── torch.mreader.1hop.word.sh
│ │ │ │ ├── torch.mreader.1hop.word.subner.sh
│ │ │ │ ├── torch.mreader.1hop.word.v2.sh
│ │ │ │ ├── torch.mreader.2hop.word.pos.labelrnn.topkattlast.sh
│ │ │ │ ├── torch.rnet.word.gate.sh
│ │ │ │ ├── torch.rnet.word.ner.sh
│ │ │ │ ├── torch.rnet.word.ner.v2.sh
│ │ │ │ ├── torch.rnet.word.ner.v3.sh
│ │ │ │ ├── torch.rnet.word.ner.v4.sh
│ │ │ │ ├── torch.rnet.word.ner.v5.sh
│ │ │ │ ├── torch.rnet.word.sh
│ │ │ │ ├── torch.rnet.word.v2.sh
│ │ │ │ └── torch.rnet.word.v4.sh
│ │ │ ├── v7/
│ │ │ │ ├── torch.mreader.1hop.word.latt.pad.sh
│ │ │ │ ├── torch.mreader.1hop.word.latt.sh
│ │ │ │ ├── torch.mreader.1hop.word.sh
│ │ │ │ ├── torch.mreader.1hop.word.v2.sh
│ │ │ │ ├── torch.mreader.1hop.word.v3.sh
│ │ │ │ └── torch.rnet.word.v2.sh
│ │ │ ├── v8/
│ │ │ │ ├── torch.lm.mreader.word.nopad.rand.sh
│ │ │ │ ├── torch.lm.mreader.word.nopad.rand2.sh
│ │ │ │ ├── torch.lm.mreader.word.nopad.sh
│ │ │ │ ├── torch.lm.mreader.word.nopad2.sh
│ │ │ │ ├── torch.lm.mreader.word.sh
│ │ │ │ ├── torch.mreader.word.aug.sh
│ │ │ │ ├── torch.mreader.word.chconv.sh
│ │ │ │ ├── torch.mreader.word.chpad.sh
│ │ │ │ ├── torch.mreader.word.dynamic.curnn.sh
│ │ │ │ ├── torch.mreader.word.dynamic.latt.nopad.sh
│ │ │ │ ├── torch.mreader.word.dynamic.latt.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.2layer.lm.rand.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.2layer.lm.rand2.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.2layer.lm.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.2layer.lm2.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.2layer.lm3.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.lm.rand.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.lm.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.lm2.sh
│ │ │ │ ├── torch.mreader.word.dynamic.nopad.sh
│ │ │ │ ├── torch.mreader.word.dynamic.sh
│ │ │ │ ├── torch.mreader.word.dynamic.transformer.sh
│ │ │ │ ├── torch.mreader.word.elmo.sh
│ │ │ │ ├── torch.mreader.word.finetune.sh
│ │ │ │ ├── torch.mreader.word.finetune2.sh
│ │ │ │ ├── torch.mreader.word.noam.sh
│ │ │ │ ├── torch.mreader.word.noam2.sh
│ │ │ │ ├── torch.mreader.word.rand.sh
│ │ │ │ └── torch.mreader.word.sh
│ │ │ └── v9/
│ │ │ ├── README.md
│ │ │ ├── tf.char.transformer.5epoch.sh
│ │ │ ├── tf.char.transformer.bert.sh
│ │ │ ├── tf.char.transformer.lm.10epoch.sh
│ │ │ ├── tf.char.transformer.lm.2.sh
│ │ │ ├── tf.char.transformer.lm.5epoch.finetune.sh
│ │ │ ├── tf.char.transformer.lm.5epoch.sh
│ │ │ ├── tf.char.transformer.lm.sh
│ │ │ ├── tf.char.transformer.rand.sh
│ │ │ ├── tf.char.transformer.sh
│ │ │ ├── tf.word.lm.2layer.sh
│ │ │ ├── tf.word.lm.rand.sh
│ │ │ ├── tf.word.lm.rnet.sh
│ │ │ ├── tf.word.lm.sh
│ │ │ ├── tf.word.mreader.sfu.sh
│ │ │ ├── tf.word.mreader.sh
│ │ │ ├── tf.word.rnet.attnodrop.sh
│ │ │ ├── tf.word.rnet.bertopt.10epoch.2.sh
│ │ │ ├── tf.word.rnet.bertopt.10epoch.sh
│ │ │ ├── tf.word.rnet.bertopt.15epoch.d5.sh
│ │ │ ├── tf.word.rnet.bertopt.5epoch.finetune.sh
│ │ │ ├── tf.word.rnet.bertopt.5epoch.sh
│ │ │ ├── tf.word.rnet.bertopt.sh
│ │ │ ├── tf.word.rnet.lm.sh
│ │ │ ├── tf.word.rnet.rand.sh
│ │ │ ├── tf.word.rnet.sh
│ │ │ ├── tf.word.transformer.5epoch.sh
│ │ │ ├── tf.word.transformer.sh
│ │ │ ├── torch.char.lm.lstm.nopad.rand.sh
│ │ │ ├── torch.char.lm.lstm.nopad.sh
│ │ │ ├── torch.char.lm.lstm.sh
│ │ │ ├── torch.char.lm.nopad.addbaike.sh
│ │ │ ├── torch.char.lm.nopad.rand.sh
│ │ │ ├── torch.char.lm.nopad.sh
│ │ │ ├── torch.char.lm.sh
│ │ │ ├── torch.char.mreader.2layer.nopad.sh
│ │ │ ├── torch.char.mreader.nopad.lm.sh
│ │ │ ├── torch.char.mreader.nopad.pos.sh
│ │ │ ├── torch.char.mreader.nopad.sh
│ │ │ ├── torch.word.lm.2layer.nopad.sh
│ │ │ ├── torch.word.lm.2layer.sh
│ │ │ ├── torch.word.lm.lstm.nopad.rand.sh
│ │ │ ├── torch.word.lm.lstm.nopad.sh
│ │ │ ├── torch.word.lm.lstm.sh
│ │ │ ├── torch.word.lm.nopad.addbaike.sh
│ │ │ ├── torch.word.lm.nopad.rand.sh
│ │ │ ├── torch.word.lm.nopad.sh
│ │ │ ├── torch.word.lm.sh
│ │ │ ├── torch.word.lstm.mreader.2layer.nopad.latt6.lm.sh
│ │ │ ├── torch.word.lstm.mreader.2layer.nopad.latt80.lm.sh
│ │ │ ├── torch.word.lstm.mreader.2layer.nopad.lm.sh
│ │ │ ├── torch.word.lstm.mreader.2layer.nopad.sh
│ │ │ ├── torch.word.mreader.2layer.lm.bertopt.sh
│ │ │ ├── torch.word.mreader.2layer.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nolatt.lm.bertopt.sh
│ │ │ ├── torch.word.mreader.2layer.nolatt.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.cw.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.labelrnn.2hop.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.labelrnn.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.latt6.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.latt80.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.2.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.3.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.4.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.5.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.beropt.d5.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.beropt.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.rand.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.lm.v2.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.nolatt.lm.bertopt.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.nolatt.lm.sh
│ │ │ ├── torch.word.mreader.2layer.nopad.sh
│ │ │ ├── torch.word.mreader.3layer.nopad.lm.sh
│ │ │ ├── torch.word.mreader.3layer.nopad.nolatt.lm.sh
│ │ │ ├── torch.word.mreader.nopad.cw.sh
│ │ │ ├── torch.word.mreader.nopad.lm.sh
│ │ │ ├── torch.word.mreader.nopad.sh
│ │ │ ├── torch.word.mreader.sh
│ │ │ ├── torch.word.rnet.2layer.nopad.lm.sh
│ │ │ ├── torch.word.rnet.nopad.lm.sh
│ │ │ ├── torch.word.rnet.nopad.sh
│ │ │ ├── torch.word.rnet.sh
│ │ │ ├── torch.word.rnn.lm.sh
│ │ │ ├── torch.word.rnn.nopad.lm.sh
│ │ │ ├── torch.word.rnn.nopad.rand.sh
│ │ │ ├── torch.word.rnn.nopad.sh
│ │ │ └── torch.word.rnn.sh
│ │ ├── train.py
│ │ ├── train.v1/
│ │ │ ├── README.md
│ │ │ ├── v1/
│ │ │ │ ├── bow.emb.fix.sh
│ │ │ │ ├── bow.emb.sh
│ │ │ │ ├── bow.emb.warm.sh
│ │ │ │ ├── bow.labelatt.emb.sh
│ │ │ │ ├── bow.labelatt.sh
│ │ │ │ ├── bow.labelatt.simple.sh
│ │ │ │ ├── bow.mdecay.sh
│ │ │ │ ├── bow.sh
│ │ │ │ ├── bow.topk-3att.labelatt.emb.sh
│ │ │ │ ├── convnet.topk-3att.labelatt.emb.sh
│ │ │ │ ├── crnn.topk-3att.labelatt.emb.sh
│ │ │ │ ├── gru.2layer.emb.sh
│ │ │ │ ├── gru.2layer.sh
│ │ │ │ ├── gru.att.sh
│ │ │ │ ├── gru.att2.sh
│ │ │ │ ├── gru.emb.finetune.later.sh
│ │ │ │ ├── gru.emb.finetune.sh
│ │ │ │ ├── gru.emb.fix.sh
│ │ │ │ ├── gru.emb.sh
│ │ │ │ ├── gru.emb.warm.sh
│ │ │ │ ├── gru.f05.sh
│ │ │ │ ├── gru.f07.sh
│ │ │ │ ├── gru.f09.sh
│ │ │ │ ├── gru.focal.sh
│ │ │ │ ├── gru.focal2.sh
│ │ │ │ ├── gru.hidden200.sh
│ │ │ │ ├── gru.label.sh
│ │ │ │ ├── gru.maxatt.2.sh
│ │ │ │ ├── gru.maxatt.sh
│ │ │ │ ├── gru.maxmean.sh
│ │ │ │ ├── gru.mdecay.f09.sh
│ │ │ │ ├── gru.mdecay.p2.sh
│ │ │ │ ├── gru.mdecay.sh
│ │ │ │ ├── gru.mdecay2.f07.sh
│ │ │ │ ├── gru.mdecay2.f09.sh
│ │ │ │ ├── gru.mdecay2.sh
│ │ │ │ ├── gru.mean.sh
│ │ │ │ ├── gru.naonly.sh
│ │ │ │ ├── gru.naratio01.sh
│ │ │ │ ├── gru.naratio1.sh
│ │ │ │ ├── gru.nodecay.sh
│ │ │ │ ├── gru.p2.sh
│ │ │ │ ├── gru.sh
│ │ │ │ ├── gru.topk-3.2layer.sh
│ │ │ │ ├── gru.topk-3.cosine.10epoch.decay05.sh
│ │ │ │ ├── gru.topk-3.cosine.10epoch.sh
│ │ │ │ ├── gru.topk-3.cosine.5epoch.decay01.sh
│ │ │ │ ├── gru.topk-3.cosine.5epoch.decay04.sh
│ │ │ │ ├── gru.topk-3.cosine.5epoch.decay08.sh
│ │ │ │ ├── gru.topk-3.cosine.5epoch.sh
│ │ │ │ ├── gru.topk-3.cosine.decay05.sh
│ │ │ │ ├── gru.topk-3.cosine.decay08.sh
│ │ │ │ ├── gru.topk-3.cosine.sh
│ │ │ │ ├── gru.topk-3.earth.na.sh
│ │ │ │ ├── gru.topk-3.earth.sh
│ │ │ │ ├── gru.topk-3.hier.sh
│ │ │ │ ├── gru.topk-3.label.sh
│ │ │ │ ├── gru.topk-3.labelatt.emb.sh
│ │ │ │ ├── gru.topk-3.labelatt.sh
│ │ │ │ ├── gru.topk-3.mlp.2.sh
│ │ │ │ ├── gru.topk-3.mlp.sh
│ │ │ │ ├── gru.topk-3.mlp05.sh
│ │ │ │ ├── gru.topk-3.mlp1.sh
│ │ │ │ ├── gru.topk-3.na.sh
│ │ │ │ ├── gru.topk-3.sh
│ │ │ │ ├── gru.topk-3att.2layer.emb.sh
│ │ │ │ ├── gru.topk-3att.2layer.hier.sh
│ │ │ │ ├── gru.topk-3att.2layer.label.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.10w.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.1w.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.2.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.mdecay.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.naonly.finetune.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.naonly.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.selfmatch.2.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.selfmatch.naratio01.finetune.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.selfmatch.naratio01.finetune2.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.selfmatch.naratio01.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.selfmatch.naratio05.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.selfmatch.naratio1.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.selfmatch.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.height20.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.mdecay.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.sfu.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.emb.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.finetune.d05.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.finetune.sh
│ │ │ │ ├── gru.topk-3att.2layer.labelatt.sh
│ │ │ │ ├── gru.topk-3att.2layer.sh
│ │ │ │ ├── gru.topk-3att.sh
│ │ │ │ ├── gru.topk-4.sh
│ │ │ │ ├── gru.topk-5.sh
│ │ │ │ ├── gru.topk-6.sh
│ │ │ │ ├── gru.topk.sh
│ │ │ │ ├── gru.weights.0.naratio01.sh
│ │ │ │ ├── gru.weights.0.sh
│ │ │ │ ├── gru.weights.location.naratio01.sh
│ │ │ │ ├── gru.weights.location.sh
│ │ │ │ ├── gru.weights.service.sh
│ │ │ │ ├── gru.weights.service.topk.sh
│ │ │ │ ├── gru.weights.service0.sh
│ │ │ │ ├── gru.weights.service0.topk.sh
│ │ │ │ ├── qanet.topk-3att.labelatt.emb.sh
│ │ │ │ └── rcnn.topk-3att.labelatt.emb.sh
│ │ │ ├── v2/
│ │ │ │ ├── bow.sh
│ │ │ │ ├── gru.1w.2layer.topk3att.labelatt20.selfmatch.hidden200.sh
│ │ │ │ ├── gru.1w.2layer.topk3att.labelatt20.selfmatch.sh
│ │ │ │ ├── gru.1w.2layer.topk3att.labelatt80.selfmatch.hidden200.sh
│ │ │ │ ├── gru.1w.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.hidden200.sh
│ │ │ │ ├── gru.1w.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.sh
│ │ │ │ ├── gru.1w.wordpretrain_glove_dianping.2layer.topk3att.labelatt80.selfmatch.hidden200.sh
│ │ │ │ ├── gru.5k.2layer.topk3att.labelatt20.selfmatch.hidden200.sh
│ │ │ │ ├── gru.5k.2layer.topk3att.labelatt20.selfmatch.sh
│ │ │ │ ├── gru.5k.2layer.topk3att.labelatt80.selfmatch.hidden200.sh
│ │ │ │ ├── gru.5k.aspect.dish.sh
│ │ │ │ ├── gru.5k.aspect.environment.sh
│ │ │ │ ├── gru.5k.aspect.location.sh
│ │ │ │ ├── gru.5k.aspect.others.sh
│ │ │ │ ├── gru.5k.aspect.price.sh
│ │ │ │ ├── gru.5k.aspect.service.sh
│ │ │ │ ├── gru.5k.keep06.sh
│ │ │ │ ├── gru.5k.keep08.sh
│ │ │ │ ├── gru.5k.keep09.sh
│ │ │ │ ├── gru.5k.labelemb.sh
│ │ │ │ ├── gru.5k.labelrnn.sh
│ │ │ │ ├── gru.5k.sfu.sh
│ │ │ │ ├── gru.5k.sh
│ │ │ │ ├── gru.5k.tanh.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt10.selfmatch.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.hidden200.sfu.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.hidden200.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.hidden300.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt40.selfmatch.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt80.selfmatch.hidden200.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.2layer.topk3att.labelatt80.selfmatch.sh
│ │ │ │ ├── gru.5k.wordpretrain_glove_dianping.3layer.topk3att.labelatt20.selfmatch.sh
│ │ │ │ ├── gru.char.2layer.topk3att.labelatt20.selfmatch.hidden200.sh
│ │ │ │ ├── gru.char.2layer.topk3att.labelatt20.selfmatch.sh
│ │ │ │ ├── gru.char.2layer.topk3att.labelatt80.selfmatch.hidden200.sh
│ │ │ │ ├── gru.char.aspect.dish.sh
│ │ │ │ ├── gru.char.aspect.environment.sh
│ │ │ │ ├── gru.char.aspect.location.sh
│ │ │ │ ├── gru.char.aspect.others.sh
│ │ │ │ ├── gru.char.aspect.price.sh
│ │ │ │ ├── gru.char.aspect.service.sh
│ │ │ │ ├── gru.char.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.hidden200.sh
│ │ │ │ ├── gru.char.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.hidden300.sh
│ │ │ │ ├── gru.char.wordpretrain_glove_dianping.2layer.topk3att.labelatt20.selfmatch.sh
│ │ │ │ ├── gru.char.wordpretrain_glove_dianping.2layer.topk3att.labelatt80.selfmatch.hidden200.sh
│ │ │ │ ├── gru.char.wordpretrain_glove_dianping.3layer.topk3att.labelatt20.selfmatch.hidden200.sh
│ │ │ │ └── gru.char.wordpretrain_glove_dianping.3layer.topk3att.labelatt20.selfmatch.sh
│ │ │ ├── v3/
│ │ │ │ ├── gru-5k-ftwiki.sh
│ │ │ │ ├── gru-5k-labelrnn.sh
│ │ │ │ ├── gru-5k-nopad.sh
│ │ │ │ ├── gru-baseline-att.sh
│ │ │ │ ├── gru-baseline-ftwiki.sh
│ │ │ │ ├── gru-baseline.sh
│ │ │ │ ├── gru.5k.mreader.2hop.sh
│ │ │ │ ├── gru.5k.mreader.sfu.2hop.sh
│ │ │ │ ├── gru.5k.mreader.sfu.sh
│ │ │ │ ├── gru.5k.mreader.sh
│ │ │ │ ├── gru.5k.sfu.sh
│ │ │ │ ├── gru.5k.sh
│ │ │ │ ├── torch-gru-baseline-att-nopad.sh
│ │ │ │ ├── torch-gru-baseline-att-pad.sh
│ │ │ │ ├── torch-gru-baseline-att-v2.sh
│ │ │ │ ├── torch-gru-baseline-nopad.sh
│ │ │ │ ├── torch-gru-baseline-pad.sh
│ │ │ │ ├── torch-gru-baseline-v2-ftwiki.sh
│ │ │ │ ├── torch-gru-baseline-v2.sh
│ │ │ │ ├── torch-gru-baseline.sh
│ │ │ │ ├── torch-gru-labelatt-nopad.sh
│ │ │ │ ├── torch-gru-labelatt-sfu-nopad.sh
│ │ │ │ ├── torch-gru-rnet-curnn-nopad.sh
│ │ │ │ ├── torch-gru-rnet-curnn-sfu-nopad.sh
│ │ │ │ ├── torch-gru-rnet-curnn-sfu.sh
│ │ │ │ ├── torch-gru-rnet-curnn.sh
│ │ │ │ ├── torch-gru-rnet-nopad.sh
│ │ │ │ ├── torch-gru-rnet-sfu-nopad.sh
│ │ │ │ ├── torch-mreader-1hop.sh
│ │ │ │ ├── torch-mreader-2hop.sh
│ │ │ │ ├── torch-mreader-labelrnn-1hop.sh
│ │ │ │ └── torch-mreader-labelrnn-2hop.sh
│ │ │ └── v4/
│ │ │ ├── gru-5k-canyin-char-addneubinary-finetune.sh
│ │ │ ├── gru-5k-canyin-char-addneubinary-trans.sh
│ │ │ ├── gru-5k-canyin-char-addneubinary.sh
│ │ │ ├── gru-5k-canyin-char.sh
│ │ │ ├── gru-5k-canyin-mix-addbinaries-trans.sh
│ │ │ ├── gru-5k-canyin-mix-addbinaries.sh
│ │ │ ├── gru-5k-canyin-mix-addnabinary-finetune-epoch5.sh
│ │ │ ├── gru-5k-canyin-mix-addnabinary-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-addnabinary.sh
│ │ │ ├── gru-5k-canyin-mix-addnaneubinary-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-addneubinary-finetune-epoch5.sh
│ │ │ ├── gru-5k-canyin-mix-addneubinary-finetune-factor2.sh
│ │ │ ├── gru-5k-canyin-mix-addneubinary-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-addneubinary-trans.sh
│ │ │ ├── gru-5k-canyin-mix-addneubinary.2.sh
│ │ │ ├── gru-5k-canyin-mix-addneubinary.sh
│ │ │ ├── gru-5k-canyin-mix-binariesonly.sh
│ │ │ ├── gru-5k-canyin-mix-cotrain.sh
│ │ │ ├── gru-5k-canyin-mix-deformcanyin.sh
│ │ │ ├── gru-5k-canyin-mix-deformdianping-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-deformdianping.sh
│ │ │ ├── gru-5k-canyin-mix-finetune-dianpingemb.sh
│ │ │ ├── gru-5k-canyin-mix-hier-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-hier-trans.sh
│ │ │ ├── gru-5k-canyin-mix-hier.sh
│ │ │ ├── gru-5k-canyin-mix-hierneu-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-neubinary-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-neubinary-trans.sh
│ │ │ ├── gru-5k-canyin-mix-neubinary.sh
│ │ │ ├── gru-5k-canyin-mix-othersAgainOnly.sh
│ │ │ ├── gru-5k-canyin-mix-othersOnly.sh
│ │ │ ├── gru-5k-canyin-mix-othersOverallOnly.sh
│ │ │ ├── gru-5k-canyin-mix-othersOverallW2.sh
│ │ │ ├── gru-5k-canyin-mix-othersOverallW5.sh
│ │ │ ├── gru-5k-canyin-mix-othersW10.sh
│ │ │ ├── gru-5k-canyin-mix-othersW2.sh
│ │ │ ├── gru-5k-canyin-mix-othersW5.sh
│ │ │ ├── gru-5k-canyin-mix-regression-finetune.sh
│ │ │ ├── gru-5k-canyin-mix-regression.sh
│ │ │ ├── gru-5k-canyin-mix-small.sh
│ │ │ ├── gru-5k-canyin-mix-trans.2.sh
│ │ │ ├── gru-5k-canyin-mix-trans.3.sh
│ │ │ ├── gru-5k-canyin-mix-trans.sh
│ │ │ ├── gru-5k-canyin-mix.sh
│ │ │ ├── gru-5k-canyin.sh
│ │ │ ├── gru-5k-hier-canyin-finetune.sh
│ │ │ ├── gru-5k-hier-canyin.sh
│ │ │ ├── gru-5k-lattonly-canyin-char.sh
│ │ │ ├── gru-5k-maxmean-canyin.sh
│ │ │ ├── gru-5k-rnetv2-canyin-char.sh
│ │ │ ├── gru-5k-rnetv2-canyin.sh
│ │ │ ├── gru-5k-rnetv2-sfu-canyin.sh
│ │ │ ├── gru-5k-rnetv3-canyin-char.sh
│ │ │ ├── gru-5k-rnetv3-canyin.sh
│ │ │ ├── gru-5k-rnetv3-selfmatchonly-canyin-char.sh
│ │ │ ├── gru-5k-rnetv4-canyin.sh
│ │ │ ├── gru-5k-selfmatchonly-canyin-char.sh
│ │ │ ├── gru-5k-topk3attmean-canyin-char.sh
│ │ │ ├── gru-5k-topk3attmean-canyin.sh
│ │ │ ├── gru-5k-topk3mean-canyin.sh
│ │ │ ├── gru.5k.sh
│ │ │ ├── rcnn-k5-canyin-mix.sh
│ │ │ ├── torch-mreader-trans.sh
│ │ │ └── torch-mreader.sh
│ │ └── train.v2/
│ │ ├── README.md
│ │ ├── v1/
│ │ │ ├── bow.emb.binary.sh
│ │ │ ├── bow.emb.sh
│ │ │ ├── bow.ori.emb.sh
│ │ │ ├── bow.ori.sh
│ │ │ ├── bow.sh
│ │ │ ├── bow.simplify.sh
│ │ │ ├── bow.testori.sh
│ │ │ ├── bow.testsimplify.sh
│ │ │ ├── gru.400.sh
│ │ │ ├── gru.char.sfu.sh
│ │ │ ├── gru.char.sh
│ │ │ ├── gru.emb.2.sh
│ │ │ ├── gru.emb.adjust.sh
│ │ │ ├── gru.emb.char.sh
│ │ │ ├── gru.emb.cw.sh
│ │ │ ├── gru.emb.decay.sh
│ │ │ ├── gru.emb.decay2.sh
│ │ │ ├── gru.emb.last.sh
│ │ │ ├── gru.emb.lastmax.sh
│ │ │ ├── gru.emb.lastmaxmean.sh
│ │ │ ├── gru.emb.loss.sh
│ │ │ ├── gru.emb.norec.sh
│ │ │ ├── gru.emb.rec.sh
│ │ │ ├── gru.emb.rec.topkattlast.sh
│ │ │ ├── gru.emb.sh
│ │ │ ├── gru.emb.smooth.sh
│ │ │ ├── gru.emb.trans.sh
│ │ │ ├── gru.lm.2.sh
│ │ │ ├── gru.lm.3.sh
│ │ │ ├── gru.lm.emb.2.sh
│ │ │ ├── gru.lm.emb.sh
│ │ │ ├── gru.lm.lr0002.sh
│ │ │ ├── gru.lm.pretrain.sh
│ │ │ ├── gru.lm.sh
│ │ │ ├── gru.ori.emb.sh
│ │ │ ├── gru.ori.sh
│ │ │ ├── gru.sh
│ │ │ ├── gru.simplify.emb.2.sh
│ │ │ ├── gru.simplify.emb.addneubinary.sh
│ │ │ ├── gru.simplify.emb.finetune.sh
│ │ │ ├── gru.simplify.emb.sh
│ │ │ ├── gru.simplify.sh
│ │ │ ├── gru.testsimplify.emb.finetune.sh
│ │ │ ├── gru.testsimplify.emb.sh
│ │ │ ├── gru.testsimplify.sh
│ │ │ ├── lstm.emb.norec.sh
│ │ │ ├── lstm.emb.rec.2layer.noconcat.sh
│ │ │ ├── lstm.emb.rec.2layer.noconcat.topkattlast.sh
│ │ │ ├── lstm.emb.rec.sh
│ │ │ ├── lstm.emb.rec.topkattlast.sh
│ │ │ ├── lstm.emb.sh
│ │ │ ├── rnet.char.ft.sh
│ │ │ ├── rnet.char.glove.sh
│ │ │ ├── torch.fastai.1layer.sh
│ │ │ ├── torch.fastai.2.sh
│ │ │ ├── torch.fastai.3.sh
│ │ │ ├── torch.fastai.4.sh
│ │ │ ├── torch.fastai.5.sh
│ │ │ ├── torch.fastai.sh
│ │ │ ├── torch.gru.2layer.sh
│ │ │ ├── torch.gru.sh
│ │ │ ├── torch.lstm.1layer.labelatt.2hop.topkattlast.sh
│ │ │ ├── torch.lstm.1layer.labelatt.topkattlast.sh
│ │ │ ├── torch.lstm.2.sh
│ │ │ ├── torch.lstm.2layer.noconcat.labelatt.gate.topkattlast.sh
│ │ │ ├── torch.lstm.2layer.noconcat.labelatt.topkattlast.sh
│ │ │ ├── torch.lstm.2layer.noconcat.sh
│ │ │ ├── torch.lstm.2layer.noconcat.topkattlast.sh
│ │ │ ├── torch.lstm.2layer.sh
│ │ │ ├── torch.lstm.evpadd.sh
│ │ │ ├── torch.lstm.max.2.sh
│ │ │ ├── torch.lstm.max.sh
│ │ │ ├── torch.lstm.padd.sh
│ │ │ ├── torch.lstm.recdrop.sh
│ │ │ ├── torch.lstm.sh
│ │ │ ├── torch.lstmV2.2.sh
│ │ │ ├── torch.lstmV2.bwdrop.sh
│ │ │ ├── torch.lstmV2.norec.2.sh
│ │ │ ├── torch.lstmV2.norec.nopad.sh
│ │ │ ├── torch.lstmV2.norec.sh
│ │ │ ├── torch.lstmV2.sh
│ │ │ ├── torch.rnet.2layer.noconcat.topkattlast.sh
│ │ │ └── torch.rnetV2.2layer.noconcat.topkattlast.sh
│ │ ├── v1.1/
│ │ │ ├── bow.emb.binary.sh
│ │ │ ├── bow.emb.sh
│ │ │ ├── bow.ori.emb.sh
│ │ │ ├── bow.ori.sh
│ │ │ ├── bow.sh
│ │ │ ├── bow.simplify.sh
│ │ │ ├── bow.testori.sh
│ │ │ ├── bow.testsimplify.sh
│ │ │ ├── gru.400.sh
│ │ │ ├── gru.char.sfu.sh
│ │ │ ├── gru.char.sh
│ │ │ ├── gru.emb.2.sh
│ │ │ ├── gru.emb.adjust.sh
│ │ │ ├── gru.emb.char.2.sh
│ │ │ ├── gru.emb.char.3.sh
│ │ │ ├── gru.emb.char.4.sh
│ │ │ ├── gru.emb.char.5.sh
│ │ │ ├── gru.emb.char.sh
│ │ │ ├── gru.emb.cw.sh
│ │ │ ├── gru.emb.decay.sh
│ │ │ ├── gru.emb.decay2.sh
│ │ │ ├── gru.emb.last.sh
│ │ │ ├── gru.emb.lastmax.sh
│ │ │ ├── gru.emb.lastmaxmean.sh
│ │ │ ├── gru.emb.loss.sh
│ │ │ ├── gru.emb.norec.sh
│ │ │ ├── gru.emb.rec.sh
│ │ │ ├── gru.emb.rec.topkattlast.sh
│ │ │ ├── gru.emb.sh
│ │ │ ├── gru.emb.smooth.sh
│ │ │ ├── gru.emb.trans.sh
│ │ │ ├── gru.lm.2.sh
│ │ │ ├── gru.lm.3.sh
│ │ │ ├── gru.lm.emb.2.sh
│ │ │ ├── gru.lm.emb.sh
│ │ │ ├── gru.lm.lr0002.sh
│ │ │ ├── gru.lm.pretrain.sh
│ │ │ ├── gru.lm.sh
│ │ │ ├── gru.ori.emb.sh
│ │ │ ├── gru.ori.sh
│ │ │ ├── gru.sh
│ │ │ ├── gru.simplify.emb.2.sh
│ │ │ ├── gru.simplify.emb.addneubinary.sh
│ │ │ ├── gru.simplify.emb.finetune.sh
│ │ │ ├── gru.simplify.emb.sh
│ │ │ ├── gru.simplify.sh
│ │ │ ├── gru.testsimplify.emb.finetune.sh
│ │ │ ├── gru.testsimplify.emb.sh
│ │ │ ├── gru.testsimplify.sh
│ │ │ ├── lstm.emb.norec.sh
│ │ │ ├── lstm.emb.rec.2layer.noconcat.sh
│ │ │ ├── lstm.emb.rec.2layer.noconcat.topkattlast.sh
│ │ │ ├── lstm.emb.rec.sh
│ │ │ ├── lstm.emb.rec.topkattlast.sh
│ │ │ ├── lstm.emb.sh
│ │ │ ├── rnet.char.ft.sh
│ │ │ ├── rnet.char.glove.sh
│ │ │ ├── torch.fastai.1layer.sh
│ │ │ ├── torch.fastai.2.sh
│ │ │ ├── torch.fastai.3.sh
│ │ │ ├── torch.fastai.4.sh
│ │ │ ├── torch.fastai.5.sh
│ │ │ ├── torch.fastai.sh
│ │ │ ├── torch.gru.2layer.sh
│ │ │ ├── torch.gru.sh
│ │ │ ├── torch.lstm.1layer.labelatt.2hop.topkattlast.sh
│ │ │ ├── torch.lstm.1layer.labelatt.topkattlast.sh
│ │ │ ├── torch.lstm.2.sh
│ │ │ ├── torch.lstm.2layer.noconcat.labelatt.gate.topkattlast.sh
│ │ │ ├── torch.lstm.2layer.noconcat.labelatt.topkattlast.sh
│ │ │ ├── torch.lstm.2layer.noconcat.sh
│ │ │ ├── torch.lstm.2layer.noconcat.topkattlast.sh
│ │ │ ├── torch.lstm.2layer.sh
│ │ │ ├── torch.lstm.evpadd.sh
│ │ │ ├── torch.lstm.max.2.sh
│ │ │ ├── torch.lstm.max.sh
│ │ │ ├── torch.lstm.padd.sh
│ │ │ ├── torch.lstm.recdrop.sh
│ │ │ ├── torch.lstm.sh
│ │ │ ├── torch.lstmV2.2.sh
│ │ │ ├── torch.lstmV2.bwdrop.sh
│ │ │ ├── torch.lstmV2.norec.2.sh
│ │ │ ├── torch.lstmV2.norec.nopad.sh
│ │ │ ├── torch.lstmV2.norec.sh
│ │ │ ├── torch.lstmV2.sh
│ │ │ ├── torch.rnet.2layer.noconcat.topkattlast.sh
│ │ │ └── torch.rnetV2.2layer.noconcat.topkattlast.sh
│ │ ├── v10/
│ │ │ ├── rnet.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ ├── v11/
│ │ │ ├── rnet.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ ├── v12/
│ │ │ ├── rnet.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ ├── v2/
│ │ │ ├── README.md
│ │ │ ├── rnet.3layer.noconcat.sh
│ │ │ ├── rnet.3layer.noconcat.topkattlast.sh
│ │ │ ├── rnet.3layer.sh
│ │ │ ├── rnet.3layer.topkattlast.sh
│ │ │ ├── rnet.noconcat.sh
│ │ │ ├── rnet.noconcat.topkattlast.sh
│ │ │ ├── rnet.sh
│ │ │ ├── rnet.topkattlast.sh
│ │ │ ├── torch.mreader.1hop.2layer.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.sh
│ │ │ ├── torch.mreader.1hop.decay09.sh
│ │ │ ├── torch.mreader.1hop.sh
│ │ │ ├── torch.mreader.1hop.topkattlast.sh
│ │ │ ├── torch.mreader.labelrnn.2hop.2layer.sh
│ │ │ ├── torch.mreader.labelrnn.2hop.2layer.topkattlast.sh
│ │ │ └── torch.mreaderV2.1hop.2layer.sh
│ │ ├── v2.old/
│ │ │ ├── README.md
│ │ │ ├── convnet.sh
│ │ │ ├── qanet.sh
│ │ │ ├── rnet.3layer.fine.lr10.sh
│ │ │ ├── rnet.3layer.fine.lr2.sh
│ │ │ ├── rnet.3layer.fine.lr4.sh
│ │ │ ├── rnet.3layer.fine.sh
│ │ │ ├── rnet.3layer.sh
│ │ │ ├── rnet.addneu.sh
│ │ │ ├── rnet.fine.addneu.sh
│ │ │ ├── rnet.fine.trans.sh
│ │ │ ├── rnet.fine.trans01.sh
│ │ │ ├── rnet.fine.trans05.sh
│ │ │ ├── rnet.fine2.addneu.sh
│ │ │ ├── rnet.hidden300.sh
│ │ │ ├── rnet.labelrnn.sh
│ │ │ ├── rnet.latt10.sh
│ │ │ ├── rnet.latt80.sh
│ │ │ ├── rnet.maxmeanatt.sh
│ │ │ ├── rnet.sh
│ │ │ ├── rnetv2.sh
│ │ │ ├── torch.mreader.1hop.labelrnn.addneu.sh
│ │ │ ├── torch.mreader.1hop.labelrnn.sh
│ │ │ ├── torch.mreader.1hop.sh
│ │ │ ├── torch.mreader.2hop.labelrnn.sh
│ │ │ └── torch.mreader.2hop.sh
│ │ ├── v3/
│ │ │ ├── README.md
│ │ │ ├── rnet.3layer.noconcat.sh
│ │ │ ├── rnet.3layer.noconcat.topkattlast.sh
│ │ │ ├── rnet.3layer.sh
│ │ │ ├── rnet.3layer.topkattlast.sh
│ │ │ ├── rnet.noconcat.lossesdecay.sh
│ │ │ ├── rnet.noconcat.sh
│ │ │ ├── rnet.noconcat.topkattlast.sh
│ │ │ ├── rnet.sh
│ │ │ ├── rnet.topkattlast.sh
│ │ │ ├── torch.mreader.1hop.2layer.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.adjf1sdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.aucsdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.f1sdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.lossesdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.sh
│ │ │ ├── torch.mreader.1hop.decay09.sh
│ │ │ ├── torch.mreader.1hop.sh
│ │ │ ├── torch.mreader.1hop.topkattlast.sh
│ │ │ ├── torch.mreader.labelrnn.2hop.2layer.sh
│ │ │ ├── torch.mreader.labelrnn.2hop.2layer.topkattlast.sh
│ │ │ └── torch.mreaderV2.1hop.2layer.sh
│ │ ├── v4/
│ │ │ ├── README.md
│ │ │ ├── rnet.3layer.noconcat.sh
│ │ │ ├── rnet.3layer.sh
│ │ │ ├── rnet.noconcat.lossesdecay.sh
│ │ │ ├── rnet.noconcat.sh
│ │ │ ├── rnet.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.adjf1sdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.aucsdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.f1sdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.lossesdecay.sh
│ │ │ ├── torch.mreader.1hop.decay09.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ ├── v5/
│ │ │ ├── README.md
│ │ │ ├── rnet.3layer.noconcat.sh
│ │ │ ├── rnet.3layer.sh
│ │ │ ├── rnet.noconcat.lossesdecay.sh
│ │ │ ├── rnet.noconcat.sh
│ │ │ ├── rnet.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.adjf1sdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.aucsdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.f1sdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.lossesdecay.sh
│ │ │ ├── torch.mreader.1hop.2layer.topkattlast.sh
│ │ │ ├── torch.mreader.1hop.decay09.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ ├── v6/
│ │ │ ├── README.md
│ │ │ ├── rnet.3layer.noconcat.sh
│ │ │ ├── rnet.3layer.sh
│ │ │ ├── rnet.noconcat.sh
│ │ │ ├── rnet.sh
│ │ │ ├── torch.mreader.1hop.decay09.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ ├── v7/
│ │ │ ├── rnet.finetune6k.sh
│ │ │ ├── rnet.fixword.sh
│ │ │ ├── rnet.sh
│ │ │ ├── torch.mreader.1hop.finetune6k.sh
│ │ │ ├── torch.mreader.1hop.fixword.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ ├── v8/
│ │ │ ├── rnet.sh
│ │ │ └── torch.mreader.1hop.sh
│ │ └── v9/
│ │ ├── rnet.sh
│ │ └── torch.mreader.1hop.sh
│ ├── common/
│ │ └── lm/
│ │ ├── README.md
│ │ ├── algos/
│ │ │ ├── loss.py
│ │ │ └── model.py
│ │ ├── dataset.py
│ │ ├── prepare/
│ │ │ ├── to-ids.glove.word.sh
│ │ │ ├── to-ids.py
│ │ │ └── to-ids.sh
│ │ ├── read-records.py
│ │ ├── train/
│ │ │ ├── char.concat.sh
│ │ │ ├── char.emb.sh
│ │ │ ├── char.sh
│ │ │ ├── char.small.sh
│ │ │ ├── v1/
│ │ │ │ ├── train.concat.sh
│ │ │ │ ├── train.emb.sh
│ │ │ │ └── train.sh
│ │ │ ├── v2/
│ │ │ │ ├── train.concat.sh
│ │ │ │ ├── train.emb.sh
│ │ │ │ └── train.sh
│ │ │ ├── word.concat.sh
│ │ │ ├── word.emb.sh
│ │ │ └── word.sh
│ │ └── train.py
│ ├── feed/
│ │ └── rank/
│ │ ├── tf/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── config.py
│ │ │ ├── data/
│ │ │ │ └── train_0
│ │ │ ├── dataset.py
│ │ │ ├── eager-train/
│ │ │ │ └── best-v0.sh
│ │ │ ├── err/
│ │ │ │ └── torch-only-train.py
│ │ │ ├── evaluate.py
│ │ │ ├── gen-records.py
│ │ │ ├── loss.py
│ │ │ ├── model.py
│ │ │ ├── mpi.py
│ │ │ ├── prepare/
│ │ │ │ ├── gen-records.sh
│ │ │ │ ├── gen-train.sh
│ │ │ │ └── gen-valid.sh
│ │ │ ├── pyt/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── dataset.py
│ │ │ │ └── model.py
│ │ │ ├── read/
│ │ │ │ ├── README.md
│ │ │ │ ├── horovod.sh
│ │ │ │ ├── read-record.sh
│ │ │ │ └── read-text.sh
│ │ │ ├── read-record.py
│ │ │ ├── read-simple.py
│ │ │ ├── read-test.py
│ │ │ ├── read-test2.py
│ │ │ ├── read-test3.py
│ │ │ ├── read-text.py
│ │ │ ├── test/
│ │ │ │ ├── read-torch-dataset.py
│ │ │ │ └── test-torch-hvd.py
│ │ │ ├── text_dataset.py
│ │ │ ├── tfrecord_dataset.py
│ │ │ ├── torch-hvd-train/
│ │ │ │ ├── best-v0-small.sh
│ │ │ │ └── best-v0.sh
│ │ │ ├── torch-hvd-train.py
│ │ │ ├── torch-only-train/
│ │ │ │ ├── baseline.sh
│ │ │ │ ├── best-v0-small.sh
│ │ │ │ └── best-v0.sh
│ │ │ ├── torch-only-train-hvd/
│ │ │ │ ├── best-v0-small.sh
│ │ │ │ └── best-v0.sh
│ │ │ ├── torch-only-train-hvd.py
│ │ │ ├── torch-only-train.py
│ │ │ ├── torch-train/
│ │ │ │ ├── baseline.sh
│ │ │ │ ├── best-v0-small.sh
│ │ │ │ └── best-v0.sh
│ │ │ ├── torch-train.py
│ │ │ ├── train/
│ │ │ │ ├── README.md
│ │ │ │ ├── allsum.sh
│ │ │ │ ├── attention.sh
│ │ │ │ ├── baseline.sh
│ │ │ │ ├── baseline2.sh
│ │ │ │ ├── baseline3.sh
│ │ │ │ ├── baseline4.sh
│ │ │ │ ├── best-v0-horovod-lp.sh
│ │ │ │ ├── best-v0-horovod-lrnoscal-prerand.sh
│ │ │ │ ├── best-v0-horovod-lrnoscale-largebuffer.sh
│ │ │ │ ├── best-v0-horovod-lrnoscale-nbp.sh
│ │ │ │ ├── best-v0-horovod-lrnoscale.sh
│ │ │ │ ├── best-v0-horovod-noshard.sh
│ │ │ │ ├── best-v0-horovod-record-lrnoscale-largebuffer.sh
│ │ │ │ ├── best-v0-horovod-record-lrnoscale.sh
│ │ │ │ ├── best-v0-horovod-record.sh
│ │ │ │ ├── best-v0-horovod-sp.sh
│ │ │ │ ├── best-v0-horovod.sh
│ │ │ │ ├── best-v0-horovod2.sh
│ │ │ │ ├── best-v0-record.sh
│ │ │ │ ├── best-v0-sp.sh
│ │ │ │ ├── best-v0.sh
│ │ │ │ ├── deep-addval.sh
│ │ │ │ ├── deep.embact.sh
│ │ │ │ ├── deep.sh
│ │ │ │ ├── dw-add.sh
│ │ │ │ ├── dw-att-fieldemb.sh
│ │ │ │ ├── dw-bigbatch.sh
│ │ │ │ ├── dw-concat.sh
│ │ │ │ ├── dw-concat2.sh
│ │ │ │ ├── dw-concat3.sh
│ │ │ │ ├── dw-concat4.sh
│ │ │ │ ├── dw-field.sh
│ │ │ │ ├── dw-fieldemb.sh
│ │ │ │ ├── dw-hidden100.sh
│ │ │ │ ├── dw-hidden64.sh
│ │ │ │ ├── dw-max-fieldemb.sh
│ │ │ │ ├── dw-max.sh
│ │ │ │ ├── dw-max2.sh
│ │ │ │ ├── dw-mean.sh
│ │ │ │ ├── dw-mean2-2.sh
│ │ │ │ ├── dw-mean2-fieldemb-mlp1layer.sh
│ │ │ │ ├── dw-mean2-fieldemb-mlp1layer2.sh
│ │ │ │ ├── dw-mean2-fieldemb-mlp2layer.sh
│ │ │ │ ├── dw-mean2-fieldemb-rankloss.sh
│ │ │ │ ├── dw-mean2-fieldemb.sh
│ │ │ │ ├── dw-mean2-fieldemb2.sh
│ │ │ │ ├── dw-mean2-rankloss.sh
│ │ │ │ ├── dw-mean2.sh
│ │ │ │ ├── dw-mean3.sh
│ │ │ │ ├── dw-new-allsum.sh
│ │ │ │ ├── dw-new.sh
│ │ │ │ ├── dw-old-allsum.sh
│ │ │ │ ├── dw-old.sh
│ │ │ │ ├── dw-record.sh
│ │ │ │ ├── dw-sum-fconcat-fieldemb-mlp.sh
│ │ │ │ ├── dw-sum-fconcat-fieldemb-mlp2.sh
│ │ │ │ ├── dw-sum-fconcat-fieldemb-mlp3.sh
│ │ │ │ ├── dw-sum-fconcat-mlp.sh
│ │ │ │ ├── dw-sum-fconcat-mlp2.sh
│ │ │ │ ├── dw-sum-fconcat.sh
│ │ │ │ ├── dw-sum-fieldemb-rankloss.sh
│ │ │ │ ├── dw-sum-fieldemb.sh
│ │ │ │ ├── dw-sum-fieldemb2.sh
│ │ │ │ ├── dw-sum-horovod-noshard.sh
│ │ │ │ ├── dw-sum-horovod.sh
│ │ │ │ ├── dw-sum-record.sh
│ │ │ │ ├── dw-sum2-fieldemb-mlp1layer.sh
│ │ │ │ ├── dw-sum2.sh
│ │ │ │ ├── dw-text.sh
│ │ │ │ ├── dw-topk.sh
│ │ │ │ ├── fconcat-record.sh
│ │ │ │ ├── fconcat.sh
│ │ │ │ ├── fconcat2.sh
│ │ │ │ ├── fconcat3.sh
│ │ │ │ ├── horovod-batch128-test.sh
│ │ │ │ ├── horovod-batch256.sh
│ │ │ │ ├── horovod-batch64-2.sh
│ │ │ │ ├── horovod-batch64-test.sh
│ │ │ │ ├── horovod-batch64.sh
│ │ │ │ ├── horovod-scale.sh
│ │ │ │ ├── horovod-small.sh
│ │ │ │ ├── horovod-test-record.sh
│ │ │ │ ├── horovod-test.sh
│ │ │ │ ├── horovod.sh
│ │ │ │ ├── horovod2.sh
│ │ │ │ ├── rankloss.sh
│ │ │ │ ├── record.sh
│ │ │ │ ├── small.sh
│ │ │ │ ├── sum.sh
│ │ │ │ ├── test1.sh
│ │ │ │ ├── test10.sh
│ │ │ │ ├── test11.sh
│ │ │ │ ├── test12.sh
│ │ │ │ ├── test2.sh
│ │ │ │ ├── test3.sh
│ │ │ │ ├── test4.sh
│ │ │ │ ├── test5.sh
│ │ │ │ ├── test6-2.sh
│ │ │ │ ├── test6-3.sh
│ │ │ │ ├── test6.sh
│ │ │ │ ├── test7.sh
│ │ │ │ ├── test8.sh
│ │ │ │ ├── test9.sh
│ │ │ │ ├── wide-deep.sh
│ │ │ │ ├── wide-deep2-bert-2epoch.sh
│ │ │ │ ├── wide-deep2-bert.sh
│ │ │ │ ├── wide-deep2-bert2-2epoch.sh
│ │ │ │ ├── wide-deep2-bert2.sh
│ │ │ │ ├── wide-deep2-lazy.sh
│ │ │ │ ├── wide-deep2.sh
│ │ │ │ ├── wide-ftrl.sh
│ │ │ │ ├── wide-noval.sh
│ │ │ │ └── wide.sh
│ │ │ └── train.py
│ │ └── v1/
│ │ └── README.md
│ ├── kaggle/
│ │ ├── blindness/
│ │ │ ├── jupter/
│ │ │ │ └── tf-image.ipynb
│ │ │ ├── keras/
│ │ │ │ ├── READEME.md
│ │ │ │ ├── aug.py
│ │ │ │ ├── check.py
│ │ │ │ ├── config.py
│ │ │ │ ├── dataset.py
│ │ │ │ ├── evaluate.py
│ │ │ │ ├── evaluate2.py
│ │ │ │ ├── fake-infer.py
│ │ │ │ ├── folds.py
│ │ │ │ ├── gen-folds.py
│ │ │ │ ├── infer.py
│ │ │ │ ├── infer.sh
│ │ │ │ ├── jupter.ipynb
│ │ │ │ ├── loss.py
│ │ │ │ ├── lr.py
│ │ │ │ ├── model.py
│ │ │ │ ├── train.py
│ │ │ │ ├── train2.py
│ │ │ │ └── util.py
│ │ │ ├── keras2/
│ │ │ │ ├── READEME.md
│ │ │ │ ├── aug.py
│ │ │ │ ├── config.py
│ │ │ │ ├── dataset.py
│ │ │ │ ├── evaluate.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── model.py
│ │ │ │ └── train.py
│ │ │ ├── keras2tf/
│ │ │ │ ├── READEME.md
│ │ │ │ ├── aug.py
│ │ │ │ ├── config.py
│ │ │ │ ├── dataset.py
│ │ │ │ ├── evaluate.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── model.py
│ │ │ │ └── train.py
│ │ │ ├── keras3/
│ │ │ │ ├── READEME.md
│ │ │ │ ├── aug.py
│ │ │ │ ├── bak/
│ │ │ │ │ └── evaluate.py
│ │ │ │ ├── config.py
│ │ │ │ ├── dataset.py
│ │ │ │ ├── evaluate.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── model.py
│ │ │ │ └── train.py
│ │ │ ├── other/
│ │ │ │ ├── APTOS 2019_ Keras Baseline.ipynb
│ │ │ │ ├── Intro APTOS Diabetic Retinopathy (EDA & Sta e77c27.ipynb
│ │ │ │ ├── keras_baseline.py
│ │ │ │ └── training-mobilenet-v2-in-4-min.py
│ │ │ ├── prepare/
│ │ │ │ ├── gen-records.py
│ │ │ │ └── gen-records.sh
│ │ │ ├── tf/
│ │ │ │ ├── dataset.py
│ │ │ │ ├── evaluate.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── model.py
│ │ │ │ ├── model_base.py
│ │ │ │ ├── train/
│ │ │ │ │ └── train.sh
│ │ │ │ └── train.py
│ │ │ └── tf2/
│ │ │ ├── dataset.py
│ │ │ ├── evaluate.py
│ │ │ ├── loss.py
│ │ │ ├── model.py
│ │ │ ├── train/
│ │ │ │ └── train.sh
│ │ │ └── train.py
│ │ ├── cifar10/
│ │ │ ├── README.md
│ │ │ ├── baseline/
│ │ │ │ ├── keras/
│ │ │ │ │ ├── KerasT-master/
│ │ │ │ │ │ ├── .idea/
│ │ │ │ │ │ │ ├── KerasT.iml
│ │ │ │ │ │ │ ├── inspectionProfiles/
│ │ │ │ │ │ │ │ └── Project_Default.xml
│ │ │ │ │ │ │ ├── misc.xml
│ │ │ │ │ │ │ ├── modules.xml
│ │ │ │ │ │ │ ├── preferred-vcs.xml
│ │ │ │ │ │ │ └── vcs.xml
│ │ │ │ │ │ ├── 4layerCNN.py
│ │ │ │ │ │ ├── 6layerCNN.py
│ │ │ │ │ │ ├── ConfusionMatrixVisualization.py
│ │ │ │ │ │ └── README.md
│ │ │ │ │ ├── hands-on-deep-learning-master/
│ │ │ │ │ │ ├── .gitignore
│ │ │ │ │ │ ├── README.md
│ │ │ │ │ │ └── cifar_image_classification/
│ │ │ │ │ │ ├── aux/
│ │ │ │ │ │ │ ├── cifar100_meta
│ │ │ │ │ │ │ ├── cifar100_to_h5py.ipynb
│ │ │ │ │ │ │ └── cifar10_to_h5py.ipynb
│ │ │ │ │ │ ├── cnn_adv.py
│ │ │ │ │ │ ├── cnn_fchollet.py
│ │ │ │ │ │ ├── cnn_pkaur.py
│ │ │ │ │ │ ├── cnn_simple.py
│ │ │ │ │ │ ├── drawings.ipynb
│ │ │ │ │ │ ├── helpers.py
│ │ │ │ │ │ ├── lr.py
│ │ │ │ │ │ ├── mlp.py
│ │ │ │ │ │ └── neptune.yaml
│ │ │ │ │ └── simple/
│ │ │ │ │ ├── cifar10.py
│ │ │ │ │ └── result.csv
│ │ │ │ └── tf/
│ │ │ │ ├── cifar10.py
│ │ │ │ ├── cifar10_estimator/
│ │ │ │ │ ├── README.md
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── cifar10.py
│ │ │ │ │ ├── cifar10_main.py
│ │ │ │ │ ├── cifar10_model.py
│ │ │ │ │ ├── cifar10_multi_gpu_train.py
│ │ │ │ │ ├── cifar10_utils.py
│ │ │ │ │ ├── cmle_config.yaml
│ │ │ │ │ ├── evaluator.py
│ │ │ │ │ ├── gen-records.sh
│ │ │ │ │ ├── generate_cifar10_tfrecords.py
│ │ │ │ │ ├── inference.py
│ │ │ │ │ ├── model_base.py
│ │ │ │ │ ├── prepare.sh
│ │ │ │ │ ├── train/
│ │ │ │ │ │ ├── readme
│ │ │ │ │ │ ├── train-2gpu-2.sh
│ │ │ │ │ │ ├── train-2gpu.sh
│ │ │ │ │ │ ├── train-4gpu.sh
│ │ │ │ │ │ ├── train-melt-2gpu.sh
│ │ │ │ │ │ ├── train-melt-decay-2gpu.sh
│ │ │ │ │ │ ├── train-melt-decay.sh
│ │ │ │ │ │ ├── train-melt-decay98.sh
│ │ │ │ │ │ ├── train-melt-momentum-batch128.sh
│ │ │ │ │ │ ├── train-melt-momentum-decay-2gpu.sh
│ │ │ │ │ │ ├── train-melt-momentum-decay-batch64.sh
│ │ │ │ │ │ ├── train-melt-momentum-decay.sh
│ │ │ │ │ │ ├── train-melt-momentum.sh
│ │ │ │ │ │ ├── train-melt-momentum2-batch128-2gpu.sh
│ │ │ │ │ │ ├── train-melt-momentum2-batch128.sh
│ │ │ │ │ │ ├── train-melt-momentum2.sh
│ │ │ │ │ │ ├── train-melt.sh
│ │ │ │ │ │ └── train.sh
│ │ │ │ │ ├── train.py
│ │ │ │ │ └── train.tfrecords
│ │ │ │ └── cifar10_estimator.v1/
│ │ │ │ ├── README.md
│ │ │ │ ├── __init__.py
│ │ │ │ ├── cifar10.py
│ │ │ │ ├── cifar10_main.py
│ │ │ │ ├── cifar10_model.py
│ │ │ │ ├── cifar10_multi_gpu_train.py
│ │ │ │ ├── cifar10_utils.py
│ │ │ │ ├── cmle_config.yaml
│ │ │ │ ├── evaluator.py
│ │ │ │ ├── gen-records.sh
│ │ │ │ ├── generate_cifar10_tfrecords.py
│ │ │ │ ├── inference.py
│ │ │ │ ├── model_base.py
│ │ │ │ ├── prepare.sh
│ │ │ │ ├── train/
│ │ │ │ │ ├── readme
│ │ │ │ │ ├── train-2gpu-2.sh
│ │ │ │ │ ├── train-2gpu.sh
│ │ │ │ │ ├── train-4gpu.sh
│ │ │ │ │ ├── train-melt-2gpu.sh
│ │ │ │ │ ├── train-melt-decay-2gpu.sh
│ │ │ │ │ ├── train-melt-decay.sh
│ │ │ │ │ ├── train-melt-decay98.sh
│ │ │ │ │ ├── train-melt-momentum-batch128.sh
│ │ │ │ │ ├── train-melt-momentum-decay-2gpu.sh
│ │ │ │ │ ├── train-melt-momentum-decay-batch64.sh
│ │ │ │ │ ├── train-melt-momentum-decay.sh
│ │ │ │ │ ├── train-melt-momentum.sh
│ │ │ │ │ ├── train-melt-momentum2-batch128-2gpu.sh
│ │ │ │ │ ├── train-melt-momentum2-batch128.sh
│ │ │ │ │ ├── train-melt-momentum2.sh
│ │ │ │ │ ├── train-melt.sh
│ │ │ │ │ └── train.sh
│ │ │ │ ├── train.py
│ │ │ │ └── train.tfrecords
│ │ │ ├── fastai/
│ │ │ │ ├── cifar10-simplenet.ipynb
│ │ │ │ └── cifar10.ipynb
│ │ │ └── tf/
│ │ │ ├── cifar10.py
│ │ │ ├── cifar10_model.py
│ │ │ ├── evaluate.py
│ │ │ ├── evaluator.py
│ │ │ ├── loss.py
│ │ │ ├── model_base.py
│ │ │ ├── train/
│ │ │ │ ├── readme
│ │ │ │ ├── train-2gpu-2.sh
│ │ │ │ ├── train-2gpu.sh
│ │ │ │ ├── train-4gpu.sh
│ │ │ │ ├── train-melt-2gpu.sh
│ │ │ │ ├── train-melt-decay-2gpu.sh
│ │ │ │ ├── train-melt-decay.sh
│ │ │ │ ├── train-melt-decay98.sh
│ │ │ │ ├── train-melt-momentum-batch128.sh
│ │ │ │ ├── train-melt-momentum-decay-2gpu.sh
│ │ │ │ ├── train-melt-momentum-decay-batch64.sh
│ │ │ │ ├── train-melt-momentum-decay.sh
│ │ │ │ ├── train-melt-momentum-decay2-2gpu.sh
│ │ │ │ ├── train-melt-momentum-decay2.sh
│ │ │ │ ├── train-melt-momentum.sh
│ │ │ │ ├── train-melt-momentum2-batch128-2gpu.sh
│ │ │ │ ├── train-melt-momentum2-batch128.sh
│ │ │ │ ├── train-melt-momentum2.sh
│ │ │ │ ├── train-melt.sh
│ │ │ │ └── train.sh
│ │ │ ├── train.py
│ │ │ └── train2.py
│ │ └── toxic/
│ │ ├── algos/
│ │ │ ├── config.py
│ │ │ └── model.py
│ │ ├── checkpoint
│ │ ├── dataset.py
│ │ ├── evaluate.py
│ │ ├── prepare/
│ │ │ ├── __init__.py
│ │ │ ├── black.csv
│ │ │ ├── config.py
│ │ │ ├── count-unks.py
│ │ │ ├── extend-table.py
│ │ │ ├── filter-special.py
│ │ │ ├── gen-correction.py
│ │ │ ├── gen-en-lang-prob.py
│ │ │ ├── gen-full-vocab.sh
│ │ │ ├── gen-lang.py
│ │ │ ├── gen-ori-vocab.sh
│ │ │ ├── gen-records-parse.py
│ │ │ ├── gen-records.py
│ │ │ ├── gen-sentences-csv.py
│ │ │ ├── gen-sentences.py
│ │ │ ├── gen-tokens.py
│ │ │ ├── gen-twitter-ori-vocab.sh
│ │ │ ├── gen-vocab-parse.py
│ │ │ ├── gen-vocab.py
│ │ │ ├── gray.csv
│ │ │ ├── merge-2emb.py
│ │ │ ├── merge-charemb.py
│ │ │ ├── merge-emb.py
│ │ │ ├── merge-glove.py
│ │ │ ├── merge-ngram-emb.py
│ │ │ ├── merge-word-ngram-emb.py
│ │ │ ├── merge-wordemb.py
│ │ │ ├── post-deal.py
│ │ │ ├── prepare-test.sh
│ │ │ ├── prepare-train.sh
│ │ │ ├── preprocess.py
│ │ │ ├── readme
│ │ │ ├── run-gloveonly.sh
│ │ │ ├── run-scratch.sh
│ │ │ ├── run-v1.sh
│ │ │ ├── run-v10-fasttext.sh
│ │ │ ├── run-v10-fttalk.sh
│ │ │ ├── run-v10.sh
│ │ │ ├── run-v11-fasttext-lower.sh
│ │ │ ├── run-v11-fasttext.sh
│ │ │ ├── run-v11-ftngram.sh
│ │ │ ├── run-v11-fttalk.sh
│ │ │ ├── run-v11-fttalk2.sh
│ │ │ ├── run-v11-fttalk5.sh
│ │ │ ├── run-v11-glove-lower.sh
│ │ │ ├── run-v11-glove.sh
│ │ │ ├── run-v11-glovetalk.sh
│ │ │ ├── run-v11-glovetwitter.sh
│ │ │ ├── run-v11-lexvec.sh
│ │ │ ├── run-v11-lexvecc.sh
│ │ │ ├── run-v11-ngram-lower.sh
│ │ │ ├── run-v11.sh
│ │ │ ├── run-v13.sh
│ │ │ ├── run-v14.sh
│ │ │ ├── run-v15-clean-lower.sh
│ │ │ ├── run-v15-clean.sh
│ │ │ ├── run-v15-fasttext-lower-allngram.sh
│ │ │ ├── run-v15-fasttext-lower-lemma.sh
│ │ │ ├── run-v15-fasttext-lower.sh
│ │ │ ├── run-v15-fasttext.sh
│ │ │ ├── run-v15-fttalk3gram-lower.sh
│ │ │ ├── run-v15-fttalk3gramall-lower.sh
│ │ │ ├── run-v15-fttalk5-lower.sh
│ │ │ ├── run-v15-fttalk5.sh
│ │ │ ├── run-v15-glove-lower-allngram.sh
│ │ │ ├── run-v15-glove-lower-lem.sh
│ │ │ ├── run-v15-glove-lower.sh
│ │ │ ├── run-v15-gloveft-lower.sh
│ │ │ ├── run-v15-glovetalk-lower.sh
│ │ │ ├── run-v15-glovetalk.sh
│ │ │ ├── run-v15-lexvecc-clean.sh
│ │ │ ├── run-v15-lexvecc.sh
│ │ │ ├── run-v15.sh
│ │ │ ├── run-v16-fasttext-lower-lemma.sh
│ │ │ ├── run-v16-fasttext-lower.sh
│ │ │ ├── run-v16-fasttext.sh
│ │ │ ├── run-v16-ftfttalk-lower.sh
│ │ │ ├── run-v16-fttalk5-lower.sh
│ │ │ ├── run-v16-glove-lower-lemma.sh
│ │ │ ├── run-v16-glove-lower.sh
│ │ │ ├── run-v16-gloveft-lower-lemma.sh
│ │ │ ├── run-v16-gloveft-lower.sh
│ │ │ ├── run-v16-gloveglovetalk-lower.sh
│ │ │ ├── run-v16-lexveccfttalk-lower.sh
│ │ │ ├── run-v16-lm.sh
│ │ │ ├── run-v3-limit1000.sh
│ │ │ ├── run-v3-limit400.sh
│ │ │ ├── run-v3.sh
│ │ │ ├── run-v4-fasttext.sh
│ │ │ ├── run-v4-limmit400.sh
│ │ │ ├── run-v4.sh
│ │ │ ├── run-v7.sh
│ │ │ ├── run-v8.sh
│ │ │ ├── run-v9-full.sh
│ │ │ ├── run-v9-glovetwitter-full.sh
│ │ │ ├── run-v9-glovetwitter.sh
│ │ │ ├── run-v9-noparse.sh
│ │ │ ├── run-v9.sh
│ │ │ ├── run.sh
│ │ │ ├── test-tokenize.py
│ │ │ ├── test-tokenize2.py
│ │ │ ├── test-tokenize3.py
│ │ │ ├── test-tokenize4.py
│ │ │ ├── test-tokenize5.py
│ │ │ ├── tokenize-corpus.py
│ │ │ ├── tokenizer-v2.py
│ │ │ ├── tokenizer-v3.py
│ │ │ ├── tokenizer.py
│ │ │ ├── toxic_words.py
│ │ │ └── white.csv
│ │ ├── read-records.py
│ │ └── train.py
│ └── pyt/
│ └── README.md
├── tests/
│ ├── horovod/
│ │ ├── allgather.py
│ │ ├── allgather2.py
│ │ ├── allgather3.py
│ │ ├── allgather4.py
│ │ ├── allreduce.py
│ │ ├── allreduce2.py
│ │ ├── bcast.py
│ │ ├── shards.py
│ │ └── simple.py
│ └── sample-balance/
│ ├── README.md
│ ├── dataset.py
│ └── read-records.py
├── third/
│ ├── __init__.py
│ └── bert/
│ ├── CONTRIBUTING.md
│ ├── LICENSE
│ ├── README.md
│ ├── __init__.py
│ ├── create_pretraining_data.py
│ ├── extract_features.py
│ ├── modeling.py
│ ├── modeling_test.py
│ ├── multilingual.md
│ ├── optimization.py
│ ├── optimization_test.py
│ ├── run-char.sh
│ ├── run_classifier.py
│ ├── run_pretraining.py
│ ├── run_squad.py
│ ├── tokenization.py
│ └── tokenization_test.py
├── tools/
│ ├── dump-word-embedding.py
│ └── show-var-of-model.py
├── utils/
│ ├── README.md
│ ├── gezi/
│ │ ├── __init__.py
│ │ ├── avg_score.py
│ │ ├── bigdata_util.py
│ │ ├── bleu.py
│ │ ├── gezi_util.py
│ │ ├── hash.py
│ │ ├── libgezi_util.py
│ │ ├── melt/
│ │ │ ├── __init__.py
│ │ │ ├── logging.py
│ │ │ ├── tfrecords.py
│ │ │ └── util.py
│ │ ├── metrics/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── bleu/
│ │ │ │ ├── LICENSE
│ │ │ │ ├── __init__.py
│ │ │ │ ├── bleu.py
│ │ │ │ └── bleu_scorer.py
│ │ │ ├── cider/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── cider.py
│ │ │ │ └── cider_scorer.py
│ │ │ ├── ciderD/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── ciderD.py
│ │ │ │ └── ciderD_scorer.py
│ │ │ ├── correlation/
│ │ │ │ ├── __init__.py
│ │ │ │ └── correlation.py
│ │ │ ├── eval.py
│ │ │ ├── meteor/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── meteor-1.5.jar
│ │ │ │ └── meteor.py
│ │ │ ├── new_cider/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── cider.py
│ │ │ │ └── cider_scorer.py
│ │ │ ├── rouge/
│ │ │ │ ├── __init__.py
│ │ │ │ └── rouge.py
│ │ │ └── tokenizer/
│ │ │ ├── __init__.py
│ │ │ ├── ptbtokenizer.py
│ │ │ └── stanford-corenlp-3.4.1.jar
│ │ ├── ngram.py
│ │ ├── nowarning.py
│ │ ├── pydict.py
│ │ ├── rank_metrics.py
│ │ ├── segment.py
│ │ ├── setup.py
│ │ ├── summary.py
│ │ ├── test/
│ │ │ ├── test_get_single_cn.py
│ │ │ └── test_ngrams.py
│ │ ├── timer.py
│ │ ├── topn.py
│ │ ├── util.py
│ │ ├── vocabulary.py
│ │ ├── word_counter.py
│ │ └── zhtools/
│ │ ├── __init__.py
│ │ ├── chconv.py
│ │ ├── langconv.py
│ │ ├── test_langconv.py
│ │ ├── xpinyin.py
│ │ └── zh_wiki.py
│ ├── lele/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── apps/
│ │ │ ├── __init__.py
│ │ │ └── train.py
│ │ ├── fastai/
│ │ │ ├── __init__.py
│ │ │ ├── core.py
│ │ │ ├── imports/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── core.py
│ │ │ │ └── torch.py
│ │ │ ├── layers.py
│ │ │ ├── text/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── models.py
│ │ │ │ └── qrnn/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── forget_mult.py
│ │ │ │ └── qrnn.py
│ │ │ └── torch_core.py
│ │ ├── layers/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── classify_layer.py
│ │ │ ├── elmo/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── classify_layer.py
│ │ │ │ ├── elmo.py
│ │ │ │ ├── embedding_layer.py
│ │ │ │ ├── encoder_base.py
│ │ │ │ ├── highway.py
│ │ │ │ ├── lstm.py
│ │ │ │ ├── lstm_cell_with_projection.py
│ │ │ │ ├── token_embedder.py
│ │ │ │ └── util.py
│ │ │ ├── elmo.py
│ │ │ ├── embedding_layer.py
│ │ │ ├── encoder_base.py
│ │ │ ├── highway.py
│ │ │ ├── layers.py
│ │ │ ├── lstm.py
│ │ │ ├── lstm_cell_with_projection.py
│ │ │ ├── token_embedder.py
│ │ │ ├── transformer/
│ │ │ │ ├── __init__.py
│ │ │ │ └── transformer.py
│ │ │ └── util.py
│ │ ├── losses/
│ │ │ ├── __init__.py
│ │ │ └── losses.py
│ │ ├── ops/
│ │ │ ├── __init__.py
│ │ │ └── ops.py
│ │ ├── training/
│ │ │ ├── __init__.py
│ │ │ └── optimizers.py
│ │ └── util.py
│ └── melt/
│ ├── __init__.py
│ ├── apps/
│ │ ├── __init__.py
│ │ ├── image_processing.py
│ │ ├── read.py
│ │ └── train.py
│ ├── cnn/
│ │ ├── __init__.py
│ │ ├── cnn.py
│ │ ├── conv2d.py
│ │ └── qanet.py
│ ├── eager/
│ │ ├── __init__.py
│ │ ├── train.py
│ │ └── util.py
│ ├── encoder/
│ │ ├── __init__.py
│ │ ├── embedding_layer.py
│ │ └── transformer.py
│ ├── flow/
│ │ ├── __init__.py
│ │ ├── dataset_flow.py
│ │ ├── flow.py
│ │ ├── test.py
│ │ ├── test_once.py
│ │ ├── train.py
│ │ └── train_once.py
│ ├── image/
│ │ ├── __init__.py
│ │ ├── image_decoder.py
│ │ ├── image_embedding.py
│ │ ├── image_model.py
│ │ └── image_processing.py
│ ├── inference/
│ │ ├── __init__.py
│ │ ├── predictor.py
│ │ └── predictor_base.py
│ ├── layers/
│ │ ├── __init__.py
│ │ ├── cnn/
│ │ │ ├── __init__.py
│ │ │ ├── cnn.py
│ │ │ ├── convnet.py
│ │ │ └── qanet.py
│ │ ├── layers.py
│ │ ├── optimizers.py
│ │ ├── rnn/
│ │ │ ├── __init__.py
│ │ │ └── rnn.py
│ │ └── transformer/
│ │ └── __init__.py
│ ├── losses/
│ │ ├── __init__.py
│ │ └── losses.py
│ ├── metrics/
│ │ ├── __init__.py
│ │ └── rank_metrics.py
│ ├── models/
│ │ ├── __init__.py
│ │ └── mlp.py
│ ├── ops/
│ │ ├── __init__.py
│ │ ├── nn_impl.py
│ │ ├── ops.py
│ │ └── sparse_ops.py
│ ├── rnn/
│ │ ├── __init__.py
│ │ └── rnn.py
│ ├── seq2seq/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── attention_decoder_fn.py
│ │ ├── attention_wrapper.py
│ │ ├── basic_decoder.py
│ │ ├── beam_decoder.py
│ │ ├── beam_decoder_fn.py
│ │ ├── beam_search.py
│ │ ├── beam_search_decoder.py
│ │ ├── decoder.py
│ │ ├── decoder_fn.py
│ │ ├── exp/
│ │ │ └── beam_decoder.py
│ │ ├── helper.py
│ │ ├── legacy/
│ │ │ ├── beam_decoder.py
│ │ │ └── beam_search_decoder.py
│ │ ├── logprobs_decoder.py
│ │ ├── loss.py
│ │ ├── official/
│ │ │ └── beam_search_decoder.py
│ │ └── seq2seq.py
│ ├── setup.py
│ ├── slim2/
│ │ ├── __init__.py
│ │ ├── base_nets_factory.py
│ │ ├── layers.py
│ │ └── preprocessing_factory.py
│ ├── tfrecords/
│ │ ├── __init__.py
│ │ ├── dataset.py
│ │ ├── dataset_decode.py
│ │ ├── decode_then_shuffle.py
│ │ ├── libsvm_decode.py
│ │ ├── read.py
│ │ ├── shuffle_then_decode.py
│ │ └── write.py
│ ├── tools/
│ │ ├── count-records.py
│ │ ├── delete-old-models.py
│ │ ├── prepare-finetune-withhistory.py
│ │ ├── prepare-finetune.py
│ │ ├── rename-variables.py
│ │ ├── reset-model-top-scope.py
│ │ ├── show-records.py
│ │ └── show-var-of-model.py
│ ├── torch/
│ │ ├── __init__.py
│ │ └── train.py
│ ├── training/
│ │ ├── __init__.py
│ │ ├── adamax.py
│ │ ├── bert/
│ │ │ ├── __init__.py
│ │ │ └── optimization.py
│ │ ├── learning_rate_decay.py
│ │ └── training.py
│ ├── util.py
│ ├── utils/
│ │ ├── __init__.py
│ │ ├── embsim.py
│ │ ├── logging.py
│ │ ├── summary.py
│ │ └── weight_decay.py
│ └── variable/
│ ├── __init__.py
│ └── variable.py
└── wenzheng/
├── __init__.py
├── embedding.py
├── encoder.py
├── pyt/
│ ├── __init__.py
│ ├── embedding.py
│ └── encoder.py
└── utils/
├── __init__.py
├── conf.py
├── ids2text.py
├── input_flags.py
├── rnn_flags.py
├── text2ids.py
└── vocabulary.py
Showing preview only (294K chars total). Download the full file or copy to clipboard to get everything.
SYMBOL INDEX (3665 symbols across 506 files)
FILE: examples/pytorch/data_parallel_tutorial.py
class RandomDataset (line 70) | class RandomDataset(Dataset):
method __init__ (line 72) | def __init__(self, size, length):
method __getitem__ (line 76) | def __getitem__(self, index):
method __len__ (line 79) | def __len__(self):
class Model (line 99) | class Model(nn.Module):
method __init__ (line 102) | def __init__(self, input_size, output_size):
method forward (line 106) | def forward(self, input):
FILE: examples/pytorch/sentiment/model.py
class SimpleGRU (line 33) | class SimpleGRU(nn.Module):
method __init__ (line 34) | def __init__(self, vocab_size, embedding_dim, n_hidden, n_out):
method forward (line 41) | def forward(self, seq, lengths, gpu=True):
method init_hidden (line 61) | def init_hidden(self, batch_size, gpu):
FILE: examples/pytorch/sentiment/train.py
function indexer (line 52) | def indexer(s):
class VectorizeData (line 59) | class VectorizeData(Dataset):
method __init__ (line 60) | def __init__(self, df_path, maxlen=10):
method __len__ (line 71) | def __len__(self):
method __getitem__ (line 74) | def __getitem__(self, idx):
method pad_data (line 80) | def pad_data(self, s):
function sort_batch (line 96) | def sort_batch(X, y, lengths):
function fit (line 102) | def fit(model, train_dl, val_dl, loss_fn, opt, epochs=3):
function main (line 161) | def main(_):
FILE: examples/tf/eager/dynamic_dense.py
class Layer (line 25) | class Layer(keras.layers.Layer):
method __init__ (line 26) | def __init__(self):
method call (line 30) | def call(self, x):
class Model (line 36) | class Model(keras.Model):
method __init__ (line 37) | def __init__(self):
method call (line 49) | def call(self, x):
FILE: examples/tf/eager/rnn_ptb.py
class RNN (line 44) | class RNN(tf.keras.Model):
method __init__ (line 50) | def __init__(self, hidden_dim, num_layers, keep_ratio):
method call (line 58) | def call(self, input_seq, training):
class Embedding (line 78) | class Embedding(layers.Layer):
method __init__ (line 81) | def __init__(self, vocab_size, embedding_dim, **kwargs):
method build (line 86) | def build(self, _):
method call (line 94) | def call(self, x):
class PTBModel (line 99) | class PTBModel(tf.keras.Model):
method __init__ (line 110) | def __init__(self,
method call (line 134) | def call(self, input_seq, training):
function clip_gradients (line 150) | def clip_gradients(grads_and_vars, clip_ratio):
function loss_fn (line 156) | def loss_fn(model, inputs, targets, training):
function _divide_into_batches (line 164) | def _divide_into_batches(data, batch_size):
function _get_batch (line 172) | def _get_batch(data, i, seq_len):
function evaluate (line 179) | def evaluate(model, data):
function train (line 195) | def train(model, optimizer, train_data, sequence_length, clip_ratio):
class Datasets (line 217) | class Datasets(object):
method __init__ (line 220) | def __init__(self, path):
method vocab_size (line 235) | def vocab_size(self):
method add (line 238) | def add(self, word):
method tokenize (line 243) | def tokenize(self, path):
function small_model (line 266) | def small_model(use_cudnn_rnn):
function large_model (line 277) | def large_model(use_cudnn_rnn):
function test_model (line 288) | def test_model(use_cudnn_rnn):
function main (line 299) | def main(_):
FILE: official/benchmark/benchmark_uploader.py
class BigQueryUploader (line 34) | class BigQueryUploader(object):
method __init__ (line 37) | def __init__(self, gcp_project=None, credentials=None):
method upload_benchmark_run_json (line 53) | def upload_benchmark_run_json(
method upload_benchmark_metric_json (line 69) | def upload_benchmark_metric_json(
method upload_benchmark_run_file (line 87) | def upload_benchmark_run_file(
method upload_metric_file (line 105) | def upload_metric_file(
method _upload_json (line 127) | def _upload_json(self, dataset_name, table_name, json_list):
FILE: official/benchmark/benchmark_uploader_main.py
function main (line 38) | def main(_):
FILE: official/benchmark/benchmark_uploader_test.py
class BigQueryUploaderTest (line 40) | class BigQueryUploaderTest(tf.test.TestCase):
method setUp (line 43) | def setUp(self, mock_bigquery):
method tearDown (line 63) | def tearDown(self):
method test_upload_benchmark_run_json (line 66) | def test_upload_benchmark_run_json(self):
method test_upload_benchmark_metric_json (line 73) | def test_upload_benchmark_metric_json(self):
method test_upload_benchmark_run_file (line 87) | def test_upload_benchmark_run_file(self):
method test_upload_metric_file (line 94) | def test_upload_metric_file(self):
FILE: official/boosted_trees/data_download.py
function _download_higgs_data_and_save_npz (line 34) | def _download_higgs_data_and_save_npz(data_dir):
function main (line 65) | def main(unused_argv):
function define_data_download_flags (line 71) | def define_data_download_flags():
FILE: official/boosted_trees/train_higgs.py
function read_higgs_data (line 48) | def read_higgs_data(data_dir, train_start, train_count, eval_start, eval...
function make_inputs_from_np_arrays (line 77) | def make_inputs_from_np_arrays(features_np, label_np):
function make_eval_inputs_from_np_arrays (line 134) | def make_eval_inputs_from_np_arrays(features_np, label_np):
function _make_csv_serving_input_receiver_fn (line 153) | def _make_csv_serving_input_receiver_fn(column_names, column_defaults):
function train_boosted_trees (line 178) | def train_boosted_trees(flags_obj):
function main (line 240) | def main(_):
function define_train_higgs_flags (line 244) | def define_train_higgs_flags():
FILE: official/boosted_trees/train_higgs_test.py
class BaseTest (line 36) | class BaseTest(tf.test.TestCase):
method setUpClass (line 40) | def setUpClass(cls): # pylint: disable=invalid-name
method setUp (line 44) | def setUp(self):
method test_read_higgs_data (line 56) | def test_read_higgs_data(self):
method test_make_inputs_from_np_arrays (line 71) | def test_make_inputs_from_np_arrays(self):
method test_end_to_end (line 118) | def test_end_to_end(self):
method test_end_to_end_with_export (line 134) | def test_end_to_end_with_export(self):
FILE: official/mnist/dataset.py
function read32 (line 30) | def read32(bytestream):
function check_image_file_header (line 36) | def check_image_file_header(filename):
function check_labels_file_header (line 52) | def check_labels_file_header(filename):
function download (line 62) | def download(directory, filename):
function dataset (line 81) | def dataset(directory, images_file, labels_file):
function train (line 109) | def train(directory):
function test (line 115) | def test(directory):
FILE: official/mnist/mnist.py
function create_model (line 33) | def create_model(data_format):
function define_mnist_flags (line 89) | def define_mnist_flags():
function model_fn (line 99) | def model_fn(features, labels, mode, params):
function validate_batch_size_for_multi_gpu (line 155) | def validate_batch_size_for_multi_gpu(batch_size):
function run_mnist (line 185) | def run_mnist(flags_obj):
function main (line 257) | def main(_):
FILE: official/mnist/mnist_eager.py
function loss (line 44) | def loss(logits, labels):
function compute_accuracy (line 50) | def compute_accuracy(logits, labels):
function train (line 58) | def train(model, optimizer, dataset, step_counter, log_interval=None):
function test (line 82) | def test(model, dataset):
function run_mnist_eager (line 100) | def run_mnist_eager(flags_obj):
function define_mnist_eager_flags (line 168) | def define_mnist_eager_flags():
function main (line 200) | def main(_):
FILE: official/mnist/mnist_eager_test.py
function device (line 27) | def device():
function data_format (line 31) | def data_format():
function random_dataset (line 35) | def random_dataset():
function train (line 42) | def train(defun=False):
function evaluate (line 53) | def evaluate(defun=False):
class MNISTTest (line 62) | class MNISTTest(tf.test.TestCase):
method test_train (line 65) | def test_train(self):
method test_evaluate (line 68) | def test_evaluate(self):
method test_train_with_defun (line 71) | def test_train_with_defun(self):
method test_evaluate_with_defun (line 74) | def test_evaluate_with_defun(self):
FILE: official/mnist/mnist_test.py
function dummy_input_fn (line 29) | def dummy_input_fn():
function make_estimator (line 35) | def make_estimator():
class Tests (line 45) | class Tests(tf.test.TestCase):
method test_mnist (line 48) | def test_mnist(self):
method mnist_model_fn_helper (line 67) | def mnist_model_fn_helper(self, mode, multi_gpu=False):
method test_mnist_model_fn_train_mode (line 94) | def test_mnist_model_fn_train_mode(self):
method test_mnist_model_fn_train_mode_multi_gpu (line 97) | def test_mnist_model_fn_train_mode_multi_gpu(self):
method test_mnist_model_fn_eval_mode (line 100) | def test_mnist_model_fn_eval_mode(self):
method test_mnist_model_fn_predict_mode (line 103) | def test_mnist_model_fn_predict_mode(self):
class Benchmarks (line 107) | class Benchmarks(tf.test.Benchmark):
method benchmark_train_step_time (line 110) | def benchmark_train_step_time(self):
FILE: official/mnist/mnist_tpu.py
function metric_fn (line 75) | def metric_fn(labels, logits):
function model_fn (line 81) | def model_fn(features, labels, mode, params):
function train_input_fn (line 114) | def train_input_fn(params):
function eval_input_fn (line 128) | def eval_input_fn(params):
function main (line 137) | def main(argv):
FILE: official/recommendation/data_download.py
function _print_ratings_description (line 59) | def _print_ratings_description(ratings):
function process_movielens (line 74) | def process_movielens(ratings, sort=True):
function load_movielens_1_million (line 92) | def load_movielens_1_million(file_name, sort=True):
function load_movielens_20_million (line 116) | def load_movielens_20_million(file_name, sort=True):
function load_file_to_df (line 144) | def load_file_to_df(file_name, sort=True):
function generate_train_eval_data (line 168) | def generate_train_eval_data(df, original_users, original_items):
function parse_file_to_csv (line 232) | def parse_file_to_csv(data_dir, dataset_name):
function make_dir (line 301) | def make_dir(file_dir):
function main (line 307) | def main(_):
function define_data_download_flags (line 346) | def define_data_download_flags():
FILE: official/recommendation/dataset.py
class NCFDataSet (line 30) | class NCFDataSet(object):
method __init__ (line 33) | def __init__(self, train_data, num_users, num_items, num_negatives,
function load_data (line 58) | def load_data(file_name):
function data_preprocessing (line 70) | def data_preprocessing(train_fname, test_fname, test_neg_fname, num_nega...
function generate_train_dataset (line 130) | def generate_train_dataset(train_data, num_items, num_negatives):
function input_fn (line 161) | def input_fn(training, batch_size, ncf_dataset, repeat=1):
FILE: official/recommendation/dataset_test.py
class DatasetTest (line 36) | class DatasetTest(tf.test.TestCase):
method test_load_data (line 38) | def test_load_data(self):
method test_data_preprocessing (line 48) | def test_data_preprocessing(self):
method test_generate_train_dataset (line 76) | def test_generate_train_dataset(self):
FILE: official/recommendation/ncf_main.py
function evaluate_model (line 49) | def evaluate_model(estimator, batch_size, num_gpus, ncf_dataset):
function convert_keras_to_estimator (line 138) | def convert_keras_to_estimator(keras_model, num_gpus, model_dir):
function per_device_batch_size (line 169) | def per_device_batch_size(batch_size, num_gpus):
function main (line 200) | def main(_):
function define_ncf_flags (line 282) | def define_ncf_flags():
FILE: official/recommendation/neumf_model.py
class NeuMF (line 42) | class NeuMF(tf.keras.models.Model):
method __init__ (line 45) | def __init__(self, num_users, num_items, mf_dim, model_layers, batch_s...
FILE: official/resnet/cifar10_download_and_extract.py
function main (line 39) | def main(_):
FILE: official/resnet/cifar10_main.py
function get_filenames (line 51) | def get_filenames(is_training, data_dir):
function parse_record (line 68) | def parse_record(raw_record, is_training):
function preprocess_image (line 91) | def preprocess_image(image, is_training):
function input_fn (line 109) | def input_fn(is_training, data_dir, batch_size, num_epochs=1):
function get_synth_input_fn (line 130) | def get_synth_input_fn():
class Cifar10Model (line 138) | class Cifar10Model(resnet_model.Model):
method __init__ (line 141) | def __init__(self, resnet_size, data_format=None, num_classes=_NUM_CLA...
function cifar10_model_fn (line 182) | def cifar10_model_fn(features, labels, mode, params):
function define_cifar_flags (line 220) | def define_cifar_flags():
function run_cifar (line 231) | def run_cifar(flags_obj):
function main (line 245) | def main(_):
FILE: official/resnet/cifar10_test.py
class BaseTest (line 36) | class BaseTest(tf.test.TestCase):
method setUpClass (line 41) | def setUpClass(cls): # pylint: disable=invalid-name
method tearDown (line 45) | def tearDown(self):
method test_dataset_input_fn (line 49) | def test_dataset_input_fn(self):
method cifar10_model_fn_helper (line 79) | def cifar10_model_fn_helper(self, mode, resnet_version, dtype):
method test_cifar10_model_fn_train_mode_v1 (line 113) | def test_cifar10_model_fn_train_mode_v1(self):
method test_cifar10_model_fn_trainmode__v2 (line 117) | def test_cifar10_model_fn_trainmode__v2(self):
method test_cifar10_model_fn_eval_mode_v1 (line 121) | def test_cifar10_model_fn_eval_mode_v1(self):
method test_cifar10_model_fn_eval_mode_v2 (line 125) | def test_cifar10_model_fn_eval_mode_v2(self):
method test_cifar10_model_fn_predict_mode_v1 (line 129) | def test_cifar10_model_fn_predict_mode_v1(self):
method test_cifar10_model_fn_predict_mode_v2 (line 133) | def test_cifar10_model_fn_predict_mode_v2(self):
method _test_cifar10model_shape (line 137) | def _test_cifar10model_shape(self, resnet_version):
method test_cifar10model_shape_v1 (line 149) | def test_cifar10model_shape_v1(self):
method test_cifar10model_shape_v2 (line 152) | def test_cifar10model_shape_v2(self):
method test_cifar10_end_to_end_synthetic_v1 (line 155) | def test_cifar10_end_to_end_synthetic_v1(self):
method test_cifar10_end_to_end_synthetic_v2 (line 161) | def test_cifar10_end_to_end_synthetic_v2(self):
method test_flag_restriction (line 167) | def test_flag_restriction(self):
FILE: official/resnet/imagenet_main.py
function get_filenames (line 49) | def get_filenames(is_training, data_dir):
function _parse_example_proto (line 61) | def _parse_example_proto(example_serialized):
function parse_record (line 131) | def parse_record(raw_record, is_training):
function input_fn (line 158) | def input_fn(is_training, data_dir, batch_size, num_epochs=1):
function get_synth_input_fn (line 191) | def get_synth_input_fn():
class ImagenetModel (line 199) | class ImagenetModel(resnet_model.Model):
method __init__ (line 202) | def __init__(self, resnet_size, data_format=None, num_classes=_NUM_CLA...
function _get_block_sizes (line 244) | def _get_block_sizes(resnet_size):
function imagenet_model_fn (line 278) | def imagenet_model_fn(features, labels, mode, params):
function define_imagenet_flags (line 302) | def define_imagenet_flags():
function run_imagenet (line 309) | def run_imagenet(flags_obj):
function main (line 323) | def main(_):
FILE: official/resnet/imagenet_preprocessing.py
function _decode_crop_and_flip (line 51) | def _decode_crop_and_flip(image_buffer, bbox, num_channels):
function _central_crop (line 100) | def _central_crop(image, crop_height, crop_width):
function _mean_image_subtraction (line 122) | def _mean_image_subtraction(image, means, num_channels):
function _smallest_size_at_least (line 156) | def _smallest_size_at_least(height, width, resize_min):
function _aspect_preserving_resize (line 187) | def _aspect_preserving_resize(image, resize_min):
function _resize_image (line 206) | def _resize_image(image, height, width):
function preprocess_image (line 226) | def preprocess_image(image_buffer, bbox, output_height, output_width,
FILE: official/resnet/imagenet_test.py
class BaseTest (line 33) | class BaseTest(tf.test.TestCase):
method setUpClass (line 36) | def setUpClass(cls): # pylint: disable=invalid-name
method tearDown (line 40) | def tearDown(self):
method _tensor_shapes_helper (line 44) | def _tensor_shapes_helper(self, resnet_size, resnet_version, dtype, wi...
method tensor_shapes_helper (line 98) | def tensor_shapes_helper(self, resnet_size, resnet_version, with_gpu=F...
method test_tensor_shapes_resnet_18_v1 (line 106) | def test_tensor_shapes_resnet_18_v1(self):
method test_tensor_shapes_resnet_18_v2 (line 109) | def test_tensor_shapes_resnet_18_v2(self):
method test_tensor_shapes_resnet_34_v1 (line 112) | def test_tensor_shapes_resnet_34_v1(self):
method test_tensor_shapes_resnet_34_v2 (line 115) | def test_tensor_shapes_resnet_34_v2(self):
method test_tensor_shapes_resnet_50_v1 (line 118) | def test_tensor_shapes_resnet_50_v1(self):
method test_tensor_shapes_resnet_50_v2 (line 121) | def test_tensor_shapes_resnet_50_v2(self):
method test_tensor_shapes_resnet_101_v1 (line 124) | def test_tensor_shapes_resnet_101_v1(self):
method test_tensor_shapes_resnet_101_v2 (line 127) | def test_tensor_shapes_resnet_101_v2(self):
method test_tensor_shapes_resnet_152_v1 (line 130) | def test_tensor_shapes_resnet_152_v1(self):
method test_tensor_shapes_resnet_152_v2 (line 133) | def test_tensor_shapes_resnet_152_v2(self):
method test_tensor_shapes_resnet_200_v1 (line 136) | def test_tensor_shapes_resnet_200_v1(self):
method test_tensor_shapes_resnet_200_v2 (line 139) | def test_tensor_shapes_resnet_200_v2(self):
method test_tensor_shapes_resnet_18_with_gpu_v1 (line 143) | def test_tensor_shapes_resnet_18_with_gpu_v1(self):
method test_tensor_shapes_resnet_18_with_gpu_v2 (line 147) | def test_tensor_shapes_resnet_18_with_gpu_v2(self):
method test_tensor_shapes_resnet_34_with_gpu_v1 (line 151) | def test_tensor_shapes_resnet_34_with_gpu_v1(self):
method test_tensor_shapes_resnet_34_with_gpu_v2 (line 155) | def test_tensor_shapes_resnet_34_with_gpu_v2(self):
method test_tensor_shapes_resnet_50_with_gpu_v1 (line 159) | def test_tensor_shapes_resnet_50_with_gpu_v1(self):
method test_tensor_shapes_resnet_50_with_gpu_v2 (line 163) | def test_tensor_shapes_resnet_50_with_gpu_v2(self):
method test_tensor_shapes_resnet_101_with_gpu_v1 (line 167) | def test_tensor_shapes_resnet_101_with_gpu_v1(self):
method test_tensor_shapes_resnet_101_with_gpu_v2 (line 171) | def test_tensor_shapes_resnet_101_with_gpu_v2(self):
method test_tensor_shapes_resnet_152_with_gpu_v1 (line 175) | def test_tensor_shapes_resnet_152_with_gpu_v1(self):
method test_tensor_shapes_resnet_152_with_gpu_v2 (line 179) | def test_tensor_shapes_resnet_152_with_gpu_v2(self):
method test_tensor_shapes_resnet_200_with_gpu_v1 (line 183) | def test_tensor_shapes_resnet_200_with_gpu_v1(self):
method test_tensor_shapes_resnet_200_with_gpu_v2 (line 187) | def test_tensor_shapes_resnet_200_with_gpu_v2(self):
method resnet_model_fn_helper (line 190) | def resnet_model_fn_helper(self, mode, resnet_version, dtype):
method test_resnet_model_fn_train_mode_v1 (line 227) | def test_resnet_model_fn_train_mode_v1(self):
method test_resnet_model_fn_train_mode_v2 (line 231) | def test_resnet_model_fn_train_mode_v2(self):
method test_resnet_model_fn_eval_mode_v1 (line 235) | def test_resnet_model_fn_eval_mode_v1(self):
method test_resnet_model_fn_eval_mode_v2 (line 239) | def test_resnet_model_fn_eval_mode_v2(self):
method test_resnet_model_fn_predict_mode_v1 (line 243) | def test_resnet_model_fn_predict_mode_v1(self):
method test_resnet_model_fn_predict_mode_v2 (line 247) | def test_resnet_model_fn_predict_mode_v2(self):
method _test_imagenetmodel_shape (line 251) | def _test_imagenetmodel_shape(self, resnet_version):
method test_imagenetmodel_shape_v1 (line 264) | def test_imagenetmodel_shape_v1(self):
method test_imagenetmodel_shape_v2 (line 267) | def test_imagenetmodel_shape_v2(self):
method test_imagenet_end_to_end_synthetic_v1 (line 270) | def test_imagenet_end_to_end_synthetic_v1(self):
method test_imagenet_end_to_end_synthetic_v2 (line 276) | def test_imagenet_end_to_end_synthetic_v2(self):
method test_imagenet_end_to_end_synthetic_v1_tiny (line 282) | def test_imagenet_end_to_end_synthetic_v1_tiny(self):
method test_imagenet_end_to_end_synthetic_v2_tiny (line 288) | def test_imagenet_end_to_end_synthetic_v2_tiny(self):
method test_imagenet_end_to_end_synthetic_v1_huge (line 294) | def test_imagenet_end_to_end_synthetic_v1_huge(self):
method test_imagenet_end_to_end_synthetic_v2_huge (line 300) | def test_imagenet_end_to_end_synthetic_v2_huge(self):
method test_flag_restriction (line 306) | def test_flag_restriction(self):
FILE: official/resnet/layer_test.py
class BaseTest (line 63) | class BaseTest(reference_data.BaseTest):
method test_name (line 67) | def test_name(self):
method _batch_norm_ops (line 70) | def _batch_norm_ops(self, test=False):
method make_projection (line 88) | def make_projection(self, filters_out, strides, data_format):
method _resnet_block_ops (line 105) | def _resnet_block_ops(self, test, batch_size, bottleneck, projection,
method test_batch_norm (line 169) | def test_batch_norm(self):
method test_block_0 (line 172) | def test_block_0(self):
method test_block_1 (line 175) | def test_block_1(self):
method test_block_2 (line 178) | def test_block_2(self):
method test_block_3 (line 181) | def test_block_3(self):
method test_block_4 (line 184) | def test_block_4(self):
method test_block_5 (line 187) | def test_block_5(self):
method test_block_6 (line 190) | def test_block_6(self):
method test_block_7 (line 193) | def test_block_7(self):
method regenerate (line 196) | def regenerate(self):
FILE: official/resnet/resnet_model.py
function batch_norm (line 47) | def batch_norm(inputs, training, data_format):
function fixed_padding (line 57) | def fixed_padding(inputs, kernel_size, data_format):
function conv2d_fixed_padding (line 84) | def conv2d_fixed_padding(inputs, filters, kernel_size, strides, data_for...
function _building_block_v1 (line 101) | def _building_block_v1(inputs, filters, training, projection_shortcut, s...
function _building_block_v2 (line 148) | def _building_block_v2(inputs, filters, training, projection_shortcut, s...
function _bottleneck_block_v1 (line 194) | def _bottleneck_block_v1(inputs, filters, training, projection_shortcut,
function _bottleneck_block_v2 (line 249) | def _bottleneck_block_v2(inputs, filters, training, projection_shortcut,
function block_layer (line 309) | def block_layer(inputs, filters, bottleneck, block_fn, blocks, strides,
class Model (line 350) | class Model(object):
method __init__ (line 353) | def __init__(self, resnet_size, bottleneck, num_classes, num_filters,
method _custom_dtype_getter (line 429) | def _custom_dtype_getter(self, getter, name, shape=None, dtype=DEFAULT...
method _model_variable_scope (line 470) | def _model_variable_scope(self):
method __call__ (line 483) | def __call__(self, inputs, training):
FILE: official/resnet/resnet_run_loop.py
function process_record_dataset (line 44) | def process_record_dataset(dataset, is_training, batch_size, shuffle_buf...
function get_synth_input_fn (line 96) | def get_synth_input_fn(height, width, num_channels, num_classes):
function learning_rate_with_decay (line 124) | def learning_rate_with_decay(
function resnet_model_fn (line 159) | def resnet_model_fn(features, labels, mode, model_class,
function per_device_batch_size (line 306) | def per_device_batch_size(batch_size, num_gpus):
function resnet_main (line 337) | def resnet_main(
function define_resnet_flags (line 452) | def define_resnet_flags(resnet_size_choices=None):
FILE: official/transformer/compute_bleu.py
class UnicodeRegex (line 40) | class UnicodeRegex(object):
method __init__ (line 43) | def __init__(self):
method property_chars (line 49) | def property_chars(self, prefix):
function bleu_tokenize (line 57) | def bleu_tokenize(string):
function bleu_wrapper (line 87) | def bleu_wrapper(ref_filename, hyp_filename, case_sensitive=False):
function main (line 103) | def main(unused_argv):
function define_compute_bleu_flags (line 113) | def define_compute_bleu_flags():
FILE: official/transformer/compute_bleu_test.py
class ComputeBleuTest (line 25) | class ComputeBleuTest(unittest.TestCase):
method _create_temp_file (line 27) | def _create_temp_file(self, text):
method test_bleu_same (line 33) | def test_bleu_same(self):
method test_bleu_same_different_case (line 42) | def test_bleu_same_different_case(self):
method test_bleu_different (line 50) | def test_bleu_different(self):
method test_bleu_tokenize (line 58) | def test_bleu_tokenize(self):
FILE: official/transformer/data_download.py
function find_file (line 88) | def find_file(path, filename, max_depth=5):
function get_raw_files (line 104) | def get_raw_files(raw_dir, data_source):
function download_report_hook (line 132) | def download_report_hook(count, block_size, total_size):
function download_from_url (line 144) | def download_from_url(path, url):
function download_and_extract (line 171) | def download_and_extract(path, url, input_filename, target_filename):
function txt_line_iterator (line 212) | def txt_line_iterator(path):
function compile_files (line 219) | def compile_files(raw_dir, raw_files, tag):
function write_file (line 250) | def write_file(writer, filename):
function encode_and_save_files (line 260) | def encode_and_save_files(
function shard_filename (line 310) | def shard_filename(path, tag, shard_num, total_shards):
function shuffle_records (line 316) | def shuffle_records(fname):
function dict_to_example (line 343) | def dict_to_example(dictionary):
function all_exist (line 351) | def all_exist(filepaths):
function make_dir (line 359) | def make_dir(path):
function main (line 365) | def main(unused_argv):
function define_data_download_flags (line 400) | def define_data_download_flags():
FILE: official/transformer/model/attention_layer.py
class Attention (line 24) | class Attention(tf.layers.Layer):
method __init__ (line 27) | def __init__(self, hidden_size, num_heads, attention_dropout, train):
method split_heads (line 46) | def split_heads(self, x):
method combine_heads (line 71) | def combine_heads(self, x):
method call (line 86) | def call(self, x, y, bias, cache=None):
class SelfAttention (line 144) | class SelfAttention(Attention):
method call (line 147) | def call(self, x, bias, cache=None):
FILE: official/transformer/model/beam_search.py
class _StateKeys (line 28) | class _StateKeys(object):
class SequenceBeamSearch (line 59) | class SequenceBeamSearch(object):
method __init__ (line 62) | def __init__(self, symbols_to_logits_fn, vocab_size, batch_size,
method search (line 72) | def search(self, initial_ids, initial_cache):
method _create_initial_state (line 96) | def _create_initial_state(self, initial_ids, initial_cache):
method _continue_search (line 164) | def _continue_search(self, state):
method _search_step (line 210) | def _search_step(self, state):
method _grow_alive_seq (line 242) | def _grow_alive_seq(self, state):
method _get_new_alive_state (line 304) | def _get_new_alive_state(self, new_seq, new_log_probs, new_cache):
method _get_new_finished_state (line 334) | def _get_new_finished_state(self, state, new_seq, new_log_probs):
function sequence_beam_search (line 386) | def sequence_beam_search(
function _log_prob_from_logits (line 419) | def _log_prob_from_logits(logits):
function _length_normalization (line 423) | def _length_normalization(alpha, length):
function _expand_to_beam_size (line 428) | def _expand_to_beam_size(tensor, beam_size):
function _shape_list (line 445) | def _shape_list(tensor):
function _get_shape_keep_last_dim (line 458) | def _get_shape_keep_last_dim(tensor):
function _flatten_beam_dim (line 470) | def _flatten_beam_dim(tensor):
function _unflatten_beam_dim (line 485) | def _unflatten_beam_dim(tensor, batch_size, beam_size):
function _gather_beams (line 501) | def _gather_beams(nested, beam_indices, batch_size, new_beam_size):
function _gather_topk_beams (line 538) | def _gather_topk_beams(nested, score_or_log_prob, batch_size, beam_size):
FILE: official/transformer/model/beam_search_test.py
class BeamSearchHelperTests (line 26) | class BeamSearchHelperTests(tf.test.TestCase):
method test_expand_to_beam_size (line 28) | def test_expand_to_beam_size(self):
method test_shape_list (line 35) | def test_shape_list(self):
method test_get_shape_keep_last_dim (line 44) | def test_get_shape_keep_last_dim(self):
method test_flatten_beam_dim (line 51) | def test_flatten_beam_dim(self):
method test_unflatten_beam_dim (line 58) | def test_unflatten_beam_dim(self):
method test_gather_beams (line 65) | def test_gather_beams(self):
method test_gather_topk_beams (line 85) | def test_gather_topk_beams(self):
FILE: official/transformer/model/embedding_layer.py
class EmbeddingSharedWeights (line 26) | class EmbeddingSharedWeights(tf.layers.Layer):
method __init__ (line 29) | def __init__(self, vocab_size, hidden_size):
method build (line 34) | def build(self, _):
method call (line 45) | def call(self, x):
method linear (line 69) | def linear(self, x):
FILE: official/transformer/model/ffn_layer.py
class FeedFowardNetwork (line 24) | class FeedFowardNetwork(tf.layers.Layer):
method __init__ (line 27) | def __init__(self, hidden_size, filter_size, relu_dropout, train):
method call (line 39) | def call(self, x, padding=None):
FILE: official/transformer/model/model_params.py
class TransformerBaseParams (line 18) | class TransformerBaseParams(object):
class TransformerBigParams (line 54) | class TransformerBigParams(TransformerBaseParams):
FILE: official/transformer/model/model_utils.py
function get_position_encoding (line 28) | def get_position_encoding(
function get_decoder_self_attention_bias (line 57) | def get_decoder_self_attention_bias(length):
function get_padding (line 77) | def get_padding(x, padding_value=0):
function get_padding_bias (line 92) | def get_padding_bias(x):
FILE: official/transformer/model/model_utils_test.py
class ModelUtilsTest (line 28) | class ModelUtilsTest(tf.test.TestCase):
method test_get_padding (line 30) | def test_get_padding(self):
method test_get_padding_bias (line 39) | def test_get_padding_bias(self):
method test_get_decoder_self_attention_bias (line 53) | def test_get_decoder_self_attention_bias(self):
FILE: official/transformer/model/transformer.py
class Transformer (line 37) | class Transformer(object):
method __init__ (line 48) | def __init__(self, params, train):
method __call__ (line 64) | def __call__(self, inputs, targets=None):
method encode (line 100) | def encode(self, inputs, attention_bias):
method decode (line 128) | def decode(self, targets, encoder_outputs, attention_bias):
method _get_symbols_to_logits_fn (line 166) | def _get_symbols_to_logits_fn(self, max_decode_length):
method predict (line 205) | def predict(self, encoder_outputs, encoder_decoder_attention_bias):
class LayerNormalization (line 245) | class LayerNormalization(tf.layers.Layer):
method __init__ (line 248) | def __init__(self, hidden_size):
method build (line 252) | def build(self, _):
method call (line 259) | def call(self, x, epsilon=1e-6):
class PrePostProcessingWrapper (line 266) | class PrePostProcessingWrapper(object):
method __init__ (line 269) | def __init__(self, layer, params, train):
method __call__ (line 277) | def __call__(self, x, *args, **kwargs):
class EncoderStack (line 290) | class EncoderStack(tf.layers.Layer):
method __init__ (line 299) | def __init__(self, params, train):
method call (line 316) | def call(self, encoder_inputs, attention_bias, inputs_padding):
class DecoderStack (line 343) | class DecoderStack(tf.layers.Layer):
method __init__ (line 354) | def __init__(self, params, train):
method call (line 372) | def call(self, decoder_inputs, encoder_outputs, decoder_self_attention...
FILE: official/transformer/transformer_main.py
function model_fn (line 64) | def model_fn(features, labels, mode, params):
function get_learning_rate (line 103) | def get_learning_rate(learning_rate, hidden_size, learning_rate_warmup_s...
function get_train_op (line 125) | def get_train_op(loss, params):
function translate_and_compute_bleu (line 155) | def translate_and_compute_bleu(estimator, subtokenizer, bleu_source, ble...
function get_global_step (line 172) | def get_global_step(estimator):
function evaluate_and_log_bleu (line 177) | def evaluate_and_log_bleu(estimator, bleu_source, bleu_ref, vocab_file_p...
function train_schedule (line 189) | def train_schedule(
function define_transformer_flags (line 315) | def define_transformer_flags():
function run_transformer (line 407) | def run_transformer(flags_obj):
function main (line 463) | def main(_):
FILE: official/transformer/translate.py
function _get_sorted_inputs (line 41) | def _get_sorted_inputs(filename):
function _encode_and_add_eos (line 67) | def _encode_and_add_eos(line, subtokenizer):
function _trim_and_decode (line 72) | def _trim_and_decode(ids, subtokenizer):
function translate_file (line 81) | def translate_file(
function translate_text (line 141) | def translate_text(estimator, subtokenizer, txt):
function main (line 156) | def main(unused_argv):
function define_translate_flags (line 197) | def define_translate_flags():
FILE: official/transformer/utils/dataset.py
function _load_records (line 70) | def _load_records(filename):
function _parse_example (line 75) | def _parse_example(serialized_example):
function _filter_max_length (line 87) | def _filter_max_length(example, max_length=256):
function _get_example_length (line 93) | def _get_example_length(example):
function _create_min_max_boundaries (line 99) | def _create_min_max_boundaries(
function _batch_examples (line 131) | def _batch_examples(dataset, batch_size, max_length):
function _read_and_batch_from_files (line 192) | def _read_and_batch_from_files(
function train_input_fn (line 237) | def train_input_fn(params):
function eval_input_fn (line 245) | def eval_input_fn(params):
FILE: official/transformer/utils/metrics.py
function _pad_tensors_to_same_length (line 39) | def _pad_tensors_to_same_length(x, y):
function padded_cross_entropy_loss (line 52) | def padded_cross_entropy_loss(logits, labels, smoothing, vocab_size):
function _convert_to_eval_metric (line 90) | def _convert_to_eval_metric(metric_fn):
function get_eval_metrics (line 112) | def get_eval_metrics(logits, labels, params):
function padded_accuracy (line 133) | def padded_accuracy(logits, labels):
function padded_accuracy_topk (line 143) | def padded_accuracy_topk(logits, labels, k):
function padded_accuracy_top5 (line 159) | def padded_accuracy_top5(logits, labels):
function padded_sequence_accuracy (line 163) | def padded_sequence_accuracy(logits, labels):
function padded_neg_log_perplexity (line 176) | def padded_neg_log_perplexity(logits, labels, vocab_size):
function bleu_score (line 182) | def bleu_score(logits, labels):
function _get_ngrams_with_counter (line 202) | def _get_ngrams_with_counter(segment, max_order):
function compute_bleu (line 222) | def compute_bleu(reference_corpus, translation_corpus, max_order=4,
function rouge_2_fscore (line 288) | def rouge_2_fscore(logits, labels):
function _get_ngrams (line 307) | def _get_ngrams(n, text):
function rouge_n (line 325) | def rouge_n(eval_sentences, ref_sentences, n=2):
function rouge_l_fscore (line 365) | def rouge_l_fscore(predictions, labels):
function rouge_l_sentence_level (line 384) | def rouge_l_sentence_level(eval_sentences, ref_sentences):
function _len_lcs (line 418) | def _len_lcs(x, y):
function _lcs (line 435) | def _lcs(x, y):
function _f_lcs (line 462) | def _f_lcs(llcs, m, n):
FILE: official/transformer/utils/tokenizer.py
class Subtokenizer (line 61) | class Subtokenizer(object):
method __init__ (line 64) | def __init__(self, vocab_file, reserved_tokens=None):
method init_from_files (line 84) | def init_from_files(
method encode (line 123) | def encode(self, raw_string, add_eos=False):
method _token_to_subtoken_ids (line 133) | def _token_to_subtoken_ids(self, token):
method decode (line 148) | def decode(self, subtokens):
method _subtoken_ids_to_tokens (line 164) | def _subtoken_ids_to_tokens(self, subtokens):
function _save_vocab_file (line 180) | def _save_vocab_file(vocab_file, subtoken_list):
function _load_vocab_file (line 187) | def _load_vocab_file(vocab_file, reserved_tokens=None):
function _native_to_unicode (line 203) | def _native_to_unicode(s):
function _unicode_to_native (line 211) | def _unicode_to_native(s):
function _split_string_to_tokens (line 219) | def _split_string_to_tokens(text):
function _join_tokens_to_string (line 238) | def _join_tokens_to_string(tokens):
function _escape_token (line 249) | def _escape_token(token, alphabet):
function _unescape_token (line 270) | def _unescape_token(token):
function _count_tokens (line 325) | def _count_tokens(files, file_byte_limit=1e6):
function _list_to_index_dict (line 362) | def _list_to_index_dict(lst):
function _split_token_to_subtokens (line 367) | def _split_token_to_subtokens(token, subtoken_dict, max_subtoken_length):
function _generate_subtokens_with_target_vocab_size (line 389) | def _generate_subtokens_with_target_vocab_size(
function _generate_alphabet_dict (line 433) | def _generate_alphabet_dict(iterable, reserved_tokens=None):
function _count_and_gen_subtokens (line 443) | def _count_and_gen_subtokens(
function _filter_and_bucket_subtokens (line 476) | def _filter_and_bucket_subtokens(subtoken_counts, min_count):
function _gen_new_subtoken_list (line 497) | def _gen_new_subtoken_list(
function _generate_subtokens (line 569) | def _generate_subtokens(
FILE: official/transformer/utils/tokenizer_test.py
class SubtokenizerTest (line 26) | class SubtokenizerTest(unittest.TestCase):
method _init_subtokenizer (line 28) | def _init_subtokenizer(self, vocab_list):
method test_encode (line 36) | def test_encode(self):
method test_decode (line 43) | def test_decode(self):
method test_subtoken_ids_to_tokens (line 50) | def test_subtoken_ids_to_tokens(self):
class StringHelperTest (line 58) | class StringHelperTest(unittest.TestCase):
method test_split_string_to_tokens (line 60) | def test_split_string_to_tokens(self):
method test_join_tokens_to_string (line 66) | def test_join_tokens_to_string(self):
method test_escape_token (line 72) | def test_escape_token(self):
method test_unescape_token (line 79) | def test_unescape_token(self):
method test_list_to_index_dict (line 86) | def test_list_to_index_dict(self):
method test_split_token_to_subtokens (line 92) | def test_split_token_to_subtokens(self):
method test_generate_alphabet_dict (line 101) | def test_generate_alphabet_dict(self):
method test_count_and_gen_subtokens (line 117) | def test_count_and_gen_subtokens(self):
method test_filter_and_bucket_subtokens (line 131) | def test_filter_and_bucket_subtokens(self):
method test_gen_new_subtoken_list (line 145) | def test_gen_new_subtoken_list(self):
method test_generate_subtokens (line 164) | def test_generate_subtokens(self):
FILE: official/utils/export/export.py
function build_tensor_serving_input_receiver_fn (line 24) | def build_tensor_serving_input_receiver_fn(shape, dtype=tf.float32,
FILE: official/utils/export/export_test.py
class ExportUtilsTest (line 26) | class ExportUtilsTest(tf.test.TestCase):
method test_build_tensor_serving_input_receiver_fn (line 29) | def test_build_tensor_serving_input_receiver_fn(self):
method test_build_tensor_serving_input_receiver_fn_batch_dtype (line 44) | def test_build_tensor_serving_input_receiver_fn_batch_dtype(self):
FILE: official/utils/flags/_base.py
function define_base (line 28) | def define_base(data_dir=True, model_dir=True, train_epochs=True,
function get_num_gpus (line 133) | def get_num_gpus(flags_obj):
FILE: official/utils/flags/_benchmark.py
function define_benchmark (line 26) | def define_benchmark(benchmark_log_dir=True, bigquery_uploader=True):
FILE: official/utils/flags/_misc.py
function define_image (line 26) | def define_image(data_format=True):
FILE: official/utils/flags/_performance.py
function get_tf_dtype (line 36) | def get_tf_dtype(flags_obj):
function get_loss_scale (line 40) | def get_loss_scale(flags_obj):
function define_performance (line 46) | def define_performance(num_parallel_calls=True, inter_op=True, intra_op=...
FILE: official/utils/flags/core.py
function set_defaults (line 37) | def set_defaults(**kwargs):
function parse_flags (line 42) | def parse_flags(argv=None):
function register_key_flags_in_core (line 48) | def register_key_flags_in_core(f):
FILE: official/utils/flags/flags_test.py
function define_flags (line 24) | def define_flags():
class BaseTester (line 31) | class BaseTester(unittest.TestCase):
method setUpClass (line 34) | def setUpClass(cls):
method test_default_setting (line 38) | def test_default_setting(self):
method test_benchmark_setting (line 61) | def test_benchmark_setting(self):
method test_booleans (line 74) | def test_booleans(self):
method test_parse_dtype_info (line 83) | def test_parse_dtype_info(self):
FILE: official/utils/logs/hooks.py
class ExamplesPerSecondHook (line 28) | class ExamplesPerSecondHook(tf.train.SessionRunHook):
method __init__ (line 37) | def __init__(self,
method begin (line 77) | def begin(self):
method before_run (line 84) | def before_run(self, run_context): # pylint: disable=unused-argument
method after_run (line 95) | def after_run(self, run_context, run_values): # pylint: disable=unuse...
FILE: official/utils/logs/hooks_helper.py
function get_train_hooks (line 38) | def get_train_hooks(name_list, **kwargs):
function get_logging_tensor_hook (line 68) | def get_logging_tensor_hook(every_n_iter=100, tensors_to_log=None, **kwa...
function get_profiler_hook (line 90) | def get_profiler_hook(save_steps=1000, **kwargs): # pylint: disable=unu...
function get_examples_per_second_hook (line 104) | def get_examples_per_second_hook(every_n_steps=100,
function get_logging_metric_hook (line 127) | def get_logging_metric_hook(tensors_to_log=None,
FILE: official/utils/logs/hooks_helper_test.py
class BaseTest (line 29) | class BaseTest(unittest.TestCase):
method test_raise_in_non_list_names (line 31) | def test_raise_in_non_list_names(self):
method test_raise_in_invalid_names (line 36) | def test_raise_in_invalid_names(self):
method validate_train_hook_name (line 41) | def validate_train_hook_name(self,
method test_get_train_hooks_logging_tensor_hook (line 51) | def test_get_train_hooks_logging_tensor_hook(self):
method test_get_train_hooks_profiler_hook (line 54) | def test_get_train_hooks_profiler_hook(self):
method test_get_train_hooks_examples_per_second_hook (line 57) | def test_get_train_hooks_examples_per_second_hook(self):
method test_get_logging_metric_hook (line 61) | def test_get_logging_metric_hook(self):
FILE: official/utils/logs/hooks_test.py
class ExamplesPerSecondHookTest (line 32) | class ExamplesPerSecondHookTest(tf.test.TestCase):
method setUp (line 45) | def setUp(self):
method test_raise_in_both_secs_and_steps (line 55) | def test_raise_in_both_secs_and_steps(self):
method test_raise_in_none_secs_and_steps (line 63) | def test_raise_in_none_secs_and_steps(self):
method _validate_log_every_n_steps (line 71) | def _validate_log_every_n_steps(self, every_n_steps, warm_steps):
method test_examples_per_sec_every_1_steps (line 109) | def test_examples_per_sec_every_1_steps(self):
method test_examples_per_sec_every_5_steps (line 113) | def test_examples_per_sec_every_5_steps(self):
method test_examples_per_sec_every_1_steps_with_warm_steps (line 117) | def test_examples_per_sec_every_1_steps_with_warm_steps(self):
method test_examples_per_sec_every_5_steps_with_warm_steps (line 121) | def test_examples_per_sec_every_5_steps_with_warm_steps(self):
method _validate_log_every_n_secs (line 125) | def _validate_log_every_n_secs(self, every_n_secs):
method test_examples_per_sec_every_1_secs (line 146) | def test_examples_per_sec_every_1_secs(self):
method test_examples_per_sec_every_5_secs (line 150) | def test_examples_per_sec_every_5_secs(self):
method _assert_metrics (line 154) | def _assert_metrics(self):
FILE: official/utils/logs/logger.py
function config_benchmark_logger (line 49) | def config_benchmark_logger(flag_obj=None):
function get_benchmark_logger (line 80) | def get_benchmark_logger():
class BaseBenchmarkLogger (line 86) | class BaseBenchmarkLogger(object):
method log_evaluation_result (line 89) | def log_evaluation_result(self, eval_results):
method log_metric (line 108) | def log_metric(self, name, value, unit=None, global_step=None, extras=...
method log_run_info (line 126) | def log_run_info(self, model_name, dataset_name, run_params):
class BenchmarkFileLogger (line 131) | class BenchmarkFileLogger(BaseBenchmarkLogger):
method __init__ (line 134) | def __init__(self, logging_dir):
method log_metric (line 140) | def log_metric(self, name, value, unit=None, global_step=None, extras=...
method log_run_info (line 165) | def log_run_info(self, model_name, dataset_name, run_params):
class BenchmarkBigQueryLogger (line 188) | class BenchmarkBigQueryLogger(BaseBenchmarkLogger):
method __init__ (line 191) | def __init__(self,
method log_metric (line 204) | def log_metric(self, name, value, unit=None, global_step=None, extras=...
method log_run_info (line 228) | def log_run_info(self, model_name, dataset_name, run_params):
function _gather_run_info (line 251) | def _gather_run_info(model_name, dataset_name, run_params):
function _process_metric_to_json (line 268) | def _process_metric_to_json(
function _collect_tensorflow_info (line 287) | def _collect_tensorflow_info(run_info):
function _collect_run_params (line 292) | def _collect_run_params(run_info, run_params):
function _collect_tensorflow_environment_variables (line 308) | def _collect_tensorflow_environment_variables(run_info):
function _collect_cpu_info (line 316) | def _collect_cpu_info(run_info):
function _collect_gpu_info (line 336) | def _collect_gpu_info(run_info):
function _collect_memory_info (line 354) | def _collect_memory_info(run_info):
function _parse_gpu_model (line 366) | def _parse_gpu_model(physical_device_desc):
function _convert_to_json_dict (line 375) | def _convert_to_json_dict(input_dict):
FILE: official/utils/logs/logger_test.py
class BenchmarkLoggerTest (line 41) | class BenchmarkLoggerTest(tf.test.TestCase):
method setUpClass (line 44) | def setUpClass(cls): # pylint: disable=invalid-name
method test_get_default_benchmark_logger (line 48) | def test_get_default_benchmark_logger(self):
method test_config_base_benchmark_logger (line 53) | def test_config_base_benchmark_logger(self):
method test_config_benchmark_file_logger (line 59) | def test_config_benchmark_file_logger(self):
method test_config_benchmark_bigquery_logger (line 69) | def test_config_benchmark_bigquery_logger(self):
class BaseBenchmarkLoggerTest (line 76) | class BaseBenchmarkLoggerTest(tf.test.TestCase):
method setUp (line 78) | def setUp(self):
method tearDown (line 89) | def tearDown(self):
method test_log_metric (line 93) | def test_log_metric(self):
class BenchmarkFileLoggerTest (line 101) | class BenchmarkFileLoggerTest(tf.test.TestCase):
method setUp (line 103) | def setUp(self):
method tearDown (line 111) | def tearDown(self):
method test_create_logging_dir (line 117) | def test_create_logging_dir(self):
method test_log_metric (line 124) | def test_log_metric(self):
method test_log_multiple_metrics (line 139) | def test_log_multiple_metrics(self):
method test_log_non_number_value (line 162) | def test_log_non_number_value(self):
method test_log_evaluation_result (line 171) | def test_log_evaluation_result(self):
method test_log_evaluation_result_with_invalid_type (line 194) | def test_log_evaluation_result_with_invalid_type(self):
method test_collect_tensorflow_info (line 203) | def test_collect_tensorflow_info(self):
method test_collect_run_params (line 210) | def test_collect_run_params(self):
method test_collect_tensorflow_environment_variables (line 236) | def test_collect_tensorflow_environment_variables(self):
method test_collect_gpu_info (line 252) | def test_collect_gpu_info(self):
method test_collect_memory_info (line 257) | def test_collect_memory_info(self):
class BenchmarkBigQueryLoggerTest (line 265) | class BenchmarkBigQueryLoggerTest(tf.test.TestCase):
method setUp (line 267) | def setUp(self):
method tearDown (line 280) | def tearDown(self):
method test_log_metric (line 286) | def test_log_metric(self):
FILE: official/utils/logs/metric_hook.py
class LoggingMetricHook (line 24) | class LoggingMetricHook(tf.train.LoggingTensorHook):
method __init__ (line 36) | def __init__(self, tensors, metric_logger=None,
method begin (line 69) | def begin(self):
method after_run (line 79) | def after_run(self, unused_run_context, run_values):
method end (line 87) | def end(self, session):
method _log_metric (line 92) | def _log_metric(self, tensor_values):
FILE: official/utils/logs/metric_hook_test.py
class LoggingMetricHookTest (line 31) | class LoggingMetricHookTest(tf.test.TestCase):
method setUp (line 34) | def setUp(self):
method tearDown (line 40) | def tearDown(self):
method test_illegal_args (line 44) | def test_illegal_args(self):
method test_print_at_end_only (line 57) | def test_print_at_end_only(self):
method test_global_step_not_found (line 80) | def test_global_step_not_found(self):
method test_log_tensors (line 90) | def test_log_tensors(self):
method _validate_print_every_n_steps (line 120) | def _validate_print_every_n_steps(self, sess, at_end):
method test_print_every_n_steps (line 155) | def test_print_every_n_steps(self):
method test_print_every_n_steps_and_end (line 162) | def test_print_every_n_steps_and_end(self):
method _validate_print_every_n_secs (line 169) | def _validate_print_every_n_secs(self, sess, at_end):
method test_print_every_n_secs (line 201) | def test_print_every_n_secs(self):
method test_print_every_n_secs_and_end (line 208) | def test_print_every_n_secs_and_end(self):
FILE: official/utils/misc/model_helpers.py
function past_stop_threshold (line 26) | def past_stop_threshold(stop_threshold, eval_metric):
FILE: official/utils/misc/model_helpers_test.py
class PastStopThresholdTest (line 26) | class PastStopThresholdTest(tf.test.TestCase):
method test_past_stop_threshold (line 29) | def test_past_stop_threshold(self):
method test_past_stop_threshold_none_false (line 39) | def test_past_stop_threshold_none_false(self):
method test_past_stop_threshold_not_number (line 47) | def test_past_stop_threshold_not_number(self):
FILE: official/utils/testing/integration.py
function run_synthetic (line 32) | def run_synthetic(main, tmp_root, extra_flags=None, synth=True, max_trai...
FILE: official/utils/testing/mock_lib.py
class MockBenchmarkLogger (line 23) | class MockBenchmarkLogger(object):
method __init__ (line 26) | def __init__(self):
method log_metric (line 29) | def log_metric(self, name, value, unit=None, global_step=None,
FILE: official/utils/testing/reference_data.py
class BaseTest (line 62) | class BaseTest(tf.test.TestCase):
method regenerate (line 65) | def regenerate(self):
method test_name (line 70) | def test_name(self):
method data_root (line 75) | def data_root(self):
method name_to_seed (line 87) | def name_to_seed(name):
method common_tensor_properties (line 105) | def common_tensor_properties(input_array):
method default_correctness_function (line 126) | def default_correctness_function(self, *args):
method _construct_and_save_reference_files (line 145) | def _construct_and_save_reference_files(
method _evaluate_test_case (line 200) | def _evaluate_test_case(self, name, graph, ops_to_eval, correctness_fu...
method _save_or_test_ops (line 269) | def _save_or_test_ops(self, name, graph, ops_to_eval=None, test=True,
class ReferenceDataActionParser (line 310) | class ReferenceDataActionParser(argparse.ArgumentParser):
method __init__ (line 313) | def __init__(self):
function main (line 323) | def main(argv, test_class):
FILE: official/utils/testing/reference_data_test.py
class GoldenBaseTest (line 38) | class GoldenBaseTest(reference_data.BaseTest):
method test_name (line 42) | def test_name(self):
method _uniform_random_ops (line 45) | def _uniform_random_ops(self, test=False, wrong_name=False, wrong_shap...
method _dense_ops (line 84) | def _dense_ops(self, test=False):
method test_uniform_random (line 102) | def test_uniform_random(self):
method test_tensor_name_error (line 105) | def test_tensor_name_error(self):
method test_tensor_shape_error (line 109) | def test_tensor_shape_error(self):
method test_bad_seed (line 115) | def test_bad_seed(self):
method test_incorrectness_function (line 120) | def test_incorrectness_function(self):
method test_dense (line 124) | def test_dense(self):
method regenerate (line 127) | def regenerate(self):
FILE: official/wide_deep/data_download.py
function _download_and_clean_file (line 41) | def _download_and_clean_file(filename, url):
function main (line 58) | def main(_):
FILE: official/wide_deep/wide_deep.py
function define_wide_deep_flags (line 52) | def define_wide_deep_flags():
function build_model_columns (line 71) | def build_model_columns():
function build_estimator (line 141) | def build_estimator(model_dir, model_type):
function input_fn (line 171) | def input_fn(data_file, num_epochs, shuffle, batch_size):
function export_model (line 199) | def export_model(model, model_type, export_dir):
function run_wide_deep (line 220) | def run_wide_deep(flags_obj):
function main (line 282) | def main(_):
FILE: official/wide_deep/wide_deep_test.py
class BaseTest (line 48) | class BaseTest(tf.test.TestCase):
method setUpClass (line 52) | def setUpClass(cls): # pylint: disable=invalid-name
method setUp (line 56) | def setUp(self):
method test_input_fn (line 71) | def test_input_fn(self):
method build_and_test_estimator (line 92) | def build_and_test_estimator(self, model_type):
method test_wide_deep_estimator_training (line 121) | def test_wide_deep_estimator_training(self):
method test_end_to_end_wide (line 124) | def test_end_to_end_wide(self):
method test_end_to_end_deep (line 132) | def test_end_to_end_deep(self):
method test_end_to_end_wide_deep (line 140) | def test_end_to_end_wide_deep(self):
FILE: projects/ai2018/binary/algos/loss.py
function calc_loss (line 29) | def calc_loss(y, y_, weights, training=False):
function calc_hier_loss (line 80) | def calc_hier_loss(y, y_, weights):
function calc_hier_neu_loss (line 93) | def calc_hier_neu_loss(y, y_, weights):
function calc_add_binary_loss (line 114) | def calc_add_binary_loss(y, y_, cids, weights):
function calc_binary_loss (line 124) | def calc_binary_loss(y, y_, cid, weights):
function calc_regression_loss (line 129) | def calc_regression_loss(y, y_, weights):
function calc_add_binaries_loss (line 133) | def calc_add_binaries_loss(y, y_, cid, weights):
function calc_binaries_only_loss (line 142) | def calc_binaries_only_loss(y, y_, cid, weights):
function criterion (line 155) | def criterion(model, x, y, training=False):
FILE: projects/ai2018/binary/algos/model.py
class ModelBase (line 38) | class ModelBase(melt.Model):
method __init__ (line 39) | def __init__(self, embedding=None, lm_model=False, use_text_encoder=Tr...
method unk_aug (line 134) | def unk_aug(self, x, x_mask=None, training=False):
class BiLanguageModel (line 167) | class BiLanguageModel(ModelBase):
method __init__ (line 168) | def __init__(self, embedding=None, lm_model=True):
method call (line 171) | def call(self, input, training=False):
class RNet (line 174) | class RNet(ModelBase):
method __init__ (line 175) | def __init__(self, embedding=None, lm_model=False):
method call (line 202) | def call(self, input, training=False):
class RNetV2 (line 270) | class RNetV2(RNet):
method __init__ (line 271) | def __init__(self, embedding=None, lm_model=False):
method call (line 301) | def call(self, input, training=False):
class RNetV3 (line 364) | class RNetV3(RNet):
method __init__ (line 365) | def __init__(self, embedding=None):
class RNetV4 (line 391) | class RNetV4(RNet):
method __init__ (line 392) | def __init__(self, embedding=None):
class MReader (line 418) | class MReader(ModelBase):
method __init__ (line 419) | def __init__(self, embedding=None):
method call (line 451) | def call(self, input, training=False):
class Transformer (line 505) | class Transformer(ModelBase):
method __init__ (line 506) | def __init__(self, embedding=None):
method restore (line 550) | def restore(self):
method call (line 566) | def call(self, input, training=False):
FILE: projects/ai2018/binary/algos/weights.py
function no_weights (line 28) | def no_weights():
function get_pos (line 31) | def get_pos(aspect):
function parse_weights (line 63) | def parse_weights():
function get_weights (line 82) | def get_weights(aspect, attr_index=None):
FILE: projects/ai2018/binary/dataset.py
class Dataset (line 34) | class Dataset(melt.tfrecords.Dataset):
method __init__ (line 35) | def __init__(self, subset='train'):
method parser (line 38) | def parser(self, example):
FILE: projects/ai2018/binary/evaluate.py
function init (line 58) | def init():
function calc_loss (line 69) | def calc_loss(labels, predicts):
function calc_auc (line 80) | def calc_auc(labels, predicts):
function evaluate (line 90) | def evaluate(labels, predicts):
FILE: projects/ai2018/binary/prepare/gen-records.py
function get_mode (line 84) | def get_mode(path):
function build_features (line 110) | def build_features(index):
function main (line 211) | def main(_):
FILE: projects/ai2018/binary/prepare/text2ids.py
function text2ids (line 31) | def text2ids(text, preprocess=True, return_words=False):
FILE: projects/ai2018/binary/read-records.py
function deal (line 52) | def deal(dataset, infos):
function main (line 64) | def main(_):
FILE: projects/ai2018/binary/torch-train.py
function main (line 38) | def main(_):
FILE: projects/ai2018/binary/torch_algos/loss.py
class Criterion (line 30) | class Criterion(object):
method __init__ (line 31) | def __init__(self):
method forward (line 34) | def forward(self, model, x, y, training=False):
FILE: projects/ai2018/binary/torch_algos/model.py
class ModelBase (line 35) | class ModelBase(nn.Module):
method __init__ (line 36) | def __init__(self, embedding=None, lm_model=False):
method unk_aug (line 108) | def unk_aug(self, x, x_mask=None):
class BiLanguageModel (line 128) | class BiLanguageModel(ModelBase):
method __init__ (line 129) | def __init__(self, embedding=None):
class RNet (line 134) | class RNet(ModelBase):
method __init__ (line 135) | def __init__(self, embedding=None):
method forward (line 181) | def forward(self, input, training=False):
class MReader (line 215) | class MReader(ModelBase):
method __init__ (line 216) | def __init__(self, embedding=None):
method forward (line 270) | def forward(self, input, training=False):
FILE: projects/ai2018/reader/algos/baseline.py
class Model (line 30) | class Model(melt.Model):
method __init__ (line 31) | def __init__(self):
method call (line 57) | def call(self, input, training=False):
class Model2 (line 85) | class Model2(melt.Model):
method __init__ (line 89) | def __init__(self):
method call (line 114) | def call(self, input, training=False):
FILE: projects/ai2018/reader/algos/loss.py
function criterion (line 29) | def criterion(model, x, y, training=False):
FILE: projects/ai2018/reader/algos/m_reader.py
class MnemonicReaderV4 (line 31) | class MnemonicReaderV4(melt.Model):
method __init__ (line 32) | def __init__(self):
method call (line 122) | def call(self, input, training=False):
class MnemonicReader (line 244) | class MnemonicReader(melt.Model):
method __init__ (line 245) | def __init__(self):
method call (line 334) | def call(self, input, training=False):
class MnemonicReaderV2 (line 464) | class MnemonicReaderV2(melt.Model):
method __init__ (line 465) | def __init__(self):
method call (line 544) | def call(self, input, training=False):
class MnemonicReaderV1 (line 658) | class MnemonicReaderV1(melt.Model):
method __init__ (line 659) | def __init__(self):
method call (line 738) | def call(self, input, training=False):
FILE: projects/ai2018/reader/algos/qcatt.py
class QCAttention (line 30) | class QCAttention(melt.Model):
method __init__ (line 31) | def __init__(self):
method call (line 57) | def call(self, input, training=False):
FILE: projects/ai2018/reader/algos/rnet.py
class RNet (line 31) | class RNet(melt.Model):
method __init__ (line 32) | def __init__(self):
method call (line 99) | def call(self, input, training=False):
FILE: projects/ai2018/reader/dataset.py
class Dataset (line 33) | class Dataset(melt.tfrecords.Dataset):
method __init__ (line 34) | def __init__(self, subset='train'):
method parser (line 47) | def parser(self, example):
FILE: projects/ai2018/reader/ensemble/ensemble-infer.py
function parse (line 26) | def parse(input):
FILE: projects/ai2018/reader/ensemble/ensemble-valid.py
function parse (line 26) | def parse(input):
FILE: projects/ai2018/reader/ensemble/ensemble.py
function parse (line 38) | def parse(input):
function blend_weights (line 41) | def blend_weights(weights, norm_factor=0.1):
function main (line 52) | def main(_):
FILE: projects/ai2018/reader/evaluate.py
function init (line 44) | def init():
function calc_acc (line 89) | def calc_acc(labels, predicts, ids, model_path):
function calc_loss (line 121) | def calc_loss(labels, predicts, ids, model_path=None):
function evaluate (line 155) | def evaluate(labels, predicts, ids=None, model_path=None):
function write (line 170) | def write(id, label, predict, out, out2=None, is_infer=False):
function valid_write (line 191) | def valid_write(id, label, predict, out):
function infer_write (line 194) | def infer_write(id, predict, out, out_debug):
FILE: projects/ai2018/reader/infer.py
function main (line 44) | def main(_):
FILE: projects/ai2018/reader/prepare.v1/gen-records.py
function get_mode (line 49) | def get_mode(path):
function is_negative (line 61) | def is_negative(candidate):
function sort_alternatives (line 69) | def sort_alternatives(alternatives, query):
function build_features (line 196) | def build_features(file_):
function main (line 308) | def main(_):
FILE: projects/ai2018/reader/prepare.v1/gen-seg.py
function seg (line 41) | def seg(text):
FILE: projects/ai2018/reader/prepare.v1/merge-emb.py
function main (line 37) | def main(_):
FILE: projects/ai2018/reader/prepare.v1/text2ids.py
function text2ids (line 32) | def text2ids(text):
FILE: projects/ai2018/reader/prepare/gen-records.py
function get_mode (line 54) | def get_mode(path):
function is_negative (line 66) | def is_negative(candidate):
function sort_alternatives (line 74) | def sort_alternatives(alternatives, query):
function get_char_ids (line 201) | def get_char_ids(words):
function get_pos_ids (line 226) | def get_pos_ids(pos):
function build_features (line 235) | def build_features(file_):
function main (line 420) | def main(_):
FILE: projects/ai2018/reader/prepare/gen-seg.py
function seg (line 42) | def seg(text):
FILE: projects/ai2018/reader/prepare/merge-emb.py
function main (line 41) | def main(_):
FILE: projects/ai2018/reader/prepare/pre-seg.py
function seg_ (line 61) | def seg_(text):
function seg (line 85) | def seg(m, out):
FILE: projects/ai2018/reader/read-records.py
function deal (line 48) | def deal(dataset, infos):
function main (line 60) | def main(_):
FILE: projects/ai2018/reader/tools/ensemble-infer.py
function parse (line 26) | def parse(input):
FILE: projects/ai2018/reader/tools/ensemble-valid.py
function parse (line 26) | def parse(input):
FILE: projects/ai2018/reader/torch-train.py
function get_num_finetune_words (line 37) | def get_num_finetune_words():
function freeze_embedding (line 44) | def freeze_embedding(self, grad_input, grad_output):
function freeze_char_embedding (line 48) | def freeze_char_embedding(self, grad_input, grad_output):
function main (line 51) | def main(_):
FILE: projects/ai2018/reader/torch_algos/baseline/baseline.py
class Bow (line 21) | class Bow(nn.Module):
method __init__ (line 22) | def __init__(self):
method forward (line 43) | def forward(self, input, training=False):
class Gru (line 55) | class Gru(nn.Module):
method __init__ (line 56) | def __init__(self):
method forward (line 110) | def forward(self, input, training=False):
class MwAN (line 153) | class MwAN(nn.Module):
method __init__ (line 154) | def __init__(self):
method initiation (line 205) | def initiation(self):
method forward (line 212) | def forward(self, x, training=False):
function criterion (line 281) | def criterion(model, x, y, training=False):
FILE: projects/ai2018/reader/torch_algos/loss.py
function criterion (line 27) | def criterion(model, x, y):
FILE: projects/ai2018/reader/torch_algos/m_reader.py
function get_mask (line 46) | def get_mask(x):
class MnemonicReaderV3 (line 54) | class MnemonicReaderV3(nn.Module):
method __init__ (line 57) | def __init__(self, args=None):
method forward (line 136) | def forward(self, inputs):
class MnemonicReader (line 188) | class MnemonicReader(nn.Module):
method __init__ (line 191) | def __init__(self, args=None):
method forward (line 272) | def forward(self, inputs):
class MnemonicReaderV1 (line 356) | class MnemonicReaderV1(nn.Module):
method __init__ (line 359) | def __init__(self, args=None):
method forward (line 422) | def forward(self, inputs):
FILE: projects/ai2018/reader/torch_algos/model.py
class ModelBase (line 43) | class ModelBase(nn.Module):
method __init__ (line 44) | def __init__(self, embedding=None, lm_model=False):
function get_mask (line 97) | def get_mask(x):
class MReader (line 104) | class MReader(ModelBase):
method __init__ (line 105) | def __init__(self, embedding=None):
method forward (line 161) | def forward(self, inputs):
FILE: projects/ai2018/reader/torch_algos/rnet.py
class Rnet (line 40) | class Rnet(nn.Module):
method __init__ (line 43) | def __init__(self, args=None):
method forward (line 131) | def forward(self, inputs):
FILE: projects/ai2018/reader/train.py
function main (line 38) | def main(_):
FILE: projects/ai2018/sentiment/algos/loss.py
function calc_loss (line 29) | def calc_loss(y, y_, weights, training=False):
function calc_hier_loss (line 80) | def calc_hier_loss(y, y_, weights):
function calc_hier_neu_loss (line 93) | def calc_hier_neu_loss(y, y_, weights):
function calc_add_binary_loss (line 114) | def calc_add_binary_loss(y, y_, cids, weights):
function calc_binary_loss (line 124) | def calc_binary_loss(y, y_, cid, weights):
function calc_regression_loss (line 129) | def calc_regression_loss(y, y_, weights):
function calc_add_binaries_loss (line 133) | def calc_add_binaries_loss(y, y_, cid, weights):
function calc_binaries_only_loss (line 142) | def calc_binaries_only_loss(y, y_, cid, weights):
function criterion (line 155) | def criterion(y, y_):
FILE: projects/ai2018/sentiment/algos/model.py
class ModelBase (line 39) | class ModelBase(melt.Model):
method __init__ (line 40) | def __init__(self, embedding=None, lm_model=False, use_text_encoder=Tr...
method unk_aug (line 135) | def unk_aug(self, x, x_mask=None, training=False):
class BiLanguageModel (line 168) | class BiLanguageModel(ModelBase):
method __init__ (line 169) | def __init__(self, embedding=None, lm_model=True):
method call (line 172) | def call(self, input, training=False):
class RNet (line 175) | class RNet(ModelBase):
method __init__ (line 176) | def __init__(self, embedding=None, lm_model=False):
method call (line 203) | def call(self, input, training=False):
class RNetV2 (line 271) | class RNetV2(RNet):
method __init__ (line 272) | def __init__(self, embedding=None, lm_model=False):
method call (line 302) | def call(self, input, training=False):
class RNetV3 (line 365) | class RNetV3(RNet):
method __init__ (line 366) | def __init__(self, embedding=None):
class RNetV4 (line 392) | class RNetV4(RNet):
method __init__ (line 393) | def __init__(self, embedding=None):
class MReader (line 419) | class MReader(ModelBase):
method __init__ (line 420) | def __init__(self, embedding=None):
method call (line 452) | def call(self, input, training=False):
class Transformer (line 506) | class Transformer(ModelBase):
method __init__ (line 507) | def __init__(self, embedding=None):
method restore (line 551) | def restore(self):
method call (line 567) | def call(self, input, training=False):
FILE: projects/ai2018/sentiment/algos/weights.py
function no_weights (line 28) | def no_weights():
function get_pos (line 31) | def get_pos(aspect):
function parse_weights (line 63) | def parse_weights():
function get_weights (line 82) | def get_weights(aspect, attr_index=None):
FILE: projects/ai2018/sentiment/analysis/analyze.py
function parse (line 70) | def parse(l):
FILE: projects/ai2018/sentiment/analysis/beam_f.py
function beam_f (line 5) | def beam_f(N, weights, y,
function seed_beam_f (line 157) | def seed_beam_f(N, weights, y,
FILE: projects/ai2018/sentiment/analysis/beam_f_utils.py
function compute_biconcave_obj (line 4) | def compute_biconcave_obj(C, u, reg):
function compute_u (line 29) | def compute_u(C, eps):
function eval_classifier (line 57) | def eval_classifier(prob_estimates, G, X, y):
function eval_conf (line 80) | def eval_conf(C): # updated to use f-score
function compute_conf_grad (line 111) | def compute_conf_grad(C, u, eps, reg): # updated to use f-score
function predict_labels (line 143) | def predict_labels(G, eta):
function compute_conf (line 171) | def compute_conf(G, eta, true_labels):
function compute_rand_conf (line 207) | def compute_rand_conf(classifiers, classifier_weights, eta, true_labels):
FILE: projects/ai2018/sentiment/analysis/correlations-filter.py
function calc_correlation (line 78) | def calc_correlation(x, y, method):
FILE: projects/ai2018/sentiment/analysis/correlations.py
function plot_confusion_matrix (line 41) | def plot_confusion_matrix(cm, classes,
function calc_correlation (line 150) | def calc_correlation(x, y, method):
FILE: projects/ai2018/sentiment/dataset.py
class Dataset (line 35) | class Dataset(melt.tfrecords.Dataset):
method __init__ (line 36) | def __init__(self, subset='train'):
method parse (line 57) | def parse(self, example):
method num_examples_per_epoch (line 136) | def num_examples_per_epoch(self, mode):
FILE: projects/ai2018/sentiment/ensemble/ensemble-cv-parallel.py
function parse (line 67) | def parse(l):
function calc_f1 (line 75) | def calc_f1(labels, predicts):
function calc_f1s (line 84) | def calc_f1s(labels, predicts):
function calc_losses (line 91) | def calc_losses(labels, predicts):
function calc_loss (line 98) | def calc_loss(labels, predicts):
function calc_aucs (line 105) | def calc_aucs(labels, predicts):
function calc_f1_alls (line 117) | def calc_f1_alls(labels, predicts):
function to_predict (line 175) | def to_predict(logits, weights=None, is_single=False, adjust=True):
function blend_weights (line 211) | def blend_weights(weights, norm_facotr):
function get_counts (line 226) | def get_counts(probs):
function adjust_probs (line 233) | def adjust_probs(probs, labels):
function grid_search_class_factors (line 251) | def grid_search_class_factors(probs, labels, weights, num_grids=10):
function main (line 287) | def main(_):
FILE: projects/ai2018/sentiment/ensemble/ensemble-cv-v1.py
function parse (line 65) | def parse(l):
function calc_f1 (line 73) | def calc_f1(labels, predicts):
function calc_f1s (line 82) | def calc_f1s(labels, predicts):
function calc_losses (line 89) | def calc_losses(labels, predicts):
function calc_aucs (line 96) | def calc_aucs(labels, predicts):
function calc_f1_alls (line 108) | def calc_f1_alls(labels, predicts):
function to_predict (line 165) | def to_predict(logits, weights=None, is_single=False, adjust=True):
function blend_weights (line 201) | def blend_weights(weights, norm_facotr):
function get_counts (line 216) | def get_counts(probs):
function adjust_probs (line 223) | def adjust_probs(probs, labels):
function grid_search_class_factors (line 243) | def grid_search_class_factors(probs, labels, weights, num_grids=10):
function main (line 280) | def main(_):
FILE: projects/ai2018/sentiment/ensemble/ensemble-cv.py
function parse (line 70) | def parse(l):
function calc_f1 (line 78) | def calc_f1(labels, predicts):
function calc_f1s (line 87) | def calc_f1s(labels, predicts):
function calc_losses (line 94) | def calc_losses(labels, predicts):
function calc_loss (line 101) | def calc_loss(labels, predicts):
function calc_aucs (line 108) | def calc_aucs(labels, predicts):
function calc_f1_alls (line 120) | def calc_f1_alls(labels, predicts):
function to_predict (line 181) | def to_predict(logits, weights=None, is_single=False, adjust=True):
function blend_byrank (line 217) | def blend_byrank(weights, norm_facotr):
function blend_byweight (line 235) | def blend_byweight(weights, norm_facotr):
function blend (line 254) | def blend(weights, norm_factor):
function get_counts (line 262) | def get_counts(probs):
function adjust_probs (line 269) | def adjust_probs(probs, labels):
function grid_search_class_factors (line 287) | def grid_search_class_factors(probs, labels, weights, num_grids=10):
function get_distribution (line 323) | def get_distribution(predicts):
function print_confusion_matrix (line 330) | def print_confusion_matrix(labels, predicts):
function main (line 335) | def main(_):
FILE: projects/ai2018/sentiment/ensemble/ensemble-hillclimb.py
function parse (line 67) | def parse(l):
function calc_f1 (line 75) | def calc_f1(labels, predicts):
function calc_f1s (line 84) | def calc_f1s(labels, predicts):
function calc_losses (line 91) | def calc_losses(labels, predicts):
function calc_aucs (line 98) | def calc_aucs(labels, predicts):
function calc_f1_alls (line 110) | def calc_f1_alls(labels, predicts):
function to_predict (line 155) | def to_predict(logits, weights=None, is_single=False, adjust=True):
function to_one_predict (line 191) | def to_one_predict(logits, label, weights=None, is_single=False, adjust=...
function blend_weights (line 225) | def blend_weights(weights, norm_facotr):
function grid_search_class_factors (line 241) | def grid_search_class_factors(probs, labels, weights, num_grids=10):
function init_hillclimb (line 277) | def init_hillclimb():
function score_ensemble (line 296) | def score_ensemble(ensemble, label):
function find_best_improvement (line 318) | def find_best_improvement(ensemble, label):
function climb (line 347) | def climb(best_ensemble, best_score, best_loss, valid_score, valid_loss):
function get_optimal_weights (line 356) | def get_optimal_weights(best_ensemble):
function main (line 365) | def main(_):
FILE: projects/ai2018/sentiment/ensemble/ensemble-infer.py
function parse (line 28) | def parse(l):
function to_predict (line 31) | def to_predict(logits):
FILE: projects/ai2018/sentiment/ensemble/ensemble-v1.py
function parse (line 65) | def parse(l):
function calc_f1 (line 73) | def calc_f1(labels, predicts):
function calc_f1s (line 82) | def calc_f1s(labels, predicts):
function calc_losses (line 89) | def calc_losses(labels, predicts):
function calc_aucs (line 96) | def calc_aucs(labels, predicts):
function calc_f1_alls (line 108) | def calc_f1_alls(labels, predicts):
function to_predict (line 152) | def to_predict(logits, weights=None, is_single=False, adjust=True):
function blend_weights (line 188) | def blend_weights(weights, norm_facotr):
function get_counts (line 203) | def get_counts(probs):
function adjust_probs (line 210) | def adjust_probs(probs, labels):
function grid_search_class_factors (line 224) | def grid_search_class_factors(probs, labels, weights, num_grids=10):
function main (line 253) | def main(_):
FILE: projects/ai2018/sentiment/ensemble/ensemble.py
function parse (line 62) | def parse(l):
function calc_f1 (line 70) | def calc_f1(labels, predicts):
function calc_f1s (line 79) | def calc_f1s(labels, predicts):
function calc_loss (line 86) | def calc_loss(labels, predicts):
function calc_losses (line 93) | def calc_losses(labels, predicts):
function calc_aucs (line 100) | def calc_aucs(labels, predicts):
function calc_f1_alls (line 112) | def calc_f1_alls(labels, predicts):
function to_predict (line 169) | def to_predict(logits, weights=None, is_single=False, adjust=True):
function blend_weights (line 205) | def blend_weights(weights, norm_facotr):
function get_counts (line 220) | def get_counts(probs):
function adjust_probs (line 227) | def adjust_probs(probs, labels):
function grid_search_class_factors (line 244) | def grid_search_class_factors(probs, labels, weights, num_grids=10):
function main (line 280) | def main(_):
FILE: projects/ai2018/sentiment/ensemble/gen-train.py
function parse (line 34) | def parse(l):
FILE: projects/ai2018/sentiment/ensemble/hillclimb-ensembling.py
function init_hillclimb (line 10) | def init_hillclimb():
function score_ensemble (line 23) | def score_ensemble(ensemble, label):
function find_best_improvement (line 37) | def find_best_improvement(ensemble, label):
function climb (line 57) | def climb(best_ensemble, best_score):
function get_optimal_weights (line 66) | def get_optimal_weights(best_ensemble):
function get_optimal_blend (line 77) | def get_optimal_blend(optimal_weights):
function get_sub_file (line 89) | def get_sub_file(num):
FILE: projects/ai2018/sentiment/ensemble/lgb-adjust.py
function parse (line 48) | def parse(l):
function is_ok (line 122) | def is_ok(factor):
FILE: projects/ai2018/sentiment/ensemble/lgb-cv.py
function evaluate_macroF1_lgb (line 123) | def evaluate_macroF1_lgb(truth, predictions):
function learning_rate_power_0997 (line 131) | def learning_rate_power_0997(current_iter):
function learning_rate_power_0997 (line 202) | def learning_rate_power_0997(current_iter):
FILE: projects/ai2018/sentiment/evaluate.py
function load_class_weights (line 66) | def load_class_weights():
function init (line 87) | def init():
function regression_to_class (line 141) | def regression_to_class(predict):
function to_class (line 151) | def to_class(predicts, thre=0.5):
function calc_f1 (line 168) | def calc_f1(labels, predicts, model_path=None, name = 'f1'):
function calc_loss (line 219) | def calc_loss(labels, predicts, model_path=None):
function calc_auc (line 249) | def calc_auc(labels, predicts, model_path=None):
function evaluate (line 286) | def evaluate(labels, predicts, ids=None, model_path=None):
function write (line 337) | def write(ids, labels, predicts, ofile, ofile2=None, is_infer=False):
function valid_write (line 379) | def valid_write(ids, labels, predicts, ofile):
function infer_write (line 382) | def infer_write(ids, predicts, ofile, ofile2):
function evaluate_file (line 386) | def evaluate_file(file):
FILE: projects/ai2018/sentiment/infer.py
function main (line 44) | def main(_):
FILE: projects/ai2018/sentiment/lm-train.py
function main (line 36) | def main(_):
FILE: projects/ai2018/sentiment/lm_dataset.py
class Dataset (line 35) | class Dataset(melt.tfrecords.Dataset):
method __init__ (line 36) | def __init__(self, subset='train'):
method parser (line 57) | def parser(self, example):
FILE: projects/ai2018/sentiment/prepare.test/filter.py
function filter_duplicate_space (line 26) | def filter_duplicate_space(text):
function filter (line 41) | def filter(x):
FILE: projects/ai2018/sentiment/prepare.test/gen-canyin.py
function main (line 71) | def main(_):
FILE: projects/ai2018/sentiment/prepare.test/gen-dianping.py
function score2class (line 53) | def score2class(score):
function main (line 88) | def main(_):
FILE: projects/ai2018/sentiment/prepare.test/gen-lm-records.py
function build_features (line 60) | def build_features(file_):
function main (line 140) | def main(_):
FILE: projects/ai2018/sentiment/prepare.test/gen-records.py
function get_mode (line 82) | def get_mode(path):
function build_features (line 106) | def build_features(index):
function main (line 257) | def main(_):
FILE: projects/ai2018/sentiment/prepare.test/gen-trans.py
function main (line 58) | def main(_):
FILE: projects/ai2018/sentiment/prepare.test/merge-emb.py
function main (line 41) | def main(_):
FILE: projects/ai2018/sentiment/prepare.test/pre-mix-seg-v1.py
function seg (line 39) | def seg(id, text, out):
function main (line 44) | def main(_):
FILE: projects/ai2018/sentiment/prepare.test/pre-mix-seg.py
function seg (line 47) | def seg(id, text, out, counter):
function main (line 67) | def main(_):
FILE: projects/ai2018/sentiment/prepare.test/pre-seg-bert.py
function seg (line 42) | def seg(id, text, out):
FILE: projects/ai2018/sentiment/prepare.test/pre-seg.py
function seg (line 58) | def seg(id, text, out, type):
FILE: projects/ai2018/sentiment/prepare.test/text2ids.py
function text2ids (line 31) | def text2ids(text, preprocess=True, return_words=False):
FILE: projects/ai2018/sentiment/prepare.testb/filter.py
function filter_duplicate_space (line 26) | def filter_duplicate_space(text):
function filter (line 41) | def filter(x):
FILE: projects/ai2018/sentiment/prepare.testb/gen-canyin.py
function main (line 71) | def main(_):
FILE: projects/ai2018/sentiment/prepare.testb/gen-dianping.py
function score2class (line 53) | def score2class(score):
function main (line 88) | def main(_):
FILE: projects/ai2018/sentiment/prepare.testb/gen-lm-records.py
function build_features (line 60) | def build_features(file_):
function main (line 140) | def main(_):
FILE: projects/ai2018/sentiment/prepare.testb/gen-records.py
function get_mode (line 82) | def get_mode(path):
function build_features (line 106) | def build_features(index):
function main (line 256) | def main(_):
FILE: projects/ai2018/sentiment/prepare.testb/gen-trans.py
function main (line 58) | def main(_):
FILE: projects/ai2018/sentiment/prepare.testb/merge-emb.py
function main (line 41) | def main(_):
FILE: projects/ai2018/sentiment/prepare.testb/pre-mix-seg-v1.py
function seg (line 39) | def seg(id, text, out):
function main (line 44) | def main(_):
FILE: projects/ai2018/sentiment/prepare.testb/pre-mix-seg.py
function seg (line 47) | def seg(id, text, out, counter):
function main (line 67) | def main(_):
FILE: projects/ai2018/sentiment/prepare.testb/pre-seg-bert.py
function seg (line 42) | def seg(id, text, out):
FILE: projects/ai2018/sentiment/prepare.testb/pre-seg.py
function seg (line 58) | def seg(id, text, out, type):
FILE: projects/ai2018/sentiment/prepare.testb/text2ids.py
function text2ids (line 31) | def text2ids(text, preprocess=True, return_words=False):
FILE: projects/ai2018/sentiment/prepare.v1/filter.py
function filter (line 26) | def filter(x):
FILE: projects/ai2018/sentiment/prepare.v1/gen-canyin.py
function main (line 71) | def main(_):
FILE: projects/ai2018/sentiment/prepare.v1/gen-char-seg-canyin.py
function seg (line 43) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v1/gen-char-seg-dianping.py
function seg (line 42) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v1/gen-char-seg-train.py
function seg (line 42) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v1/gen-char-seg.py
function seg (line 42) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v1/gen-dianping.py
function score2class (line 53) | def score2class(score):
function main (line 88) | def main(_):
FILE: projects/ai2018/sentiment/prepare.v1/gen-mix-seg-canyin.py
function seg (line 49) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v1/gen-mix-seg-dianping.py
function seg (line 49) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v1/gen-records.py
function get_mode (line 56) | def get_mode(path):
function build_features (line 78) | def build_features(index):
function main (line 158) | def main(_):
FILE: projects/ai2018/sentiment/prepare.v1/gen-seg-canyin.py
function seg (line 43) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v1/gen-seg-dianping.py
function seg (line 49) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v1/gen-seg-train.py
function seg (line 42) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v1/gen-trans.py
function main (line 58) | def main(_):
FILE: projects/ai2018/sentiment/prepare.v1/merge-emb.py
function main (line 37) | def main(_):
FILE: projects/ai2018/sentiment/prepare.v1/text2ids.py
function text2ids (line 30) | def text2ids(text):
FILE: projects/ai2018/sentiment/prepare.v2/filter.py
function filter_duplicate (line 26) | def filter_duplicate(text):
function filter (line 29) | def filter(x):
FILE: projects/ai2018/sentiment/prepare.v2/gen-canyin.py
function main (line 71) | def main(_):
FILE: projects/ai2018/sentiment/prepare.v2/gen-char-seg-canyin.py
function seg (line 43) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v2/gen-char-seg-dianping.py
function seg (line 42) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v2/gen-char-seg-train.py
function seg (line 43) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v2/gen-char-seg.py
function seg (line 42) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v2/gen-dianping.py
function score2class (line 53) | def score2class(score):
function main (line 88) | def main(_):
FILE: projects/ai2018/sentiment/prepare.v2/gen-mix-seg-canyin.py
function seg (line 51) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v2/gen-mix-seg-dianping.py
function seg (line 52) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v2/gen-mix-seg-train.py
function seg (line 52) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v2/gen-records.py
function get_mode (line 66) | def get_mode(path):
function build_features (line 88) | def build_features(index):
function main (line 208) | def main(_):
FILE: projects/ai2018/sentiment/prepare.v2/gen-seg-canyin.py
function seg (line 43) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v2/gen-seg-dianping.py
function seg (line 49) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v2/gen-seg-train.py
function seg (line 42) | def seg(text, out):
FILE: projects/ai2018/sentiment/prepare.v2/gen-trans.py
function main (line 58) | def main(_):
FILE: projects/ai2018/sentiment/prepare.v2/merge-emb.py
function main (line 37) | def main(_):
FILE: projects/ai2018/sentiment/prepare.v2/text2ids.py
function text2ids (line 31) | def text2ids(text, return_words=False):
FILE: projects/ai2018/sentiment/prepare/filter.py
function filter_duplicate_space (line 26) | def filter_duplicate_space(text):
function filter (line 41) | def filter(x):
FILE: projects/ai2018/sentiment/prepare/gen-canyin.py
function main (line 71) | def main(_):
FILE: projects/ai2018/sentiment/prepare/gen-dianping.py
function score2class (line 53) | def score2class(score):
function main (line 88) | def main(_):
FILE: projects/ai2018/sentiment/prepare/gen-lm-records.py
function build_features (line 60) | def build_features(file_):
function main (line 141) | def main(_):
FILE: projects/ai2018/sentiment/prepare/gen-records.py
function get_mode (line 84) | def get_mode(path):
function build_features (line 110) | def build_features(index):
function main (line 285) | def main(_):
FILE: projects/ai2018/sentiment/prepare/gen-trans.py
function main (line 58) | def main(_):
FILE: projects/ai2018/sentiment/prepare/merge-emb.py
function main (line 41) | def main(_):
FILE: projects/ai2018/sentiment/prepare/pre-mix-seg-v1.py
function seg (line 39) | def seg(id, text, out):
function main (line 44) | def main(_):
FILE: projects/ai2018/sentiment/prepare/pre-mix-seg.py
function seg (line 47) | def seg(id, text, out, counter):
function main (line 67) | def main(_):
FILE: projects/ai2018/sentiment/prepare/pre-seg-bert.py
function seg (line 42) | def seg(id, text, out):
FILE: projects/ai2018/sentiment/prepare/pre-seg.py
function seg (line 58) | def seg(id, text, out, type):
FILE: projects/ai2018/sentiment/prepare/text2ids.py
function text2ids (line 31) | def text2ids(text, preprocess=True, return_words=False):
FILE: projects/ai2018/sentiment/read-records.py
function deal (line 51) | def deal(dataset, infos):
function main (line 63) | def main(_):
FILE: projects/ai2018/sentiment/tools/check-emb.py
function sim (line 38) | def sim(x, y):
function main (line 48) | def main(_):
FILE: projects/ai2018/sentiment/tools/find-best-epoch.py
function parse (line 29) | def parse(x, key='adjusted_f1'):
function deal (line 48) | def deal(line):
FILE: projects/ai2018/sentiment/tools/rename-variables-finetune.py
function rename (line 22) | def rename(checkpoint_dir, dry_run):
function main (line 52) | def main(argv):
FILE: projects/ai2018/sentiment/tools/seg2corpus.py
function main (line 33) | def main(_):
FILE: projects/ai2018/sentiment/torch-infer.py
function convert (line 47) | def convert(content):
function predict (line 66) | def predict(content):
function encode (line 89) | def encode(content, aspect=-2):
function sim (line 97) | def sim(content1, content2, aspect=-2):
function main (line 103) | def main(_):
FILE: projects/ai2018/sentiment/torch-lm-train.py
function main (line 36) | def main(_):
FILE: projects/ai2018/sentiment/torch-sim.py
function convert (line 47) | def convert(content):
function predict (line 66) | def predict(content):
function encode (line 89) | def encode(content):
function sim (line 96) | def sim(content1, content2):
function main (line 102) | def main(_):
FILE: projects/ai2018/sentiment/torch-train.py
function get_num_finetune_words (line 36) | def get_num_finetune_words():
function freeze_embedding (line 42) | def freeze_embedding(self, grad_input, grad_output):
function freeze_char_embedding (line 46) | def freeze_char_embedding(self, grad_input, grad_output):
function main (line 49) | def main(_):
FILE: projects/ai2018/sentiment/torch_algos/loss.py
class Criterion (line 33) | class Criterion(object):
method __init__ (line 34) | def __init__(self, class_weights=None):
method calc_soft_label_loss (line 47) | def calc_soft_label_loss(self, y_, y, num_classes):
method forward (line 55) | def forward(self, model, x, y, training=False):
FILE: projects/ai2018/sentiment/torch_algos/model.py
class ModelBase (line 37) | class ModelBase(nn.Module):
method __init__ (line 38) | def __init__(self, embedding=None, lm_model=False):
method unk_aug (line 127) | def unk_aug(self, x, x_mask=None):
class BiLanguageModel (line 147) | class BiLanguageModel(ModelBase):
method __init__ (line 148) | def __init__(self, embedding=None):
class RNet (line 153) | class RNet(ModelBase):
method __init__ (line 154) | def __init__(self, embedding=None):
method forward (line 200) | def forward(self, input, training=False):
class MReader (line 238) | class MReader(ModelBase):
method __init__ (line 239) | def __init__(self, embedding=None):
method forward (line 293) | def forward(self, input, training=False):
class Fastai (line 354) | class Fastai(ModelBase):
method __init__ (line 355) | def __init__(self, embedding=None):
method forward (line 367) | def forward(self, input, training=False):
FILE: projects/ai2018/sentiment/train.py
function main (line 37) | def main(_):
FILE: projects/common/lm/algos/loss.py
function loss_fn (line 20) | def loss_fn(model, inputs, targets, training=False):
FILE: projects/common/lm/algos/model.py
class PTBModel (line 31) | class PTBModel(tf.keras.Model):
method __init__ (line 39) | def __init__(self,
method call (line 64) | def call(self, input_seq, training=False):
FILE: projects/common/lm/dataset.py
class Dataset (line 35) | class Dataset(melt.tfrecords.Dataset):
method __init__ (line 36) | def __init__(self, subset='train'):
method make_batch (line 46) | def make_batch(self, batch_size, filenames, bptt=None, **kwargs):
method num_examples_per_epoch (line 147) | def num_examples_per_epoch(self, mode):
FILE: projects/common/lm/prepare/to-ids.py
function main (line 32) | def main(_):
FILE: projects/common/lm/read-records.py
function main (line 31) | def main(_):
FILE: projects/common/lm/train.py
function main (line 38) | def main(_):
FILE: projects/feed/rank/tf/err/torch-only-train.py
function main (line 34) | def main(_):
FILE: projects/feed/rank/tf/evaluate.py
function evaluate (line 23) | def evaluate(y, y_):
function valid_write (line 29) | def valid_write(ids, labels, predicts, out):
FILE: projects/feed/rank/tf/gen-records.py
function get_out_file (line 36) | def get_out_file(infile):
function build_features (line 42) | def build_features(infile):
function main (line 70) | def main(_):
FILE: projects/feed/rank/tf/loss.py
function binary_crossentropy_with_ranking (line 22) | def binary_crossentropy_with_ranking(y_true, y_pred):
FILE: projects/feed/rank/tf/model.py
class Wide (line 31) | class Wide(keras.Model):
method __init__ (line 32) | def __init__(self):
method call (line 39) | def call(self, input):
class Deep (line 52) | class Deep(keras.Model):
method __init__ (line 53) | def __init__(self):
method call (line 87) | def call(self, input, training=False):
class WideDeep (line 128) | class WideDeep(keras.Model):
method __init__ (line 129) | def __init__(self):
method call (line 136) | def call(self, input, training=False):
FILE: projects/feed/rank/tf/pyt/dataset.py
class TextDataset (line 28) | class TextDataset(Dataset):
method __init__ (line 29) | def __init__(self, filename, td):
method __getitem__ (line 34) | def __getitem__(self, idx):
method __len__ (line 44) | def __len__(self):
function get_dataset (line 47) | def get_dataset(files, td):
FILE: projects/feed/rank/tf/pyt/model.py
class Wide (line 31) | class Wide(nn.Module):
method __init__ (line 32) | def __init__(self):
method forward (line 41) | def forward(self, input):
class Deep (line 57) | class Deep(nn.Module):
method __init__ (line 58) | def __init__(self):
method forward (line 82) | def forward(self, input):
class WideDeep (line 128) | class WideDeep(nn.Module):
method __init__ (line 129) | def __init__(self):
method forward (line 136) | def forward(self, input):
FILE: projects/feed/rank/tf/read-test.py
function main (line 27) | def main(_):
FILE: projects/feed/rank/tf/read-test2.py
function main (line 31) | def main(_):
FILE: projects/feed/rank/tf/read-test3.py
function main (line 34) | def main(_):
FILE: projects/feed/rank/tf/text_dataset.py
class Dataset (line 28) | class Dataset(melt.Dataset):
method __init__ (line 29) | def __init__(self, subset='train'):
method load_feature_files (line 47) | def load_feature_files(self):
method get_feat (line 68) | def get_feat(self, fields):
method parse_line (line 86) | def parse_line(self, line):
method parse_line2 (line 95) | def parse_line2(self, line):
method line_parse_ (line 104) | def line_parse_(self, line):
method parse_batch (line 117) | def parse_batch(self, feat_list, batch_size):
method batch_parse_ (line 154) | def batch_parse_(self, line, batch_size):
method parse (line 170) | def parse(self, line, batch_size):
FILE: projects/feed/rank/tf/tfrecord_dataset.py
class Dataset (line 27) | class Dataset(melt.Dataset):
method __init__ (line 28) | def __init__(self, subset='valid'):
method parse (line 33) | def parse(self, example):
FILE: projects/feed/rank/tf/torch-hvd-train.py
function train (line 55) | def train(epoch, model, loss_fn, train_loader, optimizer):
function metric_average (line 77) | def metric_average(val, name):
function test (line 83) | def test(model, loss_fn, test_loader):
function main (line 105) | def main(_):
FILE: projects/feed/rank/tf/torch-only-train-hvd.py
function main (line 45) | def main(_):
FILE: projects/feed/rank/tf/torch-only-train.py
function main (line 43) | def main(_):
FILE: projects/feed/rank/tf/torch-train.py
function main (line 34) | def main(_):
FILE: projects/feed/rank/tf/train.py
function main (line 28) | def main(_):
FILE: projects/kaggle/blindness/keras/dataset.py
class Dataset (line 31) | class Dataset(Sequence):
method __init__ (line 33) | def __init__(self, image_filenames, labels,
method __len__ (line 44) | def __len__(self):
method __getitem__ (line 47) | def __getitem__(self, idx):
method on_epoch_end (line 55) | def on_epoch_end(self):
method mix_up (line 61) | def mix_up(self, x, y):
method train_generate (line 72) | def train_generate(self, batch_x, batch_y):
method get_images (line 90) | def get_images(self, indexes):
method valid_generate (line 104) | def valid_generate(self, batch_x, batch_y):
FILE: projects/kaggle/blindness/keras/evaluate.py
function gen_confusion (line 36) | def gen_confusion(y_true, y_pred, info=''):
function to_str (line 47) | def to_str(scores):
class Evaluator (line 50) | class Evaluator(Callback):
method __init__ (line 51) | def __init__(self,
method on_epoch_end (line 68) | def on_epoch_end(self, epoch, logs={}):
FILE: projects/kaggle/blindness/keras/evaluate2.py
class QWKEvaluation (line 25) | class QWKEvaluation(Callback):
method __init__ (line 26) | def __init__(self, validation_data=(), interval=1):
method on_epoch_end (line 33) | def on_epoch_end(self, epoch, logs={}):
FILE: projects/kaggle/blindness/keras/fake-infer.py
function hack_lb (line 21) | def hack_lb(test_preds):
FILE: projects/kaggle/blindness/keras/folds.py
function get_train_valid (line 20) | def get_train_valid(x, y, fold=0, num_folds=5, random_state=2019):
FILE: projects/kaggle/blindness/keras/infer.py
class Predictor (line 33) | class Predictor():
method __init__ (line 34) | def __init__(self, model, batch_size, predict_fn=None):
method _predict (line 41) | def _predict(self):
method add (line 49) | def add(self, x):
method predict (line 54) | def predict(self):
function hack_lb (line 59) | def hack_lb(test_preds):
function main (line 70) | def main(_):
FILE: projects/kaggle/blindness/keras/loss.py
function earth_mover_loss (line 30) | def earth_mover_loss(y_true, y_pred):
function kappa_loss (line 38) | def kappa_loss(y_true, y_pred, y_pow=2, eps=1e-12, N=5, bsize=32, name='...
function get_loss (line 75) | def get_loss(loss_type=None):
FILE: projects/kaggle/blindness/keras/lr.py
class WarmUpLearningRateScheduler (line 27) | class WarmUpLearningRateScheduler(keras.callbacks.Callback):
method __init__ (line 31) | def __init__(self, warmup_batches, init_lr, verbose=0):
method on_batch_end (line 49) | def on_batch_end(self, batch, logs=None):
method on_batch_begin (line 54) | def on_batch_begin(self, batch, logs=None):
function cosine_decay_with_warmup (line 62) | def cosine_decay_with_warmup(global_step,
class WarmUpCosineDecayScheduler (line 116) | class WarmUpCosineDecayScheduler(keras.callbacks.Callback):
method __init__ (line 120) | def __init__(self,
method on_batch_end (line 153) | def on_batch_end(self, batch, logs=None):
method on_batch_begin (line 158) | def on_batch_begin(self, batch, logs=None):
FILE: projects/kaggle/blindness/keras/model.py
function create_model (line 29) | def create_model(input_shape, n_out, loss_type=''):
FILE: projects/kaggle/blindness/keras/train.py
function to_regression (line 53) | def to_regression(y):
function to_regression2 (line 58) | def to_regression2(y):
function to_ordinal (line 64) | def to_ordinal(y):
function to_ordinal2 (line 74) | def to_ordinal2(y):
function trans_y (line 79) | def trans_y(y, loss_type):
function main (line 92) | def main(_):
FILE: projects/kaggle/blindness/keras/train2.py
function get_num_gpus (line 43) | def get_num_gpus():
function main (line 54) | def main(_):
FILE: projects/kaggle/blindness/keras/util.py
function get_num_gpus (line 18) | def get_num_gpus():
FILE: projects/kaggle/blindness/keras2/evaluate.py
class QWKEvaluation (line 25) | class QWKEvaluation(Callback):
method __init__ (line 26) | def __init__(self, validation_data=(), batch_size=64, interval=1):
method on_epoch_end (line 34) | def on_epoch_end(self, epoch, logs={}):
FILE: projects/kaggle/blindness/keras2/model.py
function create_model (line 25) | def create_model(input_shape, n_out):
FILE: projects/kaggle/blindness/keras2/train.py
function get_num_gpus (line 46) | def get_num_gpus():
function get_dataset (line 57) | def get_dataset(subset, use_distortion=None):
function main (line 65) | def main(_):
FILE: projects/kaggle/blindness/keras2tf/dataset.py
class Dataset (line 26) | class Dataset(Sequence):
method __init__ (line 28) | def __init__(self, image_filenames, labels,
method __len__ (line 39) | def __len__(self):
method __getitem__ (line 42) | def __getitem__(self, idx):
method on_epoch_end (line 50) | def on_epoch_end(self):
method mix_up (line 56) | def mix_up(self, x, y):
method train_generate (line 67) | def train_generate(self, batch_x, batch_y):
method valid_generate (line 81) | def valid_generate(self, batch_x, batch_y):
FILE: projects/kaggle/blindness/keras2tf/evaluate.py
class QWKEvaluation (line 25) | class QWKEvaluation(Callback):
method __init__ (line 26) | def __init__(self, validation_data=(), batch_size=64, interval=1):
method on_epoch_end (line 34) | def on_epoch_end(self, epoch, logs={}):
FILE: projects/kaggle/blindness/keras2tf/model.py
function create_model (line 26) | def create_model(input_shape, n_out):
FILE: projects/kaggle/blindness/keras2tf/train.py
function get_num_gpus (line 44) | def get_num_gpus():
function main (line 55) | def main(_):
FILE: projects/kaggle/blindness/keras3/bak/evaluate.py
class QWKEvaluation (line 25) | class QWKEvaluation(Callback):
method __init__ (line 26) | def __init__(self, validation_data=(), batch_size=64, interval=1):
method on_epoch_end (line 34) | def on_epoch_end(self, epoch, logs={}):
FILE: projects/kaggle/blindness/keras3/dataset.py
class Dataset (line 31) | class Dataset(Sequence):
method __init__ (line 33) | def __init__(self, image_filenames, labels,
method __len__ (line 45) | def __len__(self):
method __getitem__ (line 48) | def __getitem__(self, idx):
method on_epoch_end (line 56) | def on_epoch_end(self):
method mix_up (line 62) | def mix_up(self, x, y):
method train_generate (line 73) | def train_generate(self, batch_x, batch_y):
method get_images (line 91) | def get_images(self, indexes):
method valid_generate (line 105) | def valid_generate(self, batch_x, batch_y):
FILE: projects/kaggle/blindness/keras3/evaluate.py
class QWKEvaluation (line 25) | class QWKEvaluation(Callback):
method __init__ (line 26) | def __init__(self, validation_data=(), batch_size=64, interval=1):
method on_epoch_end (line 34) | def on_epoch_end(self, epoch, logs={}):
FILE: projects/kaggle/blindness/keras3/model.py
function create_model (line 26) | def create_model(input_shape, n_out):
FILE: projects/kaggle/blindness/keras3/train.py
function get_num_gpus (line 43) | def get_num_gpus():
function main (line 54) | def main(_):
FILE: projects/kaggle/blindness/other/keras_baseline.py
function display_samples (line 95) | def display_samples(df, columns=4, rows=3):
class My_Generator (line 202) | class My_Generator(Sequence):
method __init__ (line 204) | def __init__(self, image_filenames, labels,
method __len__ (line 215) | def __len__(self):
method __getitem__ (line 218) | def __getitem__(self, idx):
method on_epoch_end (line 226) | def on_epoch_end(self):
method mix_up (line 232) | def mix_up(self, x, y):
method train_generate (line 243) | def train_generate(self, batch_x, batch_y):
method valid_generate (line 257) | def valid_generate(self, batch_x, batch_y):
function create_model (line 271) | def create_model(input_shape, n_out):
function kappa_loss (line 319) | def kappa_loss(y_true, y_pred, y_pow=2, eps=1e-12, N=5, bsize=32, name='...
class QWKEvaluation (line 361) | class QWKEvaluation(Callback):
method __init__ (line 362) | def __init__(self, validation_data=(), batch_size=64, interval=1):
method on_epoch_end (line 370) | def on_epoch_end(self, epoch, logs={}):
FILE: projects/kaggle/blindness/other/training-mobilenet-v2-in-4-min.py
function pad_and_resize (line 26) | def pad_and_resize(image_path, pad=True, desired_size=224):
function build_model (line 51) | def build_model():
FILE: projects/kaggle/blindness/prepare/gen-records.py
function convert_to_tfrecord (line 44) | def convert_to_tfrecord(input_files, output_file):
function main (line 67) | def main(data_dir):
FILE: projects/kaggle/blindness/tf/dataset.py
class DataSet (line 34) | class DataSet(object):
method __init__ (line 36) | def __init__(self, data_dir, subset='train', use_distortion=True):
method get_filenames (line 41) | def get_filenames(self):
method parser (line 47) | def parser(self, serialized_example):
method make_batch (line 84) | def make_batch(self, batch_size, filenames=None, repeat=None, initiali...
method preprocess (line 116) | def preprocess(self, image):
method num_examples_per_epoch (line 146) | def num_examples_per_epoch(subset='train'):
FILE: projects/kaggle/blindness/tf/evaluate.py
function evaluate (line 24) | def evaluate(labels, logits, ids=None):
function write (line 39) | def write(ids, labels, logits, ofile):
function valid_write (line 50) | def valid_write(ids, labels, logits, ofile):
function infer_write (line 53) | def infer_write(ids, logits, ofile):
FILE: projects/kaggle/blindness/tf/loss.py
function criterion (line 20) | def criterion(model, x, y, training=False):
FILE: projects/kaggle/blindness/tf/model.py
class BaseModel (line 26) | class BaseModel(model_base.ResNet):
method __init__ (line 28) | def __init__(self,
method init_predict (line 46) | def init_predict(self, input_data_format='channels_last'):
method forward_pass (line 58) | def forward_pass(self, x, input_data_format='channels_last'):
method predict (line 101) | def predict(self, x=None, input_data_format='channels_last'):
class Model (line 115) | class Model(tf.keras.Model):
method __init__ (line 116) | def __init__(self):
method call (line 134) | def call(self, x, input_data_format='channels_last', training=False):
FILE: projects/kaggle/blindness/tf/model_base.py
class ResNet (line 29) | class ResNet(object):
method __init__ (line 32) | def __init__(self, training, data_format, batch_norm_decay, batch_norm...
method forward_pass (line 46) | def forward_pass(self, x):
method _residual_v1 (line 50) | def _residual_v1(self,
method _residual_v2 (line 83) | def _residual_v2(self,
method _bottleneck_residual_v2 (line 120) | def _bottleneck_residual_v2(self,
method _conv (line 156) | def _conv(self, x, kernel_size, filters, strides, is_atrous=False):
method _batch_norm (line 178) | def _batch_norm(self, x):
method _relu (line 193) | def _relu(self, x):
method _fully_connected (line 196) | def _fully_connected(self, x, out_dim):
method _avg_pool (line 203) | def _avg_pool(self, x, pool_size, stride):
method _global_avg_pool (line 211) | def _global_avg_pool(self, x):
FILE: projects/kaggle/blindness/tf/train.py
function get_dataset (line 35) | def get_dataset(subset):
function main (line 44) | def main(_):
FILE: projects/kaggle/blindness/tf2/dataset.py
class DataSet (line 34) | class DataSet(object):
method __init__ (line 36) | def __init__(self, data_dir, subset='train', use_distortion=True):
method get_filenames (line 41) | def get_filenames(self):
method parser (line 47) | def parser(self, serialized_example):
method make_batch (line 87) | def make_batch(self, batch_size, filenames=None, repeat=None, initiali...
method preprocess (line 121) | def preprocess(self, image):
method num_examples_per_epoch (line 151) | def num_examples_per_epoch(subset='train'):
FILE: projects/kaggle/blindness/tf2/evaluate.py
function evaluate (line 24) | def evaluate(labels, logits, ids=None):
function write (line 39) | def write(ids, labels, logits, ofile):
function valid_write (line 50) | def valid_write(ids, labels, logits, ofile):
function infer_write (line 53) | def infer_write(ids, logits, ofile):
FILE: projects/kaggle/blindness/tf2/loss.py
function criterion (line 20) | def criterion(model, x, y, training=False):
FILE: projects/kaggle/blindness/tf2/model.py
function create_model (line 37) | def create_model(input_shape, n_out):
class Model (line 61) | class Model(tf.keras.Model):
method __init__ (line 62) | def __init__(self):
method call (line 71) | def call(self, x, training=False):
FILE: projects/kaggle/blindness/tf2/train.py
function get_dataset (line 41) | def get_dataset(subset):
function main (line 49) | def main(_):
FILE: projects/kaggle/cifar10/baseline/keras/KerasT-master/4layerCNN.py
function base_model (line 89) | def base_model():
FILE: projects/kaggle/cifar10/baseline/keras/KerasT-master/6layerCNN.py
function base_model (line 81) | def base_model():
FILE: projects/kaggle/cifar10/baseline/keras/hands-on-deep-learning-master/cifar_image_classification/helpers.py
function load_cifar10 (line 8) | def load_cifar10(filepath="/public/cifar/cifar10.h5"):
function array_2d_to_image (line 29) | def array_2d_to_image(array, autorescale=True):
function model_summary (line 37) | def model_summary(model):
class NeptuneCallback (line 55) | class NeptuneCallback(Callback):
method __init__ (line 56) | def __init__(self, x_test, y_test, images_per_epoch=-1):
method on_epoch_end (line 62) | def on_epoch_end(self, epoch, logs={}):
FILE: projects/kaggle/cifar10/baseline/keras/simple/cifar10.py
function loadData (line 44) | def loadData(path='train'):
function loadTrainValid (line 62) | def loadTrainValid(path='train'):
function loadTest (line 88) | def loadTest(path='test'):
function preprocess (line 105) | def preprocess(trainData, trainLabels=None):
FILE: projects/kaggle/cifar10/baseline/tf/cifar10.py
function load_pickle (line 36) | def load_pickle(f):
function load_CIFAR_batch (line 44) | def load_CIFAR_batch(filename):
function load_CIFAR10 (line 54) | def load_CIFAR10(ROOT):
function get_CIFAR10_data (line 69) | def get_CIFAR10_data(num_training=49000, num_validation=1000, num_test=1...
class CifarNet (line 104) | class CifarNet():
method __init__ (line 105) | def __init__(self):
method forward (line 124) | def forward(self, X, y, is_training):
method run (line 182) | def run(self, session, loss_val, Xd, yd,
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator.v1/cifar10.py
class Cifar10DataSet (line 28) | class Cifar10DataSet(object):
method __init__ (line 34) | def __init__(self, data_dir, subset='train', use_distortion=True):
method get_filenames (line 39) | def get_filenames(self):
method parser (line 45) | def parser(self, serialized_example):
method make_batch (line 71) | def make_batch(self, batch_size, repeat=None):
method preprocess (line 107) | def preprocess(self, image):
method num_examples_per_epoch (line 121) | def num_examples_per_epoch(subset='train'):
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator.v1/cifar10_main.py
function get_model_fn (line 47) | def get_model_fn(num_gpus, variable_strategy, num_workers):
function _tower_fn (line 212) | def _tower_fn(is_training, weight_decay, feature, label, data_format,
function input_fn (line 259) | def input_fn(data_dir,
function get_experiment_fn (line 301) | def get_experiment_fn(data_dir,
function main (line 371) | def main(job_dir, data_dir, num_gpus, variable_strategy,
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator.v1/cifar10_model.py
class ResNetCifar10 (line 26) | class ResNetCifar10(model_base.ResNet):
method __init__ (line 29) | def __init__(self,
method init_predict (line 47) | def init_predict(self, input_data_format='channels_last'):
method forward_pass (line 59) | def forward_pass(self, x, input_data_format='channels_last'):
method predict (line 102) | def predict(self, x=None, input_data_format='channels_last'):
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator.v1/cifar10_multi_gpu_train.py
function tower_loss (line 65) | def tower_loss(scope, images, labels):
function average_gradients (line 101) | def average_gradients(tower_grads):
function train (line 139) | def train():
function main (line 268) | def main(argv=None): # pylint: disable=unused-argument
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator.v1/cifar10_utils.py
class RunConfig (line 17) | class RunConfig(tf.contrib.learn.RunConfig):
method uid (line 18) | def uid(self, whitelist=None):
class ExamplesPerSecondHook (line 51) | class ExamplesPerSecondHook(session_run_hook.SessionRunHook):
method __init__ (line 60) | def __init__(
method begin (line 83) | def begin(self):
method before_run (line 89) | def before_run(self, run_context): # pylint: disable=unused-argument
method after_run (line 92) | def after_run(self, run_context, run_values):
function local_device_setter (line 112) | def local_device_setter(num_devices=1,
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator.v1/evaluator.py
function write (line 39) | def write(ids, predicts, model_path, labels=None, images=None, suffix='v...
function evaluate (line 67) | def evaluate(eval_ops, iterator, model_path=None, sess=None):
function inference (line 120) | def inference(ops, iterator, model_path=None, sess=None):
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator.v1/generate_cifar10_tfrecords.py
function _int64_feature (line 58) | def _int64_feature(value):
function _bytes_feature (line 62) | def _bytes_feature(value):
function convert_to_tfrecord (line 65) | def convert_to_tfrecord(input_files, output_file):
function main (line 86) | def main(data_dir):
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator.v1/model_base.py
class ResNet (line 29) | class ResNet(object):
method __init__ (line 32) | def __init__(self, is_training, data_format, batch_norm_decay, batch_n...
method forward_pass (line 46) | def forward_pass(self, x):
method _residual_v1 (line 50) | def _residual_v1(self,
method _residual_v2 (line 83) | def _residual_v2(self,
method _bottleneck_residual_v2 (line 120) | def _bottleneck_residual_v2(self,
method _conv (line 156) | def _conv(self, x, kernel_size, filters, strides, is_atrous=False):
method _batch_norm (line 178) | def _batch_norm(self, x):
method _relu (line 193) | def _relu(self, x):
method _fully_connected (line 196) | def _fully_connected(self, x, out_dim):
method _avg_pool (line 203) | def _avg_pool(self, x, pool_size, stride):
method _global_avg_pool (line 211) | def _global_avg_pool(self, x):
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator.v1/train.py
function tower_loss (line 39) | def tower_loss(model, feature, label):
function main (line 68) | def main(_):
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator/cifar10.py
class Cifar10DataSet (line 28) | class Cifar10DataSet(object):
method __init__ (line 34) | def __init__(self, data_dir, subset='train', use_distortion=True):
method get_filenames (line 39) | def get_filenames(self):
method parser (line 45) | def parser(self, serialized_example):
method make_batch (line 71) | def make_batch(self, batch_size, repeat=None):
method preprocess (line 103) | def preprocess(self, image):
method num_examples_per_epoch (line 117) | def num_examples_per_epoch(subset='train'):
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator/cifar10_main.py
function get_model_fn (line 47) | def get_model_fn(num_gpus, variable_strategy, num_workers):
function _tower_fn (line 212) | def _tower_fn(is_training, weight_decay, feature, label, data_format,
function input_fn (line 259) | def input_fn(data_dir,
function get_experiment_fn (line 301) | def get_experiment_fn(data_dir,
function main (line 371) | def main(job_dir, data_dir, num_gpus, variable_strategy,
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator/cifar10_model.py
class ResNetCifar10 (line 26) | class ResNetCifar10(model_base.ResNet):
method __init__ (line 29) | def __init__(self,
method init_predict (line 47) | def init_predict(self, input_data_format='channels_last'):
method forward_pass (line 59) | def forward_pass(self, x, input_data_format='channels_last'):
method predict (line 102) | def predict(self, x=None, input_data_format='channels_last'):
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator/cifar10_multi_gpu_train.py
function tower_loss (line 65) | def tower_loss(scope, images, labels):
function average_gradients (line 101) | def average_gradients(tower_grads):
function train (line 139) | def train():
function main (line 268) | def main(argv=None): # pylint: disable=unused-argument
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator/cifar10_utils.py
class RunConfig (line 17) | class RunConfig(tf.contrib.learn.RunConfig):
method uid (line 18) | def uid(self, whitelist=None):
class ExamplesPerSecondHook (line 51) | class ExamplesPerSecondHook(session_run_hook.SessionRunHook):
method __init__ (line 60) | def __init__(
method begin (line 83) | def begin(self):
method before_run (line 89) | def before_run(self, run_context): # pylint: disable=unused-argument
method after_run (line 92) | def after_run(self, run_context, run_values):
function local_device_setter (line 112) | def local_device_setter(num_devices=1,
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator/evaluator.py
function write (line 39) | def write(ids, predicts, model_path, labels=None, images=None, suffix='v...
function evaluate (line 67) | def evaluate(eval_ops, iterator, num_steps, num_examples, model_path=Non...
function inference (line 122) | def inference(ops, iterator, num_steps, num_examples, model_path=None, n...
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator/generate_cifar10_tfrecords.py
function _int64_feature (line 58) | def _int64_feature(value):
function _bytes_feature (line 62) | def _bytes_feature(value):
function convert_to_tfrecord (line 65) | def convert_to_tfrecord(input_files, output_file):
function main (line 86) | def main(data_dir):
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator/model_base.py
class ResNet (line 29) | class ResNet(object):
method __init__ (line 32) | def __init__(self, is_training, data_format, batch_norm_decay, batch_n...
method forward_pass (line 46) | def forward_pass(self, x):
method _residual_v1 (line 50) | def _residual_v1(self,
method _residual_v2 (line 83) | def _residual_v2(self,
method _bottleneck_residual_v2 (line 120) | def _bottleneck_residual_v2(self,
method _conv (line 156) | def _conv(self, x, kernel_size, filters, strides, is_atrous=False):
method _batch_norm (line 178) | def _batch_norm(self, x):
method _relu (line 193) | def _relu(self, x):
method _fully_connected (line 196) | def _fully_connected(self, x, out_dim):
method _avg_pool (line 203) | def _avg_pool(self, x, pool_size, stride):
method _global_avg_pool (line 211) | def _global_avg_pool(self, x):
FILE: projects/kaggle/cifar10/baseline/tf/cifar10_estimator/train.py
function tower_loss (line 40) | def tower_loss(model, feature, label):
function main (line 69) | def main(_):
FILE: projects/kaggle/cifar10/tf/cifar10.py
class Cifar10DataSet (line 34) | class Cifar10DataSet(object):
method __init__ (line 40) | def __init__(self, data_dir, subset='train', use_distortion=True):
method get_filenames (line 45) | def get_filenames(self):
method parser (line 51) | def parser(self, serialized_example):
method make_batch (line 80) | def make_batch(self, batch_size, filenames=None, repeat=None, initiali...
method preprocess (line 112) | def preprocess(self, image):
method num_examples_per_epoch (line 133) | def num_examples_per_epoch(subset='train'):
FILE: projects/kaggle/cifar10/tf/cifar10_model.py
class ResNetCifar10 (line 26) | class ResNetCifar10(model_base.ResNet):
method __init__ (line 29) | def __init__(self,
method init_predict (line 47) | def init_predict(self, input_data_format='channels_last'):
method forward_pass (line 59) | def forward_pass(self, x, input_data_format='channels_last'):
method predict (line 102) | def predict(self, x=None, input_data_format='channels_last'):
class Model (line 116) | class Model(tf.keras.Model):
method __init__ (line 117) | def __init__(self):
method call (line 135) | def call(self, x, input_data_format='channels_last', training=False):
FILE: projects/kaggle/cifar10/tf/evaluate.py
function evaluate (line 26) | def evaluate(labels, logits, ids=None):
function write (line 39) | def write(ids, labels, logits, ofile):
function valid_write (line 50) | def valid_write(ids, labels, logits, ofile):
function infer_write (line 53) | def infer_write(ids, logits, ofile):
FILE: projects/kaggle/cifar10/tf/evaluator.py
function write (line 39) | def write(ids, predicts, model_path, labels=None, images=None, suffix='v...
function evaluate (line 67) | def evaluate(eval_ops, iterator, num_steps, num_examples, model_path=Non...
function inference (line 121) | def inference(ops, iterator, num_steps, num_examples, model_path=None, n...
FILE: projects/kaggle/cifar10/tf/loss.py
function criterion (line 20) | def criterion(model, x, y, training=False):
FILE: projects/kaggle/cifar10/tf/model_base.py
class ResNet (line 29) | class ResNet(object):
method __init__ (line 32) | def __init__(self, training, data_format, batch_norm_decay, batch_norm...
method forward_pass (line 46) | def forward_pass(self, x):
method _residual_v1 (line 50) | def _residual_v1(self,
method _residual_v2 (line 83) | def _residual_v2(self,
method _bottleneck_residual_v2 (line 120) | def _bottleneck_residual_v2(self,
method _conv (line 156) | def _conv(self, x, kernel_size, filters, strides, is_atrous=False):
method _batch_norm (line 178) | def _batch_norm(self, x):
method _relu (line 193) | def _relu(self, x):
method _fully_connected (line 196) | def _fully_connected(self, x, out_dim):
method _avg_pool (line 203) | def _avg_pool(self, x, pool_size, stride):
method _global_avg_pool (line 211) | def _global_avg_pool(self, x):
FILE: projects/kaggle/cifar10/tf/train.py
function tower_loss (line 39) | def tower_loss(model, feature, label):
function main (line 68) | def main(_):
FILE: projects/kaggle/cifar10/tf/train2.py
function get_dataset (line 35) | def get_dataset(subset):
function main (line 44) | def main(_):
FILE: projects/kaggle/toxic/algos/model.py
class MyModel (line 31) | class MyModel(keras.Model):
method __init__ (line 32) | def __init__(self):
method call (line 42) | def call(self, x):
class Model (line 45) | class Model(keras.Model):
method __init__ (line 46) | def __init__(self):
method call (line 81) | def call(self, x, training=False):
function criterion (line 100) | def criterion(model, x, y, training=False):
FILE: projects/kaggle/toxic/dataset.py
class Dataset (line 36) | class Dataset(melt.tfrecords.Dataset):
method __init__ (line 37) | def __init__(self, subset='train'):
method parser (line 40) | def parser(self, example):
FILE: projects/kaggle/toxic/evaluate.py
function calc_auc (line 21) | def calc_auc(labels, predicts):
FILE: projects/kaggle/toxic/prepare/count-unks.py
function run (line 24) | def run(input):
FILE: projects/kaggle/toxic/prepare/extend-table.py
function process (line 25) | def process(x):
FILE: projects/kaggle/toxic/prepare/gen-correction.py
function get_en_token (line 32) | def get_en_token(token):
function run (line 40) | def run(file_):
FILE: projects/kaggle/toxic/prepare/gen-en-lang-prob.py
function run (line 23) | def run(file_):
FILE: projects/kaggle/toxic/prepare/gen-lang.py
function run (line 28) | def run(file_):
FILE: projects/kaggle/toxic/prepare/gen-records-parse.py
function get_id (line 79) | def get_id(word, vocab):
function get_char_id (line 85) | def get_char_id(ch, vocab):
function get_ngram_id (line 90) | def get_ngram_id(ngram, vocab):
function get_mode (line 96) | def get_mode():
function get_fold (line 105) | def get_fold(ids, index):
function build_features (line 127) | def build_features(index):
function main (line 308) | def main(_):
FILE: projects/kaggle/toxic/prepare/gen-records.py
function get_id (line 56) | def get_id(word, vocab):
function get_char_id (line 62) | def get_char_id(ch, vocab):
function build_features (line 67) | def build_features(index):
function main (line 157) | def main(_):
FILE: projects/kaggle/toxic/prepare/gen-sentences-csv.py
function run (line 24) | def run():
FILE: projects/kaggle/toxic/prepare/gen-sentences.py
function run (line 25) | def run(index):
FILE: projects/kaggle/toxic/prepare/gen-tokens.py
function tokenize (line 66) | def tokenize(index):
function run (line 79) | def run(input):
function main (line 93) | def main(_):
FILE: projects/kaggle/toxic/prepare/gen-vocab-parse.py
function tokenize (line 90) | def tokenize(index):
function run (line 151) | def run(input, count=1):
function main (line 248) | def main(_):
FILE: projects/kaggle/toxic/prepare/gen-vocab.py
function tokenize (line 63) | def tokenize(index):
function run (line 76) | def run(input, count=1):
function main (line 103) | def main(_):
FILE: projects/kaggle/toxic/prepare/merge-2emb.py
function main (line 36) | def main(_):
FILE: projects/kaggle/toxic/prepare/merge-charemb.py
function main (line 31) | def main(_):
FILE: projects/kaggle/toxic/prepare/merge-emb.py
function main (line 35) | def main(_):
FILE: projects/kaggle/toxic/prepare/merge-glove.py
function main (line 36) | def main(_):
FILE: projects/kaggle/toxic/prepare/merge-ngram-emb.py
function main (line 40) | def main(_):
FILE: projects/kaggle/toxic/prepare/merge-word-ngram-emb.py
function main (line 37) | def main(_):
FILE: projects/kaggle/toxic/prepare/merge-wordemb.py
function main (line 31) | def main(_):
FILE: projects/kaggle/toxic/prepare/preprocess.py
function normalize (line 26) | def normalize(text):
function glove_twitter_preprocess (line 87) | def glove_twitter_preprocess(text):
FILE: projects/kaggle/toxic/prepare/test-tokenize.py
function tokenize (line 20) | def tokenize(text):
FILE: projects/kaggle/toxic/prepare/test-tokenize2.py
function tokenize (line 20) | def tokenize(text):
FILE: projects/kaggle/toxic/prepare/test-tokenize3.py
function tokenize (line 20) | def tokenize(text):
FILE: projects/kaggle/toxic/prepare/test-tokenize4.py
function tokenize (line 20) | def tokenize(text):
FILE: projects/kaggle/toxic/prepare/test-tokenize5.py
function tokenize (line 20) | def tokenize(text):
FILE: projects/kaggle/toxic/prepare/tokenize-corpus.py
function tokenize (line 23) | def tokenize(file_):
FILE: projects/kaggle/toxic/prepare/tokenizer-v2.py
function init (line 57) | def init(vocab_path='/home/gezi/data/glove/glove-vocab.txt'):
function dict_has (line 78) | def dict_has(word):
function has (line 84) | def has(word):
function en_filter (line 91) | def en_filter(token):
function can_split (line 128) | def can_split(w1, w2):
function try_split (line 131) | def try_split(token):
function is_toxic (line 168) | def is_toxic(word):
function maybe_toxic (line 171) | def maybe_toxic(word):
function get_token_len (line 177) | def get_token_len(token):
function is_en (line 183) | def is_en(token):
function get_attr (line 189) | def get_attr(token,
function tokenize (line 202) | def tokenize(text):
function full_tokenize (line 278) | def full_tokenize(text):
FILE: projects/kaggle/toxic/prepare/tokenizer-v3.py
function init (line 58) | def init(vocab_path='/home/gezi/data/glove/glove-vocab.txt'):
function dict_has (line 79) | def dict_has(word):
function has (line 85) | def has(word):
function en_filter (line 93) | def en_filter(token):
function can_split (line 130) | def can_split(w1, w2):
function try_split (line 133) | def try_split(token):
function is_toxic (line 170) | def is_toxic(word):
function get_token_len (line 182) | def get_token_len(token):
function is_en (line 188) | def is_en(token):
function get_attr (line 194) | def get_attr(token,
function tokenize (line 207) | def tokenize(text):
function full_tokenize (line 291) | def full_tokenize(text):
FILE: projects/kaggle/toxic/prepare/tokenizer.py
function init (line 66) | def init(vocab_path='/home/gezi/data/glove/glove-vocab.txt'):
function dict_has (line 90) | def dict_has(word):
function has (line 96) | def has(word):
function en_filter (line 104) | def en_filter(token):
function can_split (line 141) | def can_split(w1, w2):
function try_split (line 144) | def try_split(token):
function is_toxic (line 189) | def is_toxic(word):
function get_token_len (line 201) | def get_token_len(token):
function is_en (line 206) | def is_en(token):
function get_lemma (line 215) | def get_lemma(token):
function get_attr (line 230) | def get_attr(token,
function try_correct_toxic (line 246) | def try_correct_toxic(token):
function star_inside (line 264) | def star_inside(word):
function tokenize (line 270) | def tokenize(text, lemmatization=False):
function full_tokenize (line 371) | def full_tokenize(text, lemmatization=False):
FILE: projects/kaggle/toxic/prepare/toxic_words.py
function get_toxic_words (line 32) | def get_toxic_words():
FILE: projects/kaggle/toxic/read-records.py
function main (line 40) | def main(_):
FILE: projects/kaggle/toxic/train.py
function main (line 35) | def main(_):
FILE: tests/sample-balance/dataset.py
class Dataset (line 32) | class Dataset(melt.tfrecords.Dataset):
method __init__ (line 33) | def __init__(self, subset='train'):
method parser (line 51) | def parser(self, example):
FILE: tests/sample-balance/read-records.py
function main (line 46) | def main(_):
FILE: third/bert/create_pretraining_data.py
class TrainingInstance (line 67) | class TrainingInstance(object):
method __init__ (line 70) | def __init__(self, tokens, segment_ids, masked_lm_positions, masked_lm...
method __str__ (line 78) | def __str__(self):
method __repr__ (line 91) | def __repr__(self):
function write_instance_to_example_files (line 95) | def write_instance_to_example_files(instances, tokenizer, max_seq_length,
function create_int_feature (line 168) | def create_int_feature(values):
function create_float_feature (line 173) | def create_float_feature(values):
function create_training_instances (line 178) | def create_training_instances(input_files, tokenizer, max_seq_length,
function create_instances_from_document (line 229) | def create_instances_from_document(
function create_masked_lm_predictions (line 344) | def create_masked_lm_predictions(tokens, masked_lm_prob,
function truncate_seq_pair (line 399) | def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng):
function main (line 417) | def main(_):
FILE: third/bert/extract_features.py
class InputExample (line 81) | class InputExample(object):
method __init__ (line 83) | def __init__(self, unique_id, text_a, text_b):
class InputFeatures (line 89) | class InputFeatures(object):
method __init__ (line 92) | def __init__(self, unique_id, tokens, input_ids, input_mask, input_typ...
function input_fn_builder (line 100) | def input_fn_builder(features, seq_length):
function model_fn_builder (line 148) | def model_fn_builder(bert_config, init_checkpoint, layer_indexes, use_tpu,
function convert_examples_to_features (line 201) | def convert_examples_to_features(examples, seq_length, tokenizer):
function _truncate_seq_pair (line 292) | def _truncate_seq_pair(tokens_a, tokens_b, max_length):
function read_examples (line 309) | def read_examples(input_file):
function main (line 333) | def main(_):
FILE: third/bert/modeling.py
class BertConfig (line 32) | class BertConfig(object):
method __init__ (line 35) | def __init__(self,
method from_dict (line 84) | def from_dict(cls, json_object):
method from_json_file (line 92) | def from_json_file(cls, json_file):
method to_dict (line 98) | def to_dict(self):
method to_json_string (line 103) | def to_json_string(self):
class BertModel (line 108) | class BertModel(object):
method __init__ (line 132) | def __init__(self,
method get_pooled_output (line 238) | def get_pooled_output(self):
method get_sequence_output (line 241) | def get_sequence_output(self):
method get_all_encoder_layers (line 250) | def get_all_encoder_layers(self):
method get_embedding_output (line 253) | def get_embedding_output(self):
method get_embedding_table (line 264) | def get_embedding_table(self):
function gelu (line 268) | def gelu(input_tensor):
function get_activation (line 284) | def get_activation(activation_string):
function get_assigment_map_from_checkpoint (line 321) | def get_assigment_map_from_checkpoint(tvars, init_checkpoint):
function dropout (line 348) | def dropout(input_tensor, dropout_prob):
function layer_norm (line 366) | def layer_norm(input_tensor, name=None):
function layer_norm_and_dropout (line 372) | def layer_norm_and_dropout(input_tensor, dropout_prob, name=None):
function create_initializer (line 379) | def create_initializer(initializer_range=0.02):
function embedding_lookup (line 384) | def embedding_lookup(input_ids,
function embedding_postprocessor (line 471) | def embedding_postprocessor(input_tensor,
function create_attention_mask_from_input_mask (line 576) | def create_attention_mask_from_input_mask(from_tensor, to_mask):
function attention_layer (line 610) | def attention_layer(from_tensor,
function transformer_model (line 806) | def transformer_model(input_tensor,
function get_shape_list (line 947) | def get_shape_list(tensor, expected_rank=None, name=None):
function reshape_to_matrix (line 984) | def reshape_to_matrix(input_tensor):
function reshape_from_matrix (line 998) | def reshape_from_matrix(output_tensor, orig_shape_list):
function assert_rank (line 1011) | def assert_rank(tensor, expected_rank, name=None):
FILE: third/bert/modeling_test.py
class BertModelTest (line 29) | class BertModelTest(tf.test.TestCase):
class BertModelTester (line 31) | class BertModelTester(object):
method __init__ (line 33) | def __init__(self,
method create_model (line 71) | def create_model(self):
method check_output (line 114) | def check_output(self, result):
method test_default (line 126) | def test_default(self):
method test_config_to_json_string (line 129) | def test_config_to_json_string(self):
method run_tester (line 135) | def run_tester(self, tester):
method ids_tensor (line 147) | def ids_tensor(cls, shape, vocab_size, rng=None, name=None):
method assert_all_tensors_reachable (line 162) | def assert_all_tensors_reachable(self, sess, outputs):
method get_unreachable_ops (line 193) | def get_unreachable_ops(cls, graph, outputs):
method flatten_recursive (line 256) | def flatten_recursive(cls, item):
FILE: third/bert/optimization.py
function create_optimizer (line 25) | def create_optimizer(loss, init_lr, num_train_steps, num_warmup_steps, m...
class AdamWeightDecayOptimizer (line 90) | class AdamWeightDecayOptimizer(tf.train.Optimizer):
method __init__ (line 93) | def __init__(self,
method apply_gradients (line 111) | def apply_gradients(self, grads_and_vars, global_step=None, name=None):
method _do_use_weight_decay (line 162) | def _do_use_weight_decay(self, param_name):
method _get_variable_name (line 172) | def _get_variable_name(self, param_name):
FILE: third/bert/optimization_test.py
class OptimizationTest (line 23) | class OptimizationTest(tf.test.TestCase):
method test_adam (line 25) | def test_adam(self):
FILE: third/bert/run_classifier.py
class InputExample (line 121) | class InputExample(object):
method __init__ (line 124) | def __init__(self, guid, text_a, text_b=None, label=None):
class InputFeatures (line 142) | class InputFeatures(object):
method __init__ (line 145) | def __init__(self, input_ids, input_mask, segment_ids, label_id):
class DataProcessor (line 152) | class DataProcessor(object):
method get_train_examples (line 155) | def get_train_examples(self, data_dir):
method get_dev_examples (line 159) | def get_dev_examples(self, data_dir):
method get_labels (line 163) | def get_labels(self):
method _read_tsv (line 168) | def _read_tsv(cls, input_file, quotechar=None):
class XnliProcessor (line 178) | class XnliProcessor(DataProcessor):
method __init__ (line 181) | def __init__(self):
method get_train_examples (line 184) | def get_train_examples(self, data_dir):
method get_dev_examples (line 203) | def get_dev_examples(self, data_dir):
method get_labels (line 221) | def get_labels(self):
class MnliProcessor (line 226) | class MnliProcessor(DataProcessor):
method get_train_examples (line 229) | def get_train_examples(self, data_dir):
method get_dev_examples (line 234) | def get_dev_examples(self, data_dir):
method get_labels (line 240) | def get_labels(self):
method _create_examples (line 244) | def _create_examples(self, lines, set_type):
class MrpcProcessor (line 259) | class MrpcProcessor(DataProcessor):
method get_train_examples (line 262) | def get_train_examples(self, data_dir):
method get_dev_examples (line 267) | def get_dev_examples(self, data_dir):
method get_labels (line 272) | def get_labels(self):
method _create_examples (line 276) | def _create_examples(self, lines, set_type):
class ColaProcessor (line 291) | class ColaProcessor(DataProcessor):
method get_train_examples (line 294) | def get_train_examples(self, data_dir):
method get_dev_examples (line 299) | def get_dev_examples(self, data_dir):
method get_labels (line 304) | def get_labels(self):
method _create_examples (line 308) | def _create_examples(self, lines, set_type):
function convert_examples_to_features (line 320) | def convert_examples_to_features(examples, label_list, max_seq_length,
function _truncate_seq_pair (line 427) | def _truncate_seq_pair(tokens_a, tokens_b, max_length):
function create_model (line 444) | def create_model(bert_config, is_training, input_ids, input_mask, segmen...
function model_fn_builder (line 488) | def model_fn_builder(bert_config, num_labels, init_checkpoint, learning_...
function input_fn_builder (line 572) | def input_fn_builder(input_file, seq_length, is_training, drop_remainder):
function main (line 618) | def main(_):
FILE: third/bert/run_pretraining.py
function model_fn_builder (line 109) | def model_fn_builder(bert_config, init_checkpoint, learning_rate,
function get_masked_lm_output (line 241) | def get_masked_lm_output(bert_config, input_tensor, output_weights, posi...
function get_next_sentence_output (line 286) | def get_next_sentence_output(bert_config, input_tensor, labels):
function gather_indexes (line 309) | def gather_indexes(sequence_tensor, positions):
function input_fn_builder (line 325) | def input_fn_builder(input_files,
function _decode_record (line 392) | def _decode_record(record, name_to_features):
function main (line 407) | def main(_):
FILE: third/bert/run_squad.py
class SquadExample (line 149) | class SquadExample(object):
method __init__ (line 152) | def __init__(self,
method __str__ (line 166) | def __str__(self):
method __repr__ (line 169) | def __repr__(self):
class InputFeatures (line 182) | class InputFeatures(object):
method __init__ (line 185) | def __init__(self,
function read_squad_examples (line 210) | def read_squad_examples(input_file, is_training):
function convert_examples_to_features (line 279) | def convert_examples_to_features(examples, tokenizer, max_seq_length,
function _improve_answer_span (line 433) | def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer,
function _check_is_max_context (line 470) | def _check_is_max_context(doc_spans, cur_span_index, position):
function create_model (line 507) | def create_model(bert_config, is_training, input_ids, input_mask, segmen...
function model_fn_builder (line 547) | def model_fn_builder(bert_config, init_checkpoint, learning_rate,
function input_fn_builder (line 645) | def input_fn_builder(input_file, seq_length, is_training, drop_remainder):
function write_predictions (line 699) | def write_predictions(all_examples, all_features, all_results, n_best_size,
function get_final_text (line 833) | def get_final_text(pred_text, orig_text, do_lower_case):
function _get_best_indexes (line 929) | def _get_best_indexes(logits, n_best_size):
function _compute_softmax (line 941) | def _compute_softmax(scores):
class FeatureWriter (line 964) | class FeatureWriter(object):
method __init__ (line 967) | def __init__(self, filename, is_training):
method process_feature (line 973) | def process_feature(self, feature):
method close (line 995) | def close(self):
function validate_flags_or_throw (line 999) | def validate_flags_or_throw(bert_config):
function main (line 1025) | def main(_):
FILE: third/bert/tokenization.py
function convert_to_unicode (line 27) | def convert_to_unicode(text):
function printable_text (line 47) | def printable_text(text):
function load_vocab (line 70) | def load_vocab(vocab_file):
function convert_tokens_to_ids (line 85) | def convert_tokens_to_ids(vocab, tokens):
function whitespace_tokenize (line 93) | def whitespace_tokenize(text):
class FullTokenizer (line 102) | class FullTokenizer(object):
method __init__ (line 105) | def __init__(self, vocab_file, do_lower_case=True):
method tokenize (line 110) | def tokenize(self, text):
method convert_tokens_to_ids (line 118) | def convert_tokens_to_ids(self, tokens):
class BasicTokenizer (line 122) | class BasicTokenizer(object):
method __init__ (line 125) | def __init__(self, do_lower_case=True):
method tokenize (line 133) | def tokenize(self, text):
method _run_strip_accents (line 157) | def _run_strip_accents(self, text):
method _run_split_on_punc (line 168) | def _run_split_on_punc(self, text):
method _tokenize_chinese_chars (line 188) | def _tokenize_chinese_chars(self, text):
method _is_chinese_char (line 201) | def _is_chinese_char(self, cp):
method _clean_text (line 223) | def _clean_text(self, text):
class WordpieceTokenizer (line 237) | class WordpieceTokenizer(object):
method __init__ (line 240) | def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=...
method tokenize (line 245) | def tokenize(self, text):
function _is_whitespace (line 299) | def _is_whitespace(char):
function _is_control (line 311) | def _is_control(char):
function _is_punctuation (line 323) | def _is_punctuation(char):
FILE: third/bert/tokenization_test.py
class TokenizationTest (line 26) | class TokenizationTest(tf.test.TestCase):
method test_full_tokenizer (line 28) | def test_full_tokenizer(self):
method test_chinese (line 47) | def test_chinese(self):
method test_basic_tokenizer_lower (line 54) | def test_basic_tokenizer_lower(self):
method test_basic_tokenizer_no_lower (line 62) | def test_basic_tokenizer_no_lower(self):
method test_wordpiece_tokenizer (line 69) | def test_wordpiece_tokenizer(self):
method test_convert_tokens_to_ids (line 89) | def test_convert_tokens_to_ids(self):
method test_is_whitespace (line 103) | def test_is_whitespace(self):
method test_is_control (line 113) | def test_is_control(self):
method test_is_punctuation (line 121) | def test_is_punctuation(self):
FILE: utils/gezi/avg_score.py
class AvgScore (line 15) | class AvgScore():
method __init__ (line 20) | def __init__(self):
method reset (line 23) | def reset(self):
method add (line 28) | def add(self, score):
method avg_score (line 39) | def avg_score(self):
FILE: utils/gezi/bigdata_util.py
function init (line 36) | def init():
function get_handle (line 42) | def get_handle():
function fullpath (line 46) | def fullpath(path):
function glob (line 49) | def glob(file_pattern):
function hdfs_listdir (line 65) | def hdfs_listdir(dir):
function list_files (line 74) | def list_files(input):
function is_remote_path (line 88) | def is_remote_path(path):
FILE: utils/gezi/bleu.py
function normalize (line 44) | def normalize(s):
function count_ngrams (line 63) | def count_ngrams(words, n=4):
function cook_refs (line 71) | def cook_refs(refs, n=4):
function cook_test (line 84) | def cook_test(test, (reflens, refmaxcounts), n=4):
function score_cooked (line 114) | def score_cooked(allcomps, n=4):
function score_set (line 133) | def score_set(set, testid, refids, n=4):
FILE: utils/gezi/gezi_util.py
function gprint (line 9) | def gprint(convert2utf8, *content):
function uprint (line 15) | def uprint(*content):
function toutf8 (line 18) | def toutf8(content):
function togbk (line 21) | def togbk(content):
function now_time (line 25) | def now_time():
function get_timestr (line 28) | def get_timestr(stamp):
function get_datestr (line 36) | def get_datestr(stamp):
function pretty_print (line 44) | def pretty_print(df):
function print_list (line 54) | def print_list(l, sep='|'):
function get_words (line 59) | def get_words(l, ngram, sep = '\x01'):
function get_ngram_words (line 68) | def get_ngram_words(l, ngram, sep = '\x01'):
function get_skipn_bigram (line 79) | def get_skipn_bigram(l, n, sep = '\x01'):
function get_skipn_bigram (line 88) | def get_skipn_bigram(l, li, n, sep = '\x01'):
function get_skip_bigram (line 95) | def get_skip_bigram(l, li, n, sep = '\x01'):
function h2o (line 99) | def h2o(x):
function h2o2 (line 107) | def h2o2(x):
function json2obj (line 115) | def json2obj(s):
function json2obj2 (line 122) | def json2obj2(s):
function jsonfile2obj (line 129) | def jsonfile2obj(path):
function jsonfile2obj2 (line 132) | def jsonfile2obj2(path):
function xmlfile2obj (line 135) | def xmlfile2obj(path):
function xmlfile2obj2 (line 143) | def xmlfile2obj2(path):
function dict2map (line 158) | def dict2map(dict_, map_):
function map2dict (line 162) | def map2dict(map_):
function list2vec (line 168) | def list2vec(list_, vec_):
function list2vector (line 172) | def list2vector(list_, vec_):
function vec2list (line 176) | def vec2list(vec_):
function vector2list (line 182) | def vector2list(vec_):
function get_filepaths (line 188) | def get_filepaths(directory):
function get_num_lines (line 206) | def get_num_lines(file):
FILE: utils/gezi/hash.py
function hash_str (line 21) | def hash_str(input):
function fasttext_hash (line 29) | def fasttext_hash(word):
FILE: utils/gezi/libgezi_util.py
function to_simplify_ (line 27) | def to_simplify_(sentence):
function to_simplify (line 39) | def to_simplify(sentence):
function normalize_ (line 67) | def normalize_(sentence, to_lower=True, to_simplify=True, to_half=True):
function normalize (line 79) | def normalize(sentence, to_lower=True, to_simplify=True, to_half=True):
function get_single_cns (line 110) | def get_single_cns(text):
function is_single_cn (line 113) | def is_single_cn(word):
function get_single_chars (line 117) | def get_single_chars(text):
function get_single_cns (line 122) | def get_single_cns(text):
function is_cn (line 144) | def is_cn(word):
function is_single_cn (line 149) | def is_single_cn(word):
function get_single_chars (line 152) | def get_single_chars(text):
FILE: utils/gezi/melt/logging.py
function set_hvd (line 48) | def set_hvd(hvd_):
function info (line 52) | def info(*args):
function info2 (line 56) | def info2(*args):
function fatal (line 60) | def fatal(*args):
function error (line 64) | def error(*args):
function debug (line 68) | def debug(*args):
function warn (line 72) | def warn(*args):
function warning (line 76) | def warning(*args):
class ElapsedFormatter (line 83) | class ElapsedFormatter():
method __init__ (line 84) | def __init__(self):
method format (line 87) | def format(self, record):
function _get_handler (line 96) | def _get_handler(file, formatter, split=True, split_bytime=False, mode =...
function set_dir (line 113) | def set_dir(path, file='log.html', logtostderr=True, logtofile=True, spl...
function init (line 147) | def init(path, file='log.html', logtostderr=True, logtofile=True, split=...
function vlog (line 151) | def vlog(level, msg, *args, **kwargs):
function get_verbosity (line 154) | def get_verbosity():
function set_verbosity (line 158) | def set_verbosity(verbosity):
function get_logging_file (line 162) | def get_logging_file():
FILE: utils/gezi/melt/tfrecords.py
class Writer (line 19) | class Writer(object):
method __init__ (line 20) | def __init__(self, file, buffer_size=None):
method __del__ (line 28) | def __del__(self):
method __enter__ (line 32) | def __enter__(self):
method __exit__ (line 35) | def __exit__(self, exc_type, exc_value, traceback):
method close (line 39) | def close(self):
method finalize (line 46) | def finalize(self):
method write (line 49) | def write(self, example):
method size (line 61) | def size(self):
FILE: utils/gezi/melt/util.py
function int_feature (line 19) | def int_feature(value):
function int64_feature (line 25) | def int64_feature(value):
function bytes_feature (line 31) | def bytes_feature(value):
function float_feature (line 40) | def float_feature(value):
function int64_feature_list (line 53) | def int64_feature_list(values):
function bytes_feature_list (line 58) | def bytes_feature_list(values):
function float_feature_list (line 63) | def float_feature_list(values):
FILE: utils/gezi/metrics/bleu/bleu.py
class Bleu (line 14) | class Bleu:
method __init__ (line 15) | def __init__(self, n=4):
method compute_score (line 21) | def compute_score(self, gts, res):
method method (line 47) | def method(self):
FILE: utils/gezi/metrics/bleu/bleu_scorer.py
function precook (line 23) | def precook(s, n=4, out=False):
function cook_refs (line 35) | def cook_refs(refs, eff=None, n=4): ## lhuang: oracle will call with "av...
function cook_test (line 60) | def cook_test(test, l, eff=None, n=4):
class BleuScorer (line 85) | class BleuScorer(object):
method copy (line 92) | def copy(self):
method __init__ (line 100) | def __init__(self, test=None, refs=None, n=4, special_reflen=None):
method cook_append (line 109) | def cook_append(self, test, refs):
method ratio (line 122) | def ratio(self, option=None):
method score_ratio (line 126) | def score_ratio(self, option=None):
method score_ratio_str (line 130) | def score_ratio_str(self, option=None):
method reflen (line 133) | def reflen(self, option=None):
method testlen (line 137) | def testlen(self, option=None):
method retest (line 141) | def retest(self, new_test):
method rescore (line 152) | def rescore(self, new_test):
method size (line 157) | def size(self):
method __iadd__ (line 161) | def __iadd__(self, other):
method compatible (line 175) | def compatible(self, other):
method single_reflen (line 178) | def single_reflen(self, option="average"):
method _single_reflen (line 181) | def _single_reflen(self, reflens, option=None, testlen=None):
method recompute_score (line 194) | def recompute_score(self, option=None, verbose=0):
method compute_score (line 198) | def compute_score(self, option=None, verbose=0):
FILE: utils/gezi/metrics/cider/cider.py
class Cider (line 23) | class Cider:
method __init__ (line 28) | def __init__(self, test=None, refs=None, n=4, sigma=6.0, document_freq...
method compute_score (line 47) | def compute_score(self, gts, res):
method method (line 78) | def method(self):
FILE: utils/gezi/metrics/cider/cider_scorer.py
function precook (line 12) | def precook(s, n=4, out=False):
function cook_refs (line 29) | def cook_refs(refs, n=4): ## lhuang: oracle will call with "average"
function cook_test (line 39) | def cook_test(test, n=4):
class CiderScorer (line 48) | class CiderScorer(object):
method copy (line 52) | def copy(self):
method __init__ (line 59) | def __init__(self, test=None, refs=None, n=4, sigma=6.0,
method cook_append (line 70) | def cook_append(self, test, refs):
method size (line 80) | def size(self):
method __iadd__ (line 84) | def __iadd__(self, other):
method compute_doc_freq (line 95) | def compute_doc_freq(self):
method compute_cider (line 109) | def compute_cider(self):
method compute_score (line 187) | def compute_score(self, option=None, verbose=0):
FILE: utils/gezi/metrics/ciderD/ciderD.py
class CiderD (line 13) | class CiderD:
method __init__ (line 18) | def __init__(self, n=4, sigma=6.0, df="corpus"):
method compute_score (line 26) | def compute_score(self, gts, res):
method method (line 52) | def method(self):
FILE: utils/gezi/metrics/ciderD/ciderD_scorer.py
function precook (line 13) | def precook(s, n=4, out=False):
function cook_refs (line 30) | def cook_refs(refs, n=4): ## lhuang: oracle will call with "average"
function cook_test (line 40) | def cook_test(test, n=4):
class CiderScorer (line 49) | class CiderScorer(object):
method copy (line 53) | def copy(self):
method __init__ (line 60) | def __init__(self, test=None, refs=None, n=4, sigma=6.0):
method cook_append (line 70) | def cook_append(self, test, refs):
method size (line 80) | def size(self):
method __iadd__ (line 84) | def __iadd__(self, other):
method compute_doc_freq (line 95) | def compute_doc_freq(self):
method compute_cider (line 108) | def compute_cider(self, df_mode):
method compute_score (line 189) | def compute_score(self, df_mode, option=None, verbose=0):
FILE: utils/gezi/metrics/correlation/correlation.py
function lcc (line 19) | def lcc(trues, predicts):
function srocc (line 28) | def srocc(trues, predicts):
FILE: utils/gezi/metrics/eval.py
class COCOEvalCap (line 8) | class COCOEvalCap:
method __init__ (line 9) | def __init__(self, coco, cocoRes):
method evaluate (line 17) | def evaluate(self):
method setEval (line 62) | def setEval(self, score, method):
method setImgToEvalImgs (line 65) | def setImgToEvalImgs(self, scores, imgIds, method):
method setEvalImgs (line 72) | def setEvalImgs(self):
FILE: utils/gezi/metrics/meteor/meteor.py
class Meteor (line 15) | class Meteor:
method __init__ (line 17) | def __init__(self):
method compute_score (line 28) | def compute_score(self, gts, res):
method method (line 48) | def method(self):
method _stat (line 51) | def _stat(self, hypothesis_str, reference_list):
method _score (line 58) | def _score(self, hypothesis_str, reference_list):
method __del__ (line 75) | def __del__(self):
FILE: utils/gezi/metrics/new_cider/cider.py
class Cider (line 15) | class Cider:
method __init__ (line 20) | def __init__(self, n=4, df='corpus', num_docs=30000):
method compute_score (line 26) | def compute_score(self, gts, res):
method method (line 55) | def method(self):
FILE: utils/gezi/metrics/new_cider/cider_scorer.py
function precook (line 12) | def precook(s, n=4, out=False):
function cook_refs (line 29) | def cook_refs(refs, n=4): ## lhuang: oracle will call with "average"
function cook_test (line 39) | def cook_test(test, n=4):
class CiderScorer (line 48) | class CiderScorer(object):
method copy (line 52) | def copy(self):
method __init__ (line 59) | def __init__(self, test=None, refs=None, n=4, sigma=6.0, num_imgs=3000...
method cook_append (line 71) | def cook_append(self, test, refs):
method size (line 81) | def size(self):
method __iadd__ (line 85) | def __iadd__(self, other):
method compute_doc_freq (line 96) | def compute_doc_freq(self):
method compute_cider (line 109) | def compute_cider(self, df_mode="corpus"):
method compute_score (line 188) | def compute_score(self, df_mode, option=None, verbose=0):
FILE: utils/gezi/metrics/rouge/rouge.py
function my_lcs (line 13) | def my_lcs(string, sub):
class Rouge (line 36) | class Rouge():
method __init__ (line 41) | def __init__(self):
method calc_score (line 45) | def calc_score(self, candidate, refs):
method compute_score (line 77) | def compute_score(self, gts, res):
method method (line 104) | def method(self):
FILE: utils/gezi/metrics/tokenizer/ptbtokenizer.py
class PTBTokenizer (line 24) | class PTBTokenizer:
method tokenize (line 27) | def tokenize(self, captions_for_image):
FILE: utils/gezi/ngram.py
function get_ngrams (line 29) | def get_ngrams(input, minn=3, maxn=3, start='<', end='>'):
function get_ngrams_hash (line 42) | def get_ngrams_hash(input, buckets, minn=3, maxn=6, start='<', end='>', ...
function fasttext_ids (line 48) | def fasttext_ids(word, vocab, buckets, minn=3, maxn=6, start='<', end='>'):
FILE: utils/gezi/pydict.py
class Pydict (line 11) | class Pydict :
method __init__ (line 12) | def __init__(self,path_dm) :
method search (line 19) | def search(self,query,option):
method close (line 27) | def close(self) :
FILE: utils/gezi/rank_metrics.py
function mean_reciprocal_rank (line 12) | def mean_reciprocal_rank(rs):
function r_precision (line 35) | def r_precision(r):
function precision_at_k (line 60) | def precision_at_k(r, k):
function recall_at_k (line 88) | def recall_at_k(r, k):
function average_precision (line 93) | def average_precision(r):
function mean_average_precision (line 115) | def mean_average_precision(rs):
function dcg_at_k (line 133) | def dcg_at_k(r, k, method=1):
function ndcg_at_k (line 172) | def ndcg_at_k(r, k, method=1):
class RankMetrics (line 204) | class RankMetrics():
method __init__ (line 205) | def __init__(self):
method add (line 224) | def add(self, labels):
method finalize (line 241) | def finalize(self):
method get_metrics (line 245) | def get_metrics(self):
method get_names (line 250) | def get_names(self):
class RecallMetrics (line 253) | class RecallMetrics():
method __init__ (line 254) | def __init__(self):
method add (line 265) | def add(self, labels):
method finalize (line 274) | def finalize(self):
method get_metrics (line 278) | def get_metrics(self):
method get_names (line 283) | def get_names(self):
FILE: utils/gezi/segment.py
function segment_gbk_char (line 33) | def segment_gbk_char(text, cn_only=False):
function segment_utf8_char (line 69) | def segment_utf8_char(text, cn_only=False):
function segment_utf8_pinyin (line 96) | def segment_utf8_pinyin(text, cn_only=False):
function segment_utf8_pinyin2 (line 105) | def segment_utf8_pinyin2(text, cn_only=False):
function segment_en (line 115) | def segment_en(text):
function filter_quota (line 119) | def filter_quota(text):
function tokenize (line 129) | def tokenize(text):
function init_spacy_full (line 148) | def init_spacy_full():
function doc (line 156) | def doc(text):
function tokenize_filter_empty (line 171) | def tokenize_filter_empty(text):
function init_stanford_nlp (line 199) | def init_stanford_nlp(path='/home/gezi/soft/stanford-corenlp', lang='zh'):
function remove_duplicate (line 214) | def remove_duplicate(text):
function cut (line 236) | def cut(text, type='word'):
function is_emoji_en (line 250) | def is_emoji_en(word):
function hack_emoji (line 256) | def hack_emoji(l):
function hack_emoji2 (line 270) | def hack_emoji2(l):
function merge_expression (line 299) | def merge_expression(l):
function merge_expression2 (line 318) | def merge_expression2(l):
function init_bseg (line 338) | def init_bseg(use_pos=False, use_ner=False):
function to_gbk (line 355) | def to_gbk(text):
function to_utf8 (line 358) | def to_utf8(text):
function init_sp (line 363) | def init_sp(path=None):
function word_cut (line 374) | def word_cut(text):
function pos_cut (line 424) | def pos_cut(text):
function ner_cut (line 479) | def ner_cut(text):
class JiebaSegmentor (line 552) | class JiebaSegmentor(object):
method __init__ (line 553) | def __init__(self):
method segment_basic_single (line 556) | def segment_basic_single(self, text):
method segment_basic_single_all (line 562) | def segment_basic_single_all(self, text):
method segment_full_single (line 568) | def segment_full_single(self, text):
method Segment (line 574) | def Segment(self, text, method='basic'):
class BSegmentor (line 674) | class BSegmentor(object):
method __init__ (line 675) | def __init__(self, data='./data/wordseg', conf='./conf/scw.conf'):
method segment_nodupe_noseq (line 680) | def segment_nodupe_noseq(self, text):
method Segment_nodupe_noseq (line 692) | def Segment_nodupe_noseq(self, text):
method segment_nodupe (line 695) | def segment_nodupe(self, text):
method Segment_nodupe (line 702) | def Segment_nodupe(self, text):
method segment (line 706) | def segment(self, text):
method segment_seq_all (line 729) | def segment_seq_all(self, text):
method segment_phrase (line 746) | def segment_phrase(self, text):
method segment_basic (line 749) | def segment_basic(self, text):
method segment_phrase_single (line 752) | def segment_phrase_single(self, text):
method segment_phrase_single_all (line 757) | def segment_phrase_single_all(self, text):
method segment_basic_single (line 762) | def segment_basic_single(self, text):
method segment_basic_single_all (line 767) | def segment_basic_single_all(self, text):
method segment_phrase_single_all (line 772) | def segment_phrase_single_all(self, text):
method segment_merge_newword_single (line 777) | def segment_merge_newword_single(self, text):
method Segment (line 782) | def Segment(self, text, method='default'):
function segments (line 865) | def segments(texts, segmentor):
function segments_multiprocess (line 884) | def segments_multiprocess(texts, segmentor):
FILE: utils/gezi/summary.py
class SummaryWriter (line 32) | class SummaryWriter(object):
method __init__ (line 34) | def __init__(self, log_dir):
method scalar (line 38) | def scalar(self, tag, value, step):
method image (line 44) | def image(self, tag, images, step, texts=None, bytes_input=False):
method history (line 88) | def history(self, tag, values, step, bins=1000):
FILE: utils/gezi/test/test_ngrams.py
function ngrams (line 23) | def ngrams(word, minn=3, maxn=3):
FILE: utils/gezi/timer.py
class Timer (line 22) | class Timer():
method __init__ (line 23) | def __init__(self, info='', print_before=False):
method elapsed (line 29) | def elapsed(self):
method elapsed_ms (line 35) | def elapsed_ms(self):
method print (line 39) | def print(self):
method print_elapsed (line 45) | def print_elapsed(self):
FILE: utils/gezi/topn.py
class TopN (line 20) | class TopN(object):
method __init__ (line 23) | def __init__(self, n, reverse=True):
method size (line 28) | def size(self):
method push (line 32) | def push(self, x):
method extract (line 40) | def extract(self, sort=False):
method reset (line 59) | def reset(self):
FILE: utils/gezi/util.py
function is_cn (line 31) | def is_cn(word):
function break_sentence (line 34) | def break_sentence(sentence, max_sent_len, additional=5):
function add_start_end (line 54) | def add_start_end(w, start='<S>', end='</S>'):
function str2scores (line 57) | def str2scores(l):
function get_unmodify_minutes (line 65) | def get_unmodify_minutes(file_):
function extract_emojis (line 85) | def extract_emojis(content):
function remove_emojis (line 91) | def remove_emojis(sentence):
function is_emoji (line 97) | def is_emoji(w):
function dict2namedtuple (line 101) | def dict2namedtuple(thedict, name):
function csv (line 116) | def csv(s):
function get_weights (line 121) | def get_weights(weights):
function probs_entropy (line 130) | def probs_entropy(probs):
function dist (line 135) | def dist(x,y):
function cosine (line 138) | def cosine(a, b):
function softmax (line 143) | def softmax(x, axis=-1):
function sigmoid (line 151) | def sigmoid(x):
function load_image_into_numpy_array (line 155) | def load_image_into_numpy_array(image):
function dirname (line 161) | def dirname(input):
function non_empty (line 170) | def non_empty(file):
function merge_dicts (line 173) | def merge_dicts(*dict_args):
function norm (line 184) | def norm(text):
function loggest_match (line 187) | def loggest_match(cns, vocab, encode_unk=False, unk_vocab_size=None, voc...
function loggest_match_seg (line 202) | def loggest_match_seg(word, vocab, encode_unk=False):
function index (line 213) | def index(l, val):
function to_pascal_name (line 219) | def to_pascal_name(name):
function to_gnu_name (line 224) | def to_gnu_name(name):
function pascal2gnu (line 229) | def pascal2gnu(name):
function gnu2pascal (line 243) | def gnu2pascal(name):
function is_gbk_luanma (line 265) | def is_gbk_luanma(text):
function gen_sum_list (line 268) | def gen_sum_list(l):
function add_one (line 274) | def add_one(d, word):
function pretty_floats (line 280) | def pretty_floats(values):
function get_singles (line 287) | def get_singles(l):
function is_single (line 293) | def is_single(item):
function iterable (line 296) | def iterable(item):
function is_list_or_tuple (line 303) | def is_list_or_tuple(item):
function get_value_name_list (line 306) | def get_value_name_list(values, names):
function batches (line 310) | def batches(l, batch_size):
function pad (line 320) | def pad(l, maxlen, mark=0):
function nppad (line 329) | def nppad(l, maxlen):
function try_mkdir (line 335) | def try_mkdir(dir):
function get_dir (line 340) | def get_dir(path):
function dedupe_list (line 346) | def dedupe_list(l):
function parallel_run (line 358) | def parallel_run(target, args_list, num_threads):
function multithreads_run (line 369) | def multithreads_run(target, args_list):
function is_glob_pattern (line 382) | def is_glob_pattern(input):
function file_is_empty (line 385) | def file_is_empty(path):
function list_files (line 388) | def list_files(inputs):
function sorted_ls (line 408) | def sorted_ls(path, time_descending=True):
function list_models (line 412) | def list_models(model_dir, time_descending=True):
function save_conf (line 421) | def save_conf(con):
function write_to_txt (line 432) | def write_to_txt(data, file):
function read_int_from (line 436) | def read_int_from(file, default_value=None):
function read_float_from (line 439) | def read_float_from(file, default_value=None):
function read_str_from (line 442) | def read_str_from(file, default_value=None):
function img_html (line 445) | def img_html(img):
function text_html (line 448) | def text_html(text):
function thtml (line 451) | def thtml(text):
function hprint (line 455) | def hprint(content):
function imgprint (line 458) | def imgprint(img):
function unison_shuffle (line 462) | def unison_shuffle(a, b):
function finalize_feature (line 477) | def finalize_feature(fe, mode='w', outfile='./feature_name.txt', sep='\n'):
function write_feature_names (line 486) | def write_feature_names(names, mode='a', outfile='./feature_name.txt', s...
function get_feature_names (line 491) | def get_feature_names(file_):
function read_feature_names (line 500) | def read_feature_names(file_):
function get_feature_names_dict (line 509) | def get_feature_names_dict(file_):
function read_feature_names_dict (line 520) | def read_feature_names_dict(file_):
function update_sparse_feature (line 531) | def update_sparse_feature(feature, num_pre_features):
function merge_sparse_feature (line 536) | def merge_sparse_feature(fe1, fe2, num_fe1):
function edit_distance (line 546) | def edit_distance(first,second):
function save_json (line 569) | def save_json(obj, filename):
function load_json (line 575) | def load_json(filename):
function read_json (line 582) | def read_json(filename):
function strip_suffix (line 585) | def strip_suffix(s, suf):
function log (line 590) | def log(text, array):
function log_full (line 604) | def log_full(text, array):
function env_has (line 611) | def env_has(name):
function env_get (line 614) | def env_get(name):
function env_set (line 620) | def env_set(name, val=1):
function has_env (line 623) | def has_env(name):
function get_env (line 626) | def get_env(name):
function set_env (line 632) | def set_env(name, val=1):
function env_val (line 635) | def env_val(name, default=None):
function use_matplotlib (line 638) | def use_matplotlib(backend='Agg'):
function decode (line 642) | def decode(bytes_list):
function get_fold (line 651) | def get_fold(total, num_folds, index):
function is_fold (line 663) | def is_fold(input, fold):
function to_list (line 674) | def to_list(item):
function repeat (line 679) | def repeat(iter):
FILE: utils/gezi/vocabulary.py
class Vocabulary (line 31) | class Vocabulary(object):
method __init__ (line 34) | def __init__(self,
method is_special (line 158) | def is_special(self, word):
method word_to_id (line 161) | def word_to_id(self, word):
method id (line 171) | def id(self, word):
method id_to_word (line 181) | def id_to_word(self, word_id):
method key (line 188) | def key(self, word_id):
method count (line 195) | def count(self, word_id):
method count_word (line 201) | def count_word(self, word):
method size (line 207) | def size(self):
method start_id (line 210) | def start_id(self):
method end_id (line 213) | def end_id(self):
method unk_id (line 216) | def unk_id(self):
method has (line 220) | def has(self, word):
method add (line 223) | def add(self, word):
method words (line 229) | def words(self):
FILE: utils/gezi/word_counter.py
class WordCounter (line 18) | class WordCounter(object):
method __init__ (line 19) | def __init__(self,
method add (line 32) | def add(self, word, count=1):
method save (line 36) | def save(self, filename, most_common=None, min_count=None):
FILE: utils/gezi/zhtools/chconv.py
function default_error_handler (line 11478) | def default_error_handler(char, e):
function empty_error_handler (line 11482) | def empty_error_handler(char, e):
function null_error_handler (line 11486) | def null_error_handler(char, e):
function raise_error_handler (line 11490) | def raise_error_handler(char, e):
function converter (line 11494) | def converter(text, table, errors=None):
class ConverterTest (line 11521) | class ConverterTest(unittest.TestCase):
method toU (line 11522) | def toU(self, s):
method testSimpTrad (line 11525) | def testSimpTrad(self):
method testSimpKanji (line 11535) | def testSimpKanji(self):
method testTradKanji (line 11541) | def testTradKanji(self):
FILE: utils/gezi/zhtools/langconv.py
class Node (line 46) | class Node(object):
method __init__ (line 47) | def __init__(self, from_word, to_word=None, is_tail=True,
method is_original_long_word (line 61) | def is_original_long_word(self):
method is_follow (line 64) | def is_follow(self, chars):
method __str__ (line 67) | def __str__(self):
class ConvertMap (line 73) | class ConvertMap(object):
method __init__ (line 74) | def __init__(self, name, mapping=None):
method set_convert_map (line 80) | def set_convert_map(self, mapping):
method __getitem__ (line 97) | def __getitem__(self, k):
method __contains__ (line 104) | def __contains__(self, k):
method __len__ (line 107) | def __len__(self):
class StatesMachineException (line 110) | class StatesMachineException(Exception): pass
class StatesMachine (line 112) | class StatesMachine(object):
method __init__ (line 113) | def __init__(self):
method clone (line 119) | def clone(self, pool):
method feed (line 125) | def feed(self, char, map):
method __len__ (line 180) | def __len__(self):
method __str__ (line 183) | def __str__(self):
class Converter (line 188) | class Converter(object):
method __init__ (line 189) | def __init__(self, to_encoding):
method feed (line 194) | def feed(self, char):
method _clean (line 211) | def _clean(self):
method start (line 218) | def start(self):
method end (line 222) | def end(self):
method convert (line 227) | def convert(self, string):
method get_result (line 234) | def get_result(self):
function registery (line 238) | def registery(name, mapping):
function run (line 247) | def run():
FILE: utils/gezi/zhtools/test_langconv.py
class ConvertMapTest (line 8) | class ConvertMapTest(TestCase):
method test_map (line 9) | def test_map(self):
class ConverterModelTest (line 23) | class ConverterModelTest(TestCase):
method test_1 (line 24) | def test_1(self):
method test_2 (line 34) | def test_2(self):
method test_3 (line 42) | def test_3(self):
method test_4 (line 54) | def test_4(self):
method test_5 (line 66) | def test_5(self):
method test_6 (line 76) | def test_6(self):
method test_7 (line 90) | def test_7(self):
method test_8 (line 103) | def test_8(self):
method test_9 (line 118) | def test_9(self):
method test_10 (line 130) | def test_10(self):
class ConverterTest (line 141) | class ConverterTest(TestCase):
method assertConvert (line 142) | def assertConvert(self, name, string, converted):
method assertST (line 149) | def assertST(self, trad, simp):
method test_zh1 (line 156) | def test_zh1(self):
method test_zh2 (line 164) | def test_zh2(self):
method test_zh3 (line 168) | def test_zh3(self):
method test_zh4 (line 174) | def test_zh4(self):
FILE: utils/gezi/zhtools/xpinyin.py
class Pinyin (line 23) | class Pinyin(object):
method __init__ (line 47) | def __init__(self):
method py2hz (line 59) | def py2hz(self, pinyin):
method get_pinyin (line 71) | def get_pinyin(self, chars='', splitter='', tone=False):
method get_initials (line 84) | def get_initials(self, char=''):
class PinyinTestCase (line 93) | class PinyinTestCase(unittest.TestCase):
method setUp (line 94) | def setUp(self):
method to_unicode (line 101) | def to_unicode(self, s):
method test_get_pinyin (line 106) | def test_get_pinyin(self): ## test method names begin 'test*'
method test_get_initials (line 115) | def test_get_initials(self):
method test_py2hz (line 120) | def test_py2hz(self):
FILE: utils/lele/apps/train.py
function to_torch (line 37) | def to_torch(x, y=None):
function train (line 44) | def train(model,
FILE: utils/lele/fastai/core.py
function num_cpus (line 41) | def num_cpus()->int:
function is_listy (line 48) | def is_listy(x:Any)->bool: return isinstance(x, (tuple,list))
function is_tuple (line 49) | def is_tuple(x:Any)->bool: return isinstance(x, tuple)
function noop (line 50) | def noop(x): return x
function to_int (line 52) | def to_int(b):
function ifnone (line 56) | def ifnone(a:Any,b:Any)->Any:
function uniqueify (line 60) | def uniqueify(x:Series) -> List[Any]: return list(OrderedDict.fromkeys(x...
function idx_dict (line 61) | def idx_dict(a): return {v:k for k,v in enumerate(a)}
function find_classes (line 63) | def find_classes(folder:Path)->FilePathList:
function arrays_split (line 70) | def arrays_split(mask:NPArrayMask, *arrs:NPArrayableList)->SplitArrayList:
function random_split (line 75) | def random_split(valid_pct:float, *arrs:NPArrayableList)->SplitArrayList:
function listify (line 80) | def listify(p:OptListOrItem=None, q:OptListOrItem=None):
function camel2snake (line 91) | def camel2snake(name:str)->str:
function even_mults (line 95) | def even_mults(start:float, stop:float, n:int)->np.ndarray:
function extract_kwargs (line 101) | def extract_kwargs(names:Collection[str], kwargs:KWArgs):
function partition (line 110) | def partition(a:Collection, sz:int) -> List[Collection]:
function partition_by_cores (line 114) | def partition_by_cores(a:Collection, n_cpus:int) -> List[Collection]:
function get_chunk_length (line 118) | def get_chunk_length(data:Union[PathOrStr, DataFrame, pd.io.parsers.Text...
function get_total_length (line 128) | def get_total_length(csv_name:PathOrStr, chunksize:int) -> int:
function maybe_copy (line 135) | def maybe_copy(old_fnames:Collection[PathOrStr], new_fnames:Collection[P...
function series2cat (line 142) | def series2cat(df:DataFrame, *col_names):
class ItemBase (line 146) | class ItemBase():
method device (line 150) | def device(self): pass
method data (line 153) | def data(self): pass
function download_url (line 155) | def download_url(url:str, dest:str, overwrite:bool=False)->None:
function range_of (line 171) | def range_of(x): return list(range(len(x)))
function arange_of (line 172) | def arange_of(x): return np.arange(len(x))
FILE: utils/lele/fastai/layers.py
class Lambda (line 8) | class Lambda(nn.Module):
method __init__ (line 10) | def __init__(self, func:LambdaFunc):
method forward (line 15) | def forward(self, x): return self.func(x)
function ResizeBatch (line 17) | def ResizeBatch(*size:int) -> Tensor:
function Flatten (line 21) | def Flatten()->Tensor:
function PoolFlatten (line 25) | def PoolFlatten()->nn.Sequential:
function bn_drop_lin (line 29) | def bn_drop_lin(n_in:int, n_out:int, bn:bool=True, p:float=0., actn:Opti...
function conv2d (line 37) | def conv2d(ni:int, nf:int, ks:int=3, stride:int=1, padding:int=None, bia...
function conv_layer (line 42) | def conv_layer(ni:int, nf:int, ks:int=3, stride:int=1)->nn.Sequential:
function conv2d_relu (line 49) | def conv2d_relu(ni:int, nf:int, ks:int=3, stride:int=1, padding:int=None...
function conv2d_trans (line 57) | def conv2d_trans(ni:int, nf:int, ks:int=2, stride:int=2, padding:int=0) ...
class AdaptiveConcatPool2d (line 61) | class AdaptiveConcatPool2d(nn.Module):
method __init__ (line 63) | def __init__(self, sz:Optional[int]=None):
method forward (line 68) | def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1)
class Debugger (line 70) | class Debugger(nn.Module):
method forward (line 72) | def forward(self,x:Tensor) -> Tensor:
class StdUpsample (line 76) | class StdUpsample(nn.Module):
method __init__ (line 78) | def __init__(self, n_in:int, n_out:int):
method forward (line 83) | def forward(self, x:Tensor) -> Tensor:
function std_upsample_head (line 86) | def std_upsample_head(c, *nfs:Collection[int]) -> Model:
class CrossEntropyFlat (line 94) | class CrossEntropyFlat(nn.CrossEntropyLoss):
method forward (line 96) | def forward(self, input:Tensor, target:Tensor) -> Rank0Tensor:
function simple_cnn (line 100) | def simple_cnn(actns:Collection[int], kernel_szs:Collection[int]=None,
function trunc_normal_ (line 111) | def trunc_normal_(x:Tensor, mean:float=0., std:float=1.) -> Tensor:
function get_embedding (line 116) | def get_embedding(ni:int,nf:int) -> Model:
FILE: utils/lele/fastai/text/models.py
function dropout_mask (line 9) | def dropout_mask(x:Tensor, sz:Collection[int], p:float):
class RNNDropout (line 13) | class RNNDropout(nn.Module):
method __init__ (line 16) | def __init__(self, p:float=0.5):
method forward (line 20) | def forward(self, x:Tensor) -> Tensor:
class WeightDropout (line 25) | class WeightDropout(nn.Module):
method __init__ (line 28) | def __init__(self, module:Model, weight_p:float, layer_names:Collectio...
method _setweights (line 36) | def _setweights(self):
method forward (line 42) | def forward(self, *args:ArgStar):
method reset (line 49) | def reset(self):
class EmbeddingDropout (line 55) | class EmbeddingDropout(nn.Module):
method __init__ (line 58) | def __init__(self, emb:Model, embed_p:float):
method forward (line 64) | def forward(self, words:LongTensor, scale:Optional[float]=None) -> Ten...
function _repackage_var (line 74) | def _repackage_var(h:Tensors) -> Tensors:
class RNNCore (line 78) | class RNNCore(nn.Module):
method __init__ (line 83) | def __init__(self, vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, ...
method forward (line 109) | def forward(self, input:LongTensor) -> Tuple[Tensor,Tensor]:
method _one_hidden (line 125) | def _one_hidden(self, l:int) -> Tensor:
method reset (line 130) | def reset(self):
class LinearDecoder (line 137) | class LinearDecoder(nn.Module):
method __init__ (line 142) | def __init__(self, n_out:int, n_hid:int, output_p:float, tie_encoder:M...
method forward (line 150) | def forward(self, input:Tuple[Tensor,Tensor]) -> Tuple[Tensor,Tensor,T...
class SequentialRNN (line 156) | class SequentialRNN(nn.Sequential):
method reset (line 158) | def reset(self):
class MultiBatchRNNCore (line 162) | class MultiBatchRNNCore(RNNCore):
method __init__ (line 165) | def __init__(self, bptt:int, max_seq:int, *args, **kwargs):
method concat (line 169) | def concat(self, arrs:Collection[Tensor]) -> Tensor:
method forward (line 173) | def forward(self, input:LongTensor) -> Tuple[Tensor,Tensor]:
class PoolingLinearClassifier (line 184) | class PoolingLinearClassifier(nn.Module):
method __init__ (line 187) | def __init__(self, layers:Collection[int], drops:Collection[float]):
method pool (line 195) | def pool(self, x:Tensor, bs:int, is_max:bool):
method forward (line 200) | def forward(self, input:Tuple[Tensor,Tensor]) -> Tuple[Tensor,Tensor,T...
function get_language_model (line 210) | def get_language_model(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int...
function get_rnn_classifier (line 219) | def get_rnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int,...
function classifier (line 233) | def classifier(vocab_size, n_class, bptt:int=70, max_len:int=70*20, emb_...
FILE: utils/lele/fastai/text/qrnn/forget_mult.py
class CPUForgetMult (line 76) | class CPUForgetMult(torch.nn.Module):
method __init__ (line 77) | def __init__(self):
method forward (line 80) | def forward(self, f, x, hidden_init=None):
class GPUForgetMult (line 96) | class GPUForgetMult(torch.autograd.Function):
method __init__ (line 99) | def __init__(self):
method compile (line 102) | def compile(self):
method forward (line 122) | def forward(self, f, x, hidden_init=None):
method backward (line 138) | def backward(self, grad_h):
class ForgetMult (line 158) | class ForgetMult(torch.nn.Module):
method __init__ (line 171) | def __init__(self):
method forward (line 174) | def forward(self, f, x, hidden_init=None, use_cuda=True):
FILE: utils/lele/fastai/text/qrnn/qrnn.py
class QRNNLayer (line 11) | class QRNNLayer(nn.Module):
method __init__ (line 32) | def __init__(self, input_size, hidden_size=None, save_prev_x=False, zo...
method reset (line 48) | def reset(self):
method forward (line 52) | def forward(self, X, hidden=None):
class QRNN (line 114) | class QRNN(torch.nn.Module):
method __init__ (line 137) | def __init__(self, input_size, hidden_size,
method reset (line 156) | def reset(self):
method forward (line 160) | def forward(self, input, hidden=None):
FILE: utils/lele/fastai/torch_core.py
function to_data (line 65) | def to_data(b:ItemsList):
function to_device (line 70) | def to_device(b:Tensors, device:torch.device):
function data_collate (line 76) | def data_collate(batch:ItemsList)->Tensor:
function requires_grad (line 80) | def requires_grad(m:nn.Module, b:Optional[bool]=None)->Optional[bool]:
function trainable_params (line 87) | def trainable_params(m:nn.Module)->ParamList:
function children (line 92) | def children(m:nn.Module)->ModuleList:
function num_children (line 96) | def num_children(m:nn.Module)->int:
function range_children (line 100) | def range_children(m:nn.Module)->Iterator[int]:
function first_layer (line 105) | def first_layer(m:nn.Module)->nn.Module:
function split_model_idx (line 109) | def split_model_idx(model:nn.Module, idxs:Collection[int])->ModuleList:
function split_model (line 116) | def split_model(model:nn.Module, splits:Collection[Union[Model,ModuleLis...
function split_bn_bias (line 127) | def split_bn_bias(layer_groups:ModuleList)->ModuleList:
function set_bn_eval (line 138) | def set_bn_eval(m:nn.Module)->None:
function to_half (line 145) | def to_half(b:Collection[Tensor])->Collection[Tensor]:
function bn2float (line 149) | def bn2float(module:nn.Module)->nn.Module:
function model2half (line 155) | def model2half(model:nn.Module)->nn.Module:
function cond_init (line 159) | def cond_init(m:nn.Module, init_func:LayerFunc):
function apply_leaf (line 165) | def apply_leaf(m:nn.Module, f:LayerFunc):
function apply_init (line 171) | def apply_init(m, init_func:LayerFunc):
function in_channels (line 175) | def in_channels(m:Model) -> List[int]:
function calc_loss (line 181) | def calc_loss(y_pred:Tensor, y_true:Tensor, loss_class:type=nn.CrossEntr...
function to_np (line 187) | def to_np(x): return x.cpu().numpy()
function model_type (line 189) | def model_type(dtype):
function np2model_tensor (line 194) | def np2model_tensor(a):
function show_install (line 200) | def show_install(show_nvidia_smi:bool=False):
function trange_of (line 287) | def trange_of(x): return torch.arange(len(x))
FILE: utils/lele/layers/classify_layer.py
class SoftmaxLayer (line 11) | class SoftmaxLayer(nn.Module):
method __init__ (line 13) | def __init__(self, output_dim, n_class):
method forward (line 23) | def forward(self, x, y):
class SampledSoftmaxLayer (line 34) | class SampledSoftmaxLayer(nn.Module):
method __init__ (line 38) | def __init__(self, output_dim, n_class, n_samples, use_cuda):
method forward (line 66) | def forward(self, x, y):
method update_embedding_matrix (line 87) | def update_embedding_matrix(self):
method update_negative_samples (line 106) | def update_negative_samples(self, word_inp, chars_inp, mask):
class CNNSoftmaxLayer (line 135) | class CNNSoftmaxLayer(nn.Module):
method __init__ (line 136) | def __init__(self, token_embedder, output_dim, n_class, n_samples, cor...
method forward (line 158) | def forward(self, x, y):
method update_embedding_matrix (line 179) | def update_embedding_matrix(self):
method update_negative_samples (line 213) | def update_negative_samples(self, word_inp, chars_inp, mask):
FILE: utils/lele/layers/elmo.py
class ElmobiLm (line 18) | class ElmobiLm(_EncoderBase):
method __init__ (line 19) | def __init__(self, config, use_cuda=False):
method forward (line 65) | def forward(self, inputs, mask):
method _lstm_forward (line 105) | def _lstm_forward(self,
FILE: utils/lele/layers/elmo/classify_layer.py
class SoftmaxLayer (line 11) | class SoftmaxLayer(nn.Module):
method __init__ (line 13) | def __init__(self, output_dim, n_class):
method forward (line 23) | def forward(self, x, y):
class SampledSoftmaxLayer (line 34) | class SampledSoftmaxLayer(nn.Module):
method __init__ (line 38) | def __init__(self, output_dim, n_class, n_samples, use_cuda):
method forward (line 66) | def forward(self, x, y):
method update_embedding_matrix (line 87) | def update_embedding_matrix(self):
method update_negative_samples (line 106) | def update_negative_samples(self, word_inp, chars_inp, mask):
class CNNSoftmaxLayer (line 135) | class CNNSoftmaxLayer(nn.Module):
method __init__ (line 136) | def __init__(self, token_embedder, output_dim, n_class, n_samples, cor...
method forward (line 158) | def forward(self, x, y):
method update_embedding_matrix (line 179) | def update_embedding_matrix(self):
method update_negative_samples (line 213) | def update_negative_samples(self, word_inp, chars_inp, mask):
FILE: utils/lele/layers/elmo/elmo.py
class ElmobiLm (line 18) | class ElmobiLm(_EncoderBase):
method __init__ (line 19) | def __init__(self, config, use_cuda=False):
method forward (line 65) | def forward(self, inputs, mask):
method _lstm_forward (line 105) | def _lstm_forward(self,
FILE: utils/lele/layers/elmo/embedding_layer.py
class EmbeddingLayer (line 10) | class EmbeddingLayer(nn.Module):
method __init__ (line 11) | def __init__(self, n_d, word2id, embs=None, fix_emb=True, oov='<oov>',...
method forward (line 48) | def forward(self, input_):
FILE: utils/lele/layers/elmo/encoder_base.py
class _EncoderBase (line 16) | class _EncoderBase(torch.nn.Module):
method __init__ (line 27) | def __init__(self, stateful: bool = False) -> None:
method sort_and_run_forward (line 32) | def sort_and_run_forward(self,
method _get_initial_states (line 115) | def _get_initial_states(self,
method _update_states (line 199) | def _update_states(self,
method reset_states (line 278) | def reset_states(self):
FILE: utils/lele/layers/elmo/highway.py
class Highway (line 12) | class Highway(torch.nn.Module):
method __init__ (line 30) | def __init__(self,
method forward (line 47) | def forward(self, inputs: torch.Tensor) -> torch.Tensor: # pylint: di...
FILE: utils/lele/layers/elmo/lstm.py
class LstmbiLm (line 11) | class LstmbiLm(nn.Module):
method __init__ (line 12) | def __init__(self, config, use_cuda=False):
method forward (line 25) | def forward(self, inputs):
FILE: utils/lele/layers/elmo/lstm_cell_with_projection.py
class LstmCellWithProjection (line 13) | class LstmCellWithProjection(torch.nn.Module):
method __init__ (line 53) | def __init__(self,
method reset_parameters (line 80) | def reset_parameters(self):
method forward (line 90) | def forward(self, # pylint: disable=arguments-differ
FILE: utils/lele/layers/elmo/token_embedder.py
class LstmTokenEmbedder (line 12) | class LstmTokenEmbedder(nn.Module):
method __init__ (line 13) | def __init__(self, config, word_emb_layer, char_emb_layer, use_cuda=Fa...
method forward (line 31) | def forward(self, word_inp, chars_inp, shape):
class ConvTokenEmbedder (line 50) | class ConvTokenEmbedder(nn.Module):
method __init__ (line 51) | def __init__(self, config, word_emb_layer, char_emb_layer, use_cuda):
method forward (line 89) | def forward(self, word_inp, chars_inp, shape):
FILE: utils/lele/layers/elmo/util.py
function get_lengths_from_binary_sequence_mask (line 12) | def get_lengths_from_binary_sequence_mask(mask: torch.Tensor):
function sort_batch_by_length (line 29) | def sort_batch_by_length(tensor: torch.autograd.Variable,
function get_final_encoder_states (line 72) | def get_final_encoder_states(encoder_outputs: torch.Tensor,
function get_dropout_mask (line 104) | def get_dropout_mask(dropout_probability: float, tensor_for_masking: tor...
function block_orthogonal (line 127) | def block_orthogonal(tensor: torch.Tensor,
FILE: utils/lele/layers/embedding_layer.py
class EmbeddingLayer (line 10) | class EmbeddingLayer(nn.Module):
method __init__ (line 11) | def __init__(self, n_d, word2id, embs=None, fix_emb=True, oov='<oov>',...
method forward (line 48) | def forward(self, input_):
FILE: utils/lele/layers/encoder_base.py
class _EncoderBase (line 16) | class _EncoderBase(torch.nn.Module):
method __init__ (line 27) | def __init__(self, stateful: bool = False) -> None:
method sort_and_run_forward (line 32) | def sort_and_run_forward(self,
method _get_initial_states (line 115) | def _get_initial_states(self,
method _update_states (line 199) | def _update_states(self,
method reset_states (line 278) | def reset_states(self):
FILE: utils/lele/layers/highway.py
class Highway (line 12) | class Highway(torch.nn.Module):
method __init__ (line 30) | def __init__(self,
method forward (line 47) | def forward(self, inputs: torch.Tensor) -> torch.Tensor: # pylint: di...
FILE: utils/lele/layers/layers.py
class CudnnRnn (line 25) | class CudnnRnn(nn.Module):
method __init__ (line 35) | def __init__(self, input_size, hidden_size, num_layers,
method forward (line 66) | def forward(self, x, x_mask, fw_masks=None, bw_masks=None):
class StackedBRNN (line 149) | class StackedBRNN(nn.Module):
method __init__ (line 157) | def __init__(self, input_size, hidden_size, num_layers,
method forward (line 179) | def forward(self, x, x_mask):
method _forward_unpadded (line 203) | def _forward_unpadded(self, x, x_mask):
method _forward_padded (line 246) | def _forward_padded(self, x, x_mask):
class FeedForwardNetwork (line 322) | class FeedForwardNetwork(nn.Module):
method __init__ (line 323) | def __init__(self, input_size, hidden_size, output_size, dropout_rate=0):
method forward (line 329) | def forward(self, x):
class PointerNetwork (line 335) | class PointerNetwork(nn.Module):
method __init__ (line 336) | def __init__(self, x_size, y_size, hidden_size, dropout_rate=0, cell_t...
method init_hiddens (line 346) | def init_hiddens(self, y, y_mask):
method pointer (line 351) | def pointer(self, x, state, x_mask):
method forward (line 370) | def forward(self, x, y, x_mask, y_mask):
class MemoryAnsPointer (line 378) | class MemoryAnsPointer(nn.Module):
method __init__ (line 379) | def __init__(self, x_size, y_size, hidden_size, hop=1, dropout_rate=0,...
method forward (line 395) | def forward(self, x, y, x_mask, y_mask):
class SeqAttnMatch (line 435) | class SeqAttnMatch(nn.Module):
method __init__ (line 442) | def __init__(self, input_size, identity=False):
method forward (line 449) | def forward(self, x, y, y_mask):
class DotAttention (line 483) | class DotAttention(nn.Module):
method __init__ (line 490) | def __init__(self,
method forward (line 516) | def forward(self, x, y, y_mask):
class SelfAttnMatch (line 562) | class SelfAttnMatch(nn.Module):
method __init__ (line 569) | def __init__(self, input_size, identity=False, diag=True):
method forward (line 577) | def forward(self, x, x_mask):
class BilinearSeqAttn (line 613) | class BilinearSeqAttn(nn.Module):
method __init__ (line 621) | def __init__(self, x_size, y_size, identity=False, normalize=True):
method forward (line 631) | def forward(self, x, y, x_mask):
class LinearSeqAttn (line 655) | class LinearSeqAttn(nn.Module):
method __init__ (line 661) | def __init__(self, input_size):
method forward (line 665) | def forward(self, x, x_mask):
class NonLinearSeqAttn (line 679) | class NonLinearSeqAttn(nn.Module):
method __init__ (line 685) | def __init__(self, input_size, hidden_size=128):
method forward (line 689) | def forward(self, x, x_mask):
class MaxPooling (line 702) | class MaxPooling(nn.Module):
method forward (line 703) | def forward(self, x, x_mask):
class SumPooling (line 715) | class SumPooling(nn.Module):
method forward (line 716) | def forward(self, x, x_mask):
class TopKPooling (line 726) | class TopKPooling(nn.Module):
method __init__ (line 727) | def __init__(self, top_k=2):
method forward (line 731) | def forward(self, x, x_mask):
class LastPooling (line 744) | class LastPooling(nn.Module):
method __init__ (line 745) | def __init__(self):
method forward (line 748) | def forward(self, x, x_mask):
class LinearSeqAttnPooling (line 755) | class LinearSeqAttnPooling(nn.Module):
method __init__ (line 761) | def __init__(self, input_size):
method forward (line 765) | def forward(self, x, x_mask):
class LinearSeqAttnPoolings (line 782) | class LinearSeqAttnPoolings(nn.Module):
method __init__ (line 788) | def __init__(self, input_size, num_poolings):
method forward (line 793) | def forward(self, x, x_mask):
class NonLinearSeqAttnPooling (line 812) | clas
Copy disabled (too large)
Download .json
Condensed preview — 2780 files, each showing path, character count, and a content snippet. Download the .json file for the full structured content (26,937K chars).
[
{
"path": ".gitignore",
"chars": 498,
"preview": "*.pyc\n*.so\n*.npy\n*.txt\n*.bin\n*.html\n*checkpoint.ipynb\n*.swp\n*.crc\n*.jpg\n*.jpeg\n*.pdf\n*.vshistory\n.localhistory\n.vs \n.vsc"
},
{
"path": "README.md",
"chars": 451,
"preview": "# wenzheng\nDeep learning related projects using tf(eager), pytorch(fastai,allenlp) for contests like aichallenger, kaggl"
},
{
"path": "__init__.py",
"chars": 63,
"preview": "import wenzheng\nimport third \nimport official \nimport projects\n"
},
{
"path": "examples/fastai/lm/lm.ipynb",
"chars": 15637,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n "
},
{
"path": "examples/fastai/lm/lm.py",
"chars": 1088,
"preview": "# coding: utf-8\nimport melt\n# http://docs.fast.ai/text.html\nfrom fastai.text import *\nimport html\nfrom pathlib import Pa"
},
{
"path": "examples/pytorch/data_parallel_tutorial.py",
"chars": 9977,
"preview": "\"\"\"\nOptional: Data Parallelism\n==========================\n**Authors**: `Sung Kim <https://github.com/hunkim>`_ and `Jenn"
},
{
"path": "examples/pytorch/sentiment/model.py",
"chars": 2569,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "examples/pytorch/sentiment/train.py",
"chars": 5244,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "examples/shap/explainer_1.py",
"chars": 655,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "examples/tf/eager/dynamic_dense.py",
"chars": 2240,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "examples/tf/eager/rnn_ptb.py",
"chars": 12307,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "examples/tf/eager/rnn_ptb.sh",
"chars": 72,
"preview": "python rnn_ptb.py --data-path /home/gezi/data/ptb/simple-examples/data\n"
},
{
"path": "jupyter/Untitled.ipynb",
"chars": 9442,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n "
},
{
"path": "official/.gitignore",
"chars": 22,
"preview": "MNIST-data\nlabels.txt\n"
},
{
"path": "official/Dockerfile.cpu",
"chars": 459,
"preview": "# Docker image for running examples in Tensorflow models.\n# base_image depends on whether we are running on GPUs or non-"
},
{
"path": "official/Dockerfile.gpu",
"chars": 493,
"preview": "# Docker image for running examples in Tensorflow models.\n# base_image depends on whether we are running on GPUs or non-"
},
{
"path": "official/README.md",
"chars": 2593,
"preview": "# TensorFlow Official Models\n\nThe TensorFlow official models are a collection of example models that use TensorFlow's hi"
},
{
"path": "official/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/benchmark/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/benchmark/benchmark_uploader.py",
"chars": 5573,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/benchmark/benchmark_uploader_main.py",
"chars": 2125,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/benchmark/benchmark_uploader_test.py",
"chars": 3895,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/benchmark/datastore/schema/benchmark_metric.json",
"chars": 1271,
"preview": "[\n {\n \"description\": \"The ID of the benchmark run, where this metric should tie to.\",\n \"mode\": \"REQUIRED\",\n \"n"
},
{
"path": "official/benchmark/datastore/schema/benchmark_run.json",
"chars": 9921,
"preview": "[\n {\n \"description\": \"The UUID of the run for the benchmark.\",\n \"mode\": \"REQUIRED\",\n \"name\": \"model_id\",\n \""
},
{
"path": "official/boosted_trees/README.md",
"chars": 6755,
"preview": "# Classifying Higgs boson processes in the HIGGS Data Set\n## Overview\nThe [HIGGS Data Set](https://archive.ics.uci.edu/m"
},
{
"path": "official/boosted_trees/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/boosted_trees/data_download.py",
"chars": 2694,
"preview": "\"\"\"Downloads the UCI HIGGS Dataset and prepares train data.\n\nThe details on the dataset are in https://archive.ics.uci.e"
},
{
"path": "official/boosted_trees/train_higgs.py",
"chars": 10735,
"preview": "r\"\"\"A script that builds boosted trees over higgs data.\n\nIf you haven't, please run data_download.py beforehand to prepa"
},
{
"path": "official/boosted_trees/train_higgs_test.csv",
"chars": 14612,
"preview": "1.000000000000000000e+00,8.692932128906250000e-01,-6.350818276405334473e-01,2.256902605295181274e-01,3.27470064163208007"
},
{
"path": "official/boosted_trees/train_higgs_test.py",
"chars": 6141,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/mnist/README.md",
"chars": 2972,
"preview": "# MNIST in TensorFlow\n\nThis directory builds a convolutional neural net to classify the [MNIST\ndataset](http://yann.lecu"
},
{
"path": "official/mnist/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/mnist/dataset.py",
"chars": 4116,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the "
},
{
"path": "official/mnist/mnist.py",
"chars": 9266,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the "
},
{
"path": "official/mnist/mnist_eager.py",
"chars": 7709,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/mnist/mnist_eager_test.py",
"chars": 2367,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/mnist/mnist_test.py",
"chars": 4583,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/mnist/mnist_tpu.py",
"chars": 6645,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/recommendation/README.md",
"chars": 4531,
"preview": "# Recommendation Model\n## Overview\nThis is an implementation of the Neural Collaborative Filtering (NCF) framework with "
},
{
"path": "official/recommendation/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/recommendation/constants.py",
"chars": 892,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/recommendation/data_download.py",
"chars": 13514,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/recommendation/dataset.py",
"chars": 8081,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/recommendation/dataset_test.py",
"chars": 3790,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/recommendation/ncf_main.py",
"chars": 12897,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/recommendation/neumf_model.py",
"chars": 5595,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/recommendation/unittest_data/test_eval_negative.csv",
"chars": 956,
"preview": "2784\t2659\t1705\t3316\t887\t815\t1783\t3483\t1085\t329\t1830\t1880\t2699\t3194\t3596\t2700\t3520\t2414\t757\t3610\t2651\t2187\t2274\t2949\t1753"
},
{
"path": "official/recommendation/unittest_data/test_eval_ratings.csv",
"chars": 14,
"preview": "0\t25\t1\n1\t66\t1\n"
},
{
"path": "official/recommendation/unittest_data/test_train_ratings.csv",
"chars": 1324,
"preview": "0\t21\t1\n0\t41\t1\n0\t8\t1\n0\t23\t1\n0\t43\t1\n0\t10\t1\n0\t45\t1\n0\t12\t1\n0\t47\t1\n0\t14\t1\n0\t33\t1\n0\t0\t1\n0\t35\t1\n0\t2\t1\n0\t37\t1\n0\t4\t1\n0\t39\t1\n0\t6\t1"
},
{
"path": "official/resnet/README.md",
"chars": 3124,
"preview": "# ResNet in TensorFlow\n\nDeep residual networks, or ResNets for short, provided the breakthrough idea of identity mapping"
},
{
"path": "official/resnet/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/resnet/cifar10_download_and_extract.py",
"chars": 2074,
"preview": "# Copyright 2015 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/resnet/cifar10_main.py",
"chars": 8567,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/resnet/cifar10_test.py",
"chars": 6374,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/resnet/imagenet_main.py",
"chars": 11284,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/resnet/imagenet_preprocessing.py",
"chars": 9592,
"preview": "# Copyright 2016 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/resnet/imagenet_test.py",
"chars": 12917,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/resnet/layer_test.py",
"chars": 7182,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/resnet/resnet_model.py",
"chars": 22782,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/resnet/resnet_run_loop.py",
"chars": 20432,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/README.md",
"chars": 14903,
"preview": "# Transformer Translation Model\nThis is an implementation of the Transformer translation model as described in the [Atte"
},
{
"path": "official/transformer/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/transformer/compute_bleu.py",
"chars": 4922,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/compute_bleu_test.py",
"chars": 2395,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/data_download.py",
"chars": 14804,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/model/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/transformer/model/attention_layer.py",
"chars": 5527,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/model/beam_search.py",
"chars": 21734,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/model/beam_search_test.py",
"chars": 3389,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/model/embedding_layer.py",
"chars": 3061,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/model/ffn_layer.py",
"chars": 3105,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/model/model_params.py",
"chars": 2127,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/model/model_utils.py",
"chars": 3716,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/model/model_utils_test.py",
"chars": 2525,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/model/transformer.py",
"chars": 16575,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/test_data/newstest2014.de",
"chars": 392050,
"preview": "Gutach: Noch mehr Sicherheit für Fußgänger\nSie stehen keine 100 Meter voneinander entfernt: Am Dienstag ist in Gutach di"
},
{
"path": "official/transformer/test_data/newstest2014.en",
"chars": 359706,
"preview": "Gutach: Increased safety for pedestrians\nThey are not even 100 metres apart: On Tuesday, the new B 33 pedestrian lights "
},
{
"path": "official/transformer/transformer_main.py",
"chars": 19459,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/translate.py",
"chars": 8955,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/utils/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/transformer/utils/dataset.py",
"chars": 9919,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/utils/metrics.py",
"chars": 16412,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the 'L"
},
{
"path": "official/transformer/utils/tokenizer.py",
"chars": 22490,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/transformer/utils/tokenizer_test.py",
"chars": 6314,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/utils/export/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/utils/export/export.py",
"chars": 1921,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/export/export_test.py",
"chars": 2628,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/flags/README.md",
"chars": 4313,
"preview": "# Adding Abseil (absl) flags quickstart\n## Defining a flag\nabsl flag definitions are similar to argparse, although they "
},
{
"path": "official/utils/flags/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/utils/flags/_base.py",
"chars": 5458,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/flags/_benchmark.py",
"chars": 3250,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/flags/_conventions.py",
"chars": 1419,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/flags/_misc.py",
"chars": 1715,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/flags/_performance.py",
"chars": 5432,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/flags/core.py",
"chars": 2862,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/flags/flags_test.py",
"chars": 3102,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/logs/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/utils/logs/hooks.py",
"chars": 4737,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/logs/hooks_helper.py",
"chars": 5451,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/logs/hooks_helper_test.py",
"chars": 2478,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/logs/hooks_test.py",
"chars": 5781,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/logs/logger.py",
"chars": 14119,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/logs/logger_test.py",
"chars": 11756,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/logs/metric_hook.py",
"chars": 4012,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/logs/metric_hook_test.py",
"chars": 8562,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/misc/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/utils/misc/model_helpers.py",
"chars": 1902,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/misc/model_helpers_test.py",
"chars": 2694,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/testing/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/utils/testing/integration.py",
"chars": 2125,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/testing/mock_lib.py",
"chars": 1276,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/testing/pylint.rcfile",
"chars": 4836,
"preview": "[MESSAGES CONTROL]\ndisable=R,W,\n bad-option-value\n\n[REPORTS]\n# Tells whether to display a full report or only the"
},
{
"path": "official/utils/testing/reference_data/reference_data_test/dense/results.json",
"chars": 66,
"preview": "[1, 1, 0.4701630473136902, 0.4701630473136902, 0.4701630473136902]"
},
{
"path": "official/utils/testing/reference_data/reference_data_test/dense/tf_version.json",
"chars": 51,
"preview": "[\"1.8.0-dev20180325\", \"v1.7.0-rc1-750-g6c1737e6c8\"]"
},
{
"path": "official/utils/testing/reference_data/reference_data_test/uniform_random/results.json",
"chars": 20,
"preview": "[0.9872556924819946]"
},
{
"path": "official/utils/testing/reference_data/reference_data_test/uniform_random/tf_version.json",
"chars": 51,
"preview": "[\"1.8.0-dev20180325\", \"v1.7.0-rc1-750-g6c1737e6c8\"]"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-1_width-8_channels-4/results.json",
"chars": 127,
"preview": "[32, 8, 8, 4, 0.08920872211456299, 0.8918969631195068, 4064.7060546875, 32, 4, 4, 8, 0.0, 0.10715862363576889, 2344.4775"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-1_width-8_channels-4/tf_version.json",
"chars": 48,
"preview": "[\"1.8.0-dev20180408\", \"v1.7.0-1345-gb874783ccd\"]"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-2_width-8_channels-4/results.json",
"chars": 142,
"preview": "[32, 8, 8, 4, 0.918815016746521, 0.1826801300048828, 4064.4677734375, 32, 4, 4, 8, -1.3153012990951538, 0.01124709472060"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-2_width-8_channels-4/tf_version.json",
"chars": 48,
"preview": "[\"1.8.0-dev20180408\", \"v1.7.0-1345-gb874783ccd\"]"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-1_width-8_channels-4/results.json",
"chars": 140,
"preview": "[32, 8, 8, 4, 0.1677999496459961, 0.7767924070358276, 4089.44189453125, 32, 8, 8, 4, 0.8615571856498718, 1.1359407901763"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-1_width-8_channels-4/tf_version.json",
"chars": 48,
"preview": "[\"1.8.0-dev20180408\", \"v1.7.0-1345-gb874783ccd\"]"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-2_width-8_channels-4/results.json",
"chars": 142,
"preview": "[32, 8, 8, 4, 0.8239736557006836, 0.3485994338989258, 4108.87548828125, 32, 8, 8, 4, 0.16798323392868042, -0.29753112792"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-2_width-8_channels-4/tf_version.json",
"chars": 48,
"preview": "[\"1.8.0-dev20180408\", \"v1.7.0-1345-gb874783ccd\"]"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-1_width-8_channels-4/results.json",
"chars": 127,
"preview": "[32, 8, 8, 4, 0.5349493026733398, 0.5126370191574097, 4070.01220703125, 32, 4, 4, 8, 0.0, 2.7680201530456543, 2341.23486"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-1_width-8_channels-4/tf_version.json",
"chars": 48,
"preview": "[\"1.8.0-dev20180408\", \"v1.7.0-1345-gb874783ccd\"]"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-2_width-8_channels-4/results.json",
"chars": 148,
"preview": "[32, 8, 8, 4, 0.7820245027542114, 0.8173515796661377, 4095.256591796875, 32, 4, 4, 8, 0.0679062008857727, 0.009305447340"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-2_width-8_channels-4/tf_version.json",
"chars": 48,
"preview": "[\"1.8.0-dev20180408\", \"v1.7.0-1345-gb874783ccd\"]"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_building_version-1_width-8_channels-4/results.json",
"chars": 143,
"preview": "[32, 8, 8, 4, 0.23128163814544678, 0.22117376327514648, 4100.51806640625, 32, 8, 8, 4, 1.1768392324447632, 0.27284657955"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_building_version-1_width-8_channels-4/tf_version.json",
"chars": 48,
"preview": "[\"1.8.0-dev20180408\", \"v1.7.0-1345-gb874783ccd\"]"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_building_version-2_width-8_channels-4/results.json",
"chars": 144,
"preview": "[32, 8, 8, 4, 0.7616699934005737, 0.5485763549804688, 4106.8720703125, 32, 8, 8, 4, -0.056346118450164795, 0.57926899194"
},
{
"path": "official/utils/testing/reference_data/resnet/batch-size-32_building_version-2_width-8_channels-4/tf_version.json",
"chars": 48,
"preview": "[\"1.8.0-dev20180408\", \"v1.7.0-1345-gb874783ccd\"]"
},
{
"path": "official/utils/testing/reference_data/resnet/batch_norm/results.json",
"chars": 147,
"preview": "[32, 16, 16, 3, 0.9722558259963989, 0.18413543701171875, 12374.20703125, 32, 16, 16, 3, 1.6126631498336792, -1.096894383"
},
{
"path": "official/utils/testing/reference_data/resnet/batch_norm/tf_version.json",
"chars": 48,
"preview": "[\"1.8.0-dev20180408\", \"v1.7.0-1345-gb874783ccd\"]"
},
{
"path": "official/utils/testing/reference_data.py",
"chars": 12997,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/testing/reference_data_test.py",
"chars": 4774,
"preview": "# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/utils/testing/scripts/presubmit.sh",
"chars": 2310,
"preview": "#!/bin/bash\n# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version"
},
{
"path": "official/wide_deep/README.md",
"chars": 5374,
"preview": "# Predicting Income with the Census Income Dataset\n## Overview\nThe [Census Income Data Set](https://archive.ics.uci.edu/"
},
{
"path": "official/wide_deep/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "official/wide_deep/data_download.py",
"chars": 2357,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/wide_deep/wide_deep.py",
"chars": 10056,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "official/wide_deep/wide_deep_test.csv",
"chars": 2631,
"preview": "39,State-gov,77516,Bachelors,13,Never-married,Adm-clerical,Not-in-family,,,2174,0,40,,<=50K\n50,Self-emp-not-inc,83311,Ba"
},
{
"path": "official/wide_deep/wide_deep_test.py",
"chars": 5279,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "projects/__init__.py",
"chars": 102,
"preview": "import projects.ai2018 \ntry:\n import projects.kaggle\n import projects.feed\nexcept Exception:\n pass\n"
},
{
"path": "projects/ai2018/__init__.py",
"chars": 63,
"preview": "import projects.ai2018.sentiment\nimport projects.ai2018.reader\n"
},
{
"path": "projects/ai2018/binary/__init__.py",
"chars": 41,
"preview": "import projects.ai2018.sentiment.prepare\n"
},
{
"path": "projects/ai2018/binary/algos/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "projects/ai2018/binary/algos/config.py",
"chars": 6404,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/binary/algos/loss.py",
"chars": 8281,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/binary/algos/model.py",
"chars": 25010,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/binary/algos/weights.py",
"chars": 2488,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/binary/dataset.py",
"chars": 2034,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "projects/ai2018/binary/evaluate.py",
"chars": 2390,
"preview": "#!/usr/bin/env python\n# ==============================================================================\n# \\file"
},
{
"path": "projects/ai2018/binary/prepare/config.py",
"chars": 750,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/binary/prepare/gen-records.py",
"chars": 7893,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/binary/prepare/run.sh",
"chars": 95,
"preview": "python ./gen-records.py \npython ./gen-records.py --input ./mount/data/ai2018/binary/train.txt\n"
},
{
"path": "projects/ai2018/binary/prepare/text2ids.py",
"chars": 1326,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/binary/read-records.py",
"chars": 6214,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/binary/torch-train.py",
"chars": 1560,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/binary/torch_algos/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "projects/ai2018/binary/torch_algos/loss.py",
"chars": 965,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/binary/torch_algos/model.py",
"chars": 11620,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/binary/train/train.sh",
"chars": 1948,
"preview": "base=./mount \n\nif [ $CELL ];\n then echo 'CELL:' $CELL \nelse\n CELL='lstm'\n echo 'use default CELL lstm'\nfi \ndir=$base/"
},
{
"path": "projects/ai2018/reader/README.md",
"chars": 328,
"preview": "# generate tfrecord \ngo to prepare, sh run.sh gen valid/test/train \n# dump infos.pkl \npython ./read-records.py --typ"
},
{
"path": "projects/ai2018/reader/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "projects/ai2018/reader/algos/README.md",
"chars": 403,
"preview": "./baseline.py simple rnn based with query + passage one rnn encoding \n./qcatt.py query passage separately encode and i"
},
{
"path": "projects/ai2018/reader/algos/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "projects/ai2018/reader/algos/baseline.py",
"chars": 5098,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/algos/config.py",
"chars": 3923,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/algos/loss.py",
"chars": 1088,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/algos/m_reader.py",
"chars": 39016,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/algos/model.py",
"chars": 671,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/algos/qcatt.py",
"chars": 3709,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/algos/rnet.py",
"chars": 9001,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/dataset.py",
"chars": 5122,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "projects/ai2018/reader/ensemble/ensemble-infer.py",
"chars": 1484,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/ensemble/ensemble-valid.py",
"chars": 3727,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/ensemble/ensemble.py",
"chars": 8525,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/ensemble/evaluate.py",
"chars": 1016,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/evaluate.py",
"chars": 6598,
"preview": "#!/usr/bin/env python\n# ==============================================================================\n# \\file"
},
{
"path": "projects/ai2018/reader/infer.py",
"chars": 2134,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/prepare/README.md",
"chars": 81,
"preview": "./gen-seg.py is for dureader corpus seg (~/data/dureader/raw/corpus/corpus.json)\n"
},
{
"path": "projects/ai2018/reader/prepare/gen-records.py",
"chars": 14172,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/prepare/gen-seg.py",
"chars": 1641,
"preview": "#!/usr/bin/env python\n#encoding=utf8\n# ==============================================================================\n# "
},
{
"path": "projects/ai2018/reader/prepare/merge-emb.py",
"chars": 7623,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/prepare/pre-seg.py",
"chars": 3655,
"preview": "#!/usr/bin/env python\n#encoding=utf8\n# ==============================================================================\n# "
},
{
"path": "projects/ai2018/reader/prepare/run-char-ft.sh",
"chars": 213,
"preview": "dir=./mount/temp/ai2018/reader/tfrecords/char.ft \nemb=./mount/data/my-embedding.v1/fastText-dureader-char/text.vec\nemb_d"
},
{
"path": "projects/ai2018/reader/prepare/run-char-glove.sh",
"chars": 219,
"preview": "dir=./mount/temp/ai2018/reader/tfrecords/char.glove\nemb=./mount/data/my-embedding.v1/GloVe-dureader-char/vectors.fix.txt"
},
{
"path": "projects/ai2018/reader/prepare/run-noemb.sh",
"chars": 329,
"preview": "dir=$1 \nvocab=$dir/vocab.txt \n\npython ./gen-records.py --input $dir/test.json --use_char=1 --vocab_=$vocab\npython ./gen"
},
{
"path": "projects/ai2018/reader/prepare/run-word-jieba-ft.sh",
"chars": 374,
"preview": "dir=./mount/temp/ai2018/reader/tfrecords/word.jieba.ft\nemb=./mount/data/my-embedding.v1/fastText-dureader-basic/text.vec"
},
{
"path": "projects/ai2018/reader/prepare/run-word-jieba-glove.sh",
"chars": 381,
"preview": "dir=./mount/temp/ai2018/reader/tfrecords/word.jieba.glove\nemb=./mount/data/my-embedding.v1/GloVe-dureader-basic/vectors."
},
{
"path": "projects/ai2018/reader/prepare/to-simplify-dureader.py",
"chars": 1275,
"preview": "#!/usr/bin/env python\n#encoding=utf8\n# ==============================================================================\n# "
},
{
"path": "projects/ai2018/reader/prepare/to-simplify.py",
"chars": 994,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/prepare-unkvocab.sh",
"chars": 78,
"preview": "cd ./prepare/ \nsh run-unkvocab.sh \ncd ..\npython ./read-records.py --type=dump\n"
},
{
"path": "projects/ai2018/reader/prepare.sh",
"chars": 69,
"preview": "cd ./prepare/ \nsh run.sh \ncd ..\npython ./read-records.py --type=dump\n"
},
{
"path": "projects/ai2018/reader/prepare.v1/fix-vocab.py",
"chars": 894,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/prepare.v1/gen-char-vocab.py",
"chars": 642,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/prepare.v1/gen-content.py",
"chars": 642,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
},
{
"path": "projects/ai2018/reader/prepare.v1/gen-records.py",
"chars": 10015,
"preview": "#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ======================================================================="
}
]
// ... and 2580 more files (download for full content)
About this extraction
This page contains the full source code of the chenghuige/wenzheng GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 2780 files (23.8 MB), approximately 6.4M tokens, and a symbol index with 3665 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.